From 4d6098042e0f4741b15aa349d7646df897e784d9 Mon Sep 17 00:00:00 2001 From: David Hendriks <davidhendriks93@gmail.com> Date: Sat, 25 Jan 2020 10:55:16 +0000 Subject: [PATCH] working on grid generation code. also black-formatted the whole code again --- .../utils/custom_logging_functions.py | 8 +- binarycpython/utils/functions.py | 159 ++++++++-------- binarycpython/utils/grid.py | 173 +++++++++++++----- binarycpython/utils/grid_options_defaults.py | 170 +++++------------ .../utils/probability_distributions.py | 37 ++-- binarycpython/utils/useful_functions.py | 0 examples/examples_custom_logging.py | 42 +++-- tests/function_tests.py | 14 +- tests/population/grid_tests.py | 48 ++--- tests/population/grid_tests_cmdline.py | 2 +- 10 files changed, 339 insertions(+), 314 deletions(-) delete mode 100644 binarycpython/utils/useful_functions.py diff --git a/binarycpython/utils/custom_logging_functions.py b/binarycpython/utils/custom_logging_functions.py index 06910af44..a9c0ffff6 100644 --- a/binarycpython/utils/custom_logging_functions.py +++ b/binarycpython/utils/custom_logging_functions.py @@ -266,7 +266,7 @@ def compile_shared_lib(code, sourcefile_name, outfile_name, verbose=False): print("Output of compilation command:\n{}".format(res)) -def temp_custom_logging_dir(): +def temp_dir(): """ Function to return the path the custom logging library shared object and script will be written to. @@ -293,14 +293,12 @@ def create_and_load_logging_function(custom_logging_code): # library_name = os.path.join( - temp_custom_logging_dir(), "libcustom_logging_{}.so".format(uuid.uuid4().hex) + temp_dir(), "libcustom_logging_{}.so".format(uuid.uuid4().hex) ) compile_shared_lib( custom_logging_code, - sourcefile_name=os.path.join(temp_custom_logging_dir(), "custom_logging.c"), - # outfile_name=os.path.join(temp_custom_logging_dir(), "libcustom_logging.so"), - # outfile_name=os.path.join(temp_custom_logging_dir(), "libcustom_logging_{}.so".format(random.randint(1, 100))), + sourcefile_name=os.path.join(temp_dir(), "custom_logging.c"), outfile_name=library_name, # verbose=True ) diff --git a/binarycpython/utils/functions.py b/binarycpython/utils/functions.py index dc9ad41c5..50f467ed1 100644 --- a/binarycpython/utils/functions.py +++ b/binarycpython/utils/functions.py @@ -10,90 +10,91 @@ from binarycpython.utils.custom_logging_functions import ( create_and_load_logging_function, ) + def parse_binary_c_version_info(version_info_string): version_info_dict = {} for el in version_info_string.splitlines(): el = el.strip() - if el =='': + if el == "": continue - if ' is ' in el: + if " is " in el: split = el.split(" is ") version_info_dict[split[0].strip()] = split[1].strip() else: - if el.startswith('Binary_c/nucsyn'): - version_info_dict['intro'] = el - elif el.startswith('Email'): - emails = el.split('Email ')[1].split(',') + if el.startswith("Binary_c/nucsyn"): + version_info_dict["intro"] = el + elif el.startswith("Email"): + emails = el.split("Email ")[1].split(",") cleaned_emails = [email.strip() for email in emails] - version_info_dict['emails'] = cleaned_emails - elif el.startswith('DTlimit'): - split = el.split(' : ') - version_info_dict[split[0]] = ': '.join(split[1:]) - elif el.startswith('Version'): + version_info_dict["emails"] = cleaned_emails + elif el.startswith("DTlimit"): + split = el.split(" : ") + version_info_dict[split[0]] = ": ".join(split[1:]) + elif el.startswith("Version"): split = el.split("Version ") version_number = split[1] - version_info_dict['version_number'] = version_number + version_info_dict["version_number"] = version_number elif el.startswith("git URL"): split = el.split("git URL ") git_url = split[1] - version_info_dict['git_url'] = git_url + version_info_dict["git_url"] = git_url elif el.startswith("Build: "): - split = el.split('Build: ') + split = el.split("Build: ") build = split[1] - version_info_dict['build'] = build + version_info_dict["build"] = build elif el.startswith("Compiled for "): - split = el.split('Compiled for ') + split = el.split("Compiled for ") compiled_for = split[1] - version_info_dict['compiled_for'] = compiled_for + version_info_dict["compiled_for"] = compiled_for elif el.startswith("Stack limit "): - split = el.split('Stack limit ') + split = el.split("Stack limit ") stack_limit = split[1] - version_info_dict['stack_limit'] = stack_limit - elif el.startswith('SVN URL '): - split = el.split('SVN URL ') + version_info_dict["stack_limit"] = stack_limit + elif el.startswith("SVN URL "): + split = el.split("SVN URL ") svn_url = split[1] - version_info_dict['svn_url'] = svn_url - elif el.startswith('git branch '): - split = el.split('git branch ') + version_info_dict["svn_url"] = svn_url + elif el.startswith("git branch "): + split = el.split("git branch ") git_branch = split[1] - version_info_dict['git_branch'] = git_branch - elif el.startswith('_SC_CLK_TCK'): - split = el.split(' = ') + version_info_dict["git_branch"] = git_branch + elif el.startswith("_SC_CLK_TCK"): + split = el.split(" = ") _SC_CLK_TCK = split[1] - version_info_dict['_SC_CLK_TCK'] = _SC_CLK_TCK - elif el.startswith('Random number mean '): - split = el.split('Random number mean ') + version_info_dict["_SC_CLK_TCK"] = _SC_CLK_TCK + elif el.startswith("Random number mean "): + split = el.split("Random number mean ") random_number_mean = split[1] - version_info_dict['Random number mean'] = random_number_mean - elif el.startswith('SVN revision '): - split = el.split('SVN revision ') + version_info_dict["Random number mean"] = random_number_mean + elif el.startswith("SVN revision "): + split = el.split("SVN revision ") svn_revision = split[1] - version_info_dict['svn_revision'] = svn_revision - elif el.startswith('Size of :'): - split = el.split('Size of :') + version_info_dict["svn_revision"] = svn_revision + elif el.startswith("Size of :"): + split = el.split("Size of :") data_type_sizes = split[1] - version_info_dict['data_type_sizes'] = data_type_sizes - elif el.startswith('git revision '): - split = el.split('git revision ') + version_info_dict["data_type_sizes"] = data_type_sizes + elif el.startswith("git revision "): + split = el.split("git revision ") git_revision = split[1] - version_info_dict['git_revision'] = git_revision - elif el.startswith('BINARY_C_PRE_VERSION '): - split = el.split('BINARY_C_PRE_VERSION ') + version_info_dict["git_revision"] = git_revision + elif el.startswith("BINARY_C_PRE_VERSION "): + split = el.split("BINARY_C_PRE_VERSION ") binary_c_pre_version = split[1] - version_info_dict['binary_c_pre_version'] = binary_c_pre_version - elif el.startswith('Comenv accretion:'): - split = el.split('Comenv accretion:') + version_info_dict["binary_c_pre_version"] = binary_c_pre_version + elif el.startswith("Comenv accretion:"): + split = el.split("Comenv accretion:") comenv_accretion = split[1] - version_info_dict['comenv_accretion'] = comenv_accretion - elif el.startswith('Compiled in parameters:'): - split = el.split('Compiled in parameters:') + version_info_dict["comenv_accretion"] = comenv_accretion + elif el.startswith("Compiled in parameters:"): + split = el.split("Compiled in parameters:") compiled_in_parameters = split[1] - version_info_dict['compiled_in_parameters'] = compiled_in_parameters - elif el.startswith('__short__ is'): - split = el.split('__short__ is') + version_info_dict["compiled_in_parameters"] = compiled_in_parameters + elif el.startswith("__short__ is"): + split = el.split("__short__ is") short_type = split[1] - version_info_dict['short_type'] = short_type + version_info_dict["short_type"] = short_type else: print("Still found unmatched items!:\n{}".format(repr(el))) @@ -111,33 +112,37 @@ def create_hdf5(data_dir): content_data_dir = os.listdir(data_dir) # Settings - settings_file = [file for file in content_data_dir if file.endswith('_settings.json')][0] - with open(settings_file, 'r') as f: + settings_file = [ + file for file in content_data_dir if file.endswith("_settings.json") + ][0] + with open(settings_file, "r") as f: settings_json = json.load(f) # create basename for hdf5 - base_name = settings_file.replace('_settings.json', '') + base_name = settings_file.replace("_settings.json", "") # Get data files - data_files = [el for el in content_data_dir if el.endswith('.dat')] + data_files = [el for el in content_data_dir if el.endswith(".dat")] # Create the file - hdf5_filename = os.path.join(data_dir,'{base_name}.hdf5'.format(base_name=base_name)) - f = h5py.File(hdf5_filename, 'w') + hdf5_filename = os.path.join( + data_dir, "{base_name}.hdf5".format(base_name=base_name) + ) + f = h5py.File(hdf5_filename, "w") # Create settings group - settings_grp = f.create_group('settings') + settings_grp = f.create_group("settings") # Write version_string to settings_group - settings_grp.create_dataset('used_settings', data=json.dumps(settings_json)) + settings_grp.create_dataset("used_settings", data=json.dumps(settings_json)) # Create the data group - data_grp = f.create_group('data') + data_grp = f.create_group("data") - print('Creating {}'.format(hdf5_filename)) + print("Creating {}".format(hdf5_filename)) # Write the data to the file: - # Make sure: + # Make sure: for data_file in data_files: # filename stuff filename = data_file @@ -145,8 +150,8 @@ def create_hdf5(data_dir): base_name = os.path.splitext(os.path.basename(filename))[0] # Get header info - header_name = '{base_name}_header'.format(base_name=base_name) - data_headers = np.genfromtxt(full_path, dtype='str', max_rows=1) + header_name = "{base_name}_header".format(base_name=base_name) + data_headers = np.genfromtxt(full_path, dtype="str", max_rows=1) data_headers = np.char.encode(data_headers) data_grp.create_dataset(header_name, data=data_headers) @@ -156,6 +161,7 @@ def create_hdf5(data_dir): f.close() + def get_help_super(print_help=False, return_dict=True, fail_silently=True): """ Function that first runs get_help_all, and then per argument also run the help function to get as much information as possible. @@ -175,30 +181,31 @@ def get_help_super(print_help=False, return_dict=True, fail_silently=True): # Get detailed help info detailed_help = get_help( - parameter_name, print_help=False, return_dict=True, fail_silently=fail_silently + parameter_name, + print_help=False, + return_dict=True, + fail_silently=fail_silently, ) if detailed_help: # check whether the descriptions of help_all and detailed help are the same if not fail_silently: - if ( - not parameter["description"] - == detailed_help["description"] - ): + if not parameter["description"] == detailed_help["description"]: print(json.dumps(parameter, indent=4)) ## put values into help all super dict # input type - parameter["parameter_value_input_type"] = detailed_help["parameter_value_input_type"] + parameter["parameter_value_input_type"] = detailed_help[ + "parameter_value_input_type" + ] # default parameter["default"] = detailed_help["default"] # macros - if 'macros' in detailed_help.keys(): + if "macros" in detailed_help.keys(): parameter["macros"] = detailed_help["macros"] - if print_help: # TODO: make a pretty print print(json.dumps(help_all_super_dict, indent=4)) @@ -326,6 +333,7 @@ def create_arg_string(arg_dict): arg_string = arg_string.strip() return arg_string + def get_defaults(filter_values=False): """ Function that calls the binaryc get args function and cast it into a dictionary @@ -453,9 +461,6 @@ def get_help(param_name, print_help=True, return_dict=False, fail_silently=False return None -# get_help("RLOF_method") - - def run_system(**kwargs): """ Wrapper to run a system with settings @@ -649,4 +654,4 @@ def load_logfile(logfile): rel_r2_list.append(split_line[8]) event_list.append(" ".join(split_line[9:])) - print(event_list) \ No newline at end of file + print(event_list) diff --git a/binarycpython/utils/grid.py b/binarycpython/utils/grid.py index bc35f34e4..977dd7be0 100644 --- a/binarycpython/utils/grid.py +++ b/binarycpython/utils/grid.py @@ -11,7 +11,7 @@ from binarycpython.utils.custom_logging_functions import ( autogen_C_logging_code, binary_c_log_code, create_and_load_logging_function, - temp_custom_logging_dir, + temp_dir, ) from binarycpython.utils.functions import get_defaults, parse_binary_c_version_info @@ -35,6 +35,7 @@ from binarycpython.utils.functions import get_defaults, parse_binary_c_version_i # DONE: add functionality to return the evcode_args_list # TODO: add grid generation script + class Population(object): def __init__(self): """ @@ -69,6 +70,8 @@ class Population(object): # And because binary_c uses internal defaults, its not necessary to explicitly pass them. # I do however suggest everyone to export the binary_c defaults to a file, so that you know exactly which values were the defaults. + # TODO: maybe make a set_bse option. + def set(self, **kwargs): """ Function to set the values of the population. This is the preferred method to set values of functions, as it @@ -100,7 +103,6 @@ class Population(object): ) self.custom_options[key] = kwargs[key] - def parse_cmdline(self): """ Function to handle settings values via the command line: @@ -108,24 +110,31 @@ class Population(object): """ import argparse + parser = argparse.ArgumentParser() - parser.add_argument('--cmdline', help='Setting values via the commandline. Input like --cmdline "metallicity=0.02"') + parser.add_argument( + "--cmdline", + help='Setting values via the commandline. Input like --cmdline "metallicity=0.02"', + ) args = parser.parse_args() - # How its set up now is that as input you need to give --cmdline "metallicity=0.002" # Its checked if this exists and handled accordingly. if args.cmdline: # Grab the input and split them up, while accepting only non-empty entries cmdline_args = args.cmdline - split_args = [cmdline_arg for cmdline_arg in cmdline_args.split(' ') if not cmdline_arg==''] + split_args = [ + cmdline_arg + for cmdline_arg in cmdline_args.split(" ") + if not cmdline_arg == "" + ] # Make dict and fill it cmdline_dict = {} for cmdline_arg in split_args: - split = cmdline_arg.split('=') + split = cmdline_arg.split("=") parameter = split[0] - value=split[1] + value = split[1] # Add to dict cmdline_dict[parameter] = value @@ -141,7 +150,7 @@ class Population(object): if not parameter_dict: parameter_dict = self.bse_options - argline = "binary_c " + argline = "binary_c " # TODO: check if that sort actually works for param_name in sorted(parameter_dict): argline += "{} {} ".format(param_name, parameter_dict[param_name]) @@ -155,7 +164,19 @@ class Population(object): pass - def add_grid_variable(self, name, longname, valuerange, resolution, spacingfunc, precode, probdist, dphasevol, parameter_name, condition=None): + def add_grid_variable( + self, + name, + longname, + valuerange, + resolution, + spacingfunc, + precode, + probdist, + dphasevol, + parameter_name, + condition=None, + ): """ Function to add grid variables to the grid_options. @@ -256,13 +277,13 @@ class Population(object): from binarycpython.utils.functions import get_help_all - # + # population_settings = self.return_population_settings() binary_c_defaults = self.return_binary_c_defaults() binary_c_version_info = self.return_binary_c_version_info(parsed=True) binary_c_help_all_info = get_help_all(print_help=False, return_dict=True) - # + # all_info = {} all_info["population_settings"] = population_settings all_info["binary_c_defaults"] = binary_c_defaults @@ -284,13 +305,15 @@ class Population(object): all_info = self.return_all_info() if use_datadir: - base_name = os.path.splitext(self.custom_options['base_filename'])[0] - settings_name = base_name + '_settings.json' + base_name = os.path.splitext(self.custom_options["base_filename"])[0] + settings_name = base_name + "_settings.json" # Check directory, make if necessary - os.makedirs(self.custom_options['data_dir'], exist_ok=True) + os.makedirs(self.custom_options["data_dir"], exist_ok=True) - settings_fullname = os.path.join(self.custom_options['data_dir'], settings_name) + settings_fullname = os.path.join( + self.custom_options["data_dir"], settings_name + ) # if not outfile.endswith('json'): with open(settings_fullname, "w") as f: @@ -461,8 +484,7 @@ class Population(object): ################################################### # Unordered functions - ################################################### - + ################################################### def generate_grid_code(self): """ @@ -474,64 +496,114 @@ class Population(object): # TODO: add part to handle separation if orbital_period is added """ - # Some local values code_string = "" depth = 0 - indent = ' ' + indent = " " # Import packages - code_string += "from binarycpython.utils.probability_distributions import *\n" + code_string += "import math\n" code_string += "import numpy as np\n" + code_string += "from binarycpython.utils.probability_distributions import *\n" + code_string += "from binarycpython.utils.spacing_functions import *\n" code_string += "\n\n" # Make the function code_string += "def grid_code(self):\n" - # Increase depth + # Increase depth depth += 1 + # Write some info in the function + code_string += ( + indent * depth + + "# Grid code generated on {}. This function generates the systems that will be evolved with binary_c\n\n".format() + ) + # Set some values in the generated code: - code_string += indent*depth + "starcount = 0\n" - code_string += indent*depth + "probabilities = {}\n" - code_string += indent*depth + "parameter_dict = {}\n" - + code_string += indent * depth + "starcount = 0\n" + code_string += indent * depth + "probabilities = {}\n" + code_string += indent * depth + "parameter_dict = {}\n" + # Prepare the probability - for el in sorted(self.grid_options["grid_variables"].items(), key=lambda x: x[1]['grid_variable_number']): + for el in sorted( + self.grid_options["grid_variables"].items(), + key=lambda x: x[1]["grid_variable_number"], + ): grid_variable = el[1] - code_string += indent*depth + 'probabilities["{}"] = 0\n'.format(grid_variable['parameter_name']) + code_string += indent * depth + 'probabilities["{}"] = 0\n'.format( + grid_variable["parameter_name"] + ) # Generate code print("Generating grid code") - for el in sorted(self.grid_options["grid_variables"].items(), key=lambda x: x[1]['grid_variable_number']): + for el in sorted( + self.grid_options["grid_variables"].items(), + key=lambda x: x[1]["grid_variable_number"], + ): print("Constructing/adding: {}".format(el[0])) grid_variable = el[1] # If the grid variable has a condition, write the check and the action - if grid_variable['condition']: + if grid_variable["condition"]: # Add condition check - code_string += indent * depth + 'if not {}:'.format(grid_variable['condition']) + '\n' + code_string += ( + indent * depth + + "if not {}:".format(grid_variable["condition"]) + + "\n" + ) # Add condition failed action: #TODO: add correct exception error - code_string += indent * (depth + 1) + 'print("Condition not met!")'.format(grid_variable['condition']) + '\n' - code_string += indent * (depth + 1) + 'raise ValueError'.format(grid_variable['condition']) + '\n' + code_string += ( + indent * (depth + 1) + + 'print("Condition not met!")'.format(grid_variable["condition"]) + + "\n" + ) + code_string += ( + indent * (depth + 1) + + "raise ValueError".format(grid_variable["condition"]) + + "\n" + ) # Adding for loop structure - code_string += indent * depth + 'for {} in {}:'.format(grid_variable['name'], grid_variable['spacingfunc']) + '\n' + code_string += ( + indent * depth + + "for {} in {}:".format( + grid_variable["name"], grid_variable["spacingfunc"] + ) + + "\n" + ) # Add pre-code - code_string += indent * (depth + 1) + '{}'.format(grid_variable['precode'].replace('\n', '\n'+indent * (depth + 1))) + '\n' + code_string += ( + indent * (depth + 1) + + "{}".format( + grid_variable["precode"].replace("\n", "\n" + indent * (depth + 1)) + ) + + "\n" + ) # Calculate probability - code_string += indent * (depth + 1) + 'probabilities["{}"] = {}'.format(grid_variable['parameter_name'], grid_variable['probdist']) + '\n' + code_string += ( + indent * (depth + 1) + + 'probabilities["{}"] = {}'.format( + grid_variable["parameter_name"], grid_variable["probdist"] + ) + + "\n" + ) # some testing line. # code_string += indent * (depth + 1) + 'print({})'.format(grid_variable['name']) + '\n' # Add value to dict - code_string += indent * (depth + 1) + 'parameter_dict["{}"] = {}'.format(grid_variable['parameter_name'], grid_variable['parameter_name']) + '\n' - + code_string += ( + indent * (depth + 1) + + 'parameter_dict["{}"] = {}'.format( + grid_variable["parameter_name"], grid_variable["parameter_name"] + ) + + "\n" + ) # Add some space code_string += "\n" @@ -541,19 +613,21 @@ class Population(object): # placeholder for calls to threading # starcount - code_string += indent * (depth) + 'starcount += 1\n' - code_string += indent * (depth) + 'print(probabilities)\n' + code_string += indent * (depth) + "starcount += 1\n" + code_string += indent * (depth) + "print(probabilities)\n" code_string += indent * (depth) + 'print("starcount: ", starcount)\n' - code_string += indent * (depth) + 'yield(parameter_dict)\n' + code_string += indent * (depth) + "yield(parameter_dict)\n" # Save the gridcode to the grid_options - self.grid_options['code_string'] = code_string + self.grid_options["code_string"] = code_string # Write to file - gridcode_filename = os.path.join(temp_custom_logging_dir(), 'example_grid.py') - self.grid_options['gridcode_filename'] = gridcode_filename + gridcode_filename = os.path.join( + self.grid_options["tmp_dir"], "example_grid.py" + ) + self.grid_options["gridcode_filename"] = gridcode_filename - with open(gridcode_filename, 'w') as f: + with open(gridcode_filename, "w") as f: f.write(code_string) def load_grid_function(self): @@ -561,15 +635,20 @@ class Population(object): Test function to run grid stuff. mostly to test the import """ - # Code to load the + # Code to load the import importlib.util - spec = importlib.util.spec_from_file_location("binary_c_python_grid", os.path.join(temp_custom_logging_dir(), 'example_grid.py')) + + spec = importlib.util.spec_from_file_location( + "binary_c_python_grid", + os.path.join(self.grid_options["tmp_dir"], "example_grid.py"), + ) grid_file = importlib.util.module_from_spec(spec) spec.loader.exec_module(grid_file) generator = grid_file.grid_code(self) - print(next(generator)) print(next(generator)) print(next(generator)) + + ################################################################################################ diff --git a/binarycpython/utils/grid_options_defaults.py b/binarycpython/utils/grid_options_defaults.py index 8ff1a0c32..223d0bd17 100644 --- a/binarycpython/utils/grid_options_defaults.py +++ b/binarycpython/utils/grid_options_defaults.py @@ -1,3 +1,8 @@ +import os +import sys + +from binarycpython.utils.custom_logging_functions import temp_dir + grid_options_defaults_dict = { # general "amt_cores": 1, # total amount of cores used to evolve the population @@ -19,6 +24,9 @@ grid_options_defaults_dict = { "gridcode_filename": None, # binary "binary": 0, + # Locations: + "binary_c_dir": os.environ["BINARYC_DIR"], + "tmp_dir": temp_dir() ## # return_array_refs=>1, # quicker data parsing mode # sort_args=>1, @@ -27,48 +35,25 @@ grid_options_defaults_dict = { # timeout=>15, # seconds until timeout # log_filename=>"/scratch/davidh/results_simulations/tmp/log.txt", # # current_log_filename=>"/scratch/davidh/results_simulations/tmp/grid_errors.log", - - - - - -# my $self = shift; - ############################################################ - # Set default grid properties (in %self->{_grid_options}} + # Set default grid properties (in %self->{_grid_options}} # and %{$self->{_bse_options}}) # This is the first thing that should be called by the user! ############################################################ - # # set signal handlers for timeout # $self->set_class_signal_handlers(); - # # set operating system # my $os = rob_misc::operating_system(); - - # # set tmp dir - # my $tmp = $self->tmpdir($os); - # %{$self->{_grid_options}}=( - # # save operating system # operating_system=>$os, - # # process name # process_name => 'binary_grid'.$VERSION, - - # # temp directory: - # tmp=>$tmp, - # grid_defaults_set=>1, # so we know the grid_defaults function has been called - # # grid suspend files: assume binary_c by default # suspend_files=>[$tmp.'/force_binary_c_suspend', # './force_binary_c_suspend'], - # snapshot_file=>$tmp.'/binary_c-snapshot', - - # ######################################## # # infomration about the running grid script # ######################################## @@ -78,119 +63,92 @@ grid_options_defaults_dict = { # perl_executable=>$^X, # the perl executable # command_line=>join(' ',$0,@ARGV), # full command line # process_ID=>$$, # process ID of the main perl script - # ######################################## # # GRID # ######################################## - # # if undef, generate gridcode, otherwise load the gridcode # # from this file. useful for debugging # gridcode_from_file => undef, - # # assume binary_grid perl backend by default - # backend => - # $self->{_grid_options}->{backend} // + # backend => + # $self->{_grid_options}->{backend} // # $binary_grid2::backend // # 'binary_grid::Perl', - # # custom C function for output : this automatically # # binds if a function is available. # C_logging_code => undef, # C_auto_logging => undef, - # custom_output_C_function_pointer => binary_c_function_bind(), - + # custom_output_C_function_pointer => binary_c_function_bind(), # # control flow # rungrid=>1, # usually run the grid, but can be 0 # # to skip it (e.g. for condor/slurm runs) # merge_datafiles=>'', # merge_datafiles_filelist=>'', - # # parameter space options # weight=>1.0, # weighting for the probability - # repeat=>1, # number of times to repeat each system (probability is adjusted to be 1/repeat) - - # binary=>0, # set to 0 for single stars, 1 for binaries - + # binary=>0, # set to 0 for single stars, 1 for binaries # # if use_full_resolution is 1, then run a dummy grid to # # calculate the resolution. this could be slow... # use_full_resolution => 1, - # # the probability in any distribution must be within # # this tolerance of 1.0, ignored if undef (if you want # # to run *part* of the parameter space then this *must* be undef) # probability_tolerance=>undef, - # # how to deal with a failure of the probability tolerance: # # 0 = nothing # # 1 = warning # # 2 = stop # probability_tolerance_failmode=>1, - # # add up and log system error count and probability # add_up_system_errors=>1, # log_system_errors=>1, - # # codes, paths, executables etc. - # # assume binary_c by default, and set its defaults # code=>'binary_c', # arg_prefix=>'--', - # prog=>'binary_c', # executable + # prog=>'binary_c', # executable # nice=>'nice -n +0', # nice command # ionice=>'', - # # compress output? # binary_c_compression=>0, - # # get output as array of pre-split array refs # return_array_refs=>1, - # # environment # shell_environment=>undef, # libpath=>undef, # for backwards compatibility - # # where is binary_c? need this to get the values of some counters - # rootpath=>$self->okdir($ENV{BINARY_C_ROOTPATH}) // + # rootpath=>$self->okdir($ENV{BINARY_C_ROOTPATH}) // # $self->okdir($ENV{HOME}.'/progs/stars/binary_c') // # '.' , # last option is a fallback ... will fail if it doesn't exist - - # srcpath=>$self->okdir($ENV{BINARY_C_SRCPATH}) // - # $self->okdir($ENV{BINARY_C_ROOTPATH}.'/src') // + # srcpath=>$self->okdir($ENV{BINARY_C_SRCPATH}) // + # $self->okdir($ENV{BINARY_C_ROOTPATH}.'/src') // # $self->okdir($ENV{HOME}.'/progs/stars/binary_c/src') // # './src' , # last option is fallback... will fail if it doesn't exist - # # stack size per thread in megabytes - # threads_stack_size=>50, - - # # thread sleep time between starting the evolution code and starting + # threads_stack_size=>50, + # # thread sleep time between starting the evolution code and starting # # the grid # thread_presleep=>0, - # # threads # # Max time a thread can sit looping (with calls to tbse_line) # # before a warning is issued : NB this does not catch real freezes, # # just infinite loops (which still output) # thread_max_freeze_time_before_warning=>10, - # # run all models by default: modulo=1, offset=0 # modulo=>1, - # offset=>0, - + # offset=>0, # # max number of stars on the queue # maxq_per_thread => 100, - # # data dump file : undef by default (do nothing) # results_hash_dumpfile => '', - # # compress files with bzip2 by default # compress_results_hash => 1, - # ######################################## # # Condor stuff - # ######################################## + # ######################################## # condor=>0, # 1 to use condor, 0 otherwise - # condor_command=>'',# condor command e.g. "run_flexigrid", + # condor_command=>'',# condor command e.g. "run_flexigrid", # # "join_datafiles" # condor_dir=>'', # working directory containing e.g. # # scripts, output, logs (e.g. should be NFS available to all) @@ -205,12 +163,11 @@ grid_options_defaults_dict = { # condor_universe=>'vanilla', # usually vanilla universe # condor_snapshot_on_kill=>0, # if 1 snapshot on SIGKILL before exit # condor_load_from_snapshot=>0, # if 1 check for snapshot .sv file and load it if found - # condor_checkpoint_interval=>0, # checkpoint interval (seconds) + # condor_checkpoint_interval=>0, # checkpoint interval (seconds) # condor_checkpoint_stamp_times=>0, # if 1 then files are given timestamped names (warning: lots of files!), otherwise just store the lates # condor_streams=>0, # stream stderr/stdout by default (warning: might cause heavy network load) # condor_save_joined_file=>0, # if 1 then results/joined contains the results (useful for debugging, otherwise a lot of work) # condor_requirements=>'', # used? - # # resubmit options : if the status of a condor script is # # either 'finished','submitted','running' or 'crashed', # # decide whether to resubmit it. @@ -220,13 +177,11 @@ grid_options_defaults_dict = { # condor_resubmit_submitted=>0, # condor_resubmit_running=>0, # condor_resubmit_crashed=>0, - - # ######################################## # # Slurm stuff - # ######################################## + # ######################################## # slurm=>0, # don't use slurm by default - # slurm_command=>'',# slurm command e.g. "run_flexigrid", + # slurm_command=>'',# slurm command e.g. "run_flexigrid", # # "join_datafiles" # slurm_dir=>'', # working directory containing e.g. # # scripts, output, logs (e.g. should be NFS available to all) @@ -237,7 +192,6 @@ grid_options_defaults_dict = { # slurm_postpone_join=>0, # if 1, data is not joined, e.g. if you # # want to do it off the slurm grid (e.g. with more RAM) # slurm_postpone_sbatch=>0, # if 1, don't submit, just make the script - # # (defaults to $ENV{PWD} if undef) # slurm_memory=>512, # in MB, the memory use of the job # slurm_warn_max_memory=>1024, # in MB : warn if mem req. > this @@ -249,43 +203,34 @@ grid_options_defaults_dict = { # # which means it allocates all the CPUs in a node to the job # slurm_control_CPUs=>0, # if so, leave this many for Perl control (0) # slurm_array=>undef,# override for --array, useful for rerunning jobs - # ######################################## - # # CPU + # # CPU # ######################################## # cpu_cap=>0, # if 1, limits to one CPU # cpu_affinity => 0, # do not bind to a CPU by default - # ######################################## # # Code, Timeouts, Signals # ######################################## # binary_grid_code_filtering=>1, # you want this, it's (MUCH!) faster # pre_filter_file=>undef, # dump pre filtered code to this file # post_filter_file=>undef, # dump post filtered code to this file - # timeout=>30, # timeout in seconds # timeout_vb=>0, # no timeout logging # tvb=>0, # no thread logging # nfs_sleep=>1, # time to wait for NFS to catch up with file accesses - - # # flexigrid checks the timeouts every + # # flexigrid checks the timeouts every # # flexigrid_timeout_check_interval seconds - # flexigrid_timeout_check_interval=>0.01, - + # flexigrid_timeout_check_interval=>0.01, # # this is set to 1 when the grid is finished # flexigrid_finished=>0, - # # allow signals by default # 'no signals'=>0, - # # but perhaps disable specific signals? # 'disable signal'=>{INT=>0,ALRM=>0,CONT=>0,USR1=>0,STOP=>0}, - # # dummy variables # single_star_period=>1e50, # orbital period of a single star - - # #### timers : set timers to 0 (or empty list) to ignore, - # #### NB these must be given context (e.g. main::xyz) + # #### timers : set timers to 0 (or empty list) to ignore, + # #### NB these must be given context (e.g. main::xyz) # #### for functions not in binary_grid # timers=>0, # timer_subroutines=>[ @@ -297,20 +242,16 @@ grid_options_defaults_dict = { # 'run_flexigrid_thread', # 'thread_vb' # ], - # ######################################## # # INPUT/OUTPUT # ######################################## # blocking=>undef, # not yet set - # # prepend command with stdbuf to stop buffering (if available) # stdbuf_command=>`stdbuf --version`=~/stdbuf \(GNU/ ? ' stdbuf -i0 -o0 -e0 ' : undef, - # vb=>("@ARGV"=~/\Wvb=(\d+)\W/)[0] // 0, # set to 1 (or more) for verbose output to the screen # log_dt_secs=>1, # log output to stdout~every log_dt_secs seconds # nmod=>10, # every nmod models there is output to the screen, # # if log_dt_secs has been exceeded also (ignored if 0) - # colour=>1, # set to 1 to use the ANSIColor module for colour output # log_args=>0, # do not log args in files # log_fins=>0, # log end of runs too @@ -318,114 +259,93 @@ grid_options_defaults_dict = { # save_args=>0, # do not save args in a string # log_args_dir=>$tmp, # where to output the args files # always_reopen_arg_files=>0, # if 1 then arg files are always closed and reopened (may cause a lot of disk I/O) - # lazy_arg_sending=>1, # if 1, the previous args are remembered and # # only args that changed are sent (except M1, M2 etc. which always # # need sending) - # # force output files to open on a local disk (not an NFS partion) # # not sure how to do this on another OS - # force_local_hdd_use=>($os eq 'unix'), - + # force_local_hdd_use=>($os eq 'unix'), # # for verbose output, define the newline - # # For terminals use "\x0d", for files use "\n", in the + # # For terminals use "\x0d", for files use "\n", in the # # case of multiple threads this will be set to \n # newline=> "\x0d", - # # use reset_stars_defaults # reset_stars_defaults=>1, - # # set signal captures: argument determines behaviour when the code locks up # # 0: exit # # 1: reset and try the next star (does this work?!) # alarm_procedure=>1, - # # exit on eval failure? # exit_on_eval_failure=>1, - # ## functions: these should be set by perl lexical name # ## (they are automatically converted to function pointers # ## at runtime) - # # function to be called just before a thread is created # thread_precreate_function=>undef, # thread_precreate_function_pointer=>undef, - # # function to be called just after a thread is created # # (from inside the thread just before *grid () call) # threads_entry_function=>undef, # threads_entry_function_pointer=>undef, - # # function to be called just after a thread is finished # # (from inside the thread just after *grid () call) # threads_flush_function=>undef, # threads_flush_function_pointer=>undef, - # # function to be called just after a thread is created # # (but external to the thread) # thread_postrun_function=>undef, # thread_postrun_function_pointer=>undef, - # # function to be called just before a thread join # # (external to the thread) # thread_prejoin_function=>undef, # thread_prejoin_function_pointer=>undef, - # # default to using the internal join_flexigrid_thread function # threads_join_function=>'binary_grid2::join_flexigrid_thread', # threads_join_function_pointer=>sub{return $self->join_flexigrid_thread(@_)}, - # # function to be called just after a thread join # # (external to the thread) # thread_postjoin_function=>undef, # thread_postjoin_function_pointer=>undef, - # # usually, parse_bse in the main script is called # parse_bse_function=>'main::parse_bse', # parse_bse_function_pointer=>undef, - # # if starting_snapshot_file is defined, load initial - # # values for the grid from the snapshot file rather - # # than a normal initiation: this enables you to + # # values for the grid from the snapshot file rather + # # than a normal initiation: this enables you to # # stop and start a grid # starting_snapshot_file=>undef, - # # flexigrid options # flexigrid=>{ # # there are several types of flexigrid: # # # # 'grid' is the traditional N-dimensional grid # # 'monte carlo' is a Monte Carlo simulation (may not work!) - # # 'list' takes a list of systems from + # # 'list' takes a list of systems from # # # # $self->{_grid_options}->{flexigrid}->{'list filename'} # # # # (which is a file containing list strings on each line) # # - # # or from - # # + # # or from + # # # # $self->{_grid_options}->{flexigrid}->{'list reference'} # # # # (which is a reference to a perl list) # 'grid type'=>'grid', # 'list filename'=>undef, # undef unless 'grid type' is 'list' # 'list reference'=>undef, # ditto - # 'listFP'=>undef, # file pointer : keep undef here (set automatically) - + # 'listFP'=>undef, # file pointer : keep undef here (set automatically) # }, - - # # start at this model number: handy during debugging + # # start at this model number: handy during debugging # # to skip large parts of the grid # start_at => 0 # ); # ); - # # if available, use the evcode's defaults # if(1) # { # my $evcode_args = $self->evcode_args_list(); - - # if(defined $evcode_args && + # if(defined $evcode_args && # $evcode_args && # ref $evcode_args eq 'ARRAY' && # $#{$evcode_args} > 1) @@ -457,7 +377,7 @@ grid_options_defaults_dict = { # { # #print "NEW set $1 to $2\n"; # } - # $self->{_bse_options}->{$1} = + # $self->{_bse_options}->{$1} = # $2 eq 'TRUE' ? 1 : # $2 eq 'FALSE' ? 0 : # $2; @@ -466,14 +386,12 @@ grid_options_defaults_dict = { # } # } # } - # $self->{_flexigrid} = { # count => 0, # error => 0, # failed_count => 0, # failed_prob => 0.0, # global_error_string => undef, - # # random string to ID the flexigrid # id => rob_misc::random_string(8), # modulo => 1, # run modulo n @@ -481,7 +399,7 @@ grid_options_defaults_dict = { # nextlogtime => 0, # nthreads => 1, # number of threads # # start at model offset (0-based, so first model is zero) - # offset => 0, + # offset => 0, # probtot => 0.0, # resolution=>{ # shift =>0, @@ -491,10 +409,10 @@ grid_options_defaults_dict = { # results_hash => $self->{_results}, # thread_q => undef, # threads => undef, # array of threads objects - # tstart => [gettimeofday], # flexigrid start time + # tstart => [gettimeofday], # flexigrid start time # __nvar => 0, # number of grid variables # _varstub => undef, # _lock => undef, # _evcode_pids => [], - # }; + # }; } diff --git a/binarycpython/utils/probability_distributions.py b/binarycpython/utils/probability_distributions.py index 0c6193085..5506cf4eb 100644 --- a/binarycpython/utils/probability_distributions.py +++ b/binarycpython/utils/probability_distributions.py @@ -1,29 +1,36 @@ - # TODO: make some things globally present? rob does this in his module. i guess it saves calculations but not sure if im gonna do that now + def flat(parameter): """ Dummt distribution function that returns 1 """ return 1 + def number(value): """ Dummy distribution function that returns the input """ return value + def powerlaw_constant(min_val, max_val, k): """ Function that returns the constant to normalise a powerlaw """ k1 = k + 1.0 - print("Powerlaw consts from {} to {}, k={} where k1={}".format(min_val, max_val, k, k1)) + print( + "Powerlaw consts from {} to {}, k={} where k1={}".format( + min_val, max_val, k, k1 + ) + ) - powerlaw_const = k1/(max_val**k1 - min_val**k1); + powerlaw_const = k1 / (max_val ** k1 - min_val ** k1) return powerlaw_const + def powerlaw(min_val, max_val, k, x): """ Single powerlaw with index k at x from min to max @@ -42,20 +49,24 @@ def powerlaw(min_val, max_val, k, x): # powerlaw y = const * (x ** k) - print("Power law from {} to {}: const = {}, y = {}".format(min_val, max_val, const, y)) + print( + "Power law from {} to {}: const = {}, y = {}".format( + min_val, max_val, const, y + ) + ) return y - + # sub const # { # # a constant distribution function between min=$_[0] and max=$_[1] # # $_[2] is optional (if given, perform a range check) - + # my $C= (defined($_[2]) && (($_[2]<$_[0]) || ($_[2]>$_[1]))) ? 0.0 : (1.0/($_[1]-$_[0])); # #printf "CONST $_[0] to $_[1] ($_[2]) $C\n"; # return $C; # } - + # sub initialize_three_part_power_law_consts # { # local $SIG{__DIE__} = sub { Carp::confess @_ }; @@ -84,7 +95,7 @@ def powerlaw(min_val, max_val, k, x): # { # # wrapper for KTG93 on a ln(m) grid # my $m=$_[0]; -# return ktg93(@_) * $m; +# return ktg93(@_) * $m; # } # sub setopts @@ -95,8 +106,8 @@ def powerlaw(min_val, max_val, k, x): # my $opts=$_[0]; # my $newopts=$_[1]; # if(defined $newopts) -# { -# # newopts is a hash of new options +# { +# # newopts is a hash of new options # foreach my $opt (keys %$newopts) # { # # overwrite opt @@ -189,10 +200,10 @@ def powerlaw(min_val, max_val, k, x): # # use pre-calculated consts if possible, otherwise calculate them # my $consts=$threepart_powerlaw_consts{"@_"} // # initialize_three_part_power_law_consts(@_); - + # my $p; # if($m<$_[1]) -# { +# { # $p = $m<$_[0] # ? 0.0 # : $$consts[0]*($m**$_[4]); @@ -216,7 +227,7 @@ def powerlaw(min_val, max_val, k, x): # # location (X value), mean and sigma, min and max range # my ($x,$mean,$sigma,$gmin,$gmax) = @_; - + # my $p; # if($x<$gmin || $x>$gmax) # { diff --git a/binarycpython/utils/useful_functions.py b/binarycpython/utils/useful_functions.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/examples_custom_logging.py b/examples/examples_custom_logging.py index b6dd5e8ba..4cebf2323 100644 --- a/examples/examples_custom_logging.py +++ b/examples/examples_custom_logging.py @@ -9,20 +9,29 @@ from binarycpython.utils.custom_logging_functions import ( from binarycpython.utils.grid import Population ################################################# -# File containing examples for the custom logging -# I'll put several logging snippets for different purposes in this file +# File containing examples for the custom logging +# I'll put several logging snippets for different purposes in this file # ################################################# # no logging set. pop = Population() -pop.set(M_1=10, M_2=10, separation=0, orbital_period=4530, eccentricity=0, metallicity=0.02, max_evolution_time=15000) +pop.set( + M_1=10, + M_2=10, + separation=0, + orbital_period=4530, + eccentricity=0, + metallicity=0.02, + max_evolution_time=15000, +) out = pop.evolve_single() print(out) ################################################# -# Example logging snippet for logging -pop.set(C_logging_code=""" +# Example logging snippet for logging +pop.set( + C_logging_code=""" if(stardata->star[0].stellar_type>=MS) { if (stardata->model.time < stardata->model.max_evolution_time) @@ -46,15 +55,25 @@ pop.set(C_logging_code=""" /* Kill the simulation to save time */ //stardata->model.max_evolution_time = stardata->model.time - stardata->model.dtm; }; -""") +""" +) out = pop.evolve_single() print(out) ################################################# # Example logging snippet for checking whether the system becomes a NS, and stop the evolution if so. -pop.set(M_1=100, M_2=10, separation=0, orbital_period=400530, eccentricity=0, metallicity=0.002, max_evolution_time=15000) -pop.set(C_logging_code=""" +pop.set( + M_1=100, + M_2=10, + separation=0, + orbital_period=400530, + eccentricity=0, + metallicity=0.002, + max_evolution_time=15000, +) +pop.set( + C_logging_code=""" if(stardata->star[0].stellar_type>=NS) { if (stardata->model.time < stardata->model.max_evolution_time) @@ -76,12 +95,13 @@ pop.set(C_logging_code=""" /* Kill the simulation to save time */ stardata->model.max_evolution_time = stardata->model.time - stardata->model.dtm; }; -""") +""" +) out = pop.evolve_single() -# TODO: add function that shows a +# TODO: add function that shows a # TODO: add function for compact object mergers -# TODO: add function \ No newline at end of file +# TODO: add function diff --git a/tests/function_tests.py b/tests/function_tests.py index 9e42273a9..107e19008 100644 --- a/tests/function_tests.py +++ b/tests/function_tests.py @@ -1,17 +1,11 @@ -from binarycpython.utils.functions import ( - get_help_super, - get_help_all, - get_help - ) +from binarycpython.utils.functions import get_help_super, get_help_all, get_help ############################# -# File containing some tests to function. These are not unit tests where output is compared. - - +# File containing some tests to function. These are not unit tests where output is compared. ## Help functionality print(get_help_super(print_help=True, return_dict=False, fail_silently=False)) print(get_help_all(print_help=True)) -print(get_help('M_1')) -# \ No newline at end of file +print(get_help("M_1")) +# diff --git a/tests/population/grid_tests.py b/tests/population/grid_tests.py index 3d1ba1412..2d14757c5 100644 --- a/tests/population/grid_tests.py +++ b/tests/population/grid_tests.py @@ -67,7 +67,7 @@ test_pop.set( # if (stardata->model.time < stardata->model.max_evolution_time) # { # Printf("DAVID_SCO %30.12e %g %g %g %g %d %d\\n", -# // +# // # stardata->model.time, // 1 # stardata->star[0].mass, //2 @@ -100,7 +100,7 @@ test_pop.set( # test_pop.export_all_info(outfile=os.path.join(os.getcwd(), "test_output.txt")) ################# Parse function -## Testing some stuff out with giving a parse_function. +## Testing some stuff out with giving a parse_function. # test_pop.set( # C_logging_code=""" # if(stardata->star[0].stellar_type>=NS) @@ -108,7 +108,7 @@ test_pop.set( # if (stardata->model.time < stardata->model.max_evolution_time) # { # Printf("DAVID_SCO %30.12e %g %g %g %g %d %d\\n", -# // +# // # stardata->model.time, // 1 # stardata->star[0].mass, //2 @@ -129,7 +129,7 @@ test_pop.set( # def output_lines(output): # """ -# Function that outputs the lines that were recieved from the binary_c run. +# Function that outputs the lines that were recieved from the binary_c run. # """ # return output.splitlines() @@ -137,7 +137,7 @@ test_pop.set( # # extract info from the population instance # # TODO: think about whether this is smart. Passing around this object might be an overkill -# # Get some information from the +# # Get some information from the # data_dir = self.custom_options['data_dir'] # base_filename = self.custom_options['base_filename'] # outfilename = os.path.join(data_dir, base_filename) @@ -206,30 +206,30 @@ test_pop.set( ### Grid generating testing test_pop.add_grid_variable( - name='lnm1', - longname='log primary mass', + name="lnm1", + longname="log primary mass", valuerange=[10, 20], - resolution='10', - spacingfunc='np.linspace(0.213, 10.2, 10)', - precode='M_1=math.exp(lnm1)', - probdist='flat(M_1)', + resolution="10", + spacingfunc="np.linspace(0.213, 10.2, 10)", + precode="M_1=math.exp(lnm1)", + probdist="flat(M_1)", # probdist='self.custom_options["extra_prob_function"](M_1)', - dphasevol='', - parameter_name='M_1', - condition='', + dphasevol="", + parameter_name="M_1", + condition="", ) ### Grid generating test. test_pop.add_grid_variable( - name='period', - longname='period', - valuerange=['M_1', 20], - resolution='10', - spacingfunc='np.linspace(1, 10, 10)', - precode='orbital_period = period**2', - probdist='flat(orbital_period)', - parameter_name='orbital_period', - dphasevol='', + name="period", + longname="period", + valuerange=["M_1", 20], + resolution="10", + spacingfunc="np.linspace(1, 10, 10)", + precode="orbital_period = period**2", + probdist="flat(orbital_period)", + parameter_name="orbital_period", + dphasevol="", condition='self.grid_options["binary"]==0', ) @@ -237,4 +237,4 @@ test_pop.generate_grid_code() test_pop.load_grid_function() -print(test_pop.grid_options['code_string']) \ No newline at end of file +print(test_pop.grid_options["code_string"]) diff --git a/tests/population/grid_tests_cmdline.py b/tests/population/grid_tests_cmdline.py index 7314bd1ff..eee0a2df4 100644 --- a/tests/population/grid_tests_cmdline.py +++ b/tests/population/grid_tests_cmdline.py @@ -12,4 +12,4 @@ from binarycpython.utils.functions import get_help_all, get_help test_pop = Population() -test_pop.parse_cmdline() \ No newline at end of file +test_pop.parse_cmdline() -- GitLab