diff --git a/binarycpython/utils/grid.py b/binarycpython/utils/grid.py index b781c4c8180539872723a7af0130fc99942beafc..26b5520c39050f008f90752b421bc4b443f74f6b 100644 --- a/binarycpython/utils/grid.py +++ b/binarycpython/utils/grid.py @@ -53,7 +53,7 @@ from binarycpython.utils.ensemble import ( from binarycpython.utils.dicts import ( AutoVivificationDict, merge_dicts, - keys_to_floats, + keys_to_floats ) from binarycpython.utils.population_extensions.analytics import analytics from binarycpython.utils.population_extensions.cache import cache @@ -128,6 +128,7 @@ class Population( self.caches = {} self.cached_function_cache = {} self.original_function_cache = {} + self.hostnameslist = hostnames() # Different sections of options # get binary_c defaults and create a cleaned up dict @@ -306,15 +307,18 @@ class Population( """ # Go over all the input - for key in kwargs: + for key,value in kwargs.items(): + # match to hostname if appropriate + value = self._match_arg_to_host(arg={key:value}) + # Filter out keys for the bse_options if key in self.defaults: self.verbose_print( - "adding: {}={} to BSE_options".format(key, kwargs[key]), + "adding: {}={} to BSE_options".format(key, value), self.grid_options["verbosity"], 2, ) - self.bse_options[key] = kwargs[key] + self.bse_options[key] = value # Extra check to check if the key fits one of parameter names that end with %d # TODO: abstract this function @@ -324,33 +328,33 @@ class Population( ): self.verbose_print( "adding: {}={} to BSE_options by catching the %d".format( - key, kwargs[key] + key, value ), self.grid_options["verbosity"], 1, ) - self.bse_options[key] = kwargs[key] + self.bse_options[key] = value # Filter out keys for the grid_options elif key in self.grid_options.keys(): self.verbose_print( - "adding: {}={} to grid_options".format(key, kwargs[key]), + "adding: {}={} to grid_options".format(key, value), self.grid_options["verbosity"], 1, ) - self.grid_options[key] = kwargs[key] + self.grid_options[key] = value # The of the keys go into a custom_options dict else: self.verbose_print( "<<<< Warning: Key does not match previously known parameter: \ adding: {}={} to custom_options >>>>".format( - key, kwargs[key] + key, value ), self.grid_options["verbosity"], 0, # NOTE: setting this to be 0 prevents mistakes being overlooked. ) - self.custom_options[key] = kwargs[key] + self.custom_options[key] = value def parse_cmdline(self) -> None: """ @@ -382,6 +386,9 @@ class Population( # cmdline_args = args self.grid_options["_commandline_input"] = cmdline_args + # expand args by hostname + cmdline_args = self.expand_args_by_hostname(cmdline_args) + # Make dict and fill it cmdline_dict = {} for cmdline_arg in cmdline_args: @@ -390,6 +397,7 @@ class Population( if len(split) == 2: parameter = split[0] value = split[1] + old_value_found = False # Find an old value @@ -772,7 +780,7 @@ class Population( print("NPROC found in grid options") if type(self.grid_options['num_cores']) is dict: # try to match hostname to the dict keys - hostnameslist = hostnames() + hostnameslist = self.my_hostnames() for host,ncores in self.grid_options['num_cores'].items(): # check if we are this host if host in hostnameslist: @@ -2607,3 +2615,119 @@ class Population( print(f"logged crashed process to {failed_systems_file}") f.write(f"Process {process} crashed at {now} with exit code {exitcode}.") return + + def my_hostnames(self): + if self.hostnameslist is None: + self.hostnameslist = binarycpython.utils.functions.hostnames() + return self.hostnameslist + + def expand_args_by_hostname(self,cmdline_args): + """ + Expand a set of arguments by scanning each of them + for host-specific dicts. + + Given each arg, either as a string "x=y" or dict {x:y}, + determine whether y is a dict and if so does one of the keys + match the current hostname or "default", if so use the + corresponding value for the current machine. + """ + hostnameslist = self.my_hostnames() + new_cmdline_args = None + + # loop over list of cmdline args + if isinstance(cmdline_args,list): + new_cmdline_args = [] + for cmdline_arg in cmdline_args: + new_arg = self._match_arg_to_host(arg=cmdline_arg) + new_cmdline_args.append(new_arg) + + # loop over a dict of cmdline args + elif isinstance(cmdline_args,dict): + new_cmdline_args = {} + for parameter,value in cmdline_args.items(): + new_arg = self._match_arg_to_host(arg={parameter:value}) + new_cmdline_args[parameter] = new_arg + + return new_cmdline_args + + def _match_arg_to_host(self, + arg=None, + hostnameslist=None, + vb=False): + """ + Given an arg, either as a string "x=y" or dict {x:y}, + determine whether y is a dict and if so does one of the keys + match the current hostname or "default", if so use the + corresponding value. If not, return the original arg's value. + """ + if arg is None: + return None + + if hostnameslist is None: + hostnameslist=self.my_hostnames() + + if isinstance(arg,dict): + # {parameter: value} dict arg + parameter = list(arg.keys())[0] + value = list(arg.values())[0] + argtype = 'dict' + else: + # scalar x=y arg + split = arg.split("=") + if len(split)==2: + parameter = split[0] + value = split[1] + argtype = 'list' + else: + parameter = None + value = None + argtype = None + + if parameter: + if vb: + print(f"_match_arg_to_host: {parameter} = {value} (argtype = {argtype})") + try: + if vb: + print("Try to eval",value) + if isinstance(value,str): + # string : eval it + _x = eval(value) + else: + _x = value + + # if dict, match to hostnames or 'default' + if isinstance(_x,dict): + _match = None + for host in hostnameslist: + if vb: + print(f"check host={host}") + _match = _x.get(host,None) + if _match: + if vb: + print(f"Parameter {parameter} matches a value set by host={host} -> {_match}") + break + + # no match? try 'default' + if not _match: + _match = _x.get('default',None) + + # no match? error + if not _match: + print(f"Parameter {parameter} is a dict {value} none of the keys or which matches our hostname ({hostnameslist}) or 'default'. Please update this parameter.") + sys.exit(1) + else: + if vb: + print(f"Parameter {parameter} set to value {_match}") + value = _match + except: + pass + + if argtype == 'list': + # return x=y type argument + new_arg = f"{parameter}={value}" + else: + new_arg = value + + if vb: + print("return arg",new_arg,"of type",type(value)) + return new_arg