From 2c402f9054d7cca7db0b4ddf0d452ae475aee8c8 Mon Sep 17 00:00:00 2001
From: Robert Izzard <r.izzard@surrey.ac.uk>
Date: Wed, 10 Aug 2022 19:18:46 +0100
Subject: [PATCH] add autoamtic parsing of cmdline args do dicts are expanded
by hostname.
---
binarycpython/utils/grid.py | 146 +++++++++++++++++++++++++++++++++---
1 file changed, 135 insertions(+), 11 deletions(-)
diff --git a/binarycpython/utils/grid.py b/binarycpython/utils/grid.py
index b781c4c81..26b5520c3 100644
--- a/binarycpython/utils/grid.py
+++ b/binarycpython/utils/grid.py
@@ -53,7 +53,7 @@ from binarycpython.utils.ensemble import (
from binarycpython.utils.dicts import (
AutoVivificationDict,
merge_dicts,
- keys_to_floats,
+ keys_to_floats
)
from binarycpython.utils.population_extensions.analytics import analytics
from binarycpython.utils.population_extensions.cache import cache
@@ -128,6 +128,7 @@ class Population(
self.caches = {}
self.cached_function_cache = {}
self.original_function_cache = {}
+ self.hostnameslist = hostnames()
# Different sections of options
# get binary_c defaults and create a cleaned up dict
@@ -306,15 +307,18 @@ class Population(
"""
# Go over all the input
- for key in kwargs:
+ for key,value in kwargs.items():
+ # match to hostname if appropriate
+ value = self._match_arg_to_host(arg={key:value})
+
# Filter out keys for the bse_options
if key in self.defaults:
self.verbose_print(
- "adding: {}={} to BSE_options".format(key, kwargs[key]),
+ "adding: {}={} to BSE_options".format(key, value),
self.grid_options["verbosity"],
2,
)
- self.bse_options[key] = kwargs[key]
+ self.bse_options[key] = value
# Extra check to check if the key fits one of parameter names that end with %d
# TODO: abstract this function
@@ -324,33 +328,33 @@ class Population(
):
self.verbose_print(
"adding: {}={} to BSE_options by catching the %d".format(
- key, kwargs[key]
+ key, value
),
self.grid_options["verbosity"],
1,
)
- self.bse_options[key] = kwargs[key]
+ self.bse_options[key] = value
# Filter out keys for the grid_options
elif key in self.grid_options.keys():
self.verbose_print(
- "adding: {}={} to grid_options".format(key, kwargs[key]),
+ "adding: {}={} to grid_options".format(key, value),
self.grid_options["verbosity"],
1,
)
- self.grid_options[key] = kwargs[key]
+ self.grid_options[key] = value
# The of the keys go into a custom_options dict
else:
self.verbose_print(
"<<<< Warning: Key does not match previously known parameter: \
adding: {}={} to custom_options >>>>".format(
- key, kwargs[key]
+ key, value
),
self.grid_options["verbosity"],
0, # NOTE: setting this to be 0 prevents mistakes being overlooked.
)
- self.custom_options[key] = kwargs[key]
+ self.custom_options[key] = value
def parse_cmdline(self) -> None:
"""
@@ -382,6 +386,9 @@ class Population(
# cmdline_args = args
self.grid_options["_commandline_input"] = cmdline_args
+ # expand args by hostname
+ cmdline_args = self.expand_args_by_hostname(cmdline_args)
+
# Make dict and fill it
cmdline_dict = {}
for cmdline_arg in cmdline_args:
@@ -390,6 +397,7 @@ class Population(
if len(split) == 2:
parameter = split[0]
value = split[1]
+
old_value_found = False
# Find an old value
@@ -772,7 +780,7 @@ class Population(
print("NPROC found in grid options")
if type(self.grid_options['num_cores']) is dict:
# try to match hostname to the dict keys
- hostnameslist = hostnames()
+ hostnameslist = self.my_hostnames()
for host,ncores in self.grid_options['num_cores'].items():
# check if we are this host
if host in hostnameslist:
@@ -2607,3 +2615,119 @@ class Population(
print(f"logged crashed process to {failed_systems_file}")
f.write(f"Process {process} crashed at {now} with exit code {exitcode}.")
return
+
+ def my_hostnames(self):
+ if self.hostnameslist is None:
+ self.hostnameslist = binarycpython.utils.functions.hostnames()
+ return self.hostnameslist
+
+ def expand_args_by_hostname(self,cmdline_args):
+ """
+ Expand a set of arguments by scanning each of them
+ for host-specific dicts.
+
+ Given each arg, either as a string "x=y" or dict {x:y},
+ determine whether y is a dict and if so does one of the keys
+ match the current hostname or "default", if so use the
+ corresponding value for the current machine.
+ """
+ hostnameslist = self.my_hostnames()
+ new_cmdline_args = None
+
+ # loop over list of cmdline args
+ if isinstance(cmdline_args,list):
+ new_cmdline_args = []
+ for cmdline_arg in cmdline_args:
+ new_arg = self._match_arg_to_host(arg=cmdline_arg)
+ new_cmdline_args.append(new_arg)
+
+ # loop over a dict of cmdline args
+ elif isinstance(cmdline_args,dict):
+ new_cmdline_args = {}
+ for parameter,value in cmdline_args.items():
+ new_arg = self._match_arg_to_host(arg={parameter:value})
+ new_cmdline_args[parameter] = new_arg
+
+ return new_cmdline_args
+
+ def _match_arg_to_host(self,
+ arg=None,
+ hostnameslist=None,
+ vb=False):
+ """
+ Given an arg, either as a string "x=y" or dict {x:y},
+ determine whether y is a dict and if so does one of the keys
+ match the current hostname or "default", if so use the
+ corresponding value. If not, return the original arg's value.
+ """
+ if arg is None:
+ return None
+
+ if hostnameslist is None:
+ hostnameslist=self.my_hostnames()
+
+ if isinstance(arg,dict):
+ # {parameter: value} dict arg
+ parameter = list(arg.keys())[0]
+ value = list(arg.values())[0]
+ argtype = 'dict'
+ else:
+ # scalar x=y arg
+ split = arg.split("=")
+ if len(split)==2:
+ parameter = split[0]
+ value = split[1]
+ argtype = 'list'
+ else:
+ parameter = None
+ value = None
+ argtype = None
+
+ if parameter:
+ if vb:
+ print(f"_match_arg_to_host: {parameter} = {value} (argtype = {argtype})")
+ try:
+ if vb:
+ print("Try to eval",value)
+ if isinstance(value,str):
+ # string : eval it
+ _x = eval(value)
+ else:
+ _x = value
+
+ # if dict, match to hostnames or 'default'
+ if isinstance(_x,dict):
+ _match = None
+ for host in hostnameslist:
+ if vb:
+ print(f"check host={host}")
+ _match = _x.get(host,None)
+ if _match:
+ if vb:
+ print(f"Parameter {parameter} matches a value set by host={host} -> {_match}")
+ break
+
+ # no match? try 'default'
+ if not _match:
+ _match = _x.get('default',None)
+
+ # no match? error
+ if not _match:
+ print(f"Parameter {parameter} is a dict {value} none of the keys or which matches our hostname ({hostnameslist}) or 'default'. Please update this parameter.")
+ sys.exit(1)
+ else:
+ if vb:
+ print(f"Parameter {parameter} set to value {_match}")
+ value = _match
+ except:
+ pass
+
+ if argtype == 'list':
+ # return x=y type argument
+ new_arg = f"{parameter}={value}"
+ else:
+ new_arg = value
+
+ if vb:
+ print("return arg",new_arg,"of type",type(value))
+ return new_arg
--
GitLab