From 7f200e7e5deae85230cd744c1c1ac599071612a8 Mon Sep 17 00:00:00 2001
From: Robert Izzard <r.izzard@surrey.ac.uk>
Date: Sat, 13 Nov 2021 11:05:20 +0000
Subject: [PATCH] split of grid.py's code into subclasses
---
.gitignore | 2 +
binarycpython/utils/HPC.py | 74 +
binarycpython/utils/Moe_di_Stefano_2017.py | 1207 +++++
binarycpython/utils/analytics.py | 71 +
binarycpython/utils/condor.py | 259 ++
binarycpython/utils/dataIO.py | 488 ++
binarycpython/utils/distribution_functions.py | 19 +-
binarycpython/utils/ensemble.py | 1 -
binarycpython/utils/functions.py | 447 --
binarycpython/utils/grid.py | 4123 ++---------------
binarycpython/utils/grid_logging.py | 403 ++
binarycpython/utils/grid_options_defaults.py | 1490 +++---
binarycpython/utils/gridcode.py | 997 ++++
binarycpython/utils/metadata.py | 115 +
binarycpython/utils/slurm.py | 345 ++
binarycpython/utils/version.py | 414 ++
16 files changed, 5317 insertions(+), 5138 deletions(-)
create mode 100644 binarycpython/utils/HPC.py
create mode 100644 binarycpython/utils/Moe_di_Stefano_2017.py
create mode 100644 binarycpython/utils/analytics.py
create mode 100644 binarycpython/utils/dataIO.py
create mode 100644 binarycpython/utils/grid_logging.py
create mode 100644 binarycpython/utils/gridcode.py
create mode 100644 binarycpython/utils/metadata.py
create mode 100644 binarycpython/utils/version.py
diff --git a/.gitignore b/.gitignore
index 2d6b5c906..d4c3a5e5c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -161,3 +161,5 @@ media/
db.sqlite3
*.swp
+
+*~
\ No newline at end of file
diff --git a/binarycpython/utils/HPC.py b/binarycpython/utils/HPC.py
new file mode 100644
index 000000000..5740c8136
--- /dev/null
+++ b/binarycpython/utils/HPC.py
@@ -0,0 +1,74 @@
+"""
+ Binary_c-python's HPC functions
+
+ These are common to both the slurm and condor interfaces.
+"""
+import os
+
+class HPC():
+
+ def __init__(self, **kwargs):
+ # don't do anything: we just inherit from this class
+ return
+
+ def joinfiles(self):
+ """
+ Function to load in the joinlist to an array and return it.
+ """
+ f = open(self.grid_options['joinlist'],'r',encoding='utf-8')
+ list = f.read().splitlines()
+ f.close()
+ return list
+
+ def join_from_files(self,newobj,joinfiles):
+ """
+ Merge the results from the list joinfiles into newobj.
+ """
+ for file in joinfiles:
+ print("Join data in",file)
+ self.merge_populations_from_file(newobj,
+ file)
+ return newobj
+
+ def can_join(self,joinfiles,joiningfile,vb=False):
+ """
+ Check the joinfiles to make sure they all exist
+ and their .saved equivalents also exist
+ """
+ vb = False # for debugging set this to True
+ if os.path.exists(joiningfile):
+ if vb:
+ print("cannot join: joiningfile exists at {}".format(joiningfile))
+ return False
+ elif vb:
+ print("joiningfile does not exist")
+ for file in joinfiles:
+ if vb:
+ print("check for {}".format(file))
+ if os.path.exists(file) == False:
+ if vb:
+ print("cannot join: {} does not exist".format(file))
+ return False
+ savedfile = file + '.saved'
+ if vb:
+ print("check for {}".format(savedfile))
+ if os.path.exists(savedfile) == False:
+ if vb:
+ print("cannot join: {} does not exist".format(savedfile))
+ return False
+
+ # found both files
+ if vb:
+ print("found {} and {}".format(file,savedfile))
+
+ # check for joiningfile again
+ if os.path.exists(joiningfile):
+ if vb:
+ print("cannot join: joiningfile exists at {}".format(joiningfile))
+ return False
+ elif vb:
+ print("joiningfile does not exist")
+
+ if vb:
+ print("returning True from can_join()")
+ return True
diff --git a/binarycpython/utils/Moe_di_Stefano_2017.py b/binarycpython/utils/Moe_di_Stefano_2017.py
new file mode 100644
index 000000000..7f17e6ae2
--- /dev/null
+++ b/binarycpython/utils/Moe_di_Stefano_2017.py
@@ -0,0 +1,1207 @@
+"""
+ Binary_c-python's functions to import Moe and di Stefano's data.
+"""
+
+import copy
+import json
+import os
+
+from binarycpython.utils.functions import (
+ verbose_print,
+)
+from binarycpython.utils.dicts import (
+ update_dicts,
+)
+from binarycpython.utils.distribution_functions import (
+ Moecache,
+ Moe_di_Stefano_2017_multiplicity_fractions,
+ get_max_multiplicity,
+ Arenou2010_binary_fraction,
+ raghavan2010_binary_fraction,
+ normalize_dict,
+ fill_data,
+ LOG_LN_CONVERTER,
+)
+from binarycpython.utils.grid_options_defaults import (
+ _MOE2017_VERBOSITY_LEVEL,
+)
+import binarycpython.utils.moe_di_stefano_2017_data as moe_di_stefano_2017_data
+
+class Moe_di_Stefano_2017():
+
+ def __init__(self, **kwargs):
+ # don't do anything: we just inherit from this class
+ return
+
+ def set_moe_di_stefano_settings(self, options=None):
+ """
+ Function to set user input configurations for the Moe & di Stefano methods
+
+ If nothing is passed then we just use the default options
+ """
+
+ if not options:
+ options = {}
+
+ # Take the option dictionary that was given and override.
+ options = update_dicts(self.grid_options["Moe2017_options"], options)
+ self.grid_options["Moe2017_options"] = copy.deepcopy(options)
+
+ # Write options to a file
+ os.makedirs(
+ os.path.join(self.grid_options["tmp_dir"], "moe_distefano"),
+ exist_ok=True,
+ )
+ with open(
+ os.path.join(
+ os.path.join(self.grid_options["tmp_dir"], "moe_distefano"),
+ "moeopts.dat",
+ ),
+ "w",
+ encoding='utf-8'
+ ) as f:
+ f.write(json.dumps(self.grid_options["Moe2017_options"], indent=4, ensure_ascii=False))
+ f.close()
+
+ def _load_moe_di_stefano_data(self):
+ """
+ Function to load the moe & di stefano data
+ """
+
+ # Only if the grid is loaded and Moecache contains information
+ if not self.grid_options["_loaded_Moe2017_data"]: # and not Moecache:
+
+ if self.grid_options["_Moe2017_JSON_data"]:
+ # Use the existing (perhaps modified) JSON data
+ json_data = self.grid_options["_Moe2017_JSON_data"]
+
+ else:
+ # Load the JSON data from a file
+ json_data = self.get_moe_di_stefano_dataset(
+ self.grid_options["Moe2017_options"],
+ verbosity=self.grid_options["verbosity"],
+ )
+
+ # entry of log10M1 is a list containing 1 dict.
+ # We can take the dict out of the list
+ if isinstance(json_data["log10M1"], list):
+ json_data["log10M1"] = json_data["log10M1"][0]
+
+ # save this data in case we want to modify it later
+ self.grid_options["_Moe2017_JSON_data"] = json_data
+
+ # Get all the masses
+ logmasses = sorted(json_data["log10M1"].keys())
+ if not logmasses:
+ msg = "The table does not contain masses."
+ verbose_print(
+ "\tMoe_di_Stefano_2017: {}".format(msg),
+ self.grid_options["verbosity"],
+ 0,
+ )
+ raise ValueError(msg)
+
+ # Write to file
+ os.makedirs(
+ os.path.join(self.grid_options["tmp_dir"], "moe_distefano"),
+ exist_ok=True,
+ )
+ with open(
+ os.path.join(
+ os.path.join(self.grid_options["tmp_dir"], "moe_distefano"),
+ "moe.log",
+ ),
+ "w",
+ encoding='utf-8',
+ ) as logfile:
+ logfile.write("logâ‚â‚€Masses(M☉) {}\n".format(logmasses))
+
+ # Get all the periods and see if they are all consistently present
+ logperiods = []
+ for logmass in logmasses:
+ if not logperiods:
+ logperiods = sorted(json_data["log10M1"][logmass]["logP"].keys())
+ dlog10P = float(logperiods[1]) - float(logperiods[0])
+
+ current_logperiods = sorted(json_data["log10M1"][logmass]["logP"])
+ if not (logperiods == current_logperiods):
+ msg = (
+ "Period values are not consistent throughout the dataset\logperiods = "
+ + " ".join(str(x) for x in logperiods)
+ + "\nCurrent periods = "
+ + " ".join(str(x) for x in current_logperiods)
+ )
+ verbose_print(
+ "\tMoe_di_Stefano_2017: {}".format(msg),
+ self.grid_options["verbosity"],
+ 0,
+ )
+ raise ValueError(msg)
+
+ ############################################################
+ # log10period binwidth : of course this assumes a fixed
+ # binwidth, so we check for this too.
+ for i in range(len(current_logperiods) - 1):
+ if not dlog10P == (
+ float(current_logperiods[i + 1]) - float(current_logperiods[i])
+ ):
+ msg = "Period spacing is not consistent throughout the dataset"
+ verbose_print(
+ "\tMoe_di_Stefano_2017: {}".format(msg),
+ self.grid_options["verbosity"],
+ 0,
+ )
+ raise ValueError(msg)
+
+ # save the logperiods list in the cache:
+ # this is used in the renormalization integration
+ Moecache["logperiods"] = logperiods
+
+ # Write to file
+ os.makedirs(
+ os.path.join(self.grid_options["tmp_dir"], "moe_distefano"),
+ exist_ok=True,
+ )
+ with open(
+ os.path.join(self.grid_options["tmp_dir"], "moe_distefano", "moe.log"),
+ "a",
+ encoding='utf-8'
+ ) as logfile:
+ logfile.write("logâ‚â‚€Periods(days) {}\n".format(logperiods))
+
+ # Fill the global dict
+ for logmass in logmasses:
+ # Create the multiplicity table
+ if not Moecache.get("multiplicity_table", None):
+ Moecache["multiplicity_table"] = []
+
+ # multiplicity as a function of primary mass
+ Moecache["multiplicity_table"].append(
+ [
+ float(logmass),
+ json_data["log10M1"][logmass]["f_multi"],
+ json_data["log10M1"][logmass]["single star fraction"],
+ json_data["log10M1"][logmass]["binary star fraction"],
+ json_data["log10M1"][logmass]["triple/quad star fraction"],
+ ]
+ )
+
+ ############################################################
+ # a small log10period which we can shift just outside the
+ # table to force integration out there to zero
+ epslog10P = 1e-8 * dlog10P
+
+ ############################################################
+ # loop over either binary or triple-outer periods
+ first = 1
+
+ # Go over the periods
+ for logperiod in logperiods:
+ ############################################################
+ # distributions of binary and triple star fractions
+ # as a function of mass, period.
+ #
+ # Note: these should be per unit log10P, hence we
+ # divide by dlog10P
+
+ if first:
+ first = 0
+
+ # Create the multiplicity table
+ if not Moecache.get("period_distributions", None):
+ Moecache["period_distributions"] = []
+
+ ############################################################
+ # lower bound the period distributions to zero probability
+ Moecache["period_distributions"].append(
+ [
+ float(logmass),
+ float(logperiod) - 0.5 * dlog10P - epslog10P,
+ 0.0,
+ 0.0,
+ ]
+ )
+ Moecache["period_distributions"].append(
+ [
+ float(logmass),
+ float(logperiod) - 0.5 * dlog10P,
+ json_data["log10M1"][logmass]["logP"][logperiod][
+ "normed_bin_frac_p_dist"
+ ]
+ / dlog10P,
+ json_data["log10M1"][logmass]["logP"][logperiod][
+ "normed_tripquad_frac_p_dist"
+ ]
+ / dlog10P,
+ ]
+ )
+
+ Moecache["period_distributions"].append(
+ [
+ float(logmass),
+ float(logperiod),
+ json_data["log10M1"][logmass]["logP"][logperiod][
+ "normed_bin_frac_p_dist"
+ ]
+ / dlog10P,
+ json_data["log10M1"][logmass]["logP"][logperiod][
+ "normed_tripquad_frac_p_dist"
+ ]
+ / dlog10P,
+ ]
+ )
+
+ ############################################################
+ # distributions as a function of mass, period, q
+ #
+ # First, get a list of the qs given by Moe
+ #
+ qs = sorted(json_data["log10M1"][logmass]["logP"][logperiod]["q"])
+
+ # Fill the data and 'normalise'
+ qdata = fill_data(
+ qs, json_data["log10M1"][logmass]["logP"][logperiod]["q"]
+ )
+
+ # Create the multiplicity table
+ if not Moecache.get("q_distributions", None):
+ Moecache["q_distributions"] = []
+
+ for q in qs:
+ Moecache["q_distributions"].append(
+ [float(logmass), float(logperiod), float(q), qdata[q]]
+ )
+
+ ############################################################
+ # eccentricity distributions as a function of mass, period, ecc
+ eccs = sorted(json_data["log10M1"][logmass]["logP"][logperiod]["e"])
+
+ # Fill the data and 'normalise'
+ ecc_data = fill_data(
+ eccs, json_data["log10M1"][logmass]["logP"][logperiod]["e"]
+ )
+
+ # Create the multiplicity table
+ if not Moecache.get("ecc_distributions", None):
+ Moecache["ecc_distributions"] = []
+
+ for ecc in eccs:
+ Moecache["ecc_distributions"].append(
+ [
+ float(logmass),
+ float(logperiod),
+ float(ecc),
+ ecc_data[ecc],
+ ]
+ )
+
+ ############################################################
+ # upper bound the period distributions to zero probability
+ Moecache["period_distributions"].append(
+ [
+ float(logmass),
+ float(logperiods[-1]) + 0.5 * dlog10P, # TODO: why this shift?
+ json_data["log10M1"][logmass]["logP"][logperiods[-1]][
+ "normed_bin_frac_p_dist"
+ ]
+ / dlog10P,
+ json_data["log10M1"][logmass]["logP"][logperiods[-1]][
+ "normed_tripquad_frac_p_dist"
+ ]
+ / dlog10P,
+ ]
+ )
+ Moecache["period_distributions"].append(
+ [
+ float(logmass),
+ float(logperiods[-1]) + 0.5 * dlog10P + epslog10P,
+ 0.0,
+ 0.0,
+ ]
+ )
+
+ verbose_print(
+ "\tMoe_di_Stefano_2017: Length period_distributions table: {}".format(
+ len(Moecache["period_distributions"])
+ ),
+ self.grid_options["verbosity"],
+ _MOE2017_VERBOSITY_LEVEL,
+ )
+ verbose_print(
+ "\tMoe_di_Stefano_2017: Length multiplicity table: {}".format(
+ len(Moecache["multiplicity_table"])
+ ),
+ self.grid_options["verbosity"],
+ _MOE2017_VERBOSITY_LEVEL,
+ )
+ verbose_print(
+ "\tMoe_di_Stefano_2017: Length q table: {}".format(
+ len(Moecache["q_distributions"])
+ ),
+ self.grid_options["verbosity"],
+ _MOE2017_VERBOSITY_LEVEL,
+ )
+ verbose_print(
+ "\tMoe_di_Stefano_2017: Length ecc table: {}".format(
+ len(Moecache["ecc_distributions"])
+ ),
+ self.grid_options["verbosity"],
+ _MOE2017_VERBOSITY_LEVEL,
+ )
+
+ # Write to log file
+ os.makedirs(
+ os.path.join(self.grid_options["tmp_dir"], "moe_distefano"),
+ exist_ok=True,
+ )
+ with open(
+ os.path.join(
+ os.path.join(self.grid_options["tmp_dir"], "moe_distefano"),
+ "moecache.json",
+ ),
+ "w",
+ encoding='utf-8'
+ ) as cache_filehandle:
+ cache_filehandle.write(json.dumps(Moecache, indent=4, ensure_ascii=False))
+
+ # Signal that the data has been loaded
+ self.grid_options["_loaded_Moe2017_data"] = True
+
+ def _set_moe_di_stefano_distributions(self):
+ """
+ Function to set the Moe & di Stefano distribution
+ """
+
+ ############################################################
+ # first, the multiplicity, this is 1,2,3,4, ...
+ # for singles, binaries, triples, quadruples, ...
+
+ max_multiplicity = get_max_multiplicity(
+ self.grid_options["Moe2017_options"]["multiplicity_modulator"]
+ )
+ verbose_print(
+ "\tMoe_di_Stefano_2017: Max multiplicity = {}".format(max_multiplicity),
+ self.grid_options["verbosity"],
+ _MOE2017_VERBOSITY_LEVEL,
+ )
+ ######
+ # Setting up the grid variables
+
+ # Multiplicity
+ self.add_grid_variable(
+ name="multiplicity",
+ parameter_name="multiplicity",
+ longname="multiplicity",
+ valuerange=[1, max_multiplicity],
+ samplerfunc="const_int(1, {n}, {n})".format(n=max_multiplicity),
+ precode='self.grid_options["multiplicity"] = multiplicity; self.bse_options["multiplicity"] = multiplicity; options={}'.format(
+ self.grid_options["Moe2017_options"]
+ ),
+ condition="({}[int(multiplicity)-1] > 0)".format(
+ str(self.grid_options["Moe2017_options"]["multiplicity_modulator"])
+ ),
+ gridtype="discrete",
+ probdist=1,
+ )
+
+ ############################################################
+ # always require M1, for all systems
+ #
+ # log-spaced m1 with given resolution
+ self.add_grid_variable(
+ name="lnm1",
+ parameter_name="M_1",
+ longname="Primary mass",
+ samplerfunc=self.grid_options["Moe2017_options"]["samplerfuncs"]["M"][0]
+ or "const(np.log({}), np.log({}), {})".format(
+ self.grid_options["Moe2017_options"]["ranges"]["M"][0],
+ self.grid_options["Moe2017_options"]["ranges"]["M"][1],
+ self.grid_options["Moe2017_options"]["resolutions"]["M"][0],
+ ),
+ valuerange=[
+ "np.log({})".format(
+ self.grid_options["Moe2017_options"]["ranges"]["M"][0]
+ ),
+ "np.log({})".format(
+ self.grid_options["Moe2017_options"]["ranges"]["M"][1]
+ ),
+ ],
+ gridtype="centred",
+ dphasevol="dlnm1",
+ precode='M_1 = np.exp(lnm1); options["M_1"]=M_1',
+ probdist="Moe_di_Stefano_2017_pdf({{{}, {}, {}}}, verbosity=self.grid_options['verbosity'])['total_probdens'] if multiplicity == 1 else 1".format(
+ str(dict(self.grid_options["Moe2017_options"]))[1:-1],
+ "'multiplicity': multiplicity",
+ "'M_1': M_1",
+ ),
+ )
+
+ # Go to higher multiplicities
+ if max_multiplicity >= 2:
+ # binaries: period
+ self.add_grid_variable(
+ name="log10per",
+ parameter_name="orbital_period",
+ longname="log10(Orbital_Period)",
+ probdist=1.0,
+ condition='(self.grid_options["multiplicity"] >= 2)',
+ branchpoint=1
+ if max_multiplicity > 1
+ else 0, # Signal here to put a branchpoint if we have a max multiplicity higher than 1.
+ gridtype="centred",
+ dphasevol="({} * dlog10per)".format(LOG_LN_CONVERTER),
+ valuerange=[
+ self.grid_options["Moe2017_options"]["ranges"]["logP"][0],
+ self.grid_options["Moe2017_options"]["ranges"]["logP"][1],
+ ],
+ samplerfunc=self.grid_options["Moe2017_options"]["samplerfuncs"][
+ "logP"
+ ][0]
+ or "const({}, {}, {})".format(
+ self.grid_options["Moe2017_options"]["ranges"]["logP"][0],
+ self.grid_options["Moe2017_options"]["ranges"]["logP"][1],
+ self.grid_options["Moe2017_options"]["resolutions"]["logP"][0],
+ ),
+ precode="""orbital_period = 10.0**log10per
+qmin={}/M_1
+qmax=maximum_mass_ratio_for_RLOF(M_1, orbital_period)
+""".format(
+ self.grid_options["Moe2017_options"]["Mmin"]
+ ),
+ ) # TODO: change the maximum_mass_ratio_for_RLOF
+
+ # binaries: mass ratio
+ self.add_grid_variable(
+ name="q",
+ parameter_name="M_2",
+ longname="Mass ratio",
+ valuerange=[
+ self.grid_options["Moe2017_options"]["ranges"]["q"][0]
+ if self.grid_options["Moe2017_options"]
+ .get("ranges", {})
+ .get("q", None)
+ else "options['Mmin']/M_1",
+ self.grid_options["Moe2017_options"]["ranges"]["q"][1]
+ if self.grid_options["Moe2017_options"]
+ .get("ranges", {})
+ .get("q", None)
+ else "qmax",
+ ],
+ probdist=1,
+ gridtype="centred",
+ dphasevol="dq",
+ precode="""
+M_2 = q * M_1
+sep = calc_sep_from_period(M_1, M_2, orbital_period)
+ """,
+ samplerfunc=self.grid_options["Moe2017_options"]["samplerfuncs"]["M"][1]
+ or "const({}, {}, {})".format(
+ self.grid_options["Moe2017_options"]["ranges"]["q"][0]
+ if self.grid_options["Moe2017_options"]
+ .get("ranges", {})
+ .get("q", [None, None])[0]
+ else "{}/M_1".format(self.grid_options["Moe2017_options"]["Mmin"]),
+ self.grid_options["Moe2017_options"]["ranges"]["q"][1]
+ if self.grid_options["Moe2017_options"]
+ .get("ranges", {})
+ .get("q", [None, None])[1]
+ else "qmax",
+ self.grid_options["Moe2017_options"]["resolutions"]["M"][1],
+ ),
+ )
+
+ # (optional) binaries: eccentricity
+ if self.grid_options["Moe2017_options"]["resolutions"]["ecc"][0] > 0:
+ self.add_grid_variable(
+ name="ecc",
+ parameter_name="eccentricity",
+ longname="Eccentricity",
+ probdist=1,
+ gridtype="centred",
+ dphasevol="decc",
+ precode="eccentricity=ecc",
+ valuerange=[
+ self.grid_options["Moe2017_options"]["ranges"]["ecc"][
+ 0
+ ], # Just fail if not defined.
+ self.grid_options["Moe2017_options"]["ranges"]["ecc"][1],
+ ],
+ samplerfunc=self.grid_options["Moe2017_options"]["samplerfuncs"][
+ "ecc"
+ ][0]
+ or "const({}, {}, {})".format(
+ self.grid_options["Moe2017_options"]["ranges"]["ecc"][
+ 0
+ ], # Just fail if not defined.
+ self.grid_options["Moe2017_options"]["ranges"]["ecc"][1],
+ self.grid_options["Moe2017_options"]["resolutions"]["ecc"][0],
+ ),
+ )
+
+ # Now for triples and quadruples
+ if max_multiplicity >= 3:
+ # Triple: period
+ self.add_grid_variable(
+ name="log10per2",
+ parameter_name="orbital_period_triple",
+ longname="log10(Orbital_Period2)",
+ probdist=1.0,
+ condition='(self.grid_options["multiplicity"] >= 3)',
+ branchpoint=2
+ if max_multiplicity > 2
+ else 0, # Signal here to put a branchpoint if we have a max multiplicity higher than 1.
+ gridtype="centred",
+ dphasevol="({} * dlog10per2)".format(LOG_LN_CONVERTER),
+ valuerange=[
+ self.grid_options["Moe2017_options"]["ranges"]["logP"][0],
+ self.grid_options["Moe2017_options"]["ranges"]["logP"][1],
+ ],
+ samplerfunc=self.grid_options["Moe2017_options"]["samplerfuncs"][
+ "logP"
+ ][1]
+ or "const({}, {}, {})".format(
+ self.grid_options["Moe2017_options"]["ranges"]["logP"][0],
+ self.grid_options["Moe2017_options"]["ranges"]["logP"][1],
+ self.grid_options["Moe2017_options"]["resolutions"]["logP"][1],
+ ),
+ precode="""orbital_period_triple = 10.0**log10per2
+q2min={}/(M_1+M_2)
+q2max=maximum_mass_ratio_for_RLOF(M_1+M_2, orbital_period_triple)
+ """.format(
+ self.grid_options["Moe2017_options"]["Mmin"]
+ ),
+ )
+
+ # Triples: mass ratio
+ # Note, the mass ratio is M_outer/M_inner
+ self.add_grid_variable(
+ name="q2",
+ parameter_name="M_3",
+ longname="Mass ratio outer/inner",
+ valuerange=[
+ self.grid_options["Moe2017_options"]["ranges"]["q"][0]
+ if self.grid_options["Moe2017_options"]
+ .get("ranges", {})
+ .get("q", None)
+ else "options['Mmin']/(M_1+M_2)",
+ self.grid_options["Moe2017_options"]["ranges"]["q"][1]
+ if self.grid_options["Moe2017_options"]
+ .get("ranges", {})
+ .get("q", None)
+ else "q2max",
+ ],
+ probdist=1,
+ gridtype="centred",
+ dphasevol="dq2",
+ precode="""
+M_3 = q2 * (M_1 + M_2)
+sep2 = calc_sep_from_period((M_1+M_2), M_3, orbital_period_triple)
+eccentricity2=0
+""",
+ samplerfunc=self.grid_options["Moe2017_options"]["samplerfuncs"][
+ "M"
+ ][2]
+ or "const({}, {}, {})".format(
+ self.grid_options["Moe2017_options"]["ranges"]["q"][0]
+ if self.grid_options["Moe2017_options"]
+ .get("ranges", {})
+ .get("q", None)
+ else "options['Mmin']/(M_1+M_2)",
+ self.grid_options["Moe2017_options"]["ranges"]["q"][1]
+ if self.grid_options["Moe2017_options"]
+ .get("ranges", {})
+ .get("q", None)
+ else "q2max",
+ self.grid_options["Moe2017_options"]["resolutions"]["M"][2],
+ ),
+ )
+
+ # (optional) triples: eccentricity
+ if self.grid_options["Moe2017_options"]["resolutions"]["ecc"][1] > 0:
+ self.add_grid_variable(
+ name="ecc2",
+ parameter_name="eccentricity2",
+ longname="Eccentricity of the triple",
+ probdist=1,
+ gridtype="centred",
+ dphasevol="decc2",
+ precode="eccentricity2=ecc2",
+ valuerange=[
+ self.grid_options["Moe2017_options"]["ranges"]["ecc"][
+ 0
+ ], # Just fail if not defined.
+ self.grid_options["Moe2017_options"]["ranges"]["ecc"][1],
+ ],
+ samplerfunc=self.grid_options["Moe2017_options"][
+ "samplerfuncs"
+ ]["ecc"][1]
+ or "const({}, {}, {})".format(
+ self.grid_options["Moe2017_options"]["ranges"]["ecc"][
+ 0
+ ], # Just fail if not defined.
+ self.grid_options["Moe2017_options"]["ranges"]["ecc"][1],
+ self.grid_options["Moe2017_options"]["resolutions"]["ecc"][
+ 1
+ ],
+ ),
+ )
+
+ if max_multiplicity == 4:
+ # Quadruple: period
+ self.add_grid_variable(
+ name="log10per3",
+ parameter_name="orbital_period_quadruple",
+ longname="log10(Orbital_Period3)",
+ probdist=1.0,
+ condition='(self.grid_options["multiplicity"] >= 4)',
+ branchpoint=3
+ if max_multiplicity > 3
+ else 0, # Signal here to put a branchpoint if we have a max multiplicity higher than 1.
+ gridtype="centred",
+ dphasevol="({} * dlog10per3)".format(LOG_LN_CONVERTER),
+ valuerange=[
+ self.grid_options["Moe2017_options"]["ranges"]["logP"][0],
+ self.grid_options["Moe2017_options"]["ranges"]["logP"][1],
+ ],
+ samplerfunc=self.grid_options["Moe2017_options"][
+ "samplerfuncs"
+ ]["logP"][2]
+ or "const({}, {}, {})".format(
+ self.grid_options["Moe2017_options"]["ranges"]["logP"][0],
+ self.grid_options["Moe2017_options"]["ranges"]["logP"][1],
+ self.grid_options["Moe2017_options"]["resolutions"]["logP"][
+ 2
+ ],
+ ),
+ precode="""orbital_period_quadruple = 10.0**log10per3
+q3min={}/(M_3)
+q3max=maximum_mass_ratio_for_RLOF(M_3, orbital_period_quadruple)
+ """.format(
+ self.grid_options["Moe2017_options"]["Mmin"]
+ ),
+ )
+
+ # Quadruple: mass ratio : M_outer / M_inner
+ self.add_grid_variable(
+ name="q3",
+ parameter_name="M_4",
+ longname="Mass ratio outer low/outer high",
+ valuerange=[
+ self.grid_options["Moe2017_options"]["ranges"]["q"][0]
+ if self.grid_options["Moe2017_options"]
+ .get("ranges", {})
+ .get("q", None)
+ else "options['Mmin']/(M_3)",
+ self.grid_options["Moe2017_options"]["ranges"]["q"][1]
+ if self.grid_options["Moe2017_options"]
+ .get("ranges", {})
+ .get("q", None)
+ else "q3max",
+ ],
+ probdist=1,
+ gridtype="centred",
+ dphasevol="dq3",
+ precode="""
+M_4 = q3 * M_3
+sep3 = calc_sep_from_period((M_3), M_4, orbital_period_quadruple)
+eccentricity3=0
+""",
+ samplerfunc=self.grid_options["Moe2017_options"][
+ "samplerfuncs"
+ ]["M"][3]
+ or "const({}, {}, {})".format(
+ self.grid_options["Moe2017_options"]["ranges"]["q"][0]
+ if self.grid_options["Moe2017_options"]
+ .get("ranges", {})
+ .get("q", None)
+ else "options['Mmin']/(M_3)",
+ self.grid_options["Moe2017_options"]["ranges"]["q"][1]
+ if self.grid_options["Moe2017_options"]
+ .get("ranges", {})
+ .get("q", None)
+ else "q3max",
+ self.grid_options["Moe2017_options"]["resolutions"]["M"][2],
+ ),
+ )
+
+ # (optional) triples: eccentricity
+ if (
+ self.grid_options["Moe2017_options"]["resolutions"]["ecc"][2]
+ > 0
+ ):
+ self.add_grid_variable(
+ name="ecc3",
+ parameter_name="eccentricity3",
+ longname="Eccentricity of the triple+quadruple/outer binary",
+ probdist=1,
+ gridtype="centred",
+ dphasevol="decc3",
+ precode="eccentricity3=ecc3",
+ valuerange=[
+ self.grid_options["Moe2017_options"]["ranges"]["ecc"][
+ 0
+ ], # Just fail if not defined.
+ self.grid_options["Moe2017_options"]["ranges"]["ecc"][
+ 1
+ ],
+ ],
+ samplerfunc=self.grid_options["Moe2017_options"][
+ "samplerfuncs"
+ ]["ecc"][2]
+ or "const({}, {}, {})".format(
+ self.grid_options["Moe2017_options"]["ranges"]["ecc"][
+ 0
+ ], # Just fail if not defined.
+ self.grid_options["Moe2017_options"]["ranges"]["ecc"][
+ 1
+ ],
+ self.grid_options["Moe2017_options"]["resolutions"][
+ "ecc"
+ ][2],
+ ),
+ )
+
+ # Now we are at the last part.
+ # Here we should combine all the information that we calculate and update the options
+ # dictionary. This will then be passed to the Moe_di_Stefano_2017_pdf to calculate
+ # the real probability. The trick we use is to strip the options_dict as a string
+ # and add some keys to it:
+
+ updated_options = "{{{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}}}".format(
+ str(dict(self.grid_options["Moe2017_options"]))[1:-1],
+ '"multiplicity": multiplicity',
+ '"M_1": M_1',
+ '"M_2": M_2',
+ '"M_3": M_3',
+ '"M_4": M_4',
+ '"P": orbital_period',
+ '"P2": orbital_period_triple',
+ '"P3": orbital_period_quadruple',
+ '"ecc": eccentricity',
+ '"ecc2": eccentricity2',
+ '"ecc3": eccentricity3',
+ )
+
+ probdist_addition = "Moe_di_Stefano_2017_pdf({}, verbosity=self.grid_options['verbosity'])['total_probdens']".format(
+ updated_options
+ )
+
+ # and finally the probability calculator
+ self.grid_options["_grid_variables"][self._last_grid_variable()][
+ "probdist"
+ ] = probdist_addition
+
+ verbose_print(
+ "\tMoe_di_Stefano_2017: Added final call to the pdf function",
+ self.grid_options["verbosity"],
+ _MOE2017_VERBOSITY_LEVEL,
+ )
+
+ # Signal that the MOE2017 grid has been set
+ self.grid_options["_set_Moe2017_grid"] = True
+
+ ################################################################################################
+ def Moe_di_Stefano_2017(self, options=None):
+ """
+ Function to handle setting the user input settings,
+ set up the data and load that into interpolators and
+ then set the distribution functions
+
+ Takes a dictionary as its only argument
+ """
+
+ default_options = {
+ "apply settings": True,
+ "setup grid": True,
+ "load data": True,
+ "clean cache": False,
+ "clean load flag": False,
+ "clean all": False,
+ }
+ if not options:
+ options = {}
+ options = update_dicts(default_options, options)
+
+ # clean cache?
+ if options["clean all"] or options["clean cache"]:
+ Moecache.clear()
+
+ if options["clean all"] or options["clean load flag"]:
+ self.grid_options["_loaded_Moe2017_data"] = False
+
+ # Set the user input
+ if options["apply settings"]:
+ self.set_moe_di_stefano_settings(options=options)
+
+ # Load the data
+ if options["load data"]:
+ self._load_moe_di_stefano_data()
+
+ # construct the grid here
+ if options["setup grid"]:
+ self._set_moe_di_stefano_distributions()
+
+ def _clean_interpolators(self):
+ """
+ Function to clean up the interpolators after a run
+
+ We look in the Moecache global variable for items that are interpolators.
+ Should be called by the general cleanup function AND the thread cleanup function
+ """
+
+ interpolator_keys = []
+ for key in Moecache.keys():
+ if isinstance(Moecache[key], py_rinterpolate.Rinterpolate):
+ interpolator_keys.append(key)
+
+ for key in interpolator_keys:
+ Moecache[key].destroy()
+ del Moecache[key]
+ gc.collect()
+
+ ##### Unsorted functions
+ def _calculate_multiplicity_fraction(self, system_dict):
+ """
+ Function to calculate multiplicity fraction
+
+ Makes use of the self.bse_options['multiplicity'] value. If its not set, it will raise an error
+
+ grid_options['multiplicity_fraction_function'] will be checked for the choice
+
+ TODO: add option to put a manual binary fraction in here (solve via negative numbers being the functions)
+ """
+
+ # Just return 1 if no option has been chosen
+ if self.grid_options["multiplicity_fraction_function"] in [0, "None"]:
+ verbose_print(
+ "_calculate_multiplicity_fraction: Chosen not to use any multiplicity fraction.",
+ self.grid_options["verbosity"],
+ 3,
+ )
+
+ return 1
+
+ # Raise an error if the multiplicity is not set
+ if not system_dict.get("multiplicity", None):
+ msg = "Multiplicity value has not been set. When using a specific multiplicity fraction function please set the multiplicity"
+ raise ValueError(msg)
+
+ # Go over the chosen options
+ if self.grid_options["multiplicity_fraction_function"] in [1, "Arenou2010"]:
+ # Arenou 2010 will be used
+ verbose_print(
+ "_calculate_multiplicity_fraction: Using Arenou 2010 to calculate multiplicity fractions",
+ self.grid_options["verbosity"],
+ 3,
+ )
+
+ binary_fraction = Arenou2010_binary_fraction(system_dict["M_1"])
+ multiplicity_fraction_dict = {
+ 1: 1 - binary_fraction,
+ 2: binary_fraction,
+ 3: 0,
+ 4: 0,
+ }
+
+ elif self.grid_options["multiplicity_fraction_function"] in [2, "Raghavan2010"]:
+ # Raghavan 2010 will be used
+ verbose_print(
+ "_calculate_multiplicity_fraction: Using Raghavan (2010) to calculate multiplicity fractions",
+ self.grid_options["verbosity"],
+ 3,
+ )
+
+ binary_fraction = raghavan2010_binary_fraction(system_dict["M_1"])
+ multiplicity_fraction_dict = {
+ 1: 1 - binary_fraction,
+ 2: binary_fraction,
+ 3: 0,
+ 4: 0,
+ }
+
+ elif self.grid_options["multiplicity_fraction_function"] in [3, "Moe2017"]:
+ # We need to check several things now here:
+
+ # First, are the options for the MOE2017 grid set? On start it is filled with the default settings
+ if not self.grid_options["Moe2017_options"]:
+ msg = "The MOE2017 options do not seem to be set properly. The value is {}".format(
+ self.grid_options["Moe2017_options"]
+ )
+ raise ValueError(msg)
+
+ # Second: is the Moecache filled.
+ if not Moecache:
+ verbose_print(
+ "_calculate_multiplicity_fraction: Moecache is empty. It needs to be filled with the data for the interpolators. Loading the data now",
+ self.grid_options["verbosity"],
+ 3,
+ )
+
+ # Load the data
+ self._load_moe_di_stefano_data()
+
+ # record the prev value
+ prev_M1_value_ms = self.grid_options["Moe2017_options"].get("M_1", None)
+
+ # Set value of M1 of the current system
+ self.grid_options["Moe2017_options"]["M_1"] = system_dict["M_1"]
+
+ # Calculate the multiplicity fraction
+ multiplicity_fraction_list = Moe_di_Stefano_2017_multiplicity_fractions(
+ self.grid_options["Moe2017_options"], self.grid_options["verbosity"]
+ )
+
+ # Turn into dict
+ multiplicity_fraction_dict = {
+ el + 1: multiplicity_fraction_list[el]
+ for el in range(len(multiplicity_fraction_list))
+ }
+
+ # Set the prev value back
+ self.grid_options["Moe2017_options"]["M_1"] = prev_M1_value_ms
+
+ # we don't know what to do next
+ else:
+ msg = "Chosen value for the multiplicity fraction function is not known."
+ raise ValueError(msg)
+
+ # To make sure we normalize the dictionary
+ multiplicity_fraction_dict = normalize_dict(
+ multiplicity_fraction_dict, verbosity=self.grid_options["verbosity"]
+ )
+
+ verbose_print(
+ "Multiplicity: {} multiplicity_fraction: {}".format(
+ system_dict["multiplicity"],
+ multiplicity_fraction_dict[system_dict["multiplicity"]],
+ ),
+ self.grid_options["verbosity"],
+ 3,
+ )
+
+ return multiplicity_fraction_dict[system_dict["multiplicity"]]
+
+ def get_moe_di_stefano_dataset(self, options, verbosity=0):
+ """
+ Function to get the default Moe and di Stefano dataset or accept a user input.
+
+ Returns a dict containing the (JSON) data.
+ """
+
+ json_data = None
+
+ if "JSON" in options:
+ # use the JSON data passed in
+ json_data = options["JSON"]
+
+ elif "file" in options:
+ # use the file passed in, if provided
+ if not os.path.isfile(options["file"]):
+ verbose_print(
+ "The provided 'file' Moe and de Stefano JSON file does not seem to exist at {}".format(
+ options["file"]
+ ),
+ verbosity,
+ 1,
+ )
+
+ raise ValueError
+ if not options["file"].endswith(".json"):
+ verbose_print(
+ "Provided filename is not a json file",
+ verbosity,
+ 1,
+ )
+
+ else:
+ # Read input data and Clean up the data if there are white spaces around the keys
+ with open(options["file"], "r",encoding='utf-8') as data_filehandle:
+ datafile_data = data_filehandle.read()
+ datafile_data = datafile_data.replace('" ', '"')
+ datafile_data = datafile_data.replace(' "', '"')
+ datafile_data = datafile_data.replace(' "', '"')
+ json_data = json.loads(datafile_data)
+
+ if not json_data:
+ # no JSON data or filename given, use the default 2017 dataset
+ verbose_print(
+ "Using the default Moe and de Stefano 2017 datafile",
+ verbosity,
+ 1,
+ )
+ json_data = copy.deepcopy(moe_di_stefano_2017_data.moe_di_stefano_2017_data)
+
+ return json_data
+
+ # Default options for the Moe & di Stefano grid
+ def Moe_di_Stefano_2017_default_options(self):
+ return {
+ # place holder for the JSON data to be used if a file
+ # isn't specified
+ "JSON": None,
+ # resolution data
+ "resolutions": {
+ "M": [
+ 20, # M1
+ 20, # M2 (i.e. q)
+ 0, # M3 currently unused
+ 0, # M4 currently unused
+ ],
+ "logP": [
+ 20, # P2 (binary period)
+ 0, # P3 (triple period) currently unused
+ 0, # P4 (quadruple period) currently unused
+ ],
+ "ecc": [
+ 10, # e (binary eccentricity)
+ 0, # e2 (triple eccentricity) currently unused
+ 0, # e3 (quadruple eccentricity) currently unused
+ ],
+ },
+ "samplerfuncs": {
+ "M": [None, None, None, None],
+ "logP": [None, None, None],
+ "ecc": [None, None, None],
+ },
+ "ranges": {
+ # stellar masses (Msun)
+ "M": [
+ self.minimum_stellar_mass()
+ * 1.05, # 0.08 is a tad bit above the minimum mass. Don't sample at 0.07, otherwise the first row of q values will have a phasevol of 0. Anything higher is fine.
+ 80.0, # (rather arbitrary) upper mass cutoff
+ ],
+ "q": [
+ None, # artificial qmin : set to None to use default
+ None, # artificial qmax : set to None to use default
+ ],
+ "logP": [0.0, 8.0], # 0 = log10(1 day) # 8 = log10(10^8 days)
+ "ecc": [0.0, 0.99],
+ },
+ # minimum stellar mass
+ "Mmin": self.minimum_stellar_mass(), # We take the value that binary_c has set as the default
+ # multiplicity model (as a function of log10M1)
+ #
+ # You can use 'Poisson' which uses the system multiplicity
+ # given by Moe and maps this to single/binary/triple/quad
+ # fractions.
+ #
+ # Alternatively, 'data' takes the fractions directly
+ # from the data, but then triples and quadruples are
+ # combined (and there are NO quadruples).
+ "multiplicity_model": "Poisson",
+ # multiplicity modulator:
+ # [single, binary, triple, quadruple]
+ #
+ # e.g. [1,0,0,0] for single stars only
+ # [0,1,0,0] for binary stars only
+ #
+ # defaults to [1,1,0,0] i.e. all types
+ #
+ "multiplicity_modulator": [
+ 1, # single
+ 1, # binary
+ 0, # triple
+ 0, # quadruple
+ ],
+ # given a mix of multiplicities, you can either (noting that
+ # here (S,B,T,Q) = appropriate modulator * model(S,B,T,Q) )
+ #
+ # 'norm' : normalise so the whole population is 1.0
+ # after implementing the appropriate fractions
+ # S/(S+B+T+Q), B/(S+B+T+Q), T/(S+B+T+Q), Q/(S+B+T+Q)
+ #
+ # 'raw' : stick to what is predicted, i.e.
+ # S/(S+B+T+Q), B/(S+B+T+Q), T/(S+B+T+Q), Q/(S+B+T+Q)
+ # without normalisation
+ # (in which case the total probability < 1.0 unless
+ # all you use single, binary, triple and quadruple)
+ #
+ # 'merge' : e.g. if you only have single and binary,
+ # add the triples and quadruples to the binaries, so
+ # binaries represent all multiple systems
+ # ...
+ # *** this is canonical binary population synthesis ***
+ #
+ # Note: if multiplicity_modulator == [1,1,1,1] this
+ # option does nothing (equivalent to 'raw').
+ #
+ #
+ # note: if you only set one multiplicity_modulator
+ # to 1, and all the others to 0, then normalising
+ # will mean that you effectively have the same number
+ # of stars as single, binary, triple or quad (whichever
+ # is non-zero) i.e. the multiplicity fraction is ignored.
+ # This is probably not useful except for
+ # testing purposes or comparing to old grids.
+ "normalize_multiplicities": "merge",
+ # q extrapolation (below 0.15 and above 0.9) method. We can choose from ['flat', 'linear', 'plaw2', 'nolowq']
+ "q_low_extrapolation_method": "linear",
+ "q_high_extrapolation_method": "linear",
+ }
+
+ def moe_di_stefano_default_options_description(self):
+ return {
+ "resolutions": "",
+ "ranges": "",
+ "Mmin": "Minimum stellar mass",
+ "multiplicity_model": """
+ multiplicity model (as a function of log10M1)
+
+ You can use 'Poisson' which uses the system multiplicity
+ given by Moe and maps this to single/binary/triple/quad
+ fractions.
+
+ Alternatively, 'data' takes the fractions directly
+ from the data, but then triples and quadruples are
+ combined (and there are NO quadruples).
+ """,
+ "multiplicity_modulator": """
+ [single, binary, triple, quadruple]
+
+ e.g. [1,0,0,0] for single stars only
+ [0,1,0,0] for binary stars only
+
+ defaults to [1,1,0,0] i.e. singles and binaries
+ """,
+ "normalize_multiplicities": """
+ 'norm': normalise so the whole population is 1.0
+ after implementing the appropriate fractions
+ S/(S+B+T+Q), B/(S+B+T+Q), T/(S+B+T+Q), Q/(S+B+T+Q)
+ given a mix of multiplicities, you can either (noting that
+ here (S,B,T,Q) = appropriate modulator * model(S,B,T,Q) )
+ note: if you only set one multiplicity_modulator
+ to 1, and all the others to 0, then normalising
+ will mean that you effectively have the same number
+ of stars as single, binary, triple or quad (whichever
+ is non-zero) i.e. the multiplicity fraction is ignored.
+ This is probably not useful except for
+ testing purposes or comparing to old grids.
+
+ 'raw' : stick to what is predicted, i.e.
+ S/(S+B+T+Q), B/(S+B+T+Q), T/(S+B+T+Q), Q/(S+B+T+Q)
+ without normalisation
+ (in which case the total probability < 1.0 unless
+ all you use single, binary, triple and quadruple)
+
+ 'merge' : e.g. if you only have single and binary,
+ add the triples and quadruples to the binaries, so
+ binaries represent all multiple systems
+ ...
+ *** this is canonical binary population synthesis ***
+
+ It only takes the maximum multiplicity into account,
+ i.e. it doesn't multiply the resulting array by the multiplicity modulator again.
+ This prevents the resulting array to always be 1 if only 1 multiplicity modulator element is nonzero
+
+ Note: if multiplicity_modulator == [1,1,1,1]. this option does nothing (equivalent to 'raw').
+ """,
+ "q_low_extrapolation_method": """
+ q extrapolation (below 0.15) method
+ none
+ flat
+ linear2
+ plaw2
+ nolowq
+ """,
+ "q_high_extrapolation_method": "Same as q_low_extrapolation_method",
+ }
diff --git a/binarycpython/utils/analytics.py b/binarycpython/utils/analytics.py
new file mode 100644
index 000000000..4e421d1a3
--- /dev/null
+++ b/binarycpython/utils/analytics.py
@@ -0,0 +1,71 @@
+import time
+
+class analytics():
+ def __init__(self, **kwargs):
+ return
+
+ #######################
+ # time used functions
+ #######################
+
+ def make_analytics_dict(self):
+ print("Do analytics")
+
+ analytics_dict = {}
+
+ if self.grid_options['do_analytics']:
+ # Put all interesting stuff in a variable and output that afterwards, as analytics of the run.
+ analytics_dict = {
+ "population_id": self.grid_options["_population_id"],
+ "evolution_type": self.grid_options["evolution_type"],
+ "failed_count": self.grid_options["_failed_count"],
+ "failed_prob": self.grid_options["_failed_prob"],
+ "failed_systems_error_codes": self.grid_options[
+ "_failed_systems_error_codes"
+ ].copy(),
+ "errors_exceeded": self.grid_options["_errors_exceeded"],
+ "errors_found": self.grid_options["_errors_found"],
+ "total_probability": self.grid_options["_probtot"],
+ "total_count": self.grid_options["_count"],
+ "start_timestamp": self.grid_options["_start_time_evolution"],
+ "end_timestamp": self.grid_options["_end_time_evolution"],
+ "time_elapsed" : self.time_elapsed(),
+ "total_mass_run": self.grid_options["_total_mass_run"],
+ "total_probability_weighted_mass_run": self.grid_options[
+ "_total_probability_weighted_mass_run"
+ ],
+ "zero_prob_stars_skipped": self.grid_options["_zero_prob_stars_skipped"],
+ }
+
+ if "metadata" in self.grid_ensemble_results:
+ # Add analytics dict to the metadata too:
+ self.grid_ensemble_results["metadata"].update(analytics_dict)
+ self.add_system_metadata()
+ else:
+ # use existing analytics dict
+ try:
+ analytics_dict = self.grid_ensemble_results["metadata"]
+ except:
+ analytics_dict = {} # should never happen
+
+ return analytics_dict
+
+ def time_elapsed(self):
+ """
+ Function to return how long a population object has been running.
+ """
+ for x in ["_start_time_evolution","_end_time_evolution"]:
+ if not self.grid_options[x]:
+ self.grid_options[x] = time.time()
+ return self.grid_options["_end_time_evolution"] - self.grid_options["_start_time_evolution"]
+
+ def CPU_time(self):
+ """
+ Function to return how much CPU time we've used
+ """
+ dt = self.time_elapsed()
+ try:
+ ncpus = self.grid_options['num_processes']
+ except:
+ ncpus = 1
+ return dt * ncpus
diff --git a/binarycpython/utils/condor.py b/binarycpython/utils/condor.py
index e69de29bb..6a9618f83 100644
--- a/binarycpython/utils/condor.py
+++ b/binarycpython/utils/condor.py
@@ -0,0 +1,259 @@
+"""
+ Binary_c-python's condor functions
+"""
+from binarycpython.utils.HPC import HPC
+
+class condor(HPC):
+
+ def __init__(self, **kwargs):
+ # don't do anything: we just inherit from this class
+ return
+ ###################################################
+ # CONDOR functions
+ #
+ # subroutines to run CONDOR grids
+ ###################################################
+
+ # def _condor_grid(self):
+ # """
+ # Main function that manages the CONDOR setup.
+
+ # Has three stages:
+
+ # - setup
+ # - evolve
+ # - join
+
+ # Which stage is used is determined by the value of grid_options['condor_command']:
+
+ # <empty>: the function will know its the user that executed the script and
+ # it will set up the necessary condor stuff
+
+ # 'evolve': evolve_population is called to evolve the population of stars
+
+ # 'join': We will attempt to join the output
+ # """
+
+ # # TODO: Put in function
+ # condor_version = get_condor_version()
+ # if not condor_version:
+ # verbose_print(
+ # "CONDOR: Error: No installation of condor found",
+ # self.grid_options["verbosity"],
+ # 0,
+ # )
+ # else:
+ # major_version = int(condor_version.split(".")[0])
+ # minor_version = int(condor_version.split(".")[1])
+
+ # if (major_version == 8) and (minor_version > 4):
+ # verbose_print(
+ # "CONDOR: Found version {} which is new enough".format(
+ # condor_version
+ # ),
+ # self.grid_options["verbosity"],
+ # 0,
+ # )
+ # elif major_version > 9:
+ # verbose_print(
+ # "CONDOR: Found version {} which is new enough".format(
+ # condor_version
+ # ),
+ # self.grid_options["verbosity"],
+ # 0,
+ # )
+ # else:
+ # verbose_print(
+ # "CONDOR: Found version {} which is too old (we require 8.3/8.4+)".format(
+ # condor_version
+ # ),
+ # self.grid_options["verbosity"],
+ # 0,
+ # )
+
+ # verbose_print(
+ # "Running Condor grid. command={}".format(
+ # self.grid_options["condor_command"]
+ # ),
+ # self.grid_options["verbosity"],
+ # 1,
+ # )
+ # if not self.grid_options["condor_command"]:
+ # # Setting up
+ # verbose_print(
+ # "CONDOR: Main controller script. Setting up",
+ # self.grid_options["verbosity"],
+ # 1,
+ # )
+
+ # # Set up working directories:
+ # verbose_print(
+ # "CONDOR: creating working directories",
+ # self.grid_options["verbosity"],
+ # 1,
+ # )
+ # create_directories_hpc(self.grid_options["condor_dir"])
+
+ # # Create command
+ # current_workingdir = os.getcwd()
+ # python_details = get_python_details()
+ # scriptname = path_of_calling_script()
+ # # command = "".join([
+ # # "{}".python_details['executable'],
+ # # "{}".scriptname,
+ # # "offset=$jobarrayindex",
+ # # "modulo={}".format(self.grid_options['condor_njobs']),
+ # # "vb={}".format(self.grid_options['verbosity'])
+
+ # # "results_hash_dumpfile=$self->{_grid_options}{slurm_dir}/results/$jobid.$jobarrayindex",
+ # # 'slurm_jobid='.$jobid,
+ # # 'slurm_jobarrayindex='.$jobarrayindex,
+ # # 'slurm_jobname=binary_grid_'.$jobid.'.'.$jobarrayindex,
+ # # "slurm_njobs=$njobs",
+ # # "slurm_dir=$self->{_grid_options}{slurm_dir}",
+ # # );
+
+ # # Create directory with info for the condor script. By creating this directory we also check whether all the values are set correctly
+ # # TODO: create the condor script.
+ # condor_script_options = {}
+ # # condor_script_options['n'] =
+ # condor_script_options["njobs"] = self.grid_options["condor_njobs"]
+ # condor_script_options["dir"] = self.grid_options["condor_dir"]
+ # condor_script_options["memory"] = self.grid_options["condor_memory"]
+ # condor_script_options["working_dir"] = self.grid_options[
+ # "condor_working_dir"
+ # ]
+ # condor_script_options["command"] = self.grid_options["command"]
+ # condor_script_options["streams"] = self.grid_options["streams"]
+
+ # # TODO: condor works with running an executable.
+
+ # # Create script contents
+ # condor_script_contents = ""
+ # condor_script_contents += """
+ # #################################################
+ # #
+ # # Condor script to run a binary_grid via python
+ # #
+ # #################################################
+ # """
+ # condor_script_contents += "Executable\t= {}".format(executable)
+ # condor_script_contents += "arguments\t= {}".format(arguments)
+ # condor_script_contents += "environment\t= {}".format(environment)
+ # condor_script_contents += "universe\t= {}".format(
+ # self.grid_options["condor_universe"]
+ # )
+ # condor_script_contents += "\n"
+ # condor_script_contents += "output\t= {}/stdout/$id\n".format(
+ # self.grid_options["condor_dir"]
+ # )
+ # condor_script_contents += "error\t={}/sterr/$id".format(
+ # self.grid_options["condor_dir"]
+ # )
+ # condor_script_contents += "log\t={}\n".format(
+ # self.grid_options["condor_dir"]
+ # )
+ # condor_script_contents += "initialdir\t={}\n".format(current_workingdir)
+ # condor_script_contents += "remote_initialdir\t={}\n".format(
+ # current_workingdir
+ # )
+ # condor_script_contents += "\n"
+ # condor_script_contents += "steam_output\t={}".format(stream)
+ # condor_script_contents += "steam_error\t={}".format(stream)
+ # condor_script_contents += "+WantCheckpoint = False"
+ # condor_script_contents += "\n"
+ # condor_script_contents += "request_memory\t={}".format(
+ # self.grid_options["condor_memory"]
+ # )
+ # condor_script_contents += "ImageSize\t={}".format(
+ # self.grid_options["condor_memory"]
+ # )
+ # condor_script_contents += "\n"
+
+ # if self.grid_options["condor_extra_settings"]:
+ # slurm_script_contents += "# Extra settings by user:"
+ # slurm_script_contents += "\n".join(
+ # [
+ # "{}\t={}".format(
+ # key, self.grid_options["condor_extra_settings"][key]
+ # )
+ # for key in self.grid_options["condor_extra_settings"]
+ # ]
+ # )
+
+ # condor_script_contents += "\n"
+
+ # # request_memory = $_[0]{memory}
+ # # ImageSize = $_[0]{memory}
+
+ # # Requirements = (1) \&\& (".
+ # # $self->{_grid_options}{condor_requirements}.")\n";
+
+ # #
+ # # file name: my_program.condor
+ # # Condor submit description file for my_program
+ # # Executable = my_program
+ # # Universe = vanilla
+ # # Error = logs/err.$(cluster)
+ # # Output = logs/out.$(cluster)
+ # # Log = logs/log.$(cluster)
+
+ # # should_transfer_files = YES
+ # # when_to_transfer_output = ON_EXIT
+ # # transfer_input_files = files/in1,files/in2
+
+ # # Arguments = files/in1 files/in2 files/out1
+ # # Queue
+
+ # # Write script contents to file
+ # if self.grid_options["condor_postpone_join"]:
+ # condor_script_contents += "{} rungrid=0 results_hash_dumpfile={}/results/$jobid.all condor_command=join\n".format(
+ # command, self.grid_options["condor_dir"]
+ # )
+
+ # condor_script_filename = os.path.join(
+ # self.grid_options["condor_dir"], "condor_script"
+ # )
+ # with open(condor_script_filename, "w") as condor_script_file:
+ # condor_script_file.write(condor_script_contents)
+
+ # if self.grid_options["condor_postpone_sbatch"]:
+ # # Execute or postpone the real call to sbatch
+ # submit_command = "condor_submit {}".format(condor_script_filename)
+ # verbose_print(
+ # "running condor script {}".format(condor_script_filename),
+ # self.grid_options["verbosity"],
+ # 0,
+ # )
+ # # subprocess.Popen(sbatch_command, close_fds=True)
+ # # subprocess.Popen(sbatch_command, creationflags=subprocess.DETACHED_PROCESS)
+ # verbose_print("Submitted scripts.", self.grid_options["verbosity"], 0)
+ # else:
+ # verbose_print(
+ # "Condor script is in {} but hasnt been executed".format(
+ # condor_script_filename
+ # ),
+ # self.grid_options["verbosity"],
+ # 0,
+ # )
+
+ # verbose_print("all done!", self.grid_options["verbosity"], 0)
+ # self.exit()
+
+ # elif self.grid_options["condor_command"] == "evolve":
+ # # TODO: write this function
+ # # Part to evolve the population.
+ # # TODO: decide how many CPUs
+ # verbose_print(
+ # "CONDOR: Evolving population", self.grid_options["verbosity"], 1
+ # )
+
+ # #
+ # self.evolve_population()
+
+ # elif self.grid_options["condor_command"] == "join":
+ # # TODO: write this function
+ # # Joining the output.
+ # verbose_print("CONDOR: Joining results", self.grid_options["verbosity"], 1)
+
+ # pass
diff --git a/binarycpython/utils/dataIO.py b/binarycpython/utils/dataIO.py
new file mode 100644
index 000000000..fe6cbf065
--- /dev/null
+++ b/binarycpython/utils/dataIO.py
@@ -0,0 +1,488 @@
+"""
+ Binary_c-python's data input-output (IO) functions
+"""
+
+import bz2
+import compress_pickle
+import datetime
+import gzip
+import json
+import msgpack
+import os
+import pathlib
+from typing import Union, Any
+
+from binarycpython.utils.ensemble import (
+ binaryc_json_serializer,
+ ensemble_compression,
+ ensemble_file_type,
+ extract_ensemble_json_from_string,
+ format_ensemble_results,
+)
+from binarycpython.utils.dicts import (
+ merge_dicts,
+)
+
+class dataIO():
+
+ def __init__(self, **kwargs):
+ # don't do anything: we just inherit from this class
+ return
+
+ def dir_ok(self,dir):
+ """
+ Function to test if we can read and write to a dir that must exist. Return True if all is ok, False otherwise.
+ """
+ return os.access(dir, os.F_OK) and os.access(dir, os.R_OK | os.W_OK)
+
+ def save_population_object(self,object=None,filename=None,confirmation=True,compression='gzip'):
+ """
+ Save pickled Population object to file at filename or, if filename is None, whatever is set at self.grid_options['save_population_object']
+
+ Args:
+ object : the object to be saved to the file. If object is None, use self.
+ filename : the name of the file to be saved. If not set, use self.grid_options['save_population_object']
+ confirmation : if True, a file "filename.saved" is touched just after the dump, so we know it is finished.
+
+ Compression is performed according to the filename, as stated in the
+ compress_pickle documentation at
+ https://lucianopaz.github.io/compress_pickle/html/
+
+ Shared memory, stored in the object.shared_memory dict, is not saved.
+
+ """
+ if object is None:
+ # default to using self
+ object = self
+
+ if filename is None:
+ # get filename from self
+ filename = self.grid_options['save_population_object']
+
+ if filename:
+
+ print("Save population {id}, probtot {probtot} to pickle in {filename}".format(
+ id=self.grid_options["_population_id"],
+ probtot=object.grid_options['_probtot'],
+ filename=filename))
+
+
+ # Some parts of the object cannot be pickled:
+ # remove them, and restore them after pickling
+
+ # remove shared memory
+ shared_memory = object.shared_memory
+ object.shared_memory = None
+
+ # delete system generator
+ system_generator = object.grid_options["_system_generator"]
+ object.grid_options["_system_generator"] = None
+
+ # delete _store_memaddr
+ _store_memaddr = object.grid_options['_store_memaddr']
+ object.grid_options['_store_memaddr'] = None
+
+ # delete persistent_data_memory_dict
+ persistent_data_memory_dict = object.persistent_data_memory_dict
+ object.persistent_data_memory_dict = None
+
+ # add metadata if it doesn't exist
+ if not "metadata" in object.grid_ensemble_results:
+ object.grid_ensemble_results["metadata"] = {}
+
+ # add datestamp
+ object.grid_ensemble_results["metadata"]['save_population_time'] = datetime.datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+
+ # add extra metadata
+ object.add_system_metadata()
+
+ # add max memory use
+ try:
+ self.grid_ensemble_results["metadata"]['max_memory_use'] = copy.deepcopy(sum(shared_memory["max_memory_use_per_thread"]))
+ except Exception as e:
+ print("save_population_object : Error: ",e)
+ pass
+
+ # dump pickle file
+ compress_pickle.dump(object,
+ filename,
+ pickler_method='dill')
+
+ # restore data
+ object.shared_memory = shared_memory
+ object.grid_options["_system_generator"] = system_generator
+ del object.grid_ensemble_results["metadata"]['save_population_time']
+ object.grid_options['store_memaddr'] = _store_memaddr
+ object.persistent_data_memory_dict = persistent_data_memory_dict
+
+ # touch 'saved' file
+ pathlib.Path(filename + '.saved').touch(exist_ok=True)
+
+ return
+
+ def load_population_object(self,filename):
+ """
+ returns the Population object loaded from filename
+ """
+ if filename is None:
+ obj = None
+ else:
+ try:
+ obj = compress_pickle.load(filename,
+ pickler_method='dill')
+ except Exception as e:
+ obj = None
+
+ return obj
+
+ def merge_populations(self,refpop,newpop):
+ """
+ merge newpop's results data into refpop's results data
+
+ Args:
+ refpop : the original "reference" Population object to be added to
+ newpop : Population object containing the new data
+
+ Returns:
+ nothing
+
+ Note:
+ The file should be saved using save_population_object()
+ """
+
+ # combine data
+ try:
+ refpop.grid_results = merge_dicts(refpop.grid_results,
+ newpop.grid_results)
+ except Exception as e:
+ print("Error merging grid_results:",e)
+
+ # special cases
+ try:
+ maxmem = max(refpop.grid_ensemble_results["metadata"]['max_memory_use'],
+ newpop.grid_ensemble_results["metadata"]['max_memory_use'])
+ except:
+ maxmem = 0
+
+ try:
+ # special cases:
+ # copy the settings and Xinit: these should just be overridden
+ try:
+ settings = copy.deepcopy(newpop.grid_ensemble_results["metadata"]['settings'])
+ except:
+ settings = None
+ try:
+ Xinit = copy.deepcopy(newpop.grid_ensemble_results["ensemble"]["Xinit"])
+ except:
+ Xinit = None
+
+ # merge the ensemble dicts
+ refpop.grid_ensemble_results = merge_dicts(refpop.grid_ensemble_results,
+ newpop.grid_ensemble_results)
+
+ # set special cases
+ try:
+ refpop.grid_ensemble_results["metadata"]['max_memory_use'] = maxmem
+ if settings:
+ refpop.grid_ensemble_results["metadata"]['settings'] = settings
+ if Xinit:
+ refpop.grid_ensemble_results["ensemble"]["Xinit"] = Xinit
+ except:
+ pass
+
+ except Exception as e:
+ print("Error merging grid_ensemble_results:",e)
+
+ for key in ["_probtot"]:
+ refpop.grid_options[key] += newpop.grid_options[key]
+
+ refpop.grid_options['_killed'] |= newpop.grid_options['_killed']
+
+ return
+
+ def merge_populations_from_file(self,refpop,filename):
+ """
+ Wrapper for merge_populations so it can be done directly
+ from a file.
+
+ Args:
+ refpop : the original "reference" Population object to be added to
+ filename : file containing the Population object containing the new data
+
+ Note:
+ The file should be saved using save_population_object()
+ """
+
+ newpop = self.load_population_object(filename)
+
+
+ # merge with refpop
+ try:
+ self.merge_populations(refpop,
+ newpop)
+ except Exception as e:
+ print("merge_populations gave exception",e)
+
+ return
+
+ def snapshot_filename(self):
+ """
+ Automatically choose the snapshot filename.
+ """
+ if self.grid_options['slurm'] > 0:
+ file = os.path.join(self.grid_options['slurm_dir'],
+ 'snapshots',
+ self.jobID() + '.gz')
+ else:
+ file = os.path.join(self.grid_options['tmp_dir'],
+ 'snapshot.gz')
+ return file
+
+ def load_snapshot(self,file):
+ """
+ Load a snapshot from file and set it in the preloaded_population placeholder.
+ """
+ newpop = self.load_population_object(file)
+ self.preloaded_population = newpop
+ self.grid_options['start_at'] = newpop.grid_options['start_at']
+ print("Loaded from snapshot at {file} : start at star {n}".format(
+ file=file,
+ n=self.grid_options['start_at']))
+ return
+
+ def save_snapshot(self,file=None):
+ """
+ Save the population object to a snapshot file, automatically choosing the filename if none is given.
+ """
+
+ if file == None:
+ file = self.snapshot_filename()
+ try:
+ n = self.grid_options['_count']
+ except:
+ n = '?'
+
+ print("Saving snapshot containing {} stars to {}".format(n,file))
+ self.save_population_object(object=self,
+ filename=file)
+
+ return
+
+ def write_ensemble(self, output_file, data=None, sort_keys=True, indent=4, encoding='utf-8', ensure_ascii=False):
+ """
+ write_ensemble : Write ensemble results to a file.
+
+ Args:
+ output_file : the output filename.
+
+ If the filename has an extension that we recognise,
+ e.g. .gz or .bz2, we compress the output appropriately.
+
+ The filename should contain .json or .msgpack, the two
+ currently-supported formats.
+
+ Usually you'll want to output to JSON, but we can
+ also output to msgpack.
+
+ data : the data dictionary to be converted and written to the file.
+ If not set, this defaults to self.grid_ensemble_results.
+
+ sort_keys : if True, and output is to JSON, the keys will be sorted.
+ (default: True, passed to json.dumps)
+
+ indent : number of space characters used in the JSON indent. (Default: 4,
+ passed to json.dumps)
+
+ encoding : file encoding method, usually defaults to 'utf-8'
+
+ ensure_ascii : the ensure_ascii flag passed to json.dump and/or json.dumps
+ (Default: False)
+ """
+
+ # get the file type
+ file_type = ensemble_file_type(output_file)
+
+ # choose compression algorithm based on file extension
+ compression = ensemble_compression(output_file)
+
+ # default to using grid_ensemble_results if no data is given
+ if data is None:
+ data = self.grid_ensemble_results
+
+ if not file_type:
+ print(
+ "Unable to determine file type from ensemble filename {} : it should be .json or .msgpack."
+ ).format(output_file)
+ self.exit(code=1)
+ elif file_type == "JSON":
+ # JSON output
+ if compression == "gzip":
+ # gzip
+ f = gzip.open(output_file, "wt", encoding=encoding)
+ elif compression == "bzip2":
+ # bzip2
+ f = bz2.open(output_file, "wt", encoding=encoding)
+ else:
+ # raw output (not compressed)
+ f = open(output_file, "wt", encoding=encoding)
+ f.write(json.dumps(data,
+ sort_keys=sort_keys,
+ indent=indent,
+ ensure_ascii=ensure_ascii))
+
+ elif file_type == "msgpack":
+ # msgpack output
+ if compression == "gzip":
+ f = gzip.open(output_file, "wb", encoding=encoding)
+ elif compression == "bzip2":
+ f = bz2.open(output_file, "wb", encoding=encoding)
+ else:
+ f = open(output_file, "wb", encoding=encoding)
+ msgpack.dump(data, f)
+ f.close()
+
+ print(
+ "Thread {thread}: Wrote ensemble results to file: {colour}{file}{reset} (file type {file_type}, compression {compression})".format(
+ thread=self.process_ID,
+ file=output_file,
+ colour=self.ANSI_colours["green"],
+ reset=self.ANSI_colours["reset"],
+ file_type=file_type,
+ compression=compression,
+ )
+ )
+ def write_binary_c_calls_to_file(
+ self,
+ output_dir: Union[str, None] = None,
+ output_filename: Union[str, None] = None,
+ include_defaults: bool = False,
+ encoding='utf-8'
+ ) -> None:
+ """
+ Function that loops over the grid code and writes the generated parameters to a file.
+ In the form of a command line call
+
+ Only useful when you have a variable grid as system_generator. MC wouldn't be that useful
+
+ Also, make sure that in this export there are the basic parameters
+ like m1,m2,sep, orb-per, ecc, probability etc.
+
+ On default this will write to the datadir, if it exists
+
+ Tasks:
+ - TODO: test this function
+ - TODO: make sure the binary_c_python .. output file has a unique name
+
+ Args:
+ output_dir: (optional, default = None) directory where to write the file to. If custom_options['data_dir'] is present, then that one will be used first, and then the output_dir
+ output_filename: (optional, default = None) filename of the output. If not set it will be called "binary_c_calls.txt"
+ include_defaults: (optional, default = None) whether to include the defaults of binary_c in the lines that are written. Beware that this will result in very long lines, and it might be better to just export the binary_c defaults and keep them in a separate file.
+
+ Returns:
+ filename: filename that was used to write the calls to
+ """
+
+ # Check if there is no compiled grid yet. If not, lets try to build it first.
+ if not self.grid_options["_system_generator"]:
+
+ ## check the settings:
+ if self.bse_options.get("ensemble", None):
+ if self.bse_options["ensemble"] == 1:
+ if not self.bse_options.get("ensemble_defer", 0) == 1:
+ verbose_print(
+ "Error, if you want to run an ensemble in a population, the output needs to be deferred",
+ self.grid_options["verbosity"],
+ 0,
+ )
+ raise ValueError
+
+ # Put in check
+ if len(self.grid_options["_grid_variables"]) == 0:
+ print("Error: you haven't defined any grid variables! Aborting")
+ raise ValueError
+
+ #
+ self._generate_grid_code(dry_run=False)
+
+ #
+ self._load_grid_function()
+
+ # then if the _system_generator is present, we go through it
+ if self.grid_options["_system_generator"]:
+ # Check if there is an output dir configured
+ if self.custom_options.get("data_dir", None):
+ binary_c_calls_output_dir = self.custom_options["data_dir"]
+ # otherwise check if there's one passed to the function
+ else:
+ if not output_dir:
+ print(
+ "Error. No data_dir configured and you gave no output_dir. Aborting"
+ )
+ raise ValueError
+ binary_c_calls_output_dir = output_dir
+
+ # check if there's a filename passed to the function
+ if output_filename:
+ binary_c_calls_filename = output_filename
+ # otherwise use default value
+ else:
+ binary_c_calls_filename = "binary_c_calls.txt"
+
+ binary_c_calls_full_filename = os.path.join(
+ binary_c_calls_output_dir, binary_c_calls_filename
+ )
+ print("Writing binary_c calls to {}".format(binary_c_calls_full_filename))
+
+ # Write to file
+ with open(binary_c_calls_full_filename, "w", encoding=encoding) as file:
+ # Get defaults and clean them, then overwrite them with the set values.
+ if include_defaults:
+ # TODO: make sure that the defaults here are cleaned up properly
+ cleaned_up_defaults = self.cleaned_up_defaults
+ full_system_dict = cleaned_up_defaults.copy()
+ full_system_dict.update(self.bse_options.copy())
+ else:
+ full_system_dict = self.bse_options.copy()
+
+ for system in self.grid_options["_system_generator"](self):
+ # update values with current system values
+ full_system_dict.update(system)
+
+ binary_cmdline_string = self._return_argline(full_system_dict)
+ file.write(binary_cmdline_string + "\n")
+ else:
+ print("Error. No grid function found!")
+ raise ValueError
+
+ return binary_c_calls_full_filename
+
+ def set_status(self,
+ string,
+ format_statment="process_{}.txt",
+ ID=None,
+ slurm=True,
+ condor=True):
+ """
+ function to set the status string in its appropriate file
+ """
+
+ if ID is None:
+ ID = self.process_ID
+
+ if self.grid_options['status_dir']:
+ with open(
+ os.path.join(
+ self.grid_options["status_dir"],
+ format_statment.format(ID),
+ ),
+ "w",
+ encoding='utf-8'
+ ) as f:
+ f.write(string)
+ f.close()
+
+ # custom logging functions
+ if slurm and self.grid_options['slurm'] >= 1:
+ self.set_slurm_status(string)
+# if self.grid_options['condor']==1:
+# self.set_condor_status(string)
diff --git a/binarycpython/utils/distribution_functions.py b/binarycpython/utils/distribution_functions.py
index 146d45f1e..f88492d1c 100644
--- a/binarycpython/utils/distribution_functions.py
+++ b/binarycpython/utils/distribution_functions.py
@@ -36,7 +36,9 @@ from binarycpython.utils.functions import verbose_print
from binarycpython.utils.grid_options_defaults import (
_MOE2017_VERBOSITY_LEVEL,
_MOE2017_VERBOSITY_INTERPOLATOR_LEVEL,
-)
+ _MOE2017_VERBOSITY_INTERPOLATOR_EXTRA_LEVEL,
+ )
+
###
# File containing probability distributions
@@ -975,14 +977,6 @@ def poisson(lambda_val, n, nmax=None, verbosity=0):
if distribution_constants["poisson_cache"].get(cachekey, None):
p_val = distribution_constants["poisson_cache"][cachekey]
- verbose_print(
- "\tMoe and di Stefano 2017: found cached value for poisson({}, {}, {}): {}".format(
- lambda_val, n, nmax, p_val
- ),
- verbosity,
- _MOE2017_VERBOSITY_LEVEL,
- )
-
return p_val
# Poisson distribution : note, n can be zero
@@ -1002,13 +996,6 @@ def poisson(lambda_val, n, nmax=None, verbosity=0):
distribution_constants["poisson_cache"] = {}
distribution_constants["poisson_cache"][cachekey] = p_val
- verbose_print(
- "\tMoe and di Stefano 2017: Poisson({}, {}, {}): {}".format(
- lambda_val, n, nmax, p_val
- ),
- verbosity,
- _MOE2017_VERBOSITY_LEVEL,
- )
return p_val
diff --git a/binarycpython/utils/ensemble.py b/binarycpython/utils/ensemble.py
index b61ff253d..a1d85aa9f 100644
--- a/binarycpython/utils/ensemble.py
+++ b/binarycpython/utils/ensemble.py
@@ -30,7 +30,6 @@ from binarycpython.utils.dicts import (
)
from binarycpython.utils.functions import verbose_print
-
def ensemble_setting(ensemble, parameter_name):
"""
Function to get the setting of parameter_name in the given ensemble, or return the default value.
diff --git a/binarycpython/utils/functions.py b/binarycpython/utils/functions.py
index 586a45211..75ff3df4d 100644
--- a/binarycpython/utils/functions.py
+++ b/binarycpython/utils/functions.py
@@ -41,7 +41,6 @@ import simplejson
# import orjson
import astropy.units as u
-import binarycpython.utils.moe_di_stefano_2017_data as moe_di_stefano_2017_data
from binarycpython import _binary_c_bindings
from binarycpython.utils.dicts import filter_dict, filter_dict_through_values
@@ -221,58 +220,6 @@ def get_size(obj, seen=None):
return size
-def get_moe_di_stefano_dataset(options, verbosity=0):
- """
- Function to get the default Moe and di Stefano dataset or accept a user input.
-
- Returns a dict containing the (JSON) data.
- """
-
- json_data = None
-
- if "JSON" in options:
- # use the JSON data passed in
- json_data = options["JSON"]
-
- elif "file" in options:
- # use the file passed in, if provided
- if not os.path.isfile(options["file"]):
- verbose_print(
- "The provided 'file' Moe and de Stefano JSON file does not seem to exist at {}".format(
- options["file"]
- ),
- verbosity,
- 1,
- )
-
- raise ValueError
- if not options["file"].endswith(".json"):
- verbose_print(
- "Provided filename is not a json file",
- verbosity,
- 1,
- )
-
- else:
- # Read input data and Clean up the data if there are white spaces around the keys
- with open(options["file"], "r",encoding='utf-8') as data_filehandle:
- datafile_data = data_filehandle.read()
- datafile_data = datafile_data.replace('" ', '"')
- datafile_data = datafile_data.replace(' "', '"')
- datafile_data = datafile_data.replace(' "', '"')
- json_data = json.loads(datafile_data)
-
- if not json_data:
- # no JSON data or filename given, use the default 2017 dataset
- verbose_print(
- "Using the default Moe and de Stefano 2017 datafile",
- verbosity,
- 1,
- )
- json_data = copy.deepcopy(moe_di_stefano_2017_data.moe_di_stefano_2017_data)
-
- return json_data
-
def imports():
for name, val in globals().items():
@@ -575,394 +522,6 @@ def create_hdf5(data_dir: str, name: str) -> None:
hdf5_file.close()
-########################################################
-# version_info functions
-########################################################
-
-
-def return_binary_c_version_info(parsed: bool = True) -> Union[str, dict]:
- """
- Function that returns the version information of binary_c. This function calls the function
- _binary_c_bindings.return_version_info()
-
- Args:
- parsed: Boolean flag whether to parse the version_info output of binary_c. default = False
-
- Returns:
- Either the raw string of binary_c or a parsed version of this in the form of a nested
- dictionary
- """
-
- found_prev = False
- if "BINARY_C_MACRO_HEADER" in os.environ:
- # the env var is already present. lets save that and put that back later
- found_prev = True
- prev_value = os.environ["BINARY_C_MACRO_HEADER"]
-
- #
- os.environ["BINARY_C_MACRO_HEADER"] = "macroxyz"
-
- # Get version_info
- version_info = _binary_c_bindings.return_version_info().strip()
-
- # parse if wanted
- if parsed:
- version_info = parse_binary_c_version_info(version_info)
-
- # delete value
- del os.environ["BINARY_C_MACRO_HEADER"]
-
- # put stuff back if we found a previous one
- if found_prev:
- os.environ["BINARY_C_MACRO_HEADER"] = prev_value
-
- return version_info
-
-
-def parse_binary_c_version_info(version_info_string: str) -> dict:
- """
- Function that parses the binary_c version info. Long function with a lot of branches
-
- TODO: fix this function. stuff is missing: isotopes, macros, nucleosynthesis_sources
-
- Args:
- version_info_string: raw output of version_info call to binary_c
-
- Returns:
- Parsed version of the version info, which is a dictionary containing the keys: 'isotopes' for isotope info, 'argpairs' for argument pair info (TODO: explain), 'ensembles' for ensemble settings/info, 'macros' for macros, 'elements' for atomic element info, 'DTlimit' for (TODO: explain), 'nucleosynthesis_sources' for nucleosynthesis sources, and 'miscellaneous' for all those that were not caught by the previous groups. 'git_branch', 'git_build', 'revision' and 'email' are also keys, but its clear what those contain.
- """
-
- version_info_dict = {}
-
- # Clean data and put in correct shape
- splitted = version_info_string.strip().splitlines()
- cleaned = {el.strip() for el in splitted if not el == ""}
-
- ##########################
- # Network:
- # Split off all the networks and parse the info.
-
- networks = {el for el in cleaned if el.startswith("Network ")}
- cleaned = cleaned - networks
-
- networks_dict = {}
- for el in networks:
- network_dict = {}
- split_info = el.split("Network ")[-1].strip().split("==")
-
- network_number = int(split_info[0])
- network_dict["network_number"] = network_number
-
- network_info_split = split_info[1].split(" is ")
-
- shortname = network_info_split[0].strip()
- network_dict["shortname"] = shortname
-
- if not network_info_split[1].strip().startswith(":"):
- network_split_info_extra = network_info_split[1].strip().split(":")
-
- longname = network_split_info_extra[0].strip()
- network_dict["longname"] = longname
-
- implementation = (
- network_split_info_extra[1].strip().replace("implemented in", "")
- )
- if implementation:
- network_dict["implemented_in"] = [i.strip("()") for i in implementation.strip().split()]
-
- networks_dict[network_number] = copy.deepcopy(network_dict)
- version_info_dict["networks"] = networks_dict if networks_dict else None
-
- ##########################
- # Isotopes:
- # Split off
- isotopes = {el for el in cleaned if el.startswith("Isotope ")}
- cleaned = cleaned - isotopes
-
- isotope_dict = {}
- for el in isotopes:
- split_info = el.split("Isotope ")[-1].strip().split(" is ")
-
- isotope_info = split_info[-1]
- name = isotope_info.split(" ")[0].strip()
-
- # Get details
- mass_g = float(
- isotope_info.split(",")[0].split("(")[1].split("=")[-1][:-2].strip()
- )
- mass_amu = float(
- isotope_info.split(",")[0].split("(")[-1].split("=")[-1].strip()
- )
- mass_mev = float(
- isotope_info.split(",")[-3].split("=")[-1].replace(")", "").strip()
- )
- A = int(isotope_info.split(",")[-1].strip().split("=")[-1].replace(")", ""))
- Z = int(isotope_info.split(",")[-2].strip().split("=")[-1])
-
- #
- isotope_dict[int(split_info[0])] = {
- "name": name,
- "Z": Z,
- "A": A,
- "mass_mev": mass_mev,
- "mass_g": mass_g,
- "mass_amu": mass_amu,
- }
- version_info_dict["isotopes"] = isotope_dict if isotope_dict else None
-
- ##########################
- # Arg pairs:
- # Split off
- argpairs = set([el for el in cleaned if el.startswith("ArgPair")])
- cleaned = cleaned - argpairs
-
- argpair_dict = {}
- for el in sorted(argpairs):
- split_info = el.split("ArgPair ")[-1].split(" ")
-
- if not argpair_dict.get(split_info[0], None):
- argpair_dict[split_info[0]] = {split_info[1]: split_info[2]}
- else:
- argpair_dict[split_info[0]][split_info[1]] = split_info[2]
-
- version_info_dict["argpairs"] = argpair_dict if argpair_dict else None
-
- ##########################
- # ensembles:
- # Split off
- ensembles = {el for el in cleaned if el.startswith("Ensemble")}
- cleaned = cleaned - ensembles
-
- ensemble_dict = {}
- ensemble_filter_dict = {}
- for el in ensembles:
- split_info = el.split("Ensemble ")[-1].split(" is ")
-
- if len(split_info) > 1:
- if not split_info[0].startswith("filter"):
- ensemble_dict[int(split_info[0])] = split_info[-1]
- else:
- filter_no = int(split_info[0].replace("filter ", ""))
- ensemble_filter_dict[filter_no] = split_info[-1]
-
- version_info_dict["ensembles"] = ensemble_dict if ensemble_dict else None
- version_info_dict["ensemble_filters"] = (
- ensemble_filter_dict if ensemble_filter_dict else None
- )
-
- ##########################
- # macros:
- # Split off
- macros = {el for el in cleaned if el.startswith("macroxyz")}
- cleaned = cleaned - macros
-
- param_type_dict = {
- "STRING": str,
- "FLOAT": float,
- "MACRO": str,
- "INT": int,
- "LONG_INT": int,
- "UINT": int,
- }
-
- macros_dict = {}
- for el in macros:
- split_info = el.split("macroxyz ")[-1].split(" : ")
- param_type = split_info[0]
-
- new_split = "".join(split_info[1:]).split(" is ")
- param_name = new_split[0].strip()
- param_value = " is ".join(new_split[1:])
- param_value = param_value.strip()
-
- #print("macro ",param_name,"=",param_value," float?",isfloat(param_value)," int?",isint(param_value))
-
- # If we're trying to set the value to "on", check that
- # it doesn't already exist. If it does, do nothing, as the
- # extra information is better than just "on"
- if param_name in macros_dict:
- #print("already exists (is ",macros_dict[param_name]," float? ",isfloat(macros_dict[param_name]),", int? ",isint(macros_dict[param_name]),") : check that we can improve it")
- if macros_dict[param_name] == "on":
- # update with better value
- store = True
- elif isfloat(macros_dict[param_name]) == False and isfloat(param_value) == True:
- # store the number we now have to replace the non-number we had
- store = True
- else:
- # don't override existing number
- store = False
-
- #if store:
- # print("Found improved macro value of param",param_name,", was ",macros_dict[param_name],", is",param_value)
- #else:
- # print("Cannot improve: use old value")
- else:
- store = True
-
- if store:
- # Sometimes the macros have extra information behind it.
- # Needs an update in outputting by binary_c (RGI: what does this mean David???)
- try:
- macros_dict[param_name] = param_type_dict[param_type](param_value)
- except ValueError:
- macros_dict[param_name] = str(param_value)
-
- version_info_dict["macros"] = macros_dict if macros_dict else None
-
- ##########################
- # Elements:
- # Split off:
- elements = {el for el in cleaned if el.startswith("Element")}
- cleaned = cleaned - elements
-
- # Fill dict:
- elements_dict = {}
- for el in elements:
- split_info = el.split("Element ")[-1].split(" : ")
- name_info = split_info[0].split(" is ")
-
- # get isotope info
- isotopes = {}
- if not split_info[-1][0] == "0":
- isotope_string = split_info[-1].split(" = ")[-1]
- isotopes = {
- int(split_isotope.split("=")[0]): split_isotope.split("=")[1]
- for split_isotope in isotope_string.split(" ")
- }
-
- elements_dict[int(name_info[0])] = {
- "name": name_info[-1],
- "atomic_number": int(name_info[0]),
- "amt_isotopes": len(isotopes),
- "isotopes": isotopes,
- }
- version_info_dict["elements"] = elements_dict if elements_dict else None
-
- ##########################
- # dt_limits:
- # split off
- dt_limits = {el for el in cleaned if el.startswith("DTlimit")}
- cleaned = cleaned - dt_limits
-
- # Fill dict
- dt_limits_dict = {}
- for el in dt_limits:
- split_info = el.split("DTlimit ")[-1].split(" : ")
- dt_limits_dict[split_info[1].strip()] = {
- "index": int(split_info[0]),
- "value": float(split_info[-1]),
- }
-
- version_info_dict["dt_limits"] = dt_limits_dict if dt_limits_dict else None
-
- ##############################
- # Units
- units = {el for el in cleaned if el.startswith("Unit ")}
- cleaned -= units
- units_dict={}
- for el in units:
- split_info = el.split("Unit ")[-1].split(",")
- s = split_info[0].split(" is ")
-
- if len(s)==2:
- long,short = [i.strip().strip("\"") for i in s]
- elif len(s)==1:
- long,short = None,s[0]
- else:
- print("Warning: Failed to split unit string {}".format(el))
-
- to_cgs = (split_info[1].split())[3].strip().strip("\"")
- code_units = split_info[2].split()
- code_unit_type_num = int(code_units[3].strip().strip("\""))
- code_unit_type = code_units[4].strip().strip("\"")
- code_unit_cgs_value = code_units[9].strip().strip("\"").strip(")")
- units_dict[long] = {
- "long" : long,
- "short" : short,
- "to_cgs" : to_cgs,
- "code_unit_type_num" : code_unit_type_num,
- "code_unit_type" : code_unit_type,
- "code_unit_cgs_value" : code_unit_cgs_value
- }
-
- units = {el for el in cleaned if el.startswith("Units: ")}
- cleaned -= units
- for el in units:
- el = el[7:] # removes "Units: "
- units_dict["units list"] = el.strip('Units:')
-
- version_info_dict["units"] = units_dict
-
- ##########################
- # Nucleosynthesis sources:
- # Split off
- nucsyn_sources = {el for el in cleaned if el.startswith("Nucleosynthesis")}
- cleaned -= nucsyn_sources
-
- # Fill dict
- nucsyn_sources_dict = {}
- for el in nucsyn_sources:
- split_info = el.split("Nucleosynthesis source")[-1].strip().split(" is ")
- nucsyn_sources_dict[int(split_info[0])] = split_info[-1]
-
- version_info_dict["nucleosynthesis_sources"] = (
- nucsyn_sources_dict if nucsyn_sources_dict else None
- )
-
- ##########################
- # miscellaneous:
- # All those that I didn't catch with the above filters. Could try to get some more out though.
- # TODO: filter a bit more.
-
- misc_dict = {}
-
- # Filter out git revision
- git_revision = [el for el in cleaned if el.startswith("git revision")]
- misc_dict["git_revision"] = (
- git_revision[0].split("git revision ")[-1].replace('"', "")
- )
- cleaned = cleaned - set(git_revision)
-
- # filter out git url
- git_url = [el for el in cleaned if el.startswith("git URL")]
- misc_dict["git_url"] = git_url[0].split("git URL ")[-1].replace('"', "")
- cleaned = cleaned - set(git_url)
-
- # filter out version
- version = [el for el in cleaned if el.startswith("Version")]
- misc_dict["version"] = str(version[0].split("Version ")[-1])
- cleaned = cleaned - set(version)
-
- git_branch = [el for el in cleaned if el.startswith("git branch")]
- misc_dict["git_branch"] = git_branch[0].split("git branch ")[-1].replace('"', "")
- cleaned = cleaned - set(git_branch)
-
- build = [el for el in cleaned if el.startswith("Build")]
- misc_dict["build"] = build[0].split("Build: ")[-1].replace('"', "")
- cleaned = cleaned - set(build)
-
- email = [el for el in cleaned if el.startswith("Email")]
- misc_dict["email"] = email[0].split("Email ")[-1].split(",")
- cleaned = cleaned - set(email)
-
- other_items = set([el for el in cleaned if " is " in el])
- cleaned = cleaned - other_items
-
- for el in other_items:
- split = el.split(" is ")
- key = split[0].strip()
- val = " is ".join(split[1:]).strip()
- if key in misc_dict:
- misc_dict[key + ' (alt)'] = val
- else:
- misc_dict[key] = val
-
- misc_dict["uncaught"] = list(cleaned)
-
- version_info_dict["miscellaneous"] = misc_dict if misc_dict else None
- return version_info_dict
-
-
########################################################
# binary_c output functions
########################################################
@@ -1595,9 +1154,3 @@ def load_logfile(logfile: str) -> None:
event_list.append(" ".join(split_line[9:]))
print(event_list)
-
-def dir_ok(dir):
- """
- Function to test if we can read and write to a dir that must exist. Return True if all is ok, False otherwise.
- """
- return os.access(dir, os.F_OK) and os.access(dir, os.R_OK | os.W_OK)
diff --git a/binarycpython/utils/grid.py b/binarycpython/utils/grid.py
index 5f5db48a7..1008893d5 100644
--- a/binarycpython/utils/grid.py
+++ b/binarycpython/utils/grid.py
@@ -21,40 +21,25 @@ Tasks:
"""
import argparse
-import bz2
-import compress_pickle
import copy
import datasize
import datetime
import functools
import json
import gc
-import gzip
-import importlib.util
-import lib_programname
-import logging
-import msgpack
import multiprocessing
import os
import pathlib
-import platform
-import pprint # for debugging only
import psutil
-import py_rinterpolate
import queue
-import re
-import resource
import setproctitle
import signal
-import stat
-import strip_ansi
import subprocess
import sys
import time
import traceback
import uuid
-
from collections import (
OrderedDict,
)
@@ -64,14 +49,6 @@ colorama_init()
from diskcache import Cache
from typing import Union, Any
-from binarycpython.utils.grid_options_defaults import (
- grid_options_defaults_dict,
- moe_di_stefano_default_options,
- _MOE2017_VERBOSITY_LEVEL,
- _CUSTOM_LOGGING_VERBOSITY_LEVEL,
- _LOGGER_VERBOSITY_LEVEL,
-)
-
from binarycpython.utils.custom_logging_functions import (
autogen_C_logging_code,
binary_c_log_code,
@@ -81,18 +58,13 @@ from binarycpython.utils.custom_logging_functions import (
from binarycpython.utils.functions import (
check_if_in_shell,
conv_time_units,
- dir_ok,
filter_arg_dict,
- format_number,
get_ANSI_colours,
get_defaults,
get_help_all,
- get_moe_di_stefano_dataset,
mem_use,
remove_file,
- return_binary_c_version_info,
timedelta,
- trem,
verbose_print,
)
from binarycpython.utils.ensemble import (
@@ -104,38 +76,33 @@ from binarycpython.utils.ensemble import (
)
from binarycpython.utils.dicts import (
AutoVivificationDict,
- custom_sort_dict,
merge_dicts,
- multiply_float_values,
multiply_values_dict,
- update_dicts,
keys_to_floats
)
-# from binarycpython.utils.hpc_functions import (
-# get_condor_version,
-# get_slurm_version,
-# create_directories_hpc,
-# path_of_calling_script,
-# get_python_details,
-# )
-
-from binarycpython.utils.distribution_functions import (
- Moecache,
- LOG_LN_CONVERTER,
- fill_data,
- get_max_multiplicity,
- Arenou2010_binary_fraction,
- raghavan2010_binary_fraction,
- Moe_di_Stefano_2017_multiplicity_fractions,
- normalize_dict,
-)
from binarycpython import _binary_c_bindings
-
-secs_per_day = 86400 # probably needs to go somewhere more sensible
-_count = 0 # used for file symlinking (for testing only)
-
-class Population:
+from binarycpython.utils.analytics import analytics
+from binarycpython.utils.condor import condor
+from binarycpython.utils.dataIO import dataIO
+from binarycpython.utils.grid_logging import grid_logging
+from binarycpython.utils.grid_options_defaults import grid_options_defaults
+from binarycpython.utils.gridcode import gridcode
+from binarycpython.utils.metadata import metadata
+from binarycpython.utils.Moe_di_Stefano_2017 import Moe_di_Stefano_2017
+from binarycpython.utils.slurm import slurm
+from binarycpython.utils.version import version
+
+class Population(analytics,
+ condor,
+ dataIO,
+ grid_logging,
+ grid_options_defaults,
+ gridcode,
+ metadata,
+ Moe_di_Stefano_2017,
+ slurm,
+ version):
"""
Population Object. Contains all the necessary functions to set up, run and process a
population of systems
@@ -163,7 +130,7 @@ class Population:
self.bse_options = {} # bse_options is just empty.
# Grid options
- self.grid_options = copy.deepcopy(grid_options_defaults_dict)
+ self.grid_options = copy.deepcopy(self.grid_options_defaults_dict())
# Custom options
self.custom_options = {
@@ -175,26 +142,38 @@ class Population:
self.indent_string = " "
self.code_string = ""
+ # cached value of minimum stellar mass
+ self._minimum_stellar_mass = None
+
+ # logging levels
+ self._LOGGER_VERBOSITY_LEVEL = 1
+ self._CUSTOM_LOGGING_VERBOSITY_LEVEL = 2
+
# Set the options that are passed at creation of the object
self.set(**kwargs)
# Load Moe and di Stefano options
self.grid_options["Moe2017_options"] = copy.deepcopy(
- moe_di_stefano_default_options
+ self.Moe_di_Stefano_2017_default_options()
)
# Write MOE2017 options to a file. NOTE: not sure why i put this here anymore
os.makedirs(
- os.path.join(self.grid_options["tmp_dir"], "moe_distefano"), exist_ok=True
+ os.path.join(self.grid_options["tmp_dir"],
+ "moe_distefano"), exist_ok=True
)
with open(
os.path.join(
- os.path.join(self.grid_options["tmp_dir"], "moe_distefano"),
+ os.path.join(self.grid_options["tmp_dir"],
+ "moe_distefano"),
"moeopts.dat",
),
"w",
) as f:
- json.dump(self.grid_options["Moe2017_options"], f, indent=4, ensure_ascii=False)
+ json.dump(self.grid_options["Moe2017_options"],
+ f,
+ indent=4,
+ ensure_ascii=False)
# Argline dict
self.argline_dict = {}
@@ -232,7 +211,12 @@ class Population:
def jobID(self):
- # job ID
+ """
+ Function to return the job ID number of this process
+
+ Normal processes return their process ID (PID)
+ Slurm processes return <jobid>.<jobarrayindex>
+ """
if self.grid_options['slurm'] > 0:
jobID = "{}.{}".format(self.grid_options['slurm_jobid'],
self.grid_options['slurm_jobarrayindex'])
@@ -301,7 +285,7 @@ class Population:
for key in kwargs:
# Filter out keys for the bse_options
if key in self.defaults.keys():
- verbose_print(
+ self.verbose_print(
"adding: {}={} to BSE_options".format(key, kwargs[key]),
self.grid_options["verbosity"],
2,
@@ -317,7 +301,7 @@ class Population:
for param in self.special_params
]
):
- verbose_print(
+ self.verbose_print(
"adding: {}={} to BSE_options by catching the %d".format(
key, kwargs[key]
),
@@ -328,7 +312,7 @@ class Population:
# Filter out keys for the grid_options
elif key in self.grid_options.keys():
- verbose_print(
+ self.verbose_print(
"adding: {}={} to grid_options".format(key, kwargs[key]),
self.grid_options["verbosity"],
1,
@@ -337,7 +321,7 @@ class Population:
# The of the keys go into a custom_options dict
else:
- verbose_print(
+ self.verbose_print(
"<<<< Warning: Key does not match previously known parameter: \
adding: {}={} to custom_options >>>>".format(
key, kwargs[key]
@@ -367,7 +351,7 @@ class Population:
cmdline_args = sys.argv[1:]
if cmdline_args:
- verbose_print(
+ self.verbose_print(
"Found cmdline args. Parsing them now",
self.grid_options["verbosity"],
1,
@@ -405,7 +389,7 @@ class Population:
if old_value_found:
if old_value != None:
try:
- verbose_print(
+ self.verbose_print(
"Converting type of {} from {} to {}".format(
parameter, type(value), type(old_value)
),
@@ -413,7 +397,7 @@ class Population:
2,
)
value = type(old_value)(value)
- verbose_print("Success!", self.grid_options["verbosity"], 2)
+ self.verbose_print("Success!", self.grid_options["verbosity"], 2)
except ValueError:
@@ -423,10 +407,10 @@ class Population:
try:
evaled = eval(value)
value = type(old_value)(evaled)
- verbose_print("Success! (evaled)", self.grid_options["verbosity"], 2)
+ self.verbose_print("Success! (evaled)", self.grid_options["verbosity"], 2)
except ValueError:
- verbose_print(
+ self.verbose_print(
"Tried to convert the given parameter {}/value {} to its correct type {} (from old value {}). But that wasn't possible.".format(
parameter, value, type(old_value), old_value
),
@@ -444,37 +428,6 @@ class Population:
self.set(**cmdline_dict)
- def set_status(self,
- string,
- format_statment="process_{}.txt",
- ID=None,
- slurm=True,
- condor=True):
- """
- function to set the status string in its appropriate file
- """
-
- if ID is None:
- ID = self.process_ID
-
- if self.grid_options['status_dir']:
- with open(
- os.path.join(
- self.grid_options["status_dir"],
- format_statment.format(ID),
- ),
- "w",
- encoding='utf-8'
- ) as f:
- f.write(string)
- f.close()
-
- # custom logging functions
- if slurm and self.grid_options['slurm'] >= 1:
- self.set_slurm_status(string)
-# if self.grid_options['condor']==1:
-# self.set_condor_status(string)
-
def _return_argline(self, parameter_dict=None):
"""
Function to create the string for the arg line from a parameter dict
@@ -490,259 +443,6 @@ class Population:
argline = argline.strip()
return argline
- def _last_grid_variable(self):
- """
- Function that returns the last grid variable
- (i.e. the one with the highest grid_variable_number)
- """
-
- number = len(self.grid_options["_grid_variables"])
- for grid_variable in self.grid_options["_grid_variables"]:
- if (
- self.grid_options["_grid_variables"][grid_variable][
- "grid_variable_number"
- ]
- == number - 1
- ):
- return grid_variable
-
- def update_grid_variable(self, name: str, **kwargs) -> None:
- """
- Function to update the values of a grid variable.
-
- Args:
- name:
- name of the grid variable to be changed.
- **kwargs:
- key-value pairs to override the existing grid variable data. See add_grid_variable for these names.
- """
-
- grid_variable = None
- try:
- grid_variable = self.grid_options["_grid_variables"][name]
- except KeyError:
- msg = "Unknown grid variable {} - please create it with the add_grid_variable() method.".format(
- name
- )
- raise KeyError(msg)
-
- for key, value in kwargs.items():
- grid_variable[key] = value
- verbose_print(
- "Updated grid variable: {}".format(json.dumps(grid_variable, indent=4, ensure_ascii=False)),
- self.grid_options["verbosity"],
- 1,
- )
-
- def delete_grid_variable(
- self,
- name: str,
- ) -> None:
- """
- Function to delete a grid variable with the given name.
-
- Args:
- name:
- name of the grid variable to be deleted.
- """
- try:
- del self.grid_options["_grid_variables"][name]
- verbose_print(
- "Deleted grid variable: {}".format(name),
- self.grid_options["verbosity"],
- 1,
- )
- except:
- msg = "Failed to remove grid variable {} : please check it exists.".format(
- name
- )
- raise ValueError(msg)
-
- def rename_grid_variable(self, oldname: str, newname: str) -> None:
- """
- Function to rename a grid variable.
-
- note: this does NOT alter the order
- of the self.grid_options["_grid_variables"] dictionary.
-
- The order in which the grid variables are loaded into the grid is based on their
- `grid_variable_number` property
-
- Args:
- oldname:
- old name of the grid variable
- newname:
- new name of the grid variable
- """
-
- try:
- self.grid_options["_grid_variables"][newname] = self.grid_options[
- "_grid_variables"
- ].pop(oldname)
- self.grid_options["_grid_variables"][newname]["name"] = newname
- verbose_print(
- "Rename grid variable: {} to {}".format(oldname, newname),
- self.grid_options["verbosity"],
- 1,
- )
- except:
- msg = "Failed to rename grid variable {} to {}.".format(oldname, newname)
- raise ValueError(msg)
-
- def add_grid_variable(
- self,
- name: str,
- parameter_name: str,
- longname: str,
- valuerange: Union[list, str],
- samplerfunc: str,
- probdist: str,
- dphasevol: Union[str, int] = -1,
- gridtype: str = "centred",
- branchpoint: int = 0,
- branchcode: Union[str, None] = None,
- precode: Union[str, None] = None,
- postcode: Union[str, None] = None,
- topcode: Union[str, None] = None,
- bottomcode: Union[str, None] = None,
- condition: Union[str, None] = None,
- ) -> None:
- """
- Function to add grid variables to the grid_options.
-
- The execution of the grid generation will be through a nested for loop.
- Each of the grid variables will get create a deeper for loop.
-
- The real function that generates the numbers will get written to a new file in the TMP_DIR,
- and then loaded imported and evaluated.
- beware that if you insert some destructive piece of code, it will be executed anyway.
- Use at own risk.
-
- Tasks:
- - TODO: Fix this complex function.
-
- Args:
- name:
- name of parameter used in the grid Python code.
- This is evaluated as a parameter and you can use it throughout
- the rest of the function
-
- Examples:
- name = 'lnm1'
-
- parameter_name:
- name of the parameter in binary_c
-
- This name must correspond to a Python variable of the same name,
- which is automatic if parameter_name == name.
-
- Note: if parameter_name != name, you must set a
- variable in "precode" or "postcode" to define a Python variable
- called parameter_name
-
- longname:
- Long name of parameter
-
- Examples:
- longname = 'Primary mass'
- range:
- Range of values to take. Does not get used really, the samplerfunc is used to
- get the values from
-
- Examples:
- range = [math.log(m_min), math.log(m_max)]
- samplerfunc:
- Function returning a list or numpy array of samples spaced appropriately.
- You can either use a real function, or a string representation of a function call.
-
- Examples:
- samplerfunc = "const(math.log(m_min), math.log(m_max), {})".format(resolution['M_1'])
-
- precode:
- Extra room for some code. This code will be evaluated within the loop of the
- sampling function (i.e. a value for lnm1 is chosen already)
-
- Examples:
- precode = 'M_1=math.exp(lnm1);'
- postcode:
- Code executed after the probability is calculated.
- probdist:
- Function determining the probability that gets assigned to the sampled parameter
-
- Examples:
- probdist = 'Kroupa2001(M_1)*M_1'
- dphasevol:
- part of the parameter space that the total probability is calculated with. Put to -1
- if you want to ignore any dphasevol calculations and set the value to 1
- Examples:
- dphasevol = 'dlnm1'
- condition:
- condition that has to be met in order for the grid generation to continue
- Examples:
- condition = 'self.grid_options['binary']==1'
- gridtype:
- Method on how the value range is sampled. Can be either 'edge' (steps starting at
- the lower edge of the value range) or 'centred'
- (steps starting at lower edge + 0.5 * stepsize).
-
- topcode:
- Code added at the very top of the block.
-
- bottomcode:
- Code added at the very bottom of the block.
- """
-
- # check parameters
- if False and dphasevol != -1.0 and gridtype == 'discrete':
- print("Error making grid: you have set the phasevol to be not -1 and gridtype to discrete, but a discrete grid has no phasevol calculation. You should only set the gridtype to discrete and not set the phasevol in this case.")
-
- self.exit(code=1)
-
- # Add grid_variable
- grid_variable = {
- "name": name,
- "parameter_name": parameter_name,
- "longname": longname,
- "valuerange": valuerange,
- "samplerfunc": samplerfunc,
- "precode": precode,
- "postcode": postcode,
- "probdist": probdist,
- "dphasevol": dphasevol,
- "condition": condition,
- "gridtype": gridtype,
- "branchpoint": branchpoint,
- "branchcode": branchcode,
- "topcode": topcode,
- "bottomcode": bottomcode,
- "grid_variable_number": len(self.grid_options["_grid_variables"]),
- }
-
- # Check for gridtype input
- allowed_gridtypes = [
- "edge",
- "right",
- "right edge",
- "left",
- "left edge",
- "centred",
- "centre",
- "center",
- "discrete"
- ]
- if not gridtype in allowed_gridtypes:
- msg = "Unknown gridtype {gridtype}. Please choose one of: ".format(gridtype=gridtype) + ','.join(allowed_gridtypes)
- raise ValueError(msg)
-
- # Load it into the grid_options
- self.grid_options["_grid_variables"][grid_variable["name"]] = grid_variable
-
- verbose_print(
- "Added grid variable: {}".format(json.dumps(grid_variable, indent=4, ensure_ascii=False)),
- self.grid_options["verbosity"],
- 2,
- )
-
###################################################
# Return functions
###################################################
@@ -764,15 +464,6 @@ class Population:
return options
- def return_binary_c_version_info(self, parsed=False):
- """
- Function that returns the version information of binary_c
- """
-
- version_info = return_binary_c_version_info(parsed=parsed)
-
- return version_info
-
def return_binary_c_defaults(self):
"""
Function that returns the defaults of the binary_c version that is used.
@@ -820,7 +511,7 @@ class Population:
all_info["binary_c_defaults"] = binary_c_defaults
if include_binary_c_version_info:
- binary_c_version_info = return_binary_c_version_info(parsed=True)
+ binary_c_version_info = self.return_binary_c_version_info(parsed=True)
all_info["binary_c_version_info"] = binary_c_version_info
if include_binary_c_help_all:
@@ -829,26 +520,6 @@ class Population:
return all_info
- def time_elapsed(self):
- """
- Function to return how long a population object has been running.
- """
- for x in ["_start_time_evolution","_end_time_evolution"]:
- if not self.grid_options[x]:
- self.grid_options[x] = time.time()
- return self.grid_options["_end_time_evolution"] - self.grid_options["_start_time_evolution"]
-
- def CPU_time(self):
- """
- Function to return how much CPU time we've used
- """
- dt = self.time_elapsed()
- try:
- ncpus = self.grid_options['num_processes']
- except:
- ncpus = 1
- return dt * ncpus
-
def export_all_info(
self,
use_datadir: bool = True,
@@ -923,7 +594,7 @@ class Population:
self.custom_options["data_dir"], settings_name
)
- verbose_print(
+ self.verbose_print(
"Writing settings to {}".format(settings_fullname),
self.grid_options["verbosity"],
1,
@@ -944,13 +615,13 @@ class Population:
raise ValueError
else:
- verbose_print(
+ self.verbose_print(
"Writing settings to {}".format(outfile),
self.grid_options["verbosity"],
1,
)
if not outfile.endswith("json"):
- verbose_print(
+ self.verbose_print(
"Error: outfile ({}) must end with .json".format(outfile),
self.grid_options["verbosity"],
0,
@@ -967,130 +638,6 @@ class Population:
)
return outfile
- def _boxed(self, *list, colour="yellow on black", boxchar="*", separator="\n"):
- """
- Function to output a list of strings in a single box.
-
- Args:
- list = a list of strings to be output. If these contain the separator
- (see below) these strings are split by it.
- separator = strings are split on this, default "\n"
- colour = the colour to be used, usually this is 'yellow on black'
- as set in the ANSI_colours dict
- boxchar = the character used to make the box, '*' by default
-
- Note: handles tabs (\t) badly, do not use them!
- """
- strlen = 0
- strings = []
- lengths = []
-
- # make a list of strings
- if separator:
- for l in list:
- strings += l.split(sep=separator)
- else:
- strings = list
-
- # get lengths without ANSI codes
- for string in strings:
- lengths.append(len(strip_ansi.strip_ansi(string)))
-
- # hence the max length
- strlen = max(lengths)
- strlen += strlen % 2
- header = boxchar * (4 + strlen)
-
- # start output
- out = self.ANSI_colours[colour] + header + "\n"
-
- # loop over strings to output, padding as required
- for n, string in enumerate(strings):
- if lengths[n] % 2 == 1:
- string = " " + string
- pad = " " * int((strlen - lengths[n]) / 2)
- out = out + boxchar + " " + pad + string + pad + " " + boxchar + "\n"
- # close output and return
- out = out + header + "\n" + self.ANSI_colours["reset"]
- return out
-
- def _set_custom_logging(self):
- """
- Function/routine to set all the custom logging so that the function memory pointer
- is known to the grid.
-
- When the memory adress is loaded and the library file is set we'll skip rebuilding the library
- """
-
- # Only if the values are the 'default' unset values
- if (
- self.grid_options["custom_logging_func_memaddr"] == -1
- and self.grid_options["_custom_logging_shared_library_file"] is None
- ):
- verbose_print(
- "Creating and loading custom logging functionality",
- self.grid_options["verbosity"],
- 1,
- )
- # C_logging_code gets priority of C_autogen_code
- if self.grid_options["C_logging_code"]:
- # Generate entire shared lib code around logging lines
- custom_logging_code = binary_c_log_code(
- self.grid_options["C_logging_code"],
- verbosity=self.grid_options["verbosity"]
- - (_CUSTOM_LOGGING_VERBOSITY_LEVEL - 1),
- )
-
- # Load memory address
- (
- self.grid_options["custom_logging_func_memaddr"],
- self.grid_options["_custom_logging_shared_library_file"],
- ) = create_and_load_logging_function(
- custom_logging_code,
- verbosity=self.grid_options["verbosity"]
- - (_CUSTOM_LOGGING_VERBOSITY_LEVEL - 1),
- custom_tmp_dir=self.grid_options["tmp_dir"],
- )
-
- elif self.grid_options["C_auto_logging"]:
- # Generate real logging code
- logging_line = autogen_C_logging_code(
- self.grid_options["C_auto_logging"],
- verbosity=self.grid_options["verbosity"]
- - (_CUSTOM_LOGGING_VERBOSITY_LEVEL - 1),
- )
-
- # Generate entire shared lib code around logging lines
- custom_logging_code = binary_c_log_code(
- logging_line,
- verbosity=self.grid_options["verbosity"]
- - (_CUSTOM_LOGGING_VERBOSITY_LEVEL - 1),
- )
-
- # Load memory address
- (
- self.grid_options["custom_logging_func_memaddr"],
- self.grid_options["_custom_logging_shared_library_file"],
- ) = create_and_load_logging_function(
- custom_logging_code,
- verbosity=self.grid_options["verbosity"]
- - (_CUSTOM_LOGGING_VERBOSITY_LEVEL - 1),
- custom_tmp_dir=self.grid_options["tmp_dir"],
- )
- else:
- verbose_print(
- "Custom logging library already loaded. Not setting them again.",
- self.grid_options["verbosity"],
- 1,
- )
-
- ###################################################
- # Ensemble functions
- ###################################################
-
- # Now they are stored in the _process_run_population thing.
- # Needed less code since they all
-
###################################################
# Evolution functions
###################################################
@@ -1205,7 +752,7 @@ class Population:
# check directories exist and can be written to
for dir in dirs:
path = self.grid_options[dir]
- if path != None and dir_ok(path) == False:
+ if path != None and self.dir_ok(path) == False:
print("Directory {dir} currently set to {path} cannot be written to. Please check that this directory is correct and you have write access.".format(dir=dir,path=path))
self.exit(code=1)
@@ -1214,7 +761,7 @@ class Population:
for subdir in subdirs:
path = os.path.join(self.grid_options["tmp_dir"], subdir)
os.makedirs(path,exist_ok=True)
- if dir_ok(path) == False:
+ if self.dir_ok(path) == False:
print("Sub-Directory {subdir} (in tmp_dir) currently set to {path} cannot be written to. Please check that this directory is correct and you have write access.".format(subdir=subdir,path=path))
self.exit(code=1)
@@ -1300,42 +847,8 @@ class Population:
# Execute population evolution subroutines
self._evolve_population()
- print("Do analytics")
-
- if self.grid_options['do_analytics']:
- # Put all interesting stuff in a variable and output that afterwards, as analytics of the run.
- analytics_dict = {
- "population_id": self.grid_options["_population_id"],
- "evolution_type": self.grid_options["evolution_type"],
- "failed_count": self.grid_options["_failed_count"],
- "failed_prob": self.grid_options["_failed_prob"],
- "failed_systems_error_codes": self.grid_options[
- "_failed_systems_error_codes"
- ].copy(),
- "errors_exceeded": self.grid_options["_errors_exceeded"],
- "errors_found": self.grid_options["_errors_found"],
- "total_probability": self.grid_options["_probtot"],
- "total_count": self.grid_options["_count"],
- "start_timestamp": self.grid_options["_start_time_evolution"],
- "end_timestamp": self.grid_options["_end_time_evolution"],
- "time_elapsed" : self.time_elapsed(),
- "total_mass_run": self.grid_options["_total_mass_run"],
- "total_probability_weighted_mass_run": self.grid_options[
- "_total_probability_weighted_mass_run"
- ],
- "zero_prob_stars_skipped": self.grid_options["_zero_prob_stars_skipped"],
- }
-
- if "metadata" in self.grid_ensemble_results:
- # Add analytics dict to the metadata too:
- self.grid_ensemble_results["metadata"].update(analytics_dict)
- self.add_system_metadata()
- else:
- # use existing analytics dict
- try:
- analytics_dict = self.grid_ensemble_results["metadata"]
- except:
- analytics_dict = {} # should never happen
+ # make analytics information
+ analytics_dict = self.make_analytics_dict()
if self.custom_options['save_snapshot']:
# we must save a snapshot, not the population object
@@ -1361,6 +874,8 @@ class Population:
return analytics_dict
+
+
def _evolve_population(self):
"""
Function to evolve populations. This handles the setting up, evolving
@@ -1455,11 +970,11 @@ class Population:
string2 += "\n>>> Grid was killed <<<"
self.set_status("killed")
- verbose_print(self._boxed(string1, string2), self.grid_options["verbosity"], 0)
+ self.verbose_print(self._boxed(string1, string2), self.grid_options["verbosity"], 0)
if self.grid_options["_errors_found"]:
# Some information afterwards
- verbose_print(
+ self.verbose_print(
self._boxed(
"During the run {} failed systems were found\nwith a total probability of {:g}\nwith the following unique error codes: {} ".format(
self.grid_options["_failed_count"],
@@ -1471,7 +986,7 @@ class Population:
0,
)
# Some information afterwards
- verbose_print(
+ self.verbose_print(
"The full argline commands for {} these systems have been written to {}".format(
"ALL"
if not self.grid_options["_errors_exceeded"]
@@ -1487,26 +1002,12 @@ class Population:
0,
)
else:
- verbose_print(
+ self.verbose_print(
"No failed systems were found in this run.",
self.grid_options["verbosity"],
0,
)
- def _get_stream_logger(self, level=logging.DEBUG):
- """Return logger with configured StreamHandler."""
- stream_logger = logging.getLogger("stream_logger")
- stream_logger.handlers = []
- stream_logger.setLevel(level)
- sh = logging.StreamHandler(stream=sys.stdout)
- sh.setLevel(level)
- fmt = "[%(asctime)s %(levelname)-8s %(processName)s] --- %(message)s"
- formatter = logging.Formatter(fmt)
- sh.setFormatter(formatter)
- stream_logger.addHandler(sh)
-
- return stream_logger
-
def _system_queue_filler(self, job_queue, num_processes):
"""
Function that is responsible for keeping the queue filled.
@@ -1518,7 +1019,7 @@ class Population:
"""
stream_logger = self._get_stream_logger()
- if self.grid_options["verbosity"] >= _LOGGER_VERBOSITY_LEVEL:
+ if self.grid_options["verbosity"] >= self._LOGGER_VERBOSITY_LEVEL:
stream_logger.debug(f"setting up the system_queue_filler now")
# Setup of the generator
@@ -1548,7 +1049,7 @@ class Population:
# skip systems before start_at
elif system_number < self.grid_options["start_at"]:
- verbose_print("skip system {n} because < start_at = {start}".format(
+ self.verbose_print("skip system {n} because < start_at = {start}".format(
n=system_number,
start=self.grid_options["start_at"]),
self.grid_options['verbosity'],
@@ -1559,7 +1060,7 @@ class Population:
if not (
(system_number - self.grid_options["start_at"]) % self.grid_options["modulo"] == 0
):
- verbose_print("skip system {n} because modulo {mod} == {donemod}".format(
+ self.verbose_print("skip system {n} because modulo {mod} == {donemod}".format(
n=system_number,
mod=self.grid_options["modulo"],
donemod=(system_number - self.grid_options["start_at"]) % self.grid_options["modulo"]
@@ -1580,7 +1081,7 @@ class Population:
self.grid_options['stop_queue'] = True
# Print some info
- verbose_print(
+ self.verbose_print(
"Queue produced system {}".format(system_number),
self.grid_options["verbosity"],
3,
@@ -1589,7 +1090,7 @@ class Population:
self.grid_options['_queue_done'] = True
# Send closing signal to workers. When they receive this they will terminate
- if self.grid_options["verbosity"] >= _LOGGER_VERBOSITY_LEVEL:
+ if self.grid_options["verbosity"] >= self._LOGGER_VERBOSITY_LEVEL:
stream_logger.debug(f"Signalling processes to stop") # DEBUG
if True:#not self.grid_options['stop_queue']:
@@ -1671,30 +1172,25 @@ class Population:
p.join()
print("Joined.")
- keylist = ["_failed_count",
- "_failed_prob",
- "_errors_exceeded",
- "_errors_found",
- "_probtot",
- "_count",
- "_total_mass_run",
- "_total_probability_weighted_mass_run",
- "_zero_prob_stars_skipped",
- "_killed"]
# todo: error codes
# Handle the results by merging all the dictionaries. How that merging happens exactly is
# described in the merge_dicts description.
+ #
+ # If there is a preloaded_population, we add this first,
+ # then we add the populations run just now
+
+ # 1)
+ # use preloaded population's data as a basis
+ # for our combined_output_dict
if self.preloaded_population:
- # use preloaded population's data as a basis
- # for our combined_output_dict
combined_output_dict = {
"ensemble_results" : keys_to_floats(self.preloaded_population.grid_ensemble_results),
"results": keys_to_floats(self.preloaded_population.grid_results)
}
- for x in keylist:
+ for x in self._metadata_keylist():
try:
combined_output_dict[x] = self.preloaded_population.grid_options[x]
except Exception as e:
@@ -1708,6 +1204,9 @@ class Population:
combined_output_dict['ensemble_results'] = OrderedDict()
combined_output_dict['results'] = OrderedDict()
+ # 2)
+ # combine the dicts that were output from our
+ # subprocesses
sentinel = object()
for output_dict in iter(result_queue.get, sentinel):
if output_dict:
@@ -1740,37 +1239,7 @@ class Population:
] # Ensemble results are also passed as output from that dictionary
# Add metadata
- self.grid_ensemble_results["metadata"] = {}
- self.grid_ensemble_results["metadata"]["population_id"] = self.grid_options["_population_id"]
- self.grid_ensemble_results["metadata"]["total_probability_weighted_mass"] = combined_output_dict["_total_probability_weighted_mass_run"]
- self.grid_ensemble_results["metadata"]["factored_in_probability_weighted_mass"] = False
- if self.grid_options["ensemble_factor_in_probability_weighted_mass"]:
- multiply_values_dict(
- self.grid_ensemble_results["ensemble"],
- 1.0
- / self.grid_ensemble_results["metadata"][
- "total_probability_weighted_mass"
- ],
- )
- self.grid_ensemble_results["metadata"]["factored_in_probability_weighted_mass"] = True
- self.grid_ensemble_results["metadata"]["_killed"] = self.grid_options["_killed"]
-
- # Add settings of the populations
- all_info = self.return_all_info(
- include_population_settings=True,
- include_binary_c_defaults=True,
- include_binary_c_version_info=True,
- include_binary_c_help_all=True,
- )
- self.grid_ensemble_results["metadata"]["settings"] = json.loads(
- json.dumps(all_info, default=binaryc_json_serializer, ensure_ascii=False)
- )
-
- ##############################
- # Update grid options
- for x in keylist:
- self.grid_options[x] = combined_output_dict[x]
- self.grid_options["_failed_systems_error_codes"] = list(set(combined_output_dict["_failed_systems_error_codes"]))
+ self.add_ensemble_metadata(combined_output_dict)
# if we were killed, save snapshot
if self.grid_options['save_snapshots'] and self.grid_options['_killed']:
@@ -1921,7 +1390,7 @@ class Population:
print("Set process ID",self.process_ID)
stream_logger = self._get_stream_logger()
- if self.grid_options["verbosity"] >= _LOGGER_VERBOSITY_LEVEL:
+ if self.grid_options["verbosity"] >= self._LOGGER_VERBOSITY_LEVEL:
stream_logger.debug(f"Setting up processor: process-{self.process_ID}")
# Set the process names
@@ -1935,7 +1404,7 @@ class Population:
# lets try out making stores for all the grids:
self.grid_options["_store_memaddr"] = _binary_c_bindings.return_store_memaddr()
- verbose_print(
+ self.verbose_print(
"Process {} started at {}.\tUsing store memaddr {}".format(
ID,
datetime.datetime.now().isoformat(),
@@ -1956,7 +1425,7 @@ class Population:
self.process_ID: persistent_data_memaddr
}
- verbose_print(
+ self.verbose_print(
"\tUsing persistent_data memaddr: {}".format(persistent_data_memaddr),
self.grid_options["verbosity"],
3,
@@ -2190,7 +1659,7 @@ class Population:
# Set status to finishing
self.set_status("finishing")
- if self.grid_options["verbosity"] >= _LOGGER_VERBOSITY_LEVEL:
+ if self.grid_options["verbosity"] >= self._LOGGER_VERBOSITY_LEVEL:
stream_logger.debug(f"Process-{self.process_ID} is finishing.")
###########################
@@ -2199,7 +1668,7 @@ class Population:
# if ensemble==1, then either directly write that data to a file, or combine everything into 1 file.
ensemble_json = {} # Make sure it exists already
if self.bse_options.get("ensemble", 0) == 1:
- verbose_print(
+ self.verbose_print(
"Process {}: is freeing ensemble output (using persistent_data memaddr {})".format(
ID, self.persistent_data_memory_dict[self.process_ID]
),
@@ -2214,7 +1683,7 @@ class Population:
)
if ensemble_raw_output is None:
- verbose_print(
+ self.verbose_print(
"Process {}: Warning! Ensemble output is empty. ".format(ID),
self.grid_options["verbosity"],
1,
@@ -2236,7 +1705,7 @@ class Population:
self.grid_options["_population_id"], self.process_ID
),
)
- verbose_print(
+ self.verbose_print(
"Writing process {} JSON ensemble chunk output to {} ".format(
ID, output_file
),
@@ -2248,7 +1717,7 @@ class Population:
# combine ensemble chunks
if self.grid_options["combine_ensemble_with_thread_joining"] is True:
- verbose_print(
+ self.verbose_print(
"Process {}: Extracting ensemble info from raw string".format(ID),
self.grid_options["verbosity"],
1,
@@ -2257,7 +1726,7 @@ class Population:
##########################
# Clean up and return
- verbose_print(
+ self.verbose_print(
"process {} free memory and return ".format(ID),
self.grid_options["verbosity"],
1,
@@ -2290,7 +1759,7 @@ class Population:
# thread end message
colour = "cyan on black"
- verbose_print(
+ self.verbose_print(
self._boxed(
"{colour}Process {ID} finished:\ngenerator started at {start}\ngenerator finished at {end}\ntotal: {timesecs}\nof which {binary_c_secs} with binary_c\nRan {nsystems} systems\nwith a total probability of {psystems:g}\n{failcolour}This thread had {nfail} failing systems{colour}\n{failcolour}with a total failed probability of {pfail}{colour}\n{zerocolour}Skipped a total of {nzero} zero-probability systems{zeroreset}\n{failednotice}".format(
colour=self.ANSI_colours[colour],
@@ -2364,7 +1833,7 @@ class Population:
else:
self.set_status("finished")
- verbose_print(
+ self.verbose_print(
"process {} queue put output_dict ".format(ID),
self.grid_options["verbosity"],
1,
@@ -2372,10 +1841,10 @@ class Population:
result_queue.put(output_dict)
- if self.grid_options["verbosity"] >= _LOGGER_VERBOSITY_LEVEL:
+ if self.grid_options["verbosity"] >= self._LOGGER_VERBOSITY_LEVEL:
stream_logger.debug(f"Process-{self.process_ID} is finished.")
- verbose_print(
+ self.verbose_print(
"process {} return ".format(ID),
self.grid_options["verbosity"],
1,
@@ -2405,7 +1874,7 @@ class Population:
# Get argument line and
argline = self._return_argline(self.bse_options)
- verbose_print(
+ self.verbose_print(
"Running {}".format(argline), self.grid_options["verbosity"], 1
)
@@ -2466,7 +1935,7 @@ class Population:
## check the settings and set all the warnings.
if self.bse_options.get("ensemble", None):
if not self.bse_options.get("ensemble_defer", 0) == 1:
- verbose_print(
+ self.verbose_print(
"Error, if you want to run an ensemble in a population, the output needs to be deferred. Please set 'ensemble_defer' to 1",
self.grid_options["verbosity"],
0,
@@ -2476,14 +1945,14 @@ class Population:
if not any(
[key.startswith("ensemble_filter_") for key in self.bse_options]
):
- verbose_print(
+ self.verbose_print(
"Warning: Running the ensemble without any filter requires a lot of available RAM",
self.grid_options["verbosity"],
0,
)
if self.bse_options.get("ensemble_filters_off", 0) != 1:
- verbose_print(
+ self.verbose_print(
"Warning: Running the ensemble without any filter requires a lot of available RAM",
self.grid_options["verbosity"],
0,
@@ -2491,7 +1960,7 @@ class Population:
if self.grid_options["combine_ensemble_with_thread_joining"] == False:
if not self.custom_options.get("data_dir", None):
- verbose_print(
+ self.verbose_print(
"Error: chosen to write the ensemble output directly to files but data_dir isn't set",
self.grid_options["verbosity"],
0,
@@ -2523,7 +1992,7 @@ class Population:
# Do a dry run
self._dry_run()
- verbose_print(
+ self.verbose_print(
self._boxed(
"Dry run",
"Total starcount is {starcount}".format(
@@ -2628,3266 +2097,234 @@ class Population:
self.grid_options["_system_generator"] = None
self.grid_options["_failed_systems_error_codes"] = []
- # Remove files
- # TODO: remove files
-
- # Unload functions
- # TODO: unload functions
+ def _dry_run(self):
+ """
+ Function to dry run the grid and know how many stars it will run
- # Unload/free custom_logging_code
- # TODO: cleanup custom logging code.
+ Requires the grid to be built as a dry run grid
+ """
+ self.verbose_print("Dry run of the grid", self.grid_options["verbosity"], 1)
+ system_generator = self.grid_options["_system_generator"]
+ total_starcount = system_generator(self)
+ self.grid_options["_total_starcount"] = total_starcount
###################################################
- # Grid code functions
+ # Population from file functions
#
- # Function below are used to run populations with
- # a variable grid
+ # Functions below are used to run populations from
+ # a file containing binary_c calls
###################################################
- def _gridcode_filename(self):
+ def _dry_run_source_file(self):
"""
- Returns a filename for the gridcode.
+ Function to go through the source_file and count the number of lines and the total probability
"""
- if self.grid_options['slurm'] > 0:
- filename = os.path.join(
- self.grid_options["tmp_dir"],
- "binary_c_grid_{population_id}.{jobid}.{jobarrayindex}.py".format(
- population_id=self.grid_options["_population_id"],
- jobid=self.grid_options['slurm_jobid'],
- jobarrayindex=self.grid_options['slurm_jobarrayindex'],
- )
- )
- else:
- filename = os.path.join(
- self.grid_options["tmp_dir"],
- "binary_c_grid_{population_id}.py".format(
- population_id=self.grid_options["_population_id"]
- ),
- )
- return filename
+ system_generator = self.grid_options["_system_generator"]
+ total_starcount = 0
+ total_probability = 0
+ contains_probability = False
+
+ for line in system_generator:
+ total_starcount += 1
+
+ total_starcount = system_generator(self)
+ self.grid_options["_total_starcount"] = total_starcount
- def _add_code(self, *args, indent=0):
+ def _load_source_file(self, check=False):
+ """
+ Function that loads the source_file that contains a binary_c calls
"""
- Function to add code to the grid code string
- add code to the code_string
+ if not os.path.isfile(self.grid_options["source_file_filename"]):
+ self.verbose_print("Source file doesnt exist", self.grid_options["verbosity"], 0)
- indent (=0) is added once at the beginning
- mindent (=0) is added for every line
+ self.verbose_print(
+ message="Loading source file from {}".format(
+ self.grid_options["gridcode_filename"]
+ ),
+ verbosity=self.grid_options["verbosity"],
+ minimal_verbosity=1,
+ )
- don't use both!
- """
+ # We can choose to perform a check on the source file, which checks if the lines start with 'binary_c'
+ if check:
+ source_file_check_filehandle = open(
+ self.grid_options["source_file_filename"],
+ "r",
+ encoding='utf-8'
+ )
+ for line in source_file_check_filehandle:
+ if not line.startswith("binary_c"):
+ failed = True
+ break
+ if failed:
+ self.verbose_print(
+ "Error, sourcefile contains lines that do not start with binary_c",
+ self.grid_options["verbosity"],
+ 0,
+ )
+ raise ValueError
+
+ source_file_filehandle = open(self.grid_options["source_file_filename"],
+ "r",
+ encoding='utf-8')
- indent_block = self._indent_block(indent)
- for thing in args:
- self.code_string += indent_block + thing
+ self.grid_options["_system_generator"] = source_file_filehandle
- def _indent_block(self, n=0):
- """
- return an indent block, with n extra blocks in it
- """
- return (self.indent_depth + n) * self.indent_string
+ self.verbose_print("Source file loaded", self.grid_options["verbosity"], 1)
- def _increment_indent_depth(self, delta):
+ def _dict_from_line_source_file(self, line):
"""
- increment the indent indent_depth by delta
+ Function that creates a dict from a binary_c arg line
"""
- self.indent_depth += delta
+ if line.startswith("binary_c "):
+ line = line.replace("binary_c ", "")
- def _generate_grid_code(self, dry_run=False):
- """
- Function that generates the code from which the population will be made.
+ split_line = line.split()
+ arg_dict = {}
+
+ for i in range(0, len(split_line), 2):
+ if "." in split_line[i + 1]:
+ arg_dict[split_line[i]] = float(split_line[i + 1])
+ else:
+ arg_dict[split_line[i]] = int(split_line[i + 1])
- dry_run: when True, it will return the starcount at the end so that we know
- what the total number of systems is.
+ return arg_dict
- The phasevol values are handled by generating a second array
- # TODO: Add correct logging everywhere
- # TODO: add part to handle separation if orbital_period is added. Idea. use default values
- # for orbital parameters and possibly overwrite those or something.
- # TODO: add sensible description to this function.
- # TODO: Check whether all the probability and phasevol values are correct.
- # TODO: import only the necessary packages/functions
- # TODO: Put all the masses, eccentricities and periods in there already
- # TODO: Put the certain blocks that are repeated in some sub functions
- # TODO: make sure running systems with multiplicity 3+ is also possible.
+ ###################################################
+ # Unordered functions
+ #
+ # Functions that aren't ordered yet
+ ###################################################
- Results in a generated file that contains a system_generator function.
+ def _cleanup_defaults(self):
"""
- verbose_print("Generating grid code", self.grid_options["verbosity"], 1)
-
- total_grid_variables = len(self.grid_options["_grid_variables"])
-
- self._add_code(
- # Import packages
- "import math\n",
- "import numpy as np\n",
- "from collections import OrderedDict\n",
- "from binarycpython.utils.distribution_functions import *\n",
- "from binarycpython.utils.spacing_functions import *\n",
- "from binarycpython.utils.useful_funcs import *\n",
- "\n\n",
- # Make the function
- "def grid_code(self, print_results=True):\n",
- )
+ Function to clean up the default values:
- # Increase indent_depth
- self._increment_indent_depth(+1)
-
- self._add_code(
- # Write some info in the function
- "# Grid code generated on {}\n".format(datetime.datetime.now().isoformat()),
- "# This function generates the systems that will be evolved with binary_c\n\n"
- # Set some values in the generated code:
- "# Setting initial values\n",
- "_total_starcount = 0\n",
- "starcounts = [0 for i in range({})]\n".format(total_grid_variables + 1),
- "probabilities = {}\n",
- "probabilities_list = [0 for i in range({})]\n".format(
- total_grid_variables + 1
- ),
- "probabilities_sum = [0 for i in range({})]\n".format(
- total_grid_variables + 1
- ),
- "parameter_dict = {}\n",
- "phasevol = 1\n",
- )
+ from a dictionary, removes the entries that have the following values:
+ - "NULL"
+ - ""
+ - "Function"
- # Set up the system parameters
- self._add_code(
- "M_1 = None\n",
- "M_2 = None\n",
- "M_3 = None\n",
- "M_4 = None\n",
- "orbital_period = None\n",
- "orbital_period_triple = None\n",
- "orbital_period_quadruple = None\n",
- "eccentricity = None\n",
- "eccentricity2 = None\n",
- "eccentricity3 = None\n",
- "\n",
- # Prepare the probability
- "# setting probability lists\n",
- )
+ Uses the function from utils.functions
- for grid_variable_el in sorted(
- self.grid_options["_grid_variables"].items(),
- key=lambda x: x[1]["grid_variable_number"],
- ):
- # Make probabilities dict
- grid_variable = grid_variable_el[1]
- self._add_code('probabilities["{}"] = 0\n'.format(grid_variable["name"]))
-
- #################################################################################
- # Start of code generation
- #################################################################################
- self._add_code("\n")
-
- # turn vb to True to have debugging output
- vb = False
-
- # Generate code
- for loopnr, grid_variable_el in enumerate(
- sorted(
- self.grid_options["_grid_variables"].items(),
- key=lambda x: x[1]["grid_variable_number"],
- )
- ):
- verbose_print(
- "Constructing/adding: {}".format(grid_variable_el[0]),
- self.grid_options["verbosity"],
- 2,
- )
- grid_variable = grid_variable_el[1]
-
- ####################
- # top code
- if grid_variable["topcode"]:
- self._add_code(grid_variable["topcode"])
-
- #########################
- # Setting up the for loop
- # Add comment for for loop
- self._add_code(
- "# for loop for variable {name} gridtype {gridtype}".format(
- name=grid_variable["name"],
- gridtype=grid_variable["gridtype"],
- ) + "\n",
- "sampled_values_{} = {}".format(
- grid_variable["name"], grid_variable["samplerfunc"]
- )
- + "\n")
-
- if vb:
- self._add_code(
- "print('samples','{name}',':',sampled_values_{name})\n".format(
- name=grid_variable["name"],
- )
- )
-
- if vb:
- self._add_code(
- "print('sample {name} from',sampled_values_{name})".format(
- name=grid_variable["name"]
- )
- + "\n"
- )
-
- # calculate number of values and starting location
- #
- # if we're sampling a continuous variable, we
- # have one fewer grid point than the length of the
- # sampled_values list
- if ( grid_variable["gridtype"] in ["centred", "centre", "center", "edge", "left edge", "left", "right", "right edge"] ):
- offset = -1
- elif grid_variable["gridtype"] == "discrete":
- # discrete variables sample all the points
- offset = 0
-
- start = 0
-
- # for loop over the variable
- if vb:
- self._add_code(
- "print(\"var {name} values \",sampled_values_{name},\" len \",len(sampled_values_{name})+{offset},\" gridtype {gridtype} offset {offset}\\n\")\n".format(
- name=grid_variable["name"],
- offset=offset,
- gridtype=grid_variable['gridtype'],
- )
- )
- self._add_code(
- "for {name}_sample_number in range({start},len(sampled_values_{name})+{offset}):".format(
- name=grid_variable["name"],
- offset=offset,
- start=start
- )
- + "\n"
- )
-
- self._increment_indent_depth(+1)
-
- # {}_this_index is this grid point's index
- # {}_prev_index and {}_next_index are the previous and next grid points,
- # (which can be None if there is no previous or next, or if
- # previous and next should not be used: this is deliberate)
- #
-
- if grid_variable["gridtype"] == "discrete":
- # discrete grids only care about this,
- # both prev and next should be None to
- # force errors where they are used
- self._add_code(
- "{name}_this_index = {name}_sample_number ".format(
- name=grid_variable["name"],
- ),
- )
- self._add_code(
- "\n",
- "{name}_prev_index = None if {name}_this_index == 0 else ({name}_this_index - 1) ".format(
- name=grid_variable["name"],
- ),
- "\n",
- )
- self._add_code(
- "\n",
- "{name}_next_index = None if {name}_this_index >= (len(sampled_values_{name})+{offset} - 1) else ({name}_this_index + 1)".format(
- name=grid_variable["name"],
- offset=offset
- ),
- "\n",
- )
-
- elif (grid_variable["gridtype"] in [ "centred","centre","center","edge","left","left edge" ] ):
-
- # left and centred grids
- self._add_code("if {}_sample_number == 0:\n".format(grid_variable["name"]))
- self._add_code("{}_this_index = 0;\n".format(grid_variable["name"]), indent=1)
- self._add_code("else:\n")
- self._add_code("{name}_this_index = {name}_sample_number ".format(name=grid_variable["name"]),indent=1)
- self._add_code("\n")
- self._add_code("{name}_prev_index = ({name}_this_index - 1) if {name}_this_index > 0 else None ".format(name=grid_variable["name"]))
- self._add_code("\n")
- self._add_code("{name}_next_index = {name}_this_index + 1".format(name=grid_variable["name"]))
- self._add_code("\n")
-
- elif(grid_variable["gridtype"] in [ "right", "right edge" ] ):
-
- # right edged grid
- self._add_code("if {name}_sample_number == 0:\n".format(name=grid_variable["name"]))
- self._add_code("{name}_this_index = 1;\n".format(name=grid_variable["name"]),indent=1)
- self._add_code("else:\n")
- self._add_code("{name}_this_index = {name}_sample_number + 1 ".format(name=grid_variable["name"],),indent=1)
- self._add_code("\n")
- self._add_code("{name}_prev_index = {name}_this_index - 1".format(name=grid_variable["name"]))
- self._add_code("\n")
- self._add_code("{name}_next_index = ({name}_this_index + 1) if {name}_this_index < len(sampled_values_{name}) else None".format(name=grid_variable["name"]))
- self._add_code("\n")
-
- # calculate phase volume
- if(grid_variable["dphasevol"] == -1):
- # no phase volume required so set it to 1.0
- self._add_code("dphasevol_{name} = 1.0 # 666\n".format(name=grid_variable["name"]))
-
- elif(grid_variable["gridtype"] in [ "right", "right edge" ] ):
- # right edges always have this and prev defined
- self._add_code(
- "dphasevol_{name} = (sampled_values_{name}[{name}_this_index] - sampled_values_{name}[{name}_prev_index])".format(name=grid_variable["name"])
- + "\n"
- )
- elif grid_variable["gridtype"] == "discrete":
- # discrete might have next defined, use it if we can,
- # otherwise use prev
- self._add_code(
- "dphasevol_{name} = (sampled_values_{name}[{name}_next_index] - sampled_values_{name}[{name}_this_index]) if {name}_next_index else (sampled_values_{name}[{name}_this_index] - sampled_values_{name}[{name}_prev_index])".format(name=grid_variable["name"])
- + "\n"
- )
- else:
- # left and centred always have this and next defined
- self._add_code(
- "dphasevol_{name} = (sampled_values_{name}[{name}_next_index] - sampled_values_{name}[{name}_this_index])".format(name=grid_variable["name"])
- + "\n"
- )
-
-
- ##############
- # Add phasevol check:
- self._add_code("if dphasevol_{name} <= 0:\n".format(name=grid_variable["name"]))
-
- # TODO: We might actually want to add the starcount and probability to the totals regardless.
- # n that case we need another local variable which will prevent it from being run but will track those parameters
- # Add phasevol check action:
- self._add_code(
- 'print("Grid generator: dphasevol_{name} <= 0! (this=",{name}_this_index,"=",sampled_values_{name}[{name}_this_index],", next=",{name}_next_index,"=",sampled_values_{name}[{name}_next_index],") Skipping current sample.")'.format(name=grid_variable["name"])
- + "\n",
- "continue\n",
- indent=1,
- )
-
- if vb:
- self._add_code(
- "print('sample {name} from ',sampled_values_{name},' at this=',{name}_this_index,', next=',{name}_next_index)".format(name=grid_variable["name"])
- + "\n"
- )
-
- # select sampled point location based on gridtype (left, centre or right)
- if ( grid_variable["gridtype"] in ["edge", "left", "left edge", "right", "right edge", "discrete" ] ):
- self._add_code(
- "{name} = sampled_values_{name}[{name}_this_index]".format(
- name=grid_variable["name"])
- + "\n"
- )
- elif ( grid_variable["gridtype"] in ["centred", "centre", "center"] ):
- self._add_code(
- "{name} = 0.5 * (sampled_values_{name}[{name}_next_index] + sampled_values_{name}[{name}_this_index])".format(name=grid_variable["name"])
- + "\n"
- )
- else:
- msg = "Unknown gridtype value {type}.".format(type=grid_variable['gridtype'])
- raise ValueError(msg)
-
- if vb:
- self._add_code(
- "print('hence {name} = ',{name})\n".format(
- name=grid_variable["name"]
- )
- )
-
- #################################################################################
- # Check condition and generate for loop
-
- # If the grid variable has a condition, write the check and the action
- if grid_variable["condition"]:
- self._add_code(
- # Add comment
- "# Condition for {name}\n".format(name=grid_variable["name"]),
-
- # Add condition check
- "if not {condition}:\n".format(condition=grid_variable["condition"]),
- indent=0,
- )
-
- # Add condition failed action:
- if self.grid_options["verbosity"] >= 3:
- self._add_code(
- 'print("Grid generator: Condition for {name} not met!")'.format(
- name=grid_variable["name"]
- )
- + "\n",
- "continue" + "\n",
- indent=1,
- )
- else:
- self._add_code(
- "continue" + "\n",
- indent=1,
- )
- # Add some whitespace
- self._add_code("\n")
-
- # Add some whitespace
- self._add_code("\n")
-
- #########################
- # Setting up pre-code and value in some cases
- # Add pre-code
- if grid_variable["precode"]:
- self._add_code(
- "{precode}".format(
- precode=grid_variable["precode"].replace(
- "\n", "\n" + self._indent_block(0)
- )
- )
- + "\n"
- )
-
- # Set phasevol
- self._add_code(
- "phasevol *= dphasevol_{name}\n".format(
- name=grid_variable["name"],
- )
- )
-
- #######################
- # Probabilities
- # Calculate probability
- self._add_code(
- "\n",
- "# Setting probabilities\n",
- "d{name} = dphasevol_{name} * ({probdist})".format(
- name=grid_variable["name"],
- probdist=grid_variable["probdist"],
- )
- + "\n",
- # Save probability sum
- "probabilities_sum[{n}] += d{name}".format(
- n=grid_variable["grid_variable_number"],
- name=grid_variable["name"]
- )
- + "\n",
- )
-
- if grid_variable["grid_variable_number"] == 0:
- self._add_code(
- "probabilities_list[0] = d{name}".format(name=grid_variable["name"]) + "\n"
- )
- else:
- self._add_code(
- "probabilities_list[{this}] = probabilities_list[{prev}] * d{name}".format(
- this=grid_variable["grid_variable_number"],
- prev=grid_variable["grid_variable_number"] - 1,
- name=grid_variable["name"],
- )
- + "\n"
- )
-
- ##############
- # postcode
- if grid_variable["postcode"]:
- self._add_code(
- "{postcode}".format(
- postcode=grid_variable["postcode"].replace(
- "\n", "\n" + self._indent_block(0)
- )
- )
- + "\n"
- )
-
- #######################
- # Increment starcount for this parameter
- self._add_code(
- "\n",
- "# Increment starcount for {name}\n".format(name=grid_variable["name"]),
- "starcounts[{n}] += 1".format(
- n=grid_variable["grid_variable_number"],
- )
- + "\n",
- # Add value to dict
- 'parameter_dict["{name}"] = {name}'.format(
- name=grid_variable["parameter_name"]
- )
- + "\n",
- "\n",
- )
-
- self._increment_indent_depth(-1)
-
- # The final parts of the code, where things are returned, are within the deepest loop,
- # but in some cases code from a higher loop needs to go under it again
- # SO I think its better to put an if statement here that checks
- # whether this is the last loop.
- if loopnr == len(self.grid_options["_grid_variables"]) - 1:
- self._write_gridcode_system_call(
- grid_variable,
- dry_run,
- grid_variable["branchpoint"],
- grid_variable["branchcode"],
- )
-
- # increment indent_depth
- self._increment_indent_depth(+1)
-
- ####################
- # bottom code
- if grid_variable["bottomcode"]:
- self._add_code(grid_variable["bottomcode"])
-
- self._increment_indent_depth(-1)
- self._add_code("\n")
-
- # Write parts to write below the part that yield the results.
- # this has to go in a reverse order:
- # Here comes the stuff that is put after the deepest nested part that calls returns stuff.
- # Here we will have a
- reverse_sorted_grid_variables = sorted(
- self.grid_options["_grid_variables"].items(),
- key=lambda x: x[1]["grid_variable_number"],
- reverse=True,
- )
- for loopnr, grid_variable_el in enumerate(reverse_sorted_grid_variables):
- grid_variable = grid_variable_el[1]
-
- self._increment_indent_depth(+1)
- self._add_code(
- "#" * 40 + "\n",
- "# Code below is for finalising the handling of this iteration of the parameter {name}\n".format(
- name=grid_variable["name"]
- ),
- )
-
- # Set phasevol
- # TODO: fix. this isn't supposed to be the value that we give it here. discuss
- self._add_code("phasevol /= dphasevol_{name}\n\n".format(name=grid_variable["name"]))
-
- self._increment_indent_depth(-2)
-
- # Check the branchpoint part here. The branchpoint makes sure that we can construct
- # a grid with several multiplicities and still can make the system calls for each
- # multiplicity without reconstructing the grid each time
- if grid_variable["branchpoint"] > 0:
-
- self._increment_indent_depth(+1)
-
- self._add_code(
- # Add comment
- "# Condition for branchpoint at {}".format(
- reverse_sorted_grid_variables[loopnr + 1][1]["name"]
- )
- + "\n",
- # # Add condition check
- # "if not {}:".format(grid_variable["condition"])
- # + "\n"
- # Add branchpoint
- "if multiplicity=={}:".format(grid_variable["branchpoint"]) + "\n",
- )
-
- self._write_gridcode_system_call(
- reverse_sorted_grid_variables[loopnr + 1][1],
- dry_run,
- grid_variable["branchpoint"],
- grid_variable["branchcode"],
- )
- self._increment_indent_depth(-1)
- self._add_code("\n")
-
- ###############################
- # Finalising print statements
- #
- self._increment_indent_depth(+1)
- self._add_code("\n", "#" * 40 + "\n", "if print_results:\n")
- self._add_code(
- "print('Grid has handled {starcount} stars with a total probability of {probtot:g}'.format(starcount=_total_starcount,probtot=self.grid_options['_probtot']))\n",
- indent=1,
- )
-
- ################
- # Finalising return statement for dry run.
- #
- if dry_run:
- self._add_code("return _total_starcount\n")
-
- self._increment_indent_depth(-1)
- #################################################################################
- # Stop of code generation. Here the code is saved and written
-
- # Save the grid code to the grid_options
- verbose_print(
- "Saving grid code to grid_options", self.grid_options["verbosity"], 1
- )
-
- self.grid_options["code_string"] = self.code_string
-
- # Write to file
- gridcode_filename = self._gridcode_filename()
-
- self.grid_options["gridcode_filename"] = gridcode_filename
-
- verbose_print(
- "{blue}Writing grid code to {file} [dry_run = {dry}]{reset}".format(
- blue=self.ANSI_colours["blue"],
- file=gridcode_filename,
- dry=dry_run,
- reset=self.ANSI_colours["reset"],
- ),
- self.grid_options["verbosity"],
- 1,
- )
-
- with open(gridcode_filename, "w",encoding='utf-8') as file:
- file.write(self.code_string)
-
- # perhaps create symlink
- if self.grid_options['slurm']==0 and \
- self.grid_options["symlink_latest_gridcode"]:
- global _count
- symlink = os.path.join(
- self.grid_options["tmp_dir"], "binary_c_grid-latest" + str(_count)
- )
- _count += 1
- try:
- os.unlink(symlink)
- except:
- pass
-
- try:
- os.symlink(gridcode_filename, symlink)
- verbose_print(
- "{blue}Symlinked grid code to {symlink} {reset}".format(
- blue=self.ANSI_colours["blue"],
- symlink=symlink,
- reset=self.ANSI_colours["reset"]
- ),
- self.grid_options["verbosity"],
- 1,
- )
- except OSError:
- print("symlink failed")
-
- def _write_gridcode_system_call(
- self, grid_variable, dry_run, branchpoint, branchcode
- ):
- #################################################################################
- # Here are the calls to the queuing or other solution. this part is for every system
- # Add comment
- self._increment_indent_depth(+1)
- self._add_code("#" * 40 + "\n")
-
- if branchcode:
- self._add_code("# Branch code\nif {branchcode}:\n".format(branchcode=branchcode))
-
- if branchpoint:
- self._add_code(
- "# Code below will get evaluated for every system at this level of multiplicity (last one of that being {name})\n".format(
- name=grid_variable["name"]
- )
- )
- else:
- self._add_code(
- "# Code below will get evaluated for every generated system\n"
- )
-
- # Factor in the custom weight input
- self._add_code(
- "\n",
- "# Weigh the probability by a custom weighting factor\n",
- 'probability = self.grid_options["weight"] * probabilities_list[{n}]'.format(
- n=grid_variable["grid_variable_number"]
- )
- + "\n",
- # Take into account the multiplicity fraction:
- "\n",
- "# Factor the multiplicity fraction into the probability\n",
- "probability = probability * self._calculate_multiplicity_fraction(parameter_dict)"
- + "\n",
- # Add division by number of repeats
- "\n",
- "# Divide the probability by the number of repeats\n",
- 'probability = probability / self.grid_options["repeat"]' + "\n",
- # Now we yield the system self.grid_options["repeat"] times.
- "\n",
- "# Loop over the repeats\n",
- 'for _ in range(self.grid_options["repeat"]):' + "\n",
- )
- self._add_code(
- "_total_starcount += 1\n",
- # set probability and phasevol values into the system dict
- 'parameter_dict["{p}"] = {p}'.format(p="probability") + "\n",
- 'parameter_dict["{v}"] = {v}'.format(v="phasevol") + "\n",
- # Increment total probability
- "self._increment_probtot(probability)\n",
- indent=1,
- )
-
- if not dry_run:
- # Handling of what is returned, or what is not.
- self._add_code("yield(parameter_dict)\n", indent=1)
-
- # If its a dry run, dont do anything with it
- else:
- self._add_code("pass\n", indent=1)
-
- self._add_code("#" * 40 + "\n")
-
- self._increment_indent_depth(-1)
-
- return self.code_string
-
- def _load_grid_function(self):
- """
- Function that loads the script containing the grid code.
-
- TODO: Update this description
- Test function to run grid stuff. mostly to test the import
- """
-
- # Code to load the
- verbose_print(
- message="Loading grid code function from {file}".format(
- file=self.grid_options["gridcode_filename"]
- ),
- verbosity=self.grid_options["verbosity"],
- minimal_verbosity=1,
- )
-
- spec = importlib.util.spec_from_file_location(
- "binary_c_python_grid",
- os.path.join(self.grid_options["gridcode_filename"]),
- )
- grid_file = importlib.util.module_from_spec(spec)
- spec.loader.exec_module(grid_file)
- generator = grid_file.grid_code
-
- self.grid_options["_system_generator"] = generator
-
- verbose_print("Grid code loaded", self.grid_options["verbosity"], 1)
-
- def _dry_run(self):
- """
- Function to dry run the grid and know how many stars it will run
-
- Requires the grid to be built as a dry run grid
- """
- verbose_print("Dry run of the grid", self.grid_options["verbosity"], 1)
- system_generator = self.grid_options["_system_generator"]
- total_starcount = system_generator(self)
- self.grid_options["_total_starcount"] = total_starcount
-
- def _print_info(self, run_number, total_systems, full_system_dict):
- """
- Function to print info about the current system and the progress of the grid.
-
- # color info tricks from https://ozzmaker.com/add-colour-to-text-in-python/
- https://stackoverflow.com/questions/287871/how-to-print-colored-text-in-terminal-in-python
- """
-
- # Define frequency
- if self.grid_options["verbosity"] == 1:
- print_freq = 1
- else:
- print_freq = 10
-
- if run_number % print_freq == 0:
- binary_cmdline_string = self._return_argline(full_system_dict)
- info_string = "{color_part_1} \
- {text_part_1}{end_part_1}{color_part_2} \
- {text_part_2}{end_part_2}".format(
- color_part_1="\033[1;32;41m",
- text_part_1="{}/{}".format(run_number, total_systems),
- end_part_1="\033[0m",
- color_part_2="\033[1;32;42m",
- text_part_2="{}".format(binary_cmdline_string),
- end_part_2="\033[0m",
- )
- print(info_string)
-
- ###################################################
- # Monte Carlo functions
- #
- # Functions below are used to run populations with
- # Monte Carlo
- ###################################################
-
- ###################################################
- # Population from file functions
- #
- # Functions below are used to run populations from
- # a file containing binary_c calls
- ###################################################
- def _dry_run_source_file(self):
- """
- Function to go through the source_file and count the number of lines and the total probability
- """
- system_generator = self.grid_options["_system_generator"]
- total_starcount = 0
- total_probability = 0
- contains_probability = False
-
- for line in system_generator:
- total_starcount += 1
-
- total_starcount = system_generator(self)
- self.grid_options["_total_starcount"] = total_starcount
-
- def _load_source_file(self, check=False):
- """
- Function that loads the source_file that contains a binary_c calls
- """
-
- if not os.path.isfile(self.grid_options["source_file_filename"]):
- verbose_print("Source file doesnt exist", self.grid_options["verbosity"], 0)
-
- verbose_print(
- message="Loading source file from {}".format(
- self.grid_options["gridcode_filename"]
- ),
- verbosity=self.grid_options["verbosity"],
- minimal_verbosity=1,
- )
-
- # We can choose to perform a check on the source file, which checks if the lines start with 'binary_c'
- if check:
- source_file_check_filehandle = open(
- self.grid_options["source_file_filename"],
- "r",
- encoding='utf-8'
- )
- for line in source_file_check_filehandle:
- if not line.startswith("binary_c"):
- failed = True
- break
- if failed:
- verbose_print(
- "Error, sourcefile contains lines that do not start with binary_c",
- self.grid_options["verbosity"],
- 0,
- )
- raise ValueError
-
- source_file_filehandle = open(self.grid_options["source_file_filename"],
- "r",
- encoding='utf-8')
-
- self.grid_options["_system_generator"] = source_file_filehandle
-
- verbose_print("Source file loaded", self.grid_options["verbosity"], 1)
-
- def _dict_from_line_source_file(self, line):
- """
- Function that creates a dict from a binary_c arg line
- """
- if line.startswith("binary_c "):
- line = line.replace("binary_c ", "")
-
- split_line = line.split()
- arg_dict = {}
-
- for i in range(0, len(split_line), 2):
- if "." in split_line[i + 1]:
- arg_dict[split_line[i]] = float(split_line[i + 1])
- else:
- arg_dict[split_line[i]] = int(split_line[i + 1])
-
- return arg_dict
-
- ###################################################
- # CONDOR functions
- #
- # subroutines to run CONDOR grids
- ###################################################
-
- # def _condor_grid(self):
- # """
- # Main function that manages the CONDOR setup.
-
- # Has three stages:
-
- # - setup
- # - evolve
- # - join
-
- # Which stage is used is determined by the value of grid_options['condor_command']:
-
- # <empty>: the function will know its the user that executed the script and
- # it will set up the necessary condor stuff
-
- # 'evolve': evolve_population is called to evolve the population of stars
-
- # 'join': We will attempt to join the output
- # """
-
- # # TODO: Put in function
- # condor_version = get_condor_version()
- # if not condor_version:
- # verbose_print(
- # "CONDOR: Error: No installation of condor found",
- # self.grid_options["verbosity"],
- # 0,
- # )
- # else:
- # major_version = int(condor_version.split(".")[0])
- # minor_version = int(condor_version.split(".")[1])
-
- # if (major_version == 8) and (minor_version > 4):
- # verbose_print(
- # "CONDOR: Found version {} which is new enough".format(
- # condor_version
- # ),
- # self.grid_options["verbosity"],
- # 0,
- # )
- # elif major_version > 9:
- # verbose_print(
- # "CONDOR: Found version {} which is new enough".format(
- # condor_version
- # ),
- # self.grid_options["verbosity"],
- # 0,
- # )
- # else:
- # verbose_print(
- # "CONDOR: Found version {} which is too old (we require 8.3/8.4+)".format(
- # condor_version
- # ),
- # self.grid_options["verbosity"],
- # 0,
- # )
-
- # verbose_print(
- # "Running Condor grid. command={}".format(
- # self.grid_options["condor_command"]
- # ),
- # self.grid_options["verbosity"],
- # 1,
- # )
- # if not self.grid_options["condor_command"]:
- # # Setting up
- # verbose_print(
- # "CONDOR: Main controller script. Setting up",
- # self.grid_options["verbosity"],
- # 1,
- # )
-
- # # Set up working directories:
- # verbose_print(
- # "CONDOR: creating working directories",
- # self.grid_options["verbosity"],
- # 1,
- # )
- # create_directories_hpc(self.grid_options["condor_dir"])
-
- # # Create command
- # current_workingdir = os.getcwd()
- # python_details = get_python_details()
- # scriptname = path_of_calling_script()
- # # command = "".join([
- # # "{}".python_details['executable'],
- # # "{}".scriptname,
- # # "offset=$jobarrayindex",
- # # "modulo={}".format(self.grid_options['condor_njobs']),
- # # "vb={}".format(self.grid_options['verbosity'])
-
- # # "results_hash_dumpfile=$self->{_grid_options}{slurm_dir}/results/$jobid.$jobarrayindex",
- # # 'slurm_jobid='.$jobid,
- # # 'slurm_jobarrayindex='.$jobarrayindex,
- # # 'slurm_jobname=binary_grid_'.$jobid.'.'.$jobarrayindex,
- # # "slurm_njobs=$njobs",
- # # "slurm_dir=$self->{_grid_options}{slurm_dir}",
- # # );
-
- # # Create directory with info for the condor script. By creating this directory we also check whether all the values are set correctly
- # # TODO: create the condor script.
- # condor_script_options = {}
- # # condor_script_options['n'] =
- # condor_script_options["njobs"] = self.grid_options["condor_njobs"]
- # condor_script_options["dir"] = self.grid_options["condor_dir"]
- # condor_script_options["memory"] = self.grid_options["condor_memory"]
- # condor_script_options["working_dir"] = self.grid_options[
- # "condor_working_dir"
- # ]
- # condor_script_options["command"] = self.grid_options["command"]
- # condor_script_options["streams"] = self.grid_options["streams"]
-
- # # TODO: condor works with running an executable.
-
- # # Create script contents
- # condor_script_contents = ""
- # condor_script_contents += """
- # #################################################
- # #
- # # Condor script to run a binary_grid via python
- # #
- # #################################################
- # """
- # condor_script_contents += "Executable\t= {}".format(executable)
- # condor_script_contents += "arguments\t= {}".format(arguments)
- # condor_script_contents += "environment\t= {}".format(environment)
- # condor_script_contents += "universe\t= {}".format(
- # self.grid_options["condor_universe"]
- # )
- # condor_script_contents += "\n"
- # condor_script_contents += "output\t= {}/stdout/$id\n".format(
- # self.grid_options["condor_dir"]
- # )
- # condor_script_contents += "error\t={}/sterr/$id".format(
- # self.grid_options["condor_dir"]
- # )
- # condor_script_contents += "log\t={}\n".format(
- # self.grid_options["condor_dir"]
- # )
- # condor_script_contents += "initialdir\t={}\n".format(current_workingdir)
- # condor_script_contents += "remote_initialdir\t={}\n".format(
- # current_workingdir
- # )
- # condor_script_contents += "\n"
- # condor_script_contents += "steam_output\t={}".format(stream)
- # condor_script_contents += "steam_error\t={}".format(stream)
- # condor_script_contents += "+WantCheckpoint = False"
- # condor_script_contents += "\n"
- # condor_script_contents += "request_memory\t={}".format(
- # self.grid_options["condor_memory"]
- # )
- # condor_script_contents += "ImageSize\t={}".format(
- # self.grid_options["condor_memory"]
- # )
- # condor_script_contents += "\n"
-
- # if self.grid_options["condor_extra_settings"]:
- # slurm_script_contents += "# Extra settings by user:"
- # slurm_script_contents += "\n".join(
- # [
- # "{}\t={}".format(
- # key, self.grid_options["condor_extra_settings"][key]
- # )
- # for key in self.grid_options["condor_extra_settings"]
- # ]
- # )
-
- # condor_script_contents += "\n"
-
- # # request_memory = $_[0]{memory}
- # # ImageSize = $_[0]{memory}
-
- # # Requirements = (1) \&\& (".
- # # $self->{_grid_options}{condor_requirements}.")\n";
-
- # #
- # # file name: my_program.condor
- # # Condor submit description file for my_program
- # # Executable = my_program
- # # Universe = vanilla
- # # Error = logs/err.$(cluster)
- # # Output = logs/out.$(cluster)
- # # Log = logs/log.$(cluster)
-
- # # should_transfer_files = YES
- # # when_to_transfer_output = ON_EXIT
- # # transfer_input_files = files/in1,files/in2
-
- # # Arguments = files/in1 files/in2 files/out1
- # # Queue
-
- # # Write script contents to file
- # if self.grid_options["condor_postpone_join"]:
- # condor_script_contents += "{} rungrid=0 results_hash_dumpfile={}/results/$jobid.all condor_command=join\n".format(
- # command, self.grid_options["condor_dir"]
- # )
-
- # condor_script_filename = os.path.join(
- # self.grid_options["condor_dir"], "condor_script"
- # )
- # with open(condor_script_filename, "w") as condor_script_file:
- # condor_script_file.write(condor_script_contents)
-
- # if self.grid_options["condor_postpone_sbatch"]:
- # # Execute or postpone the real call to sbatch
- # submit_command = "condor_submit {}".format(condor_script_filename)
- # verbose_print(
- # "running condor script {}".format(condor_script_filename),
- # self.grid_options["verbosity"],
- # 0,
- # )
- # # subprocess.Popen(sbatch_command, close_fds=True)
- # # subprocess.Popen(sbatch_command, creationflags=subprocess.DETACHED_PROCESS)
- # verbose_print("Submitted scripts.", self.grid_options["verbosity"], 0)
- # else:
- # verbose_print(
- # "Condor script is in {} but hasnt been executed".format(
- # condor_script_filename
- # ),
- # self.grid_options["verbosity"],
- # 0,
- # )
-
- # verbose_print("all done!", self.grid_options["verbosity"], 0)
- # self.exit()
-
- # elif self.grid_options["condor_command"] == "evolve":
- # # TODO: write this function
- # # Part to evolve the population.
- # # TODO: decide how many CPUs
- # verbose_print(
- # "CONDOR: Evolving population", self.grid_options["verbosity"], 1
- # )
-
- # #
- # self.evolve_population()
-
- # elif self.grid_options["condor_command"] == "join":
- # # TODO: write this function
- # # Joining the output.
- # verbose_print("CONDOR: Joining results", self.grid_options["verbosity"], 1)
-
- # pass
- ###################################################
- # Unordered functions
- #
- # Functions that aren't ordered yet
- ###################################################
-
- def write_ensemble(self, output_file, data=None, sort_keys=True, indent=4, encoding='utf-8', ensure_ascii=False):
- """
- write_ensemble : Write ensemble results to a file.
-
- Args:
- output_file : the output filename.
-
- If the filename has an extension that we recognise,
- e.g. .gz or .bz2, we compress the output appropriately.
-
- The filename should contain .json or .msgpack, the two
- currently-supported formats.
-
- Usually you'll want to output to JSON, but we can
- also output to msgpack.
-
- data : the data dictionary to be converted and written to the file.
- If not set, this defaults to self.grid_ensemble_results.
-
- sort_keys : if True, and output is to JSON, the keys will be sorted.
- (default: True, passed to json.dumps)
-
- indent : number of space characters used in the JSON indent. (Default: 4,
- passed to json.dumps)
-
- encoding : file encoding method, usually defaults to 'utf-8'
-
- ensure_ascii : the ensure_ascii flag passed to json.dump and/or json.dumps
- (Default: False)
- """
-
- # get the file type
- file_type = ensemble_file_type(output_file)
-
- # choose compression algorithm based on file extension
- compression = ensemble_compression(output_file)
-
- # default to using grid_ensemble_results if no data is given
- if data is None:
- data = self.grid_ensemble_results
-
- if not file_type:
- print(
- "Unable to determine file type from ensemble filename {} : it should be .json or .msgpack."
- ).format(output_file)
- self.exit(code=1)
- elif file_type == "JSON":
- # JSON output
- if compression == "gzip":
- # gzip
- f = gzip.open(output_file, "wt", encoding=encoding)
- elif compression == "bzip2":
- # bzip2
- f = bz2.open(output_file, "wt", encoding=encoding)
- else:
- # raw output (not compressed)
- f = open(output_file, "wt", encoding=encoding)
- f.write(json.dumps(data,
- sort_keys=sort_keys,
- indent=indent,
- ensure_ascii=ensure_ascii))
-
- elif file_type == "msgpack":
- # msgpack output
- if compression == "gzip":
- f = gzip.open(output_file, "wb", encoding=encoding)
- elif compression == "bzip2":
- f = bz2.open(output_file, "wb", encoding=encoding)
- else:
- f = open(output_file, "wb", encoding=encoding)
- msgpack.dump(data, f)
- f.close()
-
- print(
- "Thread {thread}: Wrote ensemble results to file: {colour}{file}{reset} (file type {file_type}, compression {compression})".format(
- thread=self.process_ID,
- file=output_file,
- colour=self.ANSI_colours["green"],
- reset=self.ANSI_colours["reset"],
- file_type=file_type,
- compression=compression,
- )
- )
-
- ############################################################
- def write_binary_c_calls_to_file(
- self,
- output_dir: Union[str, None] = None,
- output_filename: Union[str, None] = None,
- include_defaults: bool = False,
- encoding='utf-8'
- ) -> None:
- """
- Function that loops over the grid code and writes the generated parameters to a file.
- In the form of a command line call
-
- Only useful when you have a variable grid as system_generator. MC wouldn't be that useful
-
- Also, make sure that in this export there are the basic parameters
- like m1,m2,sep, orb-per, ecc, probability etc.
-
- On default this will write to the datadir, if it exists
-
- Tasks:
- - TODO: test this function
- - TODO: make sure the binary_c_python .. output file has a unique name
-
- Args:
- output_dir: (optional, default = None) directory where to write the file to. If custom_options['data_dir'] is present, then that one will be used first, and then the output_dir
- output_filename: (optional, default = None) filename of the output. If not set it will be called "binary_c_calls.txt"
- include_defaults: (optional, default = None) whether to include the defaults of binary_c in the lines that are written. Beware that this will result in very long lines, and it might be better to just export the binary_c defaults and keep them in a separate file.
-
- Returns:
- filename: filename that was used to write the calls to
- """
-
- # Check if there is no compiled grid yet. If not, lets try to build it first.
- if not self.grid_options["_system_generator"]:
-
- ## check the settings:
- if self.bse_options.get("ensemble", None):
- if self.bse_options["ensemble"] == 1:
- if not self.bse_options.get("ensemble_defer", 0) == 1:
- verbose_print(
- "Error, if you want to run an ensemble in a population, the output needs to be deferred",
- self.grid_options["verbosity"],
- 0,
- )
- raise ValueError
-
- # Put in check
- if len(self.grid_options["_grid_variables"]) == 0:
- print("Error: you haven't defined any grid variables! Aborting")
- raise ValueError
-
- #
- self._generate_grid_code(dry_run=False)
-
- #
- self._load_grid_function()
-
- # then if the _system_generator is present, we go through it
- if self.grid_options["_system_generator"]:
- # Check if there is an output dir configured
- if self.custom_options.get("data_dir", None):
- binary_c_calls_output_dir = self.custom_options["data_dir"]
- # otherwise check if there's one passed to the function
- else:
- if not output_dir:
- print(
- "Error. No data_dir configured and you gave no output_dir. Aborting"
- )
- raise ValueError
- binary_c_calls_output_dir = output_dir
-
- # check if there's a filename passed to the function
- if output_filename:
- binary_c_calls_filename = output_filename
- # otherwise use default value
- else:
- binary_c_calls_filename = "binary_c_calls.txt"
-
- binary_c_calls_full_filename = os.path.join(
- binary_c_calls_output_dir, binary_c_calls_filename
- )
- print("Writing binary_c calls to {}".format(binary_c_calls_full_filename))
-
- # Write to file
- with open(binary_c_calls_full_filename, "w", encoding=encoding) as file:
- # Get defaults and clean them, then overwrite them with the set values.
- if include_defaults:
- # TODO: make sure that the defaults here are cleaned up properly
- cleaned_up_defaults = self.cleaned_up_defaults
- full_system_dict = cleaned_up_defaults.copy()
- full_system_dict.update(self.bse_options.copy())
- else:
- full_system_dict = self.bse_options.copy()
-
- for system in self.grid_options["_system_generator"](self):
- # update values with current system values
- full_system_dict.update(system)
-
- binary_cmdline_string = self._return_argline(full_system_dict)
- file.write(binary_cmdline_string + "\n")
- else:
- print("Error. No grid function found!")
- raise ValueError
-
- return binary_c_calls_full_filename
-
- def _cleanup_defaults(self):
- """
- Function to clean up the default values:
-
- from a dictionary, removes the entries that have the following values:
- - "NULL"
- - ""
- - "Function"
-
- Uses the function from utils.functions
-
- TODO: Rethink this functionality. seems a bit double, could also be just outside of the class
- """
-
- binary_c_defaults = self.return_binary_c_defaults().copy()
- cleaned_dict = filter_arg_dict(binary_c_defaults)
-
- return cleaned_dict
-
- def _clean_up_custom_logging(self, evol_type):
- """
- Function to clean up the custom logging.
- Has two types:
- 'single':
- - removes the compiled shared library
- (which name is stored in grid_options['_custom_logging_shared_library_file'])
- - TODO: unloads/frees the memory allocated to that shared library
- (which is stored in grid_options['custom_logging_func_memaddr'])
- - sets both to None
- 'multiple':
- - TODO: make this and design this
- """
-
- if evol_type == "single":
- verbose_print(
- "Cleaning up the custom logging stuff. type: single",
- self.grid_options["verbosity"],
- 1,
- )
-
- # TODO: Explicitly unload the library
-
- # Reset the memory adress location
- self.grid_options["custom_logging_func_memaddr"] = -1
-
- # remove shared library files
- if self.grid_options["_custom_logging_shared_library_file"]:
- remove_file(
- self.grid_options["_custom_logging_shared_library_file"],
- self.grid_options["verbosity"],
- )
- self.grid_options["_custom_logging_shared_library_file"] = None
-
- if evol_type == "population":
- verbose_print(
- "Cleaning up the custom logging stuffs. type: population",
- self.grid_options["verbosity"],
- 1,
- )
-
- # TODO: make sure that these also work. not fully sure if necessary tho.
- # whether its a single file, or a dict of files/mem addresses
-
- if evol_type == "MC":
- pass
-
- def _increment_probtot(self, prob):
- """
- Function to add to the total probability. For now not used
- """
-
- self.grid_options["_probtot"] += prob
-
- def _increment_count(self):
- """
- Function to add to the total number of stars. For now not used
- """
- self.grid_options["_count"] += 1
-
- def _set_loggers(self):
- """
- Function to set the loggers for the execution of the grid
- """
-
- # Set log file
- binary_c_logfile = self.grid_options["log_file"]
-
- # Create directory
- os.makedirs(os.path.dirname(binary_c_logfile), exist_ok=True)
-
- # Set up logger
- self.logger = logging.getLogger("binary_c_python_logger")
- self.logger.setLevel(self.grid_options["verbosity"])
-
- # Reset handlers
- self.logger.handlers = []
-
- # Set formatting of output
- log_formatter = logging.Formatter(
- "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
- )
-
- # Make and add file handlers
- # make handler for output to file
- handler_file = logging.FileHandler(filename=os.path.join(binary_c_logfile))
- handler_file.setFormatter(log_formatter)
- handler_file.setLevel(logging.INFO)
-
- # Make handler for output to stdout
- handler_stdout = logging.StreamHandler(sys.stdout)
- handler_stdout.setFormatter(log_formatter)
- handler_stdout.setLevel(logging.INFO)
-
- # Add the loggers
- self.logger.addHandler(handler_file)
- self.logger.addHandler(handler_stdout)
-
- def _check_binary_c_error(self, binary_c_output, system_dict):
- """
- Function to check whether binary_c throws an error and handle accordingly.
- """
-
- if binary_c_output:
- if (binary_c_output.splitlines()[0].startswith("SYSTEM_ERROR")) or (
- binary_c_output.splitlines()[-1].startswith("SYSTEM_ERROR")
- ):
- verbose_print(
- "FAILING SYSTEM FOUND",
- self.grid_options["verbosity"],
- 0,
- )
-
- # Keep track of the amount of failed systems and their error codes
- self.grid_options["_failed_prob"] += system_dict.get("probability", 1)
- self.grid_options["_failed_count"] += 1
- self.grid_options["_errors_found"] = True
-
- # Try catching the error code and keep track of the unique ones.
- try:
- error_code = int(
- binary_c_output.splitlines()[0]
- .split("with error code")[-1]
- .split(":")[0]
- .strip()
- )
-
- if (
- not error_code
- in self.grid_options["_failed_systems_error_codes"]
- ):
- self.grid_options["_failed_systems_error_codes"].append(
- error_code
- )
- except ValueError:
- verbose_print(
- "Failed to extract the error-code",
- self.grid_options["verbosity"],
- 1,
- )
-
- # Check if we have exceeded the number of errors
- if (
- self.grid_options["_failed_count"]
- > self.grid_options["failed_systems_threshold"]
- ):
- if not self.grid_options["_errors_exceeded"]:
- verbose_print(
- self._boxed(
- "Process {} exceeded the maximum ({}) number of failing systems. Stopped logging them to files now".format(
- self.process_ID,
- self.grid_options["failed_systems_threshold"],
- )
- ),
- self.grid_options["verbosity"],
- 1,
- )
- self.grid_options["_errors_exceeded"] = True
-
- # If not, write the failing systems to files unique to each process
- else:
- # Write arg lines to file
- argstring = self._return_argline(system_dict)
- with open(
- os.path.join(
- self.grid_options["tmp_dir"],
- "failed_systems",
- "process_{}.txt".format(self.process_ID),
- ),
- "a+",
- encoding='utf-8'
- ) as f:
- f.write(argstring + "\n")
- f.close()
- else:
- verbose_print(
- "binary_c output nothing - this is strange. If there is ensemble output being generated then this is fine.",
- self.grid_options["verbosity"],
- 3,
- )
-
- def set_moe_di_stefano_settings(self, options=None):
- """
- Function to set user input configurations for the Moe & di Stefano methods
-
- If nothing is passed then we just use the default options
- """
-
- if not options:
- options = {}
-
- # Take the option dictionary that was given and override.
- options = update_dicts(self.grid_options["Moe2017_options"], options)
- self.grid_options["Moe2017_options"] = copy.deepcopy(options)
-
- # Write options to a file
- os.makedirs(
- os.path.join(self.grid_options["tmp_dir"], "moe_distefano"),
- exist_ok=True,
- )
- with open(
- os.path.join(
- os.path.join(self.grid_options["tmp_dir"], "moe_distefano"),
- "moeopts.dat",
- ),
- "w",
- encoding='utf-8'
- ) as f:
- f.write(json.dumps(self.grid_options["Moe2017_options"], indent=4, ensure_ascii=False))
- f.close()
-
- def _load_moe_di_stefano_data(self):
- """
- Function to load the moe & di stefano data
- """
-
- # Only if the grid is loaded and Moecache contains information
- if not self.grid_options["_loaded_Moe2017_data"]: # and not Moecache:
-
- if self.grid_options["_Moe2017_JSON_data"]:
- # Use the existing (perhaps modified) JSON data
- json_data = self.grid_options["_Moe2017_JSON_data"]
-
- else:
- # Load the JSON data from a file
- json_data = get_moe_di_stefano_dataset(
- self.grid_options["Moe2017_options"],
- verbosity=self.grid_options["verbosity"],
- )
-
- # entry of log10M1 is a list containing 1 dict.
- # We can take the dict out of the list
- if isinstance(json_data["log10M1"], list):
- json_data["log10M1"] = json_data["log10M1"][0]
-
- # save this data in case we want to modify it later
- self.grid_options["_Moe2017_JSON_data"] = json_data
-
- # Get all the masses
- logmasses = sorted(json_data["log10M1"].keys())
- if not logmasses:
- msg = "The table does not contain masses."
- verbose_print(
- "\tMoe_di_Stefano_2017: {}".format(msg),
- self.grid_options["verbosity"],
- 0,
- )
- raise ValueError(msg)
-
- # Write to file
- os.makedirs(
- os.path.join(self.grid_options["tmp_dir"], "moe_distefano"),
- exist_ok=True,
- )
- with open(
- os.path.join(
- os.path.join(self.grid_options["tmp_dir"], "moe_distefano"),
- "moe.log",
- ),
- "w",
- encoding='utf-8',
- ) as logfile:
- logfile.write("logâ‚â‚€Masses(M☉) {}\n".format(logmasses))
-
- # Get all the periods and see if they are all consistently present
- logperiods = []
- for logmass in logmasses:
- if not logperiods:
- logperiods = sorted(json_data["log10M1"][logmass]["logP"].keys())
- dlog10P = float(logperiods[1]) - float(logperiods[0])
-
- current_logperiods = sorted(json_data["log10M1"][logmass]["logP"])
- if not (logperiods == current_logperiods):
- msg = (
- "Period values are not consistent throughout the dataset\logperiods = "
- + " ".join(str(x) for x in logperiods)
- + "\nCurrent periods = "
- + " ".join(str(x) for x in current_logperiods)
- )
- verbose_print(
- "\tMoe_di_Stefano_2017: {}".format(msg),
- self.grid_options["verbosity"],
- 0,
- )
- raise ValueError(msg)
-
- ############################################################
- # log10period binwidth : of course this assumes a fixed
- # binwidth, so we check for this too.
- for i in range(len(current_logperiods) - 1):
- if not dlog10P == (
- float(current_logperiods[i + 1]) - float(current_logperiods[i])
- ):
- msg = "Period spacing is not consistent throughout the dataset"
- verbose_print(
- "\tMoe_di_Stefano_2017: {}".format(msg),
- self.grid_options["verbosity"],
- 0,
- )
- raise ValueError(msg)
-
- # save the logperiods list in the cache:
- # this is used in the renormalization integration
- Moecache["logperiods"] = logperiods
-
- # Write to file
- os.makedirs(
- os.path.join(self.grid_options["tmp_dir"], "moe_distefano"),
- exist_ok=True,
- )
- with open(
- os.path.join(self.grid_options["tmp_dir"], "moe_distefano", "moe.log"),
- "a",
- encoding='utf-8'
- ) as logfile:
- logfile.write("logâ‚â‚€Periods(days) {}\n".format(logperiods))
-
- # Fill the global dict
- for logmass in logmasses:
- # Create the multiplicity table
- if not Moecache.get("multiplicity_table", None):
- Moecache["multiplicity_table"] = []
-
- # multiplicity as a function of primary mass
- Moecache["multiplicity_table"].append(
- [
- float(logmass),
- json_data["log10M1"][logmass]["f_multi"],
- json_data["log10M1"][logmass]["single star fraction"],
- json_data["log10M1"][logmass]["binary star fraction"],
- json_data["log10M1"][logmass]["triple/quad star fraction"],
- ]
- )
-
- ############################################################
- # a small log10period which we can shift just outside the
- # table to force integration out there to zero
- epslog10P = 1e-8 * dlog10P
-
- ############################################################
- # loop over either binary or triple-outer periods
- first = 1
-
- # Go over the periods
- for logperiod in logperiods:
- ############################################################
- # distributions of binary and triple star fractions
- # as a function of mass, period.
- #
- # Note: these should be per unit log10P, hence we
- # divide by dlog10P
-
- if first:
- first = 0
-
- # Create the multiplicity table
- if not Moecache.get("period_distributions", None):
- Moecache["period_distributions"] = []
-
- ############################################################
- # lower bound the period distributions to zero probability
- Moecache["period_distributions"].append(
- [
- float(logmass),
- float(logperiod) - 0.5 * dlog10P - epslog10P,
- 0.0,
- 0.0,
- ]
- )
- Moecache["period_distributions"].append(
- [
- float(logmass),
- float(logperiod) - 0.5 * dlog10P,
- json_data["log10M1"][logmass]["logP"][logperiod][
- "normed_bin_frac_p_dist"
- ]
- / dlog10P,
- json_data["log10M1"][logmass]["logP"][logperiod][
- "normed_tripquad_frac_p_dist"
- ]
- / dlog10P,
- ]
- )
-
- Moecache["period_distributions"].append(
- [
- float(logmass),
- float(logperiod),
- json_data["log10M1"][logmass]["logP"][logperiod][
- "normed_bin_frac_p_dist"
- ]
- / dlog10P,
- json_data["log10M1"][logmass]["logP"][logperiod][
- "normed_tripquad_frac_p_dist"
- ]
- / dlog10P,
- ]
- )
-
- ############################################################
- # distributions as a function of mass, period, q
- #
- # First, get a list of the qs given by Moe
- #
- qs = sorted(json_data["log10M1"][logmass]["logP"][logperiod]["q"])
-
- # Fill the data and 'normalise'
- qdata = fill_data(
- qs, json_data["log10M1"][logmass]["logP"][logperiod]["q"]
- )
-
- # Create the multiplicity table
- if not Moecache.get("q_distributions", None):
- Moecache["q_distributions"] = []
-
- for q in qs:
- Moecache["q_distributions"].append(
- [float(logmass), float(logperiod), float(q), qdata[q]]
- )
-
- ############################################################
- # eccentricity distributions as a function of mass, period, ecc
- eccs = sorted(json_data["log10M1"][logmass]["logP"][logperiod]["e"])
-
- # Fill the data and 'normalise'
- ecc_data = fill_data(
- eccs, json_data["log10M1"][logmass]["logP"][logperiod]["e"]
- )
-
- # Create the multiplicity table
- if not Moecache.get("ecc_distributions", None):
- Moecache["ecc_distributions"] = []
-
- for ecc in eccs:
- Moecache["ecc_distributions"].append(
- [
- float(logmass),
- float(logperiod),
- float(ecc),
- ecc_data[ecc],
- ]
- )
-
- ############################################################
- # upper bound the period distributions to zero probability
- Moecache["period_distributions"].append(
- [
- float(logmass),
- float(logperiods[-1]) + 0.5 * dlog10P, # TODO: why this shift?
- json_data["log10M1"][logmass]["logP"][logperiods[-1]][
- "normed_bin_frac_p_dist"
- ]
- / dlog10P,
- json_data["log10M1"][logmass]["logP"][logperiods[-1]][
- "normed_tripquad_frac_p_dist"
- ]
- / dlog10P,
- ]
- )
- Moecache["period_distributions"].append(
- [
- float(logmass),
- float(logperiods[-1]) + 0.5 * dlog10P + epslog10P,
- 0.0,
- 0.0,
- ]
- )
-
- verbose_print(
- "\tMoe_di_Stefano_2017: Length period_distributions table: {}".format(
- len(Moecache["period_distributions"])
- ),
- self.grid_options["verbosity"],
- _MOE2017_VERBOSITY_LEVEL,
- )
- verbose_print(
- "\tMoe_di_Stefano_2017: Length multiplicity table: {}".format(
- len(Moecache["multiplicity_table"])
- ),
- self.grid_options["verbosity"],
- _MOE2017_VERBOSITY_LEVEL,
- )
- verbose_print(
- "\tMoe_di_Stefano_2017: Length q table: {}".format(
- len(Moecache["q_distributions"])
- ),
- self.grid_options["verbosity"],
- _MOE2017_VERBOSITY_LEVEL,
- )
- verbose_print(
- "\tMoe_di_Stefano_2017: Length ecc table: {}".format(
- len(Moecache["ecc_distributions"])
- ),
- self.grid_options["verbosity"],
- _MOE2017_VERBOSITY_LEVEL,
- )
-
- # Write to log file
- os.makedirs(
- os.path.join(self.grid_options["tmp_dir"], "moe_distefano"),
- exist_ok=True,
- )
- with open(
- os.path.join(
- os.path.join(self.grid_options["tmp_dir"], "moe_distefano"),
- "moecache.json",
- ),
- "w",
- encoding='utf-8'
- ) as cache_filehandle:
- cache_filehandle.write(json.dumps(Moecache, indent=4, ensure_ascii=False))
-
- # Signal that the data has been loaded
- self.grid_options["_loaded_Moe2017_data"] = True
-
- def _set_moe_di_stefano_distributions(self):
- """
- Function to set the Moe & di Stefano distribution
- """
-
- ############################################################
- # first, the multiplicity, this is 1,2,3,4, ...
- # for singles, binaries, triples, quadruples, ...
-
- max_multiplicity = get_max_multiplicity(
- self.grid_options["Moe2017_options"]["multiplicity_modulator"]
- )
- verbose_print(
- "\tMoe_di_Stefano_2017: Max multiplicity = {}".format(max_multiplicity),
- self.grid_options["verbosity"],
- _MOE2017_VERBOSITY_LEVEL,
- )
- ######
- # Setting up the grid variables
-
- # Multiplicity
- self.add_grid_variable(
- name="multiplicity",
- parameter_name="multiplicity",
- longname="multiplicity",
- valuerange=[1, max_multiplicity],
- samplerfunc="const_int(1, {n}, {n})".format(n=max_multiplicity),
- precode='self.grid_options["multiplicity"] = multiplicity; self.bse_options["multiplicity"] = multiplicity; options={}'.format(
- self.grid_options["Moe2017_options"]
- ),
- condition="({}[int(multiplicity)-1] > 0)".format(
- str(self.grid_options["Moe2017_options"]["multiplicity_modulator"])
- ),
- gridtype="discrete",
- probdist=1,
- )
-
- ############################################################
- # always require M1, for all systems
- #
- # log-spaced m1 with given resolution
- self.add_grid_variable(
- name="lnm1",
- parameter_name="M_1",
- longname="Primary mass",
- samplerfunc=self.grid_options["Moe2017_options"]["samplerfuncs"]["M"][0]
- or "const(np.log({}), np.log({}), {})".format(
- self.grid_options["Moe2017_options"]["ranges"]["M"][0],
- self.grid_options["Moe2017_options"]["ranges"]["M"][1],
- self.grid_options["Moe2017_options"]["resolutions"]["M"][0],
- ),
- valuerange=[
- "np.log({})".format(
- self.grid_options["Moe2017_options"]["ranges"]["M"][0]
- ),
- "np.log({})".format(
- self.grid_options["Moe2017_options"]["ranges"]["M"][1]
- ),
- ],
- gridtype="centred",
- dphasevol="dlnm1",
- precode='M_1 = np.exp(lnm1); options["M_1"]=M_1',
- probdist="Moe_di_Stefano_2017_pdf({{{}, {}, {}}}, verbosity=self.grid_options['verbosity'])['total_probdens'] if multiplicity == 1 else 1".format(
- str(dict(self.grid_options["Moe2017_options"]))[1:-1],
- "'multiplicity': multiplicity",
- "'M_1': M_1",
- ),
- )
-
- # Go to higher multiplicities
- if max_multiplicity >= 2:
- # binaries: period
- self.add_grid_variable(
- name="log10per",
- parameter_name="orbital_period",
- longname="log10(Orbital_Period)",
- probdist=1.0,
- condition='(self.grid_options["multiplicity"] >= 2)',
- branchpoint=1
- if max_multiplicity > 1
- else 0, # Signal here to put a branchpoint if we have a max multiplicity higher than 1.
- gridtype="centred",
- dphasevol="({} * dlog10per)".format(LOG_LN_CONVERTER),
- valuerange=[
- self.grid_options["Moe2017_options"]["ranges"]["logP"][0],
- self.grid_options["Moe2017_options"]["ranges"]["logP"][1],
- ],
- samplerfunc=self.grid_options["Moe2017_options"]["samplerfuncs"][
- "logP"
- ][0]
- or "const({}, {}, {})".format(
- self.grid_options["Moe2017_options"]["ranges"]["logP"][0],
- self.grid_options["Moe2017_options"]["ranges"]["logP"][1],
- self.grid_options["Moe2017_options"]["resolutions"]["logP"][0],
- ),
- precode="""orbital_period = 10.0**log10per
-qmin={}/M_1
-qmax=maximum_mass_ratio_for_RLOF(M_1, orbital_period)
-""".format(
- self.grid_options["Moe2017_options"]["Mmin"]
- ),
- ) # TODO: change the maximum_mass_ratio_for_RLOF
-
- # binaries: mass ratio
- self.add_grid_variable(
- name="q",
- parameter_name="M_2",
- longname="Mass ratio",
- valuerange=[
- self.grid_options["Moe2017_options"]["ranges"]["q"][0]
- if self.grid_options["Moe2017_options"]
- .get("ranges", {})
- .get("q", None)
- else "options['Mmin']/M_1",
- self.grid_options["Moe2017_options"]["ranges"]["q"][1]
- if self.grid_options["Moe2017_options"]
- .get("ranges", {})
- .get("q", None)
- else "qmax",
- ],
- probdist=1,
- gridtype="centred",
- dphasevol="dq",
- precode="""
-M_2 = q * M_1
-sep = calc_sep_from_period(M_1, M_2, orbital_period)
- """,
- samplerfunc=self.grid_options["Moe2017_options"]["samplerfuncs"]["M"][1]
- or "const({}, {}, {})".format(
- self.grid_options["Moe2017_options"]["ranges"]["q"][0]
- if self.grid_options["Moe2017_options"]
- .get("ranges", {})
- .get("q", [None, None])[0]
- else "{}/M_1".format(self.grid_options["Moe2017_options"]["Mmin"]),
- self.grid_options["Moe2017_options"]["ranges"]["q"][1]
- if self.grid_options["Moe2017_options"]
- .get("ranges", {})
- .get("q", [None, None])[1]
- else "qmax",
- self.grid_options["Moe2017_options"]["resolutions"]["M"][1],
- ),
- )
-
- # (optional) binaries: eccentricity
- if self.grid_options["Moe2017_options"]["resolutions"]["ecc"][0] > 0:
- self.add_grid_variable(
- name="ecc",
- parameter_name="eccentricity",
- longname="Eccentricity",
- probdist=1,
- gridtype="centred",
- dphasevol="decc",
- precode="eccentricity=ecc",
- valuerange=[
- self.grid_options["Moe2017_options"]["ranges"]["ecc"][
- 0
- ], # Just fail if not defined.
- self.grid_options["Moe2017_options"]["ranges"]["ecc"][1],
- ],
- samplerfunc=self.grid_options["Moe2017_options"]["samplerfuncs"][
- "ecc"
- ][0]
- or "const({}, {}, {})".format(
- self.grid_options["Moe2017_options"]["ranges"]["ecc"][
- 0
- ], # Just fail if not defined.
- self.grid_options["Moe2017_options"]["ranges"]["ecc"][1],
- self.grid_options["Moe2017_options"]["resolutions"]["ecc"][0],
- ),
- )
-
- # Now for triples and quadruples
- if max_multiplicity >= 3:
- # Triple: period
- self.add_grid_variable(
- name="log10per2",
- parameter_name="orbital_period_triple",
- longname="log10(Orbital_Period2)",
- probdist=1.0,
- condition='(self.grid_options["multiplicity"] >= 3)',
- branchpoint=2
- if max_multiplicity > 2
- else 0, # Signal here to put a branchpoint if we have a max multiplicity higher than 1.
- gridtype="centred",
- dphasevol="({} * dlog10per2)".format(LOG_LN_CONVERTER),
- valuerange=[
- self.grid_options["Moe2017_options"]["ranges"]["logP"][0],
- self.grid_options["Moe2017_options"]["ranges"]["logP"][1],
- ],
- samplerfunc=self.grid_options["Moe2017_options"]["samplerfuncs"][
- "logP"
- ][1]
- or "const({}, {}, {})".format(
- self.grid_options["Moe2017_options"]["ranges"]["logP"][0],
- self.grid_options["Moe2017_options"]["ranges"]["logP"][1],
- self.grid_options["Moe2017_options"]["resolutions"]["logP"][1],
- ),
- precode="""orbital_period_triple = 10.0**log10per2
-q2min={}/(M_1+M_2)
-q2max=maximum_mass_ratio_for_RLOF(M_1+M_2, orbital_period_triple)
- """.format(
- self.grid_options["Moe2017_options"]["Mmin"]
- ),
- )
-
- # Triples: mass ratio
- # Note, the mass ratio is M_outer/M_inner
- self.add_grid_variable(
- name="q2",
- parameter_name="M_3",
- longname="Mass ratio outer/inner",
- valuerange=[
- self.grid_options["Moe2017_options"]["ranges"]["q"][0]
- if self.grid_options["Moe2017_options"]
- .get("ranges", {})
- .get("q", None)
- else "options['Mmin']/(M_1+M_2)",
- self.grid_options["Moe2017_options"]["ranges"]["q"][1]
- if self.grid_options["Moe2017_options"]
- .get("ranges", {})
- .get("q", None)
- else "q2max",
- ],
- probdist=1,
- gridtype="centred",
- dphasevol="dq2",
- precode="""
-M_3 = q2 * (M_1 + M_2)
-sep2 = calc_sep_from_period((M_1+M_2), M_3, orbital_period_triple)
-eccentricity2=0
-""",
- samplerfunc=self.grid_options["Moe2017_options"]["samplerfuncs"][
- "M"
- ][2]
- or "const({}, {}, {})".format(
- self.grid_options["Moe2017_options"]["ranges"]["q"][0]
- if self.grid_options["Moe2017_options"]
- .get("ranges", {})
- .get("q", None)
- else "options['Mmin']/(M_1+M_2)",
- self.grid_options["Moe2017_options"]["ranges"]["q"][1]
- if self.grid_options["Moe2017_options"]
- .get("ranges", {})
- .get("q", None)
- else "q2max",
- self.grid_options["Moe2017_options"]["resolutions"]["M"][2],
- ),
- )
-
- # (optional) triples: eccentricity
- if self.grid_options["Moe2017_options"]["resolutions"]["ecc"][1] > 0:
- self.add_grid_variable(
- name="ecc2",
- parameter_name="eccentricity2",
- longname="Eccentricity of the triple",
- probdist=1,
- gridtype="centred",
- dphasevol="decc2",
- precode="eccentricity2=ecc2",
- valuerange=[
- self.grid_options["Moe2017_options"]["ranges"]["ecc"][
- 0
- ], # Just fail if not defined.
- self.grid_options["Moe2017_options"]["ranges"]["ecc"][1],
- ],
- samplerfunc=self.grid_options["Moe2017_options"][
- "samplerfuncs"
- ]["ecc"][1]
- or "const({}, {}, {})".format(
- self.grid_options["Moe2017_options"]["ranges"]["ecc"][
- 0
- ], # Just fail if not defined.
- self.grid_options["Moe2017_options"]["ranges"]["ecc"][1],
- self.grid_options["Moe2017_options"]["resolutions"]["ecc"][
- 1
- ],
- ),
- )
-
- if max_multiplicity == 4:
- # Quadruple: period
- self.add_grid_variable(
- name="log10per3",
- parameter_name="orbital_period_quadruple",
- longname="log10(Orbital_Period3)",
- probdist=1.0,
- condition='(self.grid_options["multiplicity"] >= 4)',
- branchpoint=3
- if max_multiplicity > 3
- else 0, # Signal here to put a branchpoint if we have a max multiplicity higher than 1.
- gridtype="centred",
- dphasevol="({} * dlog10per3)".format(LOG_LN_CONVERTER),
- valuerange=[
- self.grid_options["Moe2017_options"]["ranges"]["logP"][0],
- self.grid_options["Moe2017_options"]["ranges"]["logP"][1],
- ],
- samplerfunc=self.grid_options["Moe2017_options"][
- "samplerfuncs"
- ]["logP"][2]
- or "const({}, {}, {})".format(
- self.grid_options["Moe2017_options"]["ranges"]["logP"][0],
- self.grid_options["Moe2017_options"]["ranges"]["logP"][1],
- self.grid_options["Moe2017_options"]["resolutions"]["logP"][
- 2
- ],
- ),
- precode="""orbital_period_quadruple = 10.0**log10per3
-q3min={}/(M_3)
-q3max=maximum_mass_ratio_for_RLOF(M_3, orbital_period_quadruple)
- """.format(
- self.grid_options["Moe2017_options"]["Mmin"]
- ),
- )
-
- # Quadruple: mass ratio : M_outer / M_inner
- self.add_grid_variable(
- name="q3",
- parameter_name="M_4",
- longname="Mass ratio outer low/outer high",
- valuerange=[
- self.grid_options["Moe2017_options"]["ranges"]["q"][0]
- if self.grid_options["Moe2017_options"]
- .get("ranges", {})
- .get("q", None)
- else "options['Mmin']/(M_3)",
- self.grid_options["Moe2017_options"]["ranges"]["q"][1]
- if self.grid_options["Moe2017_options"]
- .get("ranges", {})
- .get("q", None)
- else "q3max",
- ],
- probdist=1,
- gridtype="centred",
- dphasevol="dq3",
- precode="""
-M_4 = q3 * M_3
-sep3 = calc_sep_from_period((M_3), M_4, orbital_period_quadruple)
-eccentricity3=0
-""",
- samplerfunc=self.grid_options["Moe2017_options"][
- "samplerfuncs"
- ]["M"][3]
- or "const({}, {}, {})".format(
- self.grid_options["Moe2017_options"]["ranges"]["q"][0]
- if self.grid_options["Moe2017_options"]
- .get("ranges", {})
- .get("q", None)
- else "options['Mmin']/(M_3)",
- self.grid_options["Moe2017_options"]["ranges"]["q"][1]
- if self.grid_options["Moe2017_options"]
- .get("ranges", {})
- .get("q", None)
- else "q3max",
- self.grid_options["Moe2017_options"]["resolutions"]["M"][2],
- ),
- )
-
- # (optional) triples: eccentricity
- if (
- self.grid_options["Moe2017_options"]["resolutions"]["ecc"][2]
- > 0
- ):
- self.add_grid_variable(
- name="ecc3",
- parameter_name="eccentricity3",
- longname="Eccentricity of the triple+quadruple/outer binary",
- probdist=1,
- gridtype="centred",
- dphasevol="decc3",
- precode="eccentricity3=ecc3",
- valuerange=[
- self.grid_options["Moe2017_options"]["ranges"]["ecc"][
- 0
- ], # Just fail if not defined.
- self.grid_options["Moe2017_options"]["ranges"]["ecc"][
- 1
- ],
- ],
- samplerfunc=self.grid_options["Moe2017_options"][
- "samplerfuncs"
- ]["ecc"][2]
- or "const({}, {}, {})".format(
- self.grid_options["Moe2017_options"]["ranges"]["ecc"][
- 0
- ], # Just fail if not defined.
- self.grid_options["Moe2017_options"]["ranges"]["ecc"][
- 1
- ],
- self.grid_options["Moe2017_options"]["resolutions"][
- "ecc"
- ][2],
- ),
- )
-
- # Now we are at the last part.
- # Here we should combine all the information that we calculate and update the options
- # dictionary. This will then be passed to the Moe_di_Stefano_2017_pdf to calculate
- # the real probability. The trick we use is to strip the options_dict as a string
- # and add some keys to it:
-
- updated_options = "{{{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}}}".format(
- str(dict(self.grid_options["Moe2017_options"]))[1:-1],
- '"multiplicity": multiplicity',
- '"M_1": M_1',
- '"M_2": M_2',
- '"M_3": M_3',
- '"M_4": M_4',
- '"P": orbital_period',
- '"P2": orbital_period_triple',
- '"P3": orbital_period_quadruple',
- '"ecc": eccentricity',
- '"ecc2": eccentricity2',
- '"ecc3": eccentricity3',
- )
-
- probdist_addition = "Moe_di_Stefano_2017_pdf({}, verbosity=self.grid_options['verbosity'])['total_probdens']".format(
- updated_options
- )
-
- # and finally the probability calculator
- self.grid_options["_grid_variables"][self._last_grid_variable()][
- "probdist"
- ] = probdist_addition
-
- verbose_print(
- "\tMoe_di_Stefano_2017: Added final call to the pdf function",
- self.grid_options["verbosity"],
- _MOE2017_VERBOSITY_LEVEL,
- )
-
- # Signal that the MOE2017 grid has been set
- self.grid_options["_set_Moe2017_grid"] = True
-
- ################################################################################################
- def Moe_di_Stefano_2017(self, options=None):
- """
- Function to handle setting the user input settings,
- set up the data and load that into interpolators and
- then set the distribution functions
-
- Takes a dictionary as its only argument
- """
-
- default_options = {
- "apply settings": True,
- "setup grid": True,
- "load data": True,
- "clean cache": False,
- "clean load flag": False,
- "clean all": False,
- }
- if not options:
- options = {}
- options = update_dicts(default_options, options)
-
- # clean cache?
- if options["clean all"] or options["clean cache"]:
- Moecache.clear()
-
- if options["clean all"] or options["clean load flag"]:
- self.grid_options["_loaded_Moe2017_data"] = False
-
- # Set the user input
- if options["apply settings"]:
- self.set_moe_di_stefano_settings(options=options)
-
- # Load the data
- if options["load data"]:
- self._load_moe_di_stefano_data()
-
- # construct the grid here
- if options["setup grid"]:
- self._set_moe_di_stefano_distributions()
-
- def _clean_interpolators(self):
- """
- Function to clean up the interpolators after a run
-
- We look in the Moecache global variable for items that are interpolators.
- Should be called by the general cleanup function AND the thread cleanup function
- """
-
- interpolator_keys = []
- for key in Moecache.keys():
- if isinstance(Moecache[key], py_rinterpolate.Rinterpolate):
- interpolator_keys.append(key)
-
- for key in interpolator_keys:
- Moecache[key].destroy()
- del Moecache[key]
- gc.collect()
-
- ##### Unsorted functions
- def _calculate_multiplicity_fraction(self, system_dict):
- """
- Function to calculate multiplicity fraction
-
- Makes use of the self.bse_options['multiplicity'] value. If its not set, it will raise an error
-
- grid_options['multiplicity_fraction_function'] will be checked for the choice
-
- TODO: add option to put a manual binary fraction in here (solve via negative numbers being the functions)
- """
-
- # Just return 1 if no option has been chosen
- if self.grid_options["multiplicity_fraction_function"] in [0, "None"]:
- verbose_print(
- "_calculate_multiplicity_fraction: Chosen not to use any multiplicity fraction.",
- self.grid_options["verbosity"],
- 3,
- )
-
- return 1
-
- # Raise an error if the multiplicity is not set
- if not system_dict.get("multiplicity", None):
- msg = "Multiplicity value has not been set. When using a specific multiplicity fraction function please set the multiplicity"
- raise ValueError(msg)
-
- # Go over the chosen options
- if self.grid_options["multiplicity_fraction_function"] in [1, "Arenou2010"]:
- # Arenou 2010 will be used
- verbose_print(
- "_calculate_multiplicity_fraction: Using Arenou 2010 to calculate multiplicity fractions",
- self.grid_options["verbosity"],
- 3,
- )
-
- binary_fraction = Arenou2010_binary_fraction(system_dict["M_1"])
- multiplicity_fraction_dict = {
- 1: 1 - binary_fraction,
- 2: binary_fraction,
- 3: 0,
- 4: 0,
- }
-
- elif self.grid_options["multiplicity_fraction_function"] in [2, "Raghavan2010"]:
- # Raghavan 2010 will be used
- verbose_print(
- "_calculate_multiplicity_fraction: Using Raghavan (2010) to calculate multiplicity fractions",
- self.grid_options["verbosity"],
- 3,
- )
-
- binary_fraction = raghavan2010_binary_fraction(system_dict["M_1"])
- multiplicity_fraction_dict = {
- 1: 1 - binary_fraction,
- 2: binary_fraction,
- 3: 0,
- 4: 0,
- }
-
- elif self.grid_options["multiplicity_fraction_function"] in [3, "Moe2017"]:
- # We need to check several things now here:
-
- # First, are the options for the MOE2017 grid set? On start it is filled with the default settings
- if not self.grid_options["Moe2017_options"]:
- msg = "The MOE2017 options do not seem to be set properly. The value is {}".format(
- self.grid_options["Moe2017_options"]
- )
- raise ValueError(msg)
-
- # Second: is the Moecache filled.
- if not Moecache:
- verbose_print(
- "_calculate_multiplicity_fraction: Moecache is empty. It needs to be filled with the data for the interpolators. Loading the data now",
- self.grid_options["verbosity"],
- 3,
- )
-
- # Load the data
- self._load_moe_di_stefano_data()
-
- # record the prev value
- prev_M1_value_ms = self.grid_options["Moe2017_options"].get("M_1", None)
-
- # Set value of M1 of the current system
- self.grid_options["Moe2017_options"]["M_1"] = system_dict["M_1"]
-
- # Calculate the multiplicity fraction
- multiplicity_fraction_list = Moe_di_Stefano_2017_multiplicity_fractions(
- self.grid_options["Moe2017_options"], self.grid_options["verbosity"]
- )
-
- # Turn into dict
- multiplicity_fraction_dict = {
- el + 1: multiplicity_fraction_list[el]
- for el in range(len(multiplicity_fraction_list))
- }
-
- # Set the prev value back
- self.grid_options["Moe2017_options"]["M_1"] = prev_M1_value_ms
-
- # we don't know what to do next
- else:
- msg = "Chosen value for the multiplicity fraction function is not known."
- raise ValueError(msg)
-
- # To make sure we normalize the dictionary
- multiplicity_fraction_dict = normalize_dict(
- multiplicity_fraction_dict, verbosity=self.grid_options["verbosity"]
- )
-
- verbose_print(
- "Multiplicity: {} multiplicity_fraction: {}".format(
- system_dict["multiplicity"],
- multiplicity_fraction_dict[system_dict["multiplicity"]],
- ),
- self.grid_options["verbosity"],
- 3,
- )
-
- return multiplicity_fraction_dict[system_dict["multiplicity"]]
-
- ######################
- # Status logging
-
- def vb1print(self, ID, now, system_number, system_dict):
- """
- Verbosity-level 1 printing, to keep an eye on a grid.
- Arguments:
- ID: thread ID for debugging (int)
- now: the time now as a UNIX-style epoch in seconds (float)
- system_number: the system number
-
- TODO: add information about the number of cores. the TPR shows the dt/dn but i want to see the number per core too
- """
-
- # calculate estimated time of arrive (eta and eta_secs), time per run (tpr)
- localtime = time.localtime(now)
-
- # calculate stats
- n = self.shared_memory["n_saved_log_stats"].value
- if n < 2:
- # simple 1-system calculation: inaccurate
- # but best for small n
- dt = now - self.shared_memory["prev_log_time"][0]
- dn = system_number - self.shared_memory["prev_log_system_number"][0]
- else:
- # average over n_saved_log_stats
- dt = (
- self.shared_memory["prev_log_time"][0]
- - self.shared_memory["prev_log_time"][n - 1]
- )
- dn = (
- self.shared_memory["prev_log_system_number"][0]
- - self.shared_memory["prev_log_system_number"][n - 1]
- )
-
- eta, units, tpr, eta_secs = trem(
- dt, system_number, dn, self.grid_options["_total_starcount"]
- )
-
- # compensate for multithreading and modulo
- tpr *= self.grid_options["num_processes"] * self.grid_options["modulo"]
-
- if eta_secs < secs_per_day:
- fintime = time.localtime(now + eta_secs)
- etf = "{hours:02d}:{minutes:02d}:{seconds:02d}".format(
- hours=fintime.tm_hour, minutes=fintime.tm_min, seconds=fintime.tm_sec
- )
- else:
- d = int(eta_secs / secs_per_day)
- if d == 1:
- etf = "Tomorrow"
- else:
- etf = "In {} days".format(d)
-
- # modulo information
- if self.grid_options["modulo"] == 1:
- modulo = "" # usual case
- else:
- modulo = "%" + str(self.grid_options["modulo"])
-
- # add up memory use from each thread
- total_mem_use = sum(self.shared_memory["memory_use_per_thread"])
-
- # make a string to describe the system e.g. M1, M2, etc.
- system_string = ""
-
- # use the multiplicity if given
- if "multiplicity" in system_dict:
- nmult = int(system_dict["multiplicity"])
- else:
- nmult = 4
-
- # masses
- for i in range(nmult):
- i1 = str(i + 1)
- if "M_" + i1 in system_dict:
- system_string += (
- "M{}=".format(i1) + format_number(system_dict["M_" + i1]) + " "
- )
-
- # separation and orbital period
- if "separation" in system_dict:
- system_string += "a=" + format_number(system_dict["separation"])
- if "orbital_period" in system_dict:
- system_string += "P=" + format_number(system_dict["orbital_period"])
-
- # do the print
- verbose_print(
- "{opening_colour}{system_number}/{total_starcount}{modulo} {pc_colour}{pc_complete:5.1f}% complete {time_colour}{hours:02d}:{minutes:02d}:{seconds:02d} {ETA_colour}ETA={ETA:7.1f}{units} tpr={tpr:2.2e} {ETF_colour}ETF={ETF} {mem_use_colour}mem:{mem_use:.1f}MB {system_string_colour}{system_string}{closing_colour}".format(
- opening_colour=self.ANSI_colours["reset"]
- + self.ANSI_colours["yellow on black"],
- system_number=system_number,
- total_starcount=self.grid_options["_total_starcount"],
- modulo=modulo,
- pc_colour=self.ANSI_colours["blue on black"],
- pc_complete=(100.0 * system_number)
- / (1.0 * self.grid_options["_total_starcount"])
- if self.grid_options["_total_starcount"]
- else -1,
- time_colour=self.ANSI_colours["green on black"],
- hours=localtime.tm_hour,
- minutes=localtime.tm_min,
- seconds=localtime.tm_sec,
- ETA_colour=self.ANSI_colours["red on black"],
- ETA=eta,
- units=units,
- tpr=tpr,
- ETF_colour=self.ANSI_colours["blue"],
- ETF=etf,
- mem_use_colour=self.ANSI_colours["magenta"],
- mem_use=total_mem_use,
- system_string_colour=self.ANSI_colours["yellow"],
- system_string=system_string,
- closing_colour=self.ANSI_colours["reset"],
- ),
- self.grid_options["verbosity"],
- 1,
- )
-
- def vb2print(self, system_dict, cmdline_string):
- print(
- "Running this system now on thread {ID}\n{blue}{cmdline}{reset}\n".format(
- ID=self.process_ID,
- blue=self.ANSI_colours["blue"],
- cmdline=cmdline_string,
- reset=self.ANSI_colours["reset"],
- )
- )
-
-############################################################
-# Slurm support
-############################################################
-
- def slurmpath(self,path):
- # return the full slurm directory path
- return os.path.abspath(os.path.join(self.grid_options['slurm_dir'],path))
-
- def slurm_status_file(self,
- jobid=None,
- jobarrayindex=None):
- """
- Return the slurm status file corresponding to the jobid and jobarrayindex, which default to grid_options slurm_jobid and slurm_jobarrayindex, respectively.
- """
- if jobid is None:
- jobid= self.grid_options['slurm_jobid']
- if jobarrayindex is None:
- jobarrayindex = self.grid_options['slurm_jobarrayindex']
- if jobid and jobarrayindex:
- return os.path.join(self.slurmpath('status'),
- self.grid_options['slurm_jobid'] + '.' + self.grid_options['slurm_jobarrayindex'])
- else:
- return None
-
- def set_slurm_status(self,string):
- """
- Set the slurm status corresponing to the self object, which should have slurm_jobid and slurm_jobarrayindex set.
- """
- # save slurm jobid to file
- idfile = os.path.join(self.grid_options["slurm_dir"],
- "jobid")
- if not os.path.exists(idfile):
- with open(idfile,"w",encoding='utf-8') as fjobid:
- fjobid.write("{jobid}\n".format(jobid=self.grid_options['slurm_jobid']))
- fjobid.close()
-
- # save slurm status
- file = self.slurm_status_file()
- if file:
- with open(file,'w',encoding='utf-8') as f:
- f.write(string)
- f.close()
- return
-
- def get_slurm_status(self,
- jobid=None,
- jobarrayindex=None):
- """
- Get and return the slurm status corresponing to the self object, or jobid.jobarrayindex if they are passed in. If no status is found, returns None.
- """
- if jobid is None:
- jobid = self.grid_options['slurm_jobid']
- if jobarrayindex is None:
- jobarrayindex = self.grid_options['slurm_jobarrayindex']
-
- if jobid is None or jobarrayindex is None :
- return None
-
- path = pathlib.Path(self.slurm_status_file(jobid=jobid,
- jobarrayindex=jobarrayindex))
- if path:
- return path.read_text().strip()
- else:
- return None
-
- def slurm_outfile(self):
- """
- return a standard filename for the slurm chunk files
- """
- file = "{jobid}.{jobarrayindex}.gz".format(
- jobid = self.grid_options['slurm_jobid'],
- jobarrayindex = self.grid_options['slurm_jobarrayindex']
- )
- return os.path.abspath(os.path.join(self.grid_options['slurm_dir'],
- 'results',
- file))
-
- def make_slurm_dirs(self):
-
- # make the slurm directories
- if not self.grid_options['slurm_dir']:
- print("You must set self.grid_options['slurm_dir'] to a directory which we can use to set up binary_c-python's Slurm files. This should be unique to your set of grids.")
- os.exit()
-
- # make a list of directories, these contain the various slurm
- # output, status files, etc.
- dirs = []
- for dir in ['stdout','stderr','results','status','snapshots']:
- dirs.append(self.slurmpath(dir))
-
- # make the directories: we do not allow these to already exist
- # as the slurm directory should be a fresh location for each set of jobs
- for dir in dirs:
- try:
- pathlib.Path(self.slurmpath(dir)).mkdir(exist_ok=False,
- parents=True)
- except:
- print("Tried to make the directory {dir} but it already exists. When you launch a set of binary_c jobs on Slurm, you need to set your slurm_dir to be a fresh directory with no contents.".format(dir=dir))
- self.exit(code=1)
-
- # check that they have been made and exist: we need this
- # because on network mounts (NFS) there's often a delay between the mkdir
- # above and the actual directory being made. This shouldn't be too long...
- fail = True
- count = 0
- count_warn = 10
- while fail is True:
- fail = False
- count += 1
- if count > count_warn:
- print("Warning: Have been waiting about {} seconds for Slurm directories to be made, there seems to be significant delay...".format(count))
- for dir in dirs:
- if os.path.isdir(dir) is False:
- fail = True
- time.sleep(1)
- break # break the "for dir in dirs:"
-
- def slurm_grid(self):
- """
- function to be called when running grids when grid_options['slurm']>=1
-
- if grid_options['slurm']==1, we set up the slurm script and launch the jobs, then exit.
- if grid_options['slurm']==2, we are being called from the jobs to run the grids
- """
-
- # if slurm=1, we should have no evolution type, we just
- # set up a load of slurm scripts and get them evolving
- # in slurm array
- if self.grid_options['slurm'] == 1:
- self.grid_options['evolution_type'] = None
-
- if self.grid_options['evolution_type'] == 'grid':
- # run a grid of stars only, leaving the results
- # in a file
-
- # set output file
- self.grid_options['save_population_object'] = slurm_outfile()
-
- return self.evolve()
- elif self.grid_options['evolution_type'] == 'join':
- # should not happen!
- return
- else:
- # setup and launch slurm jobs
- self.make_slurm_dirs()
-
- # check we're not using too much RAM
- if datasize.DataSize(self.grid_options['slurm_memory']) > datasize.DataSize(self.grid_options['slurm_warn_max_memory']):
- print("WARNING: you want to use {} MB of RAM : this is unlikely to be correct. If you believe it is, set slurm_warn_max_memory to something very large (it is currently {} MB)\n".format(
- self.grid_options['slurm_memory'],
- self.grid_options['slurm_warn_max_memory']))
- self.exit(code=1)
-
- # set up slurm_array
- if not self.grid_options['slurm_array_max_jobs']:
- self.grid_options['slurm_array_max_jobs'] = self.grid_options['slurm_njobs']
- slurm_array = self.grid_options['slurm_array'] or "1-{njobs}\%{max_jobs}".format(
- njobs=self.grid_options['slurm_njobs'],
- max_jobs=self.grid_options['slurm_array_max_jobs'])
-
- # get job id (might be passed in)
- jobid = self.grid_options['slurm_jobid'] if self.grid_options['slurm_jobid'] != "" else '$SLURM_ARRAY_JOB_ID'
-
- # get job array index
- jobarrayindex = self.grid_options['slurm_jobarrayindex']
- if jobarrayindex is None:
- jobarrayindex = '$SLURM_ARRAY_TASK_ID'
-
- if self.grid_options['slurm_njobs'] == 0:
- print("binary_c-python Slurm : You must set grid_option slurm_njobs to be non-zero")
- self.exit(code=1)
-
- # build the grid command
- grid_command = [
- os.path.join("/usr","bin","env"),
- sys.executable,
- str(lib_programname.get_path_executed_script()),
- ] + sys.argv[1:] + [
- 'slurm=2',
- 'start_at=' + str(jobarrayindex) + '-1', # do we need the -1?
- 'modulo=' + str(self.grid_options['slurm_njobs']),
- 'slurm_njobs=' + str(self.grid_options['slurm_njobs']),
- 'slurm_dir=' + self.grid_options['slurm_dir'],
- 'verbosity=' + str(self.grid_options['verbosity']),
- 'num_cores=' + str(self.grid_options['num_processes'])
- ]
-
- grid_command = ' '.join(grid_command)
-
- # make slurm script
- scriptpath = self.slurmpath('slurm_script')
- try:
- script = open(scriptpath,'w',encoding='utf-8')
- except IOError:
- print("Could not open Slurm script at {path} for writing: please check you have set {slurm_dir} correctly (it is currently {slurm_dir} and can write to this directory.".format(path=scriptpath,
- slurm_dir = self.grid_options['slurm_dir']))
-
- # the joinfile contains the list of chunk files to be joined
- joinfile = "{slurm_dir}/results/{jobid}.all".format(
- slurm_dir=self.grid_options['slurm_dir'],
- jobid=jobid
- )
-
- lines = [
- "#!/bin/bash\n",
- "# Slurm file for binary_grid2 and slurm\n",
- "#SBATCH --error={slurm_dir}/stderr/\%A.\%a\n".format(slurm_dir=self.grid_options['slurm_dir']),
- "#SBATCH --output={slurm_dir}/stdout/\%A.\%a\n".format(slurm_dir=self.grid_options['slurm_dir']),
- "#SBATCH --job-name={slurm_jobname}\n".format(slurm_jobname=self.grid_options['slurm_jobname']),
- "#SBATCH --partition={slurm_partition}\n".format(slurm_partition=self.grid_options['slurm_partition']),
- "#SBATCH --time={slurm_time}\n".format(slurm_time=self.grid_options['slurm_time']),
- "#SBATCH --mem={slurm_memory}\n".format(slurm_memory=self.grid_options['slurm_memory']),
- "#SBATCH --ntasks={slurm_ntasks}\n".format(slurm_ntasks=self.grid_options['slurm_ntasks']),
- "#SBATCH --array={slurm_array}\n".format(slurm_array=slurm_array),
- "#SBATCH --cpus-per-task={ncpus}\n".format(ncpus=self.grid_options['num_processes'])
- ]
-
- for key in self.grid_options['slurm_extra_settings']:
- lines += [ "#SBATCH --{key} = {value}\n".format(
- key=key,
- value=self.grid_options['slurm_extra_settings'][key]
- )]
-
- # save original command line, working directory, time
- lines += [
- "\nexport BINARY_C_PYTHON_ORIGINAL_CMD_LINE={cmdline}".format(cmdline=repr(self.grid_options['command_line'])),
- "\nexport BINARY_C_PYTHON_ORIGINAL_WD=`pwd`",
- "\nexport BINARY_C_PYTHON_ORIGINAL_SUBMISSION_TIME=`date`",
- ]
-
-
- lines += [
- "\n# set status to \"running\"\n",
- "echo \"running\" > {slurm_dir}/status/{jobid}.{jobarrayindex}\n\n".format(slurm_dir=self.grid_options['slurm_dir'],
- jobid=jobid,
- jobarrayindex=jobarrayindex),
- "\n# make list of files\n",
- "\necho {slurm_dir}/results/{jobid}.{jobarrayindex}.gz >> {slurm_dir}/results/{jobid}.all\n".format(slurm_dir=self.grid_options['slurm_dir'],
- jobid=jobid,
- jobarrayindex=jobarrayindex,
- joinfile=joinfile),
- "\n# run grid of stars and, if this returns 0, set status to finished\n",
-
- # note: the next line ends in &&
- "{grid_command} evolution_type=grid slurm_jobid={jobid} slurm_jobarrayindex={jobarrayindex} save_population_object={slurm_dir}/results/{jobid}.{jobarrayindex}.gz && echo -n \"finished\" > {slurm_dir}/status/{jobid}.{jobarrayindex} && \\\n".format(
- slurm_dir=self.grid_options['slurm_dir'],
- jobid=jobid,
- jobarrayindex=jobarrayindex,
- grid_command=grid_command),
- ]
-
- if not self.grid_options['slurm_postpone_join']:
- lines += [
- # the following line also ends in && so that if one fails, the rest
- # also fail
- "echo && echo \"Checking if we can join...\" && echo && \\\n",
- "{grid_command} slurm=2 evolution_type=join joinlist={joinfile} slurm_jobid={jobid} slurm_jobarrayindex={jobarrayindex}\n\n".format(
- grid_command=grid_command,
- joinfile=joinfile,
- jobid=jobid,
- jobarrayindex=jobarrayindex
- )]
-
- # write to script, close it and make it executable by
- # all (so the slurm user can pick it up)
- script.writelines(lines)
- script.close()
- os.chmod(scriptpath,
- stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC | \
- stat.S_IRGRP | stat.S_IXGRP | \
- stat.S_IROTH | stat.S_IXOTH)
-
- if not self.grid_options['slurm_postpone_sbatch']:
- # call sbatch to launch the jobs
- cmd = [self.grid_options['slurm_sbatch'], scriptpath]
- pipes = subprocess.Popen(cmd,
- stdout = subprocess.PIPE,
- stderr = subprocess.PIPE)
- std_out, std_err = pipes.communicate()
- if pipes.returncode != 0:
- # an error happened!
- err_msg = "{red}{err}\nReturn Code: {code}{reset}".format(err=std_err.strip(),
- code=pipes.returncode,
- red=self.ANSI_colours["red"],
- reset=self.ANSI_colours["reset"],)
- raise Exception(err_msg)
-
- elif len(std_err):
- print("{red}{err}{reset}".format(red=self.ANSI_colours["red"],
- reset=self.ANSI_colours["reset"],
- err=std_err.strip().decode('utf-8')))
-
- print("{yellow}{out}{reset}".format(yellow=self.ANSI_colours["yellow"],
- reset=self.ANSI_colours["reset"],
- out=std_out.strip().decode('utf-8')))
- else:
- # just say we would have (use this for testing)
- print("Slurm script is at {path} but has not been launched".format(path=scriptpath))
+ TODO: Rethink this functionality. seems a bit double, could also be just outside of the class
+ """
+ binary_c_defaults = self.return_binary_c_defaults().copy()
+ cleaned_dict = filter_arg_dict(binary_c_defaults)
- # some messages to the user, then return
- if self.grid_options['slurm_postpone_sbatch'] == 1:
- print("Slurm script written, but launching the jobs with sbatch was postponed.")
- else:
- print("Slurm jobs launched")
- print("All done in slurm_grid().")
- return
+ return cleaned_dict
- def save_population_object(self,object=None,filename=None,confirmation=True,compression='gzip'):
+ def _increment_probtot(self, prob):
"""
- Save pickled Population object to file at filename or, if filename is None, whatever is set at self.grid_options['save_population_object']
-
- Args:
- object : the object to be saved to the file. If object is None, use self.
- filename : the name of the file to be saved. If not set, use self.grid_options['save_population_object']
- confirmation : if True, a file "filename.saved" is touched just after the dump, so we know it is finished.
-
- Compression is performed according to the filename, as stated in the
- compress_pickle documentation at
- https://lucianopaz.github.io/compress_pickle/html/
-
- Shared memory, stored in the object.shared_memory dict, is not saved.
-
+ Function to add to the total probability. For now not used
"""
- if object is None:
- # default to using self
- object = self
-
- if filename is None:
- # get filename from self
- filename = self.grid_options['save_population_object']
-
- if filename:
-
- print("Save population {id}, probtot {probtot} to pickle in {filename}".format(
- id=self.grid_options["_population_id"],
- probtot=object.grid_options['_probtot'],
- filename=filename))
-
-
- # Some parts of the object cannot be pickled:
- # remove them, and restore them after pickling
-
- # remove shared memory
- shared_memory = object.shared_memory
- object.shared_memory = None
-
- # delete system generator
- system_generator = object.grid_options["_system_generator"]
- object.grid_options["_system_generator"] = None
-
- # delete _store_memaddr
- _store_memaddr = object.grid_options['_store_memaddr']
- object.grid_options['_store_memaddr'] = None
-
- # delete persistent_data_memory_dict
- persistent_data_memory_dict = object.persistent_data_memory_dict
- object.persistent_data_memory_dict = None
-
- # add metadata if it doesn't exist
- if not "metadata" in object.grid_ensemble_results:
- object.grid_ensemble_results["metadata"] = {}
- # add datestamp
- object.grid_ensemble_results["metadata"]['save_population_time'] = datetime.datetime.now().strftime("%m/%d/%Y %H:%M:%S")
-
- # add extra metadata
- object.add_system_metadata()
-
- # add max memory use
- try:
- self.grid_ensemble_results["metadata"]['max_memory_use'] = copy.deepcopy(sum(shared_memory["max_memory_use_per_thread"]))
- except Exception as e:
- print("save_population_object : Error: ",e)
- pass
-
- # dump pickle file
- compress_pickle.dump(object,
- filename,
- pickler_method='dill')
-
- # restore data
- object.shared_memory = shared_memory
- object.grid_options["_system_generator"] = system_generator
- del object.grid_ensemble_results["metadata"]['save_population_time']
- object.grid_options['store_memaddr'] = _store_memaddr
- object.persistent_data_memory_dict = persistent_data_memory_dict
-
- # touch 'saved' file
- pathlib.Path(filename + '.saved').touch(exist_ok=True)
-
- return
+ self.grid_options["_probtot"] += prob
- def load_population_object(self,filename):
+ def _increment_count(self):
"""
- returns the Population object loaded from filename
+ Function to add to the total number of stars. For now not used
"""
- if filename is None:
- obj = None
- else:
- try:
- obj = compress_pickle.load(filename,
- pickler_method='dill')
- except Exception as e:
- obj = None
-
- return obj
-
+ self.grid_options["_count"] += 1
- def merge_populations(self,refpop,newpop):
+ def was_killed(self):
"""
- merge newpop's results data into refpop's results data
-
- Args:
- refpop : the original "reference" Population object to be added to
- newpop : Population object containing the new data
-
- Returns:
- nothing
-
- Note:
- The file should be saved using save_population_object()
+ Function to determine if the process was killed. Returns True if so, false otherwise.
"""
-
- # combine data
- try:
- refpop.grid_results = merge_dicts(refpop.grid_results,
- newpop.grid_results)
- except Exception as e:
- print("Error merging grid_results:",e)
-
- # special cases
+ killed = self.grid_options['_killed']
try:
- maxmem = max(refpop.grid_ensemble_results["metadata"]['max_memory_use'],
- newpop.grid_ensemble_results["metadata"]['max_memory_use'])
+ killed = killed or self.grid_results['_killed']
except:
- maxmem = 0
-
- try:
- # special cases:
- # copy the settings and Xinit: these should just be overridden
- try:
- settings = copy.deepcopy(newpop.grid_ensemble_results["metadata"]['settings'])
- except:
- settings = None
- try:
- Xinit = copy.deepcopy(newpop.grid_ensemble_results["ensemble"]["Xinit"])
- except:
- Xinit = None
-
- # merge the ensemble dicts
- refpop.grid_ensemble_results = merge_dicts(refpop.grid_ensemble_results,
- newpop.grid_ensemble_results)
-
- # set special cases
- try:
- refpop.grid_ensemble_results["metadata"]['max_memory_use'] = maxmem
- if settings:
- refpop.grid_ensemble_results["metadata"]['settings'] = settings
- if Xinit:
- refpop.grid_ensemble_results["ensemble"]["Xinit"] = Xinit
- except:
- pass
-
- except Exception as e:
- print("Error merging grid_ensemble_results:",e)
-
- for key in ["_probtot"]:
- refpop.grid_options[key] += newpop.grid_options[key]
-
- refpop.grid_options['_killed'] |= newpop.grid_options['_killed']
-
- return
-
- def merge_populations_from_file(self,refpop,filename):
- """
- Wrapper for merge_populations so it can be done directly
- from a file.
-
- Args:
- refpop : the original "reference" Population object to be added to
- filename : file containing the Population object containing the new data
-
- Note:
- The file should be saved using save_population_object()
- """
-
- newpop = self.load_population_object(filename)
-
-
- # merge with refpop
- try:
- self.merge_populations(refpop,
- newpop)
- except Exception as e:
- print("merge_populations gave exception",e)
-
- return
-
- def joinfiles(self):
- """
- Function to load in the joinlist to an array and return it.
- """
- f = open(self.grid_options['joinlist'],'r',encoding='utf-8')
- list = f.read().splitlines()
- f.close()
- return list
-
- def join_from_files(self,newobj,joinfiles):
- """
- Merge the results from the list joinfiles into newobj.
- """
- for file in joinfiles:
- print("Join data in",file)
- self.merge_populations_from_file(newobj,
- file)
- return newobj
-
- def can_join(self,joinfiles,joiningfile,vb=False):
- """
- Check the joinfiles to make sure they all exist
- and their .saved equivalents also exist
- """
- vb = False # for debugging set this to True
- if os.path.exists(joiningfile):
- if vb:
- print("cannot join: joiningfile exists at {}".format(joiningfile))
- return False
- elif vb:
- print("joiningfile does not exist")
- for file in joinfiles:
- if vb:
- print("check for {}".format(file))
- if os.path.exists(file) == False:
- if vb:
- print("cannot join: {} does not exist".format(file))
- return False
- savedfile = file + '.saved'
- if vb:
- print("check for {}".format(savedfile))
- if os.path.exists(savedfile) == False:
- if vb:
- print("cannot join: {} does not exist".format(savedfile))
- return False
-
- # found both files
- if vb:
- print("found {} and {}".format(file,savedfile))
-
- # check for joiningfile again
- if os.path.exists(joiningfile):
- if vb:
- print("cannot join: joiningfile exists at {}".format(joiningfile))
- return False
- elif vb:
- print("joiningfile does not exist")
-
- if vb:
- print("returning True from can_join()")
- return True
-
- def add_system_metadata(self):
- """
- Add system's metadata to the grid_ensemble_results, and
- add some system information to metadata.
- """
- # add metadata if it doesn't exist
- if not "metadata" in self.grid_ensemble_results:
- self.grid_ensemble_results["metadata"] = {}
-
- # add date
- self.grid_ensemble_results["metadata"]['date'] = datetime.datetime.now().strftime("%m/%d/%Y %H:%M:%S")
-
- # add platform and build information
- try:
- self.grid_ensemble_results["metadata"]['platform'] = platform.platform()
- self.grid_ensemble_results["metadata"]['platform_uname'] = list(platform.uname())
- self.grid_ensemble_results["metadata"]['platform_machine'] = platform.machine()
- self.grid_ensemble_results["metadata"]['platform_node'] = platform.node()
- self.grid_ensemble_results["metadata"]['platform_release'] = platform.release()
- self.grid_ensemble_results["metadata"]['platform_version'] = platform.version()
- self.grid_ensemble_results["metadata"]['platform_processor'] = platform.processor()
- self.grid_ensemble_results["metadata"]['platform_python_build'] = ' '.join(platform.python_build())
- self.grid_ensemble_results["metadata"]['platform_python_version'] = platform.python_version()
- except Exception as e:
- print("platform call failed:",e)
- pass
-
- try:
- self.grid_ensemble_results["metadata"]['hostname'] = platform.uname()[1]
- except Exception as e:
- print("platform call failed:",e)
- pass
-
- try:
- self.grid_ensemble_results["metadata"]['duration'] = self.time_elapsed()
- except Exception as e:
- print("Failure to calculate time elapsed:",e)
pass
-
try:
- self.grid_ensemble_results["metadata"]['CPU_time'] = self.CPU_time()
- except Exception as e:
- print("Failure to calculate CPU time consumed:",e)
+ killed = killed or self.grid_ensemble_results['metadata']['_killed']
+ except:
pass
- return
-
-
- def snapshot_filename(self):
- """
- Automatically choose the snapshot filename.
- """
- if self.grid_options['slurm'] > 0:
- file = os.path.join(self.grid_options['slurm_dir'],
- 'snapshots',
- self.jobID() + '.gz')
- else:
- file = os.path.join(self.grid_options['tmp_dir'],
- 'snapshot.gz')
- return file
+ return killed
- def load_snapshot(self,file):
- """
- Load a snapshot from file and set it in the preloaded_population placeholder.
- """
- newpop = self.load_population_object(file)
- self.preloaded_population = newpop
- self.grid_options['start_at'] = newpop.grid_options['start_at']
- print("Loaded from snapshot at {file} : start at star {n}".format(
- file=file,
- n=self.grid_options['start_at']))
- return
- def save_snapshot(self,file=None):
+ def _check_binary_c_error(self, binary_c_output, system_dict):
"""
- Save the population object to a snapshot file, automatically choosing the filename if none is given.
+ Function to check whether binary_c throws an error and handle accordingly.
"""
- if file == None:
- file = self.snapshot_filename()
- try:
- n = self.grid_options['_count']
- except:
- n = '?'
+ if binary_c_output:
+ if (binary_c_output.splitlines()[0].startswith("SYSTEM_ERROR")) or (
+ binary_c_output.splitlines()[-1].startswith("SYSTEM_ERROR")
+ ):
+ self.verbose_print(
+ "FAILING SYSTEM FOUND",
+ self.grid_options["verbosity"],
+ 0,
+ )
- print("Saving snapshot containing {} stars to {}".format(n,file))
- self.save_population_object(object=self,
- filename=file)
+ # Keep track of the amount of failed systems and their error codes
+ self.grid_options["_failed_prob"] += system_dict.get("probability", 1)
+ self.grid_options["_failed_count"] += 1
+ self.grid_options["_errors_found"] = True
- return
+ # Try catching the error code and keep track of the unique ones.
+ try:
+ error_code = int(
+ binary_c_output.splitlines()[0]
+ .split("with error code")[-1]
+ .split(":")[0]
+ .strip()
+ )
- def was_killed(self):
- # determine if the process was killed
- killed = self.grid_options['_killed']
- try:
- killed = killed or self.grid_results['_killed']
- except:
- pass
- try:
- killed = killed or self.grid_ensemble_results['metadata']['_killed']
- except:
- pass
- return killed
+ if (
+ not error_code
+ in self.grid_options["_failed_systems_error_codes"]
+ ):
+ self.grid_options["_failed_systems_error_codes"].append(
+ error_code
+ )
+ except ValueError:
+ self.verbose_print(
+ "Failed to extract the error-code",
+ self.grid_options["verbosity"],
+ 1,
+ )
- def slurm_jobid_from_dir(self,dir):
- """
- Return the Slurm jobid from a slurm directory, passed in
- """
- file = os.path.join(dir,'jobid')
- f = open(file,"r",encoding='utf-8')
- if not f:
- print("Error: could not open {} to read the Slurm jobid of the directory {}".format(file,dir))
- sys.exit(code=1)
- oldjobid = f.read().strip()
- f.close()
- if not oldjobid:
- print("Error: could not find jobid in {}".format(self.grid_options['slurm_restart_dir']))
- self.exit(code=1)
+ # Check if we have exceeded the number of errors
+ if (
+ self.grid_options["_failed_count"]
+ > self.grid_options["failed_systems_threshold"]
+ ):
+ if not self.grid_options["_errors_exceeded"]:
+ self.verbose_print(
+ self._boxed(
+ "Process {} exceeded the maximum ({}) number of failing systems. Stopped logging them to files now".format(
+ self.process_ID,
+ self.grid_options["failed_systems_threshold"],
+ )
+ ),
+ self.grid_options["verbosity"],
+ 1,
+ )
+ self.grid_options["_errors_exceeded"] = True
- return oldjobid
+ # If not, write the failing systems to files unique to each process
+ else:
+ # Write arg lines to file
+ argstring = self._return_argline(system_dict)
+ with open(
+ os.path.join(
+ self.grid_options["tmp_dir"],
+ "failed_systems",
+ "process_{}.txt".format(self.process_ID),
+ ),
+ "a+",
+ encoding='utf-8'
+ ) as f:
+ f.write(argstring + "\n")
+ f.close()
+ else:
+ self.verbose_print(
+ "binary_c output nothing - this is strange. If there is ensemble output being generated then this is fine.",
+ self.grid_options["verbosity"],
+ 3,
+ )
diff --git a/binarycpython/utils/grid_logging.py b/binarycpython/utils/grid_logging.py
new file mode 100644
index 000000000..c97a91484
--- /dev/null
+++ b/binarycpython/utils/grid_logging.py
@@ -0,0 +1,403 @@
+"""
+ Binary_c-python's grid logging functions.
+"""
+
+import logging
+import os
+import strip_ansi
+import sys
+import time
+
+import binarycpython.utils.functions
+from binarycpython.utils.functions import (
+ format_number,
+ trem,
+ verbose_print
+ )
+from binarycpython.utils.grid_options_defaults import (
+ secs_per_day
+ )
+
+class grid_logging():
+
+ def __init__(self, **kwargs):
+ # don't do anything: we just inherit from this class
+ return
+
+ def _set_custom_logging(self):
+ """
+ Function/routine to set all the custom logging so that the function memory pointer
+ is known to the grid.
+
+ When the memory adress is loaded and the library file is set we'll skip rebuilding the library
+ """
+
+ # Only if the values are the 'default' unset values
+ if (
+ self.grid_options["custom_logging_func_memaddr"] == -1
+ and self.grid_options["_custom_logging_shared_library_file"] is None
+ ):
+ self.verbose_print(
+ "Creating and loading custom logging functionality",
+ self.grid_options["verbosity"],
+ 1,
+ )
+ # C_logging_code gets priority of C_autogen_code
+ if self.grid_options["C_logging_code"]:
+ # Generate entire shared lib code around logging lines
+ custom_logging_code = binary_c_log_code(
+ self.grid_options["C_logging_code"],
+ verbosity=self.grid_options["verbosity"]
+ - (self._CUSTOM_LOGGING_VERBOSITY_LEVEL - 1),
+ )
+
+ # Load memory address
+ (
+ self.grid_options["custom_logging_func_memaddr"],
+ self.grid_options["_custom_logging_shared_library_file"],
+ ) = create_and_load_logging_function(
+ custom_logging_code,
+ verbosity=self.grid_options["verbosity"]
+ - (self._CUSTOM_LOGGING_VERBOSITY_LEVEL - 1),
+ custom_tmp_dir=self.grid_options["tmp_dir"],
+ )
+
+ elif self.grid_options["C_auto_logging"]:
+ # Generate real logging code
+ logging_line = autogen_C_logging_code(
+ self.grid_options["C_auto_logging"],
+ verbosity=self.grid_options["verbosity"]
+ - (self._CUSTOM_LOGGING_VERBOSITY_LEVEL - 1),
+ )
+
+ # Generate entire shared lib code around logging lines
+ custom_logging_code = binary_c_log_code(
+ logging_line,
+ verbosity=self.grid_options["verbosity"]
+ - (self._CUSTOM_LOGGING_VERBOSITY_LEVEL - 1),
+ )
+
+ # Load memory address
+ (
+ self.grid_options["custom_logging_func_memaddr"],
+ self.grid_options["_custom_logging_shared_library_file"],
+ ) = create_and_load_logging_function(
+ custom_logging_code,
+ verbosity=self.grid_options["verbosity"]
+ - (self._CUSTOM_LOGGING_VERBOSITY_LEVEL - 1),
+ custom_tmp_dir=self.grid_options["tmp_dir"],
+ )
+ else:
+ self.verbose_print(
+ "Custom logging library already loaded. Not setting them again.",
+ self.grid_options["verbosity"],
+ 1,
+ )
+
+ def _print_info(self, run_number, total_systems, full_system_dict):
+ """
+ Function to print info about the current system and the progress of the grid.
+
+ # color info tricks from https://ozzmaker.com/add-colour-to-text-in-python/
+ https://stackoverflow.com/questions/287871/how-to-print-colored-text-in-terminal-in-python
+ """
+
+ # Define frequency
+ if self.grid_options["verbosity"] == 1:
+ print_freq = 1
+ else:
+ print_freq = 10
+
+ if run_number % print_freq == 0:
+ binary_cmdline_string = self._return_argline(full_system_dict)
+ info_string = "{color_part_1} \
+ {text_part_1}{end_part_1}{color_part_2} \
+ {text_part_2}{end_part_2}".format(
+ color_part_1="\033[1;32;41m",
+ text_part_1="{}/{}".format(run_number, total_systems),
+ end_part_1="\033[0m",
+ color_part_2="\033[1;32;42m",
+ text_part_2="{}".format(binary_cmdline_string),
+ end_part_2="\033[0m",
+ )
+ print(info_string)
+
+ def _set_loggers(self):
+ """
+ Function to set the loggers for the execution of the grid
+ """
+
+ # Set log file
+ binary_c_logfile = self.grid_options["log_file"]
+
+ # Create directory
+ os.makedirs(os.path.dirname(binary_c_logfile), exist_ok=True)
+
+ # Set up logger
+ self.logger = logging.getLogger("binary_c_python_logger")
+ self.logger.setLevel(self.grid_options["verbosity"])
+
+ # Reset handlers
+ self.logger.handlers = []
+
+ # Set formatting of output
+ log_formatter = logging.Formatter(
+ "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+ )
+
+ # Make and add file handlers
+ # make handler for output to file
+ handler_file = logging.FileHandler(filename=os.path.join(binary_c_logfile))
+ handler_file.setFormatter(log_formatter)
+ handler_file.setLevel(logging.INFO)
+
+ # Make handler for output to stdout
+ handler_stdout = logging.StreamHandler(sys.stdout)
+ handler_stdout.setFormatter(log_formatter)
+ handler_stdout.setLevel(logging.INFO)
+
+ # Add the loggers
+ self.logger.addHandler(handler_file)
+ self.logger.addHandler(handler_stdout)
+
+ ######################
+ # Status logging
+
+ def vb1print(self, ID, now, system_number, system_dict):
+ """
+ Verbosity-level 1 printing, to keep an eye on a grid.
+ Arguments:
+ ID: thread ID for debugging (int)
+ now: the time now as a UNIX-style epoch in seconds (float)
+ system_number: the system number
+
+ TODO: add information about the number of cores. the TPR shows the dt/dn but i want to see the number per core too
+ """
+
+ # calculate estimated time of arrive (eta and eta_secs), time per run (tpr)
+ localtime = time.localtime(now)
+
+ # calculate stats
+ n = self.shared_memory["n_saved_log_stats"].value
+ if n < 2:
+ # simple 1-system calculation: inaccurate
+ # but best for small n
+ dt = now - self.shared_memory["prev_log_time"][0]
+ dn = system_number - self.shared_memory["prev_log_system_number"][0]
+ else:
+ # average over n_saved_log_stats
+ dt = (
+ self.shared_memory["prev_log_time"][0]
+ - self.shared_memory["prev_log_time"][n - 1]
+ )
+ dn = (
+ self.shared_memory["prev_log_system_number"][0]
+ - self.shared_memory["prev_log_system_number"][n - 1]
+ )
+
+ eta, units, tpr, eta_secs = trem(
+ dt, system_number, dn, self.grid_options["_total_starcount"]
+ )
+
+ # compensate for multithreading and modulo
+ tpr *= self.grid_options["num_processes"] * self.grid_options["modulo"]
+
+ if eta_secs < secs_per_day:
+ fintime = time.localtime(now + eta_secs)
+ etf = "{hours:02d}:{minutes:02d}:{seconds:02d}".format(
+ hours=fintime.tm_hour, minutes=fintime.tm_min, seconds=fintime.tm_sec
+ )
+ else:
+ d = int(eta_secs / secs_per_day)
+ if d == 1:
+ etf = "Tomorrow"
+ else:
+ etf = "In {} days".format(d)
+
+ # modulo information
+ if self.grid_options["modulo"] == 1:
+ modulo = "" # usual case
+ else:
+ modulo = "%" + str(self.grid_options["modulo"])
+
+ # add up memory use from each thread
+ total_mem_use = sum(self.shared_memory["memory_use_per_thread"])
+
+ # make a string to describe the system e.g. M1, M2, etc.
+ system_string = ""
+
+ # use the multiplicity if given
+ if "multiplicity" in system_dict:
+ nmult = int(system_dict["multiplicity"])
+ else:
+ nmult = 4
+
+ # masses
+ for i in range(nmult):
+ i1 = str(i + 1)
+ if "M_" + i1 in system_dict:
+ system_string += (
+ "M{}=".format(i1) + format_number(system_dict["M_" + i1]) + " "
+ )
+
+ # separation and orbital period
+ if "separation" in system_dict:
+ system_string += "a=" + format_number(system_dict["separation"])
+ if "orbital_period" in system_dict:
+ system_string += "P=" + format_number(system_dict["orbital_period"])
+
+ # do the print
+ self.verbose_print(
+ "{opening_colour}{system_number}/{total_starcount}{modulo} {pc_colour}{pc_complete:5.1f}% complete {time_colour}{hours:02d}:{minutes:02d}:{seconds:02d} {ETA_colour}ETA={ETA:7.1f}{units} tpr={tpr:2.2e} {ETF_colour}ETF={ETF} {mem_use_colour}mem:{mem_use:.1f}MB {system_string_colour}{system_string}{closing_colour}".format(
+ opening_colour=self.ANSI_colours["reset"]
+ + self.ANSI_colours["yellow on black"],
+ system_number=system_number,
+ total_starcount=self.grid_options["_total_starcount"],
+ modulo=modulo,
+ pc_colour=self.ANSI_colours["blue on black"],
+ pc_complete=(100.0 * system_number)
+ / (1.0 * self.grid_options["_total_starcount"])
+ if self.grid_options["_total_starcount"]
+ else -1,
+ time_colour=self.ANSI_colours["green on black"],
+ hours=localtime.tm_hour,
+ minutes=localtime.tm_min,
+ seconds=localtime.tm_sec,
+ ETA_colour=self.ANSI_colours["red on black"],
+ ETA=eta,
+ units=units,
+ tpr=tpr,
+ ETF_colour=self.ANSI_colours["blue"],
+ ETF=etf,
+ mem_use_colour=self.ANSI_colours["magenta"],
+ mem_use=total_mem_use,
+ system_string_colour=self.ANSI_colours["yellow"],
+ system_string=system_string,
+ closing_colour=self.ANSI_colours["reset"],
+ ),
+ self.grid_options["verbosity"],
+ 1,
+ )
+
+ def vb2print(self, system_dict, cmdline_string):
+ print(
+ "Running this system now on thread {ID}\n{blue}{cmdline}{reset}\n".format(
+ ID=self.process_ID,
+ blue=self.ANSI_colours["blue"],
+ cmdline=cmdline_string,
+ reset=self.ANSI_colours["reset"],
+ )
+ )
+
+ def verbose_print(self,*args,**kwargs):
+ binarycpython.utils.functions.verbose_print(*args,**kwargs)
+
+
+ def _boxed(self, *list, colour="yellow on black", boxchar="*", separator="\n"):
+ """
+ Function to output a list of strings in a single box.
+
+ Args:
+ list = a list of strings to be output. If these contain the separator
+ (see below) these strings are split by it.
+ separator = strings are split on this, default "\n"
+ colour = the colour to be used, usually this is 'yellow on black'
+ as set in the ANSI_colours dict
+ boxchar = the character used to make the box, '*' by default
+
+ Note: handles tabs (\t) badly, do not use them!
+ """
+ strlen = 0
+ strings = []
+ lengths = []
+
+ # make a list of strings
+ if separator:
+ for l in list:
+ strings += l.split(sep=separator)
+ else:
+ strings = list
+
+ # get lengths without ANSI codes
+ for string in strings:
+ lengths.append(len(strip_ansi.strip_ansi(string)))
+
+ # hence the max length
+ strlen = max(lengths)
+ strlen += strlen % 2
+ header = boxchar * (4 + strlen)
+
+ # start output
+ out = self.ANSI_colours[colour] + header + "\n"
+
+ # loop over strings to output, padding as required
+ for n, string in enumerate(strings):
+ if lengths[n] % 2 == 1:
+ string = " " + string
+ pad = " " * int((strlen - lengths[n]) / 2)
+ out = out + boxchar + " " + pad + string + pad + " " + boxchar + "\n"
+ # close output and return
+ out = out + header + "\n" + self.ANSI_colours["reset"]
+ return out
+
+
+ def _get_stream_logger(self, level=logging.DEBUG):
+ """Return logger with configured StreamHandler."""
+ stream_logger = logging.getLogger("stream_logger")
+ stream_logger.handlers = []
+ stream_logger.setLevel(level)
+ sh = logging.StreamHandler(stream=sys.stdout)
+ sh.setLevel(level)
+ fmt = "[%(asctime)s %(levelname)-8s %(processName)s] --- %(message)s"
+ formatter = logging.Formatter(fmt)
+ sh.setFormatter(formatter)
+ stream_logger.addHandler(sh)
+
+ return stream_logger
+
+ def _clean_up_custom_logging(self, evol_type):
+ """
+ Function to clean up the custom logging.
+ Has two types:
+ 'single':
+ - removes the compiled shared library
+ (which name is stored in grid_options['_custom_logging_shared_library_file'])
+ - TODO: unloads/frees the memory allocated to that shared library
+ (which is stored in grid_options['custom_logging_func_memaddr'])
+ - sets both to None
+ 'multiple':
+ - TODO: make this and design this
+ """
+
+ if evol_type == "single":
+ self.verbose_print(
+ "Cleaning up the custom logging stuff. type: single",
+ self.grid_options["verbosity"],
+ 1,
+ )
+
+ # TODO: Explicitly unload the library
+
+ # Reset the memory adress location
+ self.grid_options["custom_logging_func_memaddr"] = -1
+
+ # remove shared library files
+ if self.grid_options["_custom_logging_shared_library_file"]:
+ remove_file(
+ self.grid_options["_custom_logging_shared_library_file"],
+ self.grid_options["verbosity"],
+ )
+ self.grid_options["_custom_logging_shared_library_file"] = None
+
+ if evol_type == "population":
+ self.verbose_print(
+ "Cleaning up the custom logging stuffs. type: population",
+ self.grid_options["verbosity"],
+ 1,
+ )
+
+ # TODO: make sure that these also work. not fully sure if necessary tho.
+ # whether its a single file, or a dict of files/mem addresses
+
+ if evol_type == "MC":
+ pass
diff --git a/binarycpython/utils/grid_options_defaults.py b/binarycpython/utils/grid_options_defaults.py
index 00ad9a376..323b387d4 100644
--- a/binarycpython/utils/grid_options_defaults.py
+++ b/binarycpython/utils/grid_options_defaults.py
@@ -18,855 +18,683 @@ import os
import sys
from binarycpython.utils.custom_logging_functions import temp_dir
-from binarycpython.utils.functions import return_binary_c_version_info
-_LOGGER_VERBOSITY_LEVEL = 1
-_CUSTOM_LOGGING_VERBOSITY_LEVEL = 2
_MOE2017_VERBOSITY_LEVEL = 5
_MOE2017_VERBOSITY_INTERPOLATOR_LEVEL = 6
_MOE2017_VERBOSITY_INTERPOLATOR_EXTRA_LEVEL = 7
-
-# Options dict
-grid_options_defaults_dict = {
- ##########################
- # general (or unordered..)
- ##########################
- "num_cores": 1, # total number of cores used to evolve the population
- "num_cores_available": None, # set automatically, not by the user
- "parse_function": None, # Function to parse the output with.
- "multiplicity_fraction_function": 0, # Which multiplicity fraction function to use. 0: None, 1: Arenou 2010, 2: Rhagavan 2010, 3: Moe and di Stefano 2017
- "tmp_dir": temp_dir(), # Setting the temp dir of the program
- "cache_dir" : None, # Cache location
- "status_dir" : None, #
- "_main_pid": -1, # Placeholder for the main process id of the run.
- "save_ensemble_chunks": True, # Force the ensemble chunk to be saved even if we are joining a thread (just in case the joining fails)
- "combine_ensemble_with_thread_joining": True, # Flag on whether to combine everything and return it to the user or if false: write it to data_dir/ensemble_output_{population_id}_{thread_id}.json
- "_commandline_input": "",
- "log_runtime_systems": 0, # whether to log the runtime of the systems (1 file per thread. stored in the tmp_dir)
- "_actually_evolve_system": True, # Whether to actually evolve the systems of just act as if. for testing. used in _process_run_population_grid
- "max_queue_size": 0, # Maximum size of the system call queue. Set to 0 for this to be calculated automatically
- "run_zero_probability_system": True, # Whether to run the zero probability systems
- "_zero_prob_stars_skipped": 0,
- "ensemble_factor_in_probability_weighted_mass": False, # Whether to multiply the ensemble results by 1/probability_weighted_mass
- "do_dry_run": True, # Whether to do a dry run to calculate the total probability for this run
- "custom_generator": None, # Place for the custom system generator
- "exit_after_dry_run": False, # Exit after dry run?
-
- #####################
- # System information
- #####################
- "command_line": ' '.join(sys.argv),
- "original_command_line": os.getenv('BINARY_C_PYTHON_ORIGINAL_CMD_LINE'),
- "working_diretory": os.getcwd(),
- "original_working_diretory": os.getenv('BINARY_C_PYTHON_ORIGINAL_WD'),
- "start_time": datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"),
- "original_submission_time" : os.getenv('BINARY_C_PYTHON_ORIGINAL_SUBMISSION_TIME'),
-
- ##########################
- # Execution log:
- ##########################
- "verbosity": 0, # Level of verbosity of the simulation
- "log_file": os.path.join(
- temp_dir(), "binary_c_python.log"
- ), # Set to None to not log to file. The directory will be created
- "log_dt": 5, # time between vb=1 logging outputs
- "n_logging_stats": 50, # number of logging stats used to calculate time remaining (etc.) default = 50
- ##########################
- # binary_c files
- ##########################
- "_binary_c_executable": os.path.join(
- os.environ["BINARY_C"], "binary_c"
- ), # TODO: make this more robust
- "_binary_c_shared_library": os.path.join(
- os.environ["BINARY_C"], "src", "libbinary_c.so"
- ), # TODO: make this more robust
- "_binary_c_config_executable": os.path.join(
- os.environ["BINARY_C"], "binary_c-config"
- ), # TODO: make this more robust
- "_binary_c_dir": os.environ["BINARY_C"],
- ##########################
- # Moe and di Stefano (2017) internal settings
- ##########################
- "_loaded_Moe2017_data": False, # Holds flag whether the Moe and di Stefano (2017) data is loaded into memory
- "_set_Moe2017_grid": False, # Whether the Moe and di Stefano (2017) grid has been loaded
- "Moe2017_options": None, # Holds the Moe and di Stefano (2017) options.
- "_Moe2017_JSON_data": None, # Stores the data
- ##########################
- # Custom logging
- ##########################
- "C_auto_logging": None, # Should contain a dictionary where the keys are they headers
- # and the values are lists of parameters that should be logged.
- # This will get parsed by autogen_C_logging_code in custom_logging_functions.py
- "C_logging_code": None, # Should contain a string which holds the logging code.
- "custom_logging_func_memaddr": -1, # Contains the custom_logging functions memory address
- "_custom_logging_shared_library_file": None, # file containing the .so file
- ##########################
- # Store pre-loading:
- ##########################
- "_store_memaddr": -1, # Contains the store object memory address, useful for pre loading.
- # defaults to -1 and isn't used if that's the default then.
- ##########################
- # Log args: logging of arguments
- ##########################
- "log_args": 0, # unused
- "log_args_dir": "/tmp/", # unused
- ##########################
- # Population evolution
- ##########################
- ## General
- "evolution_type": "grid", # Flag for type of population evolution
- "_evolution_type_options": [
- "grid",
- "custom_generator",
- ], # available choices for type of population evolution. # TODO: fill later with Monte Carlo, source file
- "_system_generator": None, # value that holds the function that generates the system
- # (result of building the grid script)
- "source_file_filename": None, # filename for the source
- "_count": 0, # count of systems
- "_total_starcount": 0, # Total count of systems in this generator
- "_probtot": 0, # total probability
- "weight": 1.0, # weighting for the probability
- "repeat": 1, # number of times to repeat each system (probability is adjusted to be 1/repeat)
- "_start_time_evolution": 0, # Start time of the grid
- "_end_time_evolution": 0, # end time of the grid
- "_errors_found": False, # Flag whether there are any errors from binary_c
- "_errors_exceeded": False, # Flag whether the number of errors have exceeded the limit
- "_failed_count": 0, # number of failed systems
- "_failed_prob": 0, # Summed probability of failed systems
- "failed_systems_threshold": 20, # Maximum failed systems per process allowed to fail before the process stops logging the failing systems.
- "_failed_systems_error_codes": [], # List to store the unique error codes
- "_population_id": 0, # Random id of this grid/population run, Unique code for the population. Should be set only once by the controller process.
- "_total_mass_run": 0, # To count the total mass that thread/process has ran
- "_total_probability_weighted_mass_run": 0, # To count the total mass * probability for each system that thread/process has ran
- "modulo": 1, # run modulo n of the grid.
- "start_at": 0, # start at the first model
- ## Grid type evolution
- "_grid_variables": {}, # grid variables
- "gridcode_filename": None, # filename of gridcode
- "symlink_latest_gridcode": True, # symlink to latest gridcode
- "save_population_object" : None, # filename to which we should save a pickled grid object as the final thing we do
- 'joinlist' : None,
- 'do_analytics' : True, # if True, calculate analytics prior to return
- 'save_snapshots' : True, # if True, save snapshots on SIGINT
- 'restore_from_snapshot_file' : None, # file to restore from
- 'restore_from_snapshot_dir' : None, # dir to restore from
- 'exit_code' : 0, # return code
- 'stop_queue' : False,
- '_killed' : False,
- '_queue_done' : False,
- ## Monte carlo type evolution
- # TODO: make MC options
- ## Evolution from source file
- # TODO: make run from sourcefile options.
- ## Other no yet implemented parts for the population evolution part
- # # start at this model number: handy during debugging
- # # to skip large parts of the grid
- # start_at => 0
- # global_error_string => undef,
- # monitor_files => [],
- # nextlogtime => 0,
- # nthreads => 1, # number of threads
- # # start at model offset (0-based, so first model is zero)
- # offset => 0,
- # resolution=>{
- # shift =>0,
- # previous=>0,
- # n =>{} # per-variable resolution
- # },
- # thread_q => undef,
- # threads => undef, # array of threads objects
- # tstart => [gettimeofday], # flexigrid start time
- # __nvar => 0, # number of grid variables
- # _varstub => undef,
- # _lock => undef,
- # _evcode_pids => [],
- # };
- ########################################
- # Slurm stuff
- ########################################
- "slurm": 0, # dont use the slurm by default, 0 = no slurm, 1 = launch slurm jobs, 2 = run slurm jobs
- "slurm_ntasks": 1, # CPUs required per array job: usually only need this to be 1
- "slurm_dir": "", # working directory containing scripts output logs etc.
- "slurm_njobs": 0, # number of scripts; set to 0 as default
- "slurm_jobid": "", # slurm job id (%A)
- "slurm_memory": '512MB', # memory required for the job
- "slurm_warn_max_memory": '1024MB', # warn if we set it to more than this (usually by accident)
- "slurm_postpone_join": 0, # if 1 do not join on slurm, join elsewhere. want to do it off the slurm grid (e.g. with more RAM)
- "slurm_jobarrayindex": None, # slurm job array index (%a)
- "slurm_jobname": "binary_grid", # default
- "slurm_partition": None,
- "slurm_time": 0, # total time. 0 = infinite time
- "slurm_postpone_sbatch": 0, # if 1: don't submit, just make the script
- "slurm_array": None, # override for --array, useful for rerunning jobs
- "slurm_array_max_jobs" : None, # override for the max number of concurrent array jobs
- "slurm_extra_settings": {}, # Dictionary of extra settings for Slurm to put in its launch script.
- "slurm_sbatch": "sbatch", # sbatch command
- "slurm_restart_dir" : None, # restart Slurm jobs from this directory
- ########################################
- # Condor stuff
- ########################################
- "condor": 0, # 1 to use condor, 0 otherwise
- "condor_command": "", # condor command e.g. "evolve", "join"
- "condor_dir": "", # working directory containing e.g. scripts, output, logs (e.g. should be NFS available to all)
- "condor_njobs": "", # number of scripts/jobs that CONDOR will run in total
- "condor_jobid": "", # condor job id
- "condor_postpone_join": 0, # if 1, data is not joined, e.g. if you want to do it off the condor grid (e.g. with more RAM)
- # "condor_join_machine": None, # if defined then this is the machine on which the join command should be launched (must be sshable and not postponed)
- "condor_join_pwd": "", # directory the join should be in (defaults to $ENV{PWD} if undef)
- "condor_memory": 1024, # in MB, the memory use (ImageSize) of the job
- "condor_universe": "vanilla", # usually vanilla universe
- "condor_extra_settings": {}, # Place to put extra configuration for the CONDOR submit file. The key and value of the dict will become the key and value of the line in te slurm batch file. Will be put in after all the other settings (and before the command). Take care not to overwrite something without really meaning to do so.
- # snapshots and checkpoints
- 'condor_snapshot_on_kill':0, # if 1 snapshot on SIGKILL before exit
- 'condor_load_from_snapshot':0, # if 1 check for snapshot .sv file and load it if found
- 'condor_checkpoint_interval':0, # checkpoint interval (seconds)
- 'condor_checkpoint_stamp_times':0, # if 1 then files are given timestamped names
- # (warning: lots of files!), otherwise just store the lates
- 'condor_streams':0, # stream stderr/stdout by default (warning: might cause heavy network load)
- 'condor_save_joined_file':0, # if 1 then results/joined contains the results
- # (useful for debugging, otherwise a lot of work)
- 'condor_requirements':'', # used?
- # # resubmit options : if the status of a condor script is
- # # either 'finished','submitted','running' or 'crashed',
- # # decide whether to resubmit it.
- # # NB Normally the status is empty, e.g. on the first run.
- # # These are for restarting runs.
- # condor_resubmit_finished:0,
- 'condor_resubmit_submitted':0,
- 'condor_resubmit_running':0,
- 'condor_resubmit_crashed':0,
- ##########################
- # Unordered. Need to go through this. Copied from the perl implementation.
- ##########################
- ##
- # return_array_refs:1, # quicker data parsing mode
- # sort_args:1,
- # save_args:1,
- # nice:'nice -n +20', # nice command e.g. 'nice -n +10' or ''
- # timeout:15, # seconds until timeout
- # log_filename:"/scratch/davidh/results_simulations/tmp/log.txt",
- # # current_log_filename:"/scratch/davidh/results_simulations/tmp/grid_errors.log",
- ############################################################
- # Set default grid properties (in %self->{_grid_options}}
- # and %{$self->{_bse_options}})
- # This is the first thing that should be called by the user!
- ############################################################
- # # set signal handlers for timeout
- # $self->set_class_signal_handlers();
- # # set operating system
- # my $os = rob_misc::operating_system();
- # %{$self->{_grid_options}}=(
- # # save operating system
- # operating_system:$os,
- # # process name
- # process_name : 'binary_grid'.$VERSION,
- # grid_defaults_set:1, # so we know the grid_defaults function has been called
- # # grid suspend files: assume binary_c by default
- # suspend_files:[$tmp.'/force_binary_c_suspend',
- # './force_binary_c_suspend'],
- # snapshot_file:$tmp.'/binary_c-snapshot',
- # ########################################
- # # infomration about the running grid script
- # ########################################
- # working_directory:cwd(), # the starting directory
- # perlscript:$0, # the name of the perlscript
- # perlscript_arguments:join(' ',@ARGV), # arguments as a string
- # perl_executable:$^X, # the perl executable
- # command_line:join(' ',$0,@ARGV), # full command line
- # process_ID:$$, # process ID of the main perl script
- # ########################################
- # # GRID
- # ########################################
- # # if undef, generate gridcode, otherwise load the gridcode
- # # from this file. useful for debugging
- # gridcode_from_file : undef,
- # # assume binary_grid perl backend by default
- # backend :
- # $self->{_grid_options}->{backend} //
- # $binary_grid2::backend //
- # 'binary_grid::Perl',
- # # custom C function for output : this automatically
- # # binds if a function is available.
- # C_logging_code : undef,
- # C_auto_logging : undef,
- # custom_output_C_function_pointer : binary_c_function_bind(),
- # # control flow
- 'rungrid' : 1, # usually run the grid, but can be 0
- # # to skip it (e.g. for condor/slurm runs)
- # merge_datafiles:'',
- # merge_datafiles_filelist:'',
- # # parameter space options
- # binary:0, # set to 0 for single stars, 1 for binaries
- # # if use_full_resolution is 1, then run a dummy grid to
- # # calculate the resolution. this could be slow...
- # use_full_resolution : 1,
- # # the probability in any distribution must be within
- # # this tolerance of 1.0, ignored if undef (if you want
- # # to run *part* of the parameter space then this *must* be undef)
- # probability_tolerance:undef,
- # # how to deal with a failure of the probability tolerance:
- # # 0 = nothing
- # # 1 = warning
- # # 2 = stop
- # probability_tolerance_failmode:1,
- # # add up and log system error count and probability
- # add_up_system_errors:1,
- # log_system_errors:1,
- # # codes, paths, executables etc.
- # # assume binary_c by default, and set its defaults
- # code:'binary_c',
- # arg_prefix:'--',
- # prog:'binary_c', # executable
- # nice:'nice -n +0', # nice command
- # ionice:'',
- # # compress output?
- # binary_c_compression:0,
- # # get output as array of pre-split array refs
- # return_array_refs:1,
- # # environment
- # shell_environment:undef,
- # libpath:undef, # for backwards compatibility
- # # where is binary_c? need this to get the values of some counters
- # rootpath:$self->okdir($ENV{BINARY_C_ROOTPATH}) //
- # $self->okdir($ENV{HOME}.'/progs/stars/binary_c') //
- # '.' , # last option is a fallback ... will fail if it doesn't exist
- # srcpath:$self->okdir($ENV{BINARY_C_SRCPATH}) //
- # $self->okdir($ENV{BINARY_C_ROOTPATH}.'/src') //
- # $self->okdir($ENV{HOME}.'/progs/stars/binary_c/src') //
- # './src' , # last option is fallback... will fail if it doesn't exist
- # # stack size per thread in megabytes
- # threads_stack_size:50,
- # # thread sleep time between starting the evolution code and starting
- # # the grid
- # thread_presleep:0,
- # # threads
- # # Max time a thread can sit looping (with calls to tbse_line)
- # # before a warning is issued : NB this does not catch real freezes,
- # # just infinite loops (which still output)
- # thread_max_freeze_time_before_warning:10,
- # # run all models by default: modulo=1, offset=0
- # modulo:1,
- # offset:0,
- # # max number of stars on the queue
- # maxq_per_thread : 100,
- # # data dump file : undef by default (do nothing)
- # results_hash_dumpfile : '',
- # # compress files with bzip2 by default
- # compress_results_hash : 1,
- # ########################################
- # # CPU
- # ########################################
- # cpu_cap:0, # if 1, limits to one CPU
- # cpu_affinity : 0, # do not bind to a CPU by default
- # ########################################
- # # Code, Timeouts, Signals
- # ########################################
- # binary_grid_code_filtering:1, # you want this, it's (MUCH!) faster
- # pre_filter_file:undef, # dump pre filtered code to this file
- # post_filter_file:undef, # dump post filtered code to this file
- # timeout:30, # timeout in seconds
- # timeout_vb:0, # no timeout logging
- # tvb:0, # no thread logging
- # nfs_sleep:1, # time to wait for NFS to catch up with file accesses
- # # flexigrid checks the timeouts every
- # # flexigrid_timeout_check_interval seconds
- # flexigrid_timeout_check_interval:0.01,
- # # this is set to 1 when the grid is finished
- # flexigrid_finished:0,
- # # allow signals by default
- # 'no signals':0,
- # # but perhaps disable specific signals?
- # 'disable signal':{INT:0,ALRM:0,CONT:0,USR1:0,STOP:0},
- # # dummy variables
- # single_star_period:1e50, # orbital period of a single star
- # #### timers : set timers to 0 (or empty list) to ignore,
- # #### NB these must be given context (e.g. main::xyz)
- # #### for functions not in binary_grid
- # timers:0,
- # timer_subroutines:[
- # # this is a suggested default list
- # 'flexigrid',
- # 'set_next_alarm',
- # 'vbout',
- # 'vbout_fast',
- # 'run_flexigrid_thread',
- # 'thread_vb'
- # ],
- # ########################################
- # # INPUT/OUTPUT
- # ########################################
- # blocking:undef, # not yet set
- # # prepend command with stdbuf to stop buffering (if available)
- # stdbuf_command:`stdbuf --version`=~/stdbuf \(GNU/ ? ' stdbuf -i0 -o0 -e0 ' : undef,
- # vb:("@ARGV"=~/\Wvb=(\d+)\W/)[0] // 0, # set to 1 (or more) for verbose output to the screen
- # log_dt_secs:1, # log output to stdout~every log_dt_secs seconds
- # nmod:10, # every nmod models there is output to the screen,
- # # if log_dt_secs has been exceeded also (ignored if 0)
- # colour:1, # set to 1 to use the ANSIColor module for colour output
- # log_args:0, # do not log args in files
- # log_fins:0, # log end of runs too
- # sort_args:0, # do not sort args
- # save_args:0, # do not save args in a string
- # log_args_dir:$tmp, # where to output the args files
- # always_reopen_arg_files:0, # if 1 then arg files are always closed and reopened
- # (may cause a lot of disk I/O)
- # lazy_arg_sending:1, # if 1, the previous args are remembered and
- # # only args that changed are sent (except M1, M2 etc. which always
- # # need sending)
- # # force output files to open on a local disk (not an NFS partion)
- # # not sure how to do this on another OS
- # force_local_hdd_use:($os eq 'unix'),
- # # for verbose output, define the newline
- # # For terminals use "\x0d", for files use "\n", in the
- # # case of multiple threads this will be set to \n
- # newline: "\x0d",
- # # use reset_stars_defaults
- # reset_stars_defaults:1,
- # # set signal captures: argument determines behaviour when the code locks up
- # # 0: exit
- # # 1: reset and try the next star (does this work?!)
- # alarm_procedure:1,
- # # exit on eval failure?
- # exit_on_eval_failure:1,
- # ## functions: these should be set by perl lexical name
- # ## (they are automatically converted to function pointers
- # ## at runtime)
- # # function to be called just before a thread is created
- # thread_precreate_function:undef,
- # thread_precreate_function_pointer:undef,
- # # function to be called just after a thread is created
- # # (from inside the thread just before *grid () call)
- # threads_entry_function:undef,
- # threads_entry_function_pointer:undef,
- # # function to be called just after a thread is finished
- # # (from inside the thread just after *grid () call)
- # threads_flush_function:undef,
- # threads_flush_function_pointer:undef,
- # # function to be called just after a thread is created
- # # (but external to the thread)
- # thread_postrun_function:undef,
- # thread_postrun_function_pointer:undef,
- # # function to be called just before a thread join
- # # (external to the thread)
- # thread_prejoin_function:undef,
- # thread_prejoin_function_pointer:undef,
- # # default to using the internal join_flexigrid_thread function
- # threads_join_function:'binary_grid2::join_flexigrid_thread',
- # threads_join_function_pointer:sub{return $self->join_flexigrid_thread(@_)},
- # # function to be called just after a thread join
- # # (external to the thread)
- # thread_postjoin_function:undef,
- # thread_postjoin_function_pointer:undef,
- # # usually, parse_bse in the main script is called
- # parse_bse_function:'main::parse_bse',
- # parse_bse_function_pointer:undef,
- # # if starting_snapshot_file is defined, load initial
- # # values for the grid from the snapshot file rather
- # # than a normal initiation: this enables you to
- # # stop and start a grid
- # starting_snapshot_file:undef,
-}
-
-# Grid containing the descriptions of the options # TODO: add input types for all of them
-grid_options_descriptions = {
- "tmp_dir": "Directory where certain types of output are stored. The grid code is stored in that directory, as well as the custom logging libraries. Log files and other diagnostics will usually be written to this location, unless specified otherwise", # TODO: improve this
- "status_dir" : "Directory where grid status is stored",
- "_binary_c_dir": "Director where binary_c is stored. This options are not really used",
- "_binary_c_config_executable": "Full path of the binary_c-config executable. This options is not used in the population object.",
- "_binary_c_executable": "Full path to the binary_c executable. This options is not used in the population object.",
- "_binary_c_shared_library": "Full path to the libbinary_c file. This options is not used in the population object",
- "verbosity": "Verbosity of the population code. Default is 0, by which only errors will be printed. Higher values will show more output, which is good for debugging.",
- # deprecated: "binary": "Set this to 1 if the population contains binaries. Input: int", # TODO: write what effect this has.
- "num_cores": "The number of cores that the population grid will use. You can set this manually by entering an integer great than 0. When 0 uses all logical cores. When -1 uses all physical cores. Input: int",
- "num_processes" : "Number of processes launched by multiprocessing. This should be set automatically by binary_c-python, not by the user.",
- "_start_time_evolution": "Variable storing the start timestamp of the population evolution. Set by the object itself.", # TODO: make sure this is logged to a file
- "_end_time_evolution": "Variable storing the end timestamp of the population evolution. Set by the object itself", # TODO: make sure this is logged to a file
- "_total_starcount": "Variable storing the total number of systems in the generator. Used and set by the population object.",
- "_custom_logging_shared_library_file": "filename for the custom_logging shared library. Used and set by the population object",
- "_errors_found": "Variable storing a Boolean flag whether errors by binary_c are encountered.",
- "_errors_exceeded": "Variable storing a Boolean flag whether the number of errors was higher than the set threshold (failed_systems_threshold). If True, then the command line arguments of the failing systems will not be stored in the failed_system_log files.",
- "source_file_filename": "Variable containing the source file containing lines of binary_c command line calls. These all have to start with binary_c.", # TODO: Expand
- "C_auto_logging": "Dictionary containing parameters to be logged by binary_c. The structure of this dictionary is as follows: the key is used as the headline which the user can then catch. The value at that key is a list of binary_c system parameters (like star[0].mass)",
- "C_logging_code": "Variable to store the exact code that is used for the custom_logging. In this way the user can do more complex logging, as well as putting these logging strings in files.",
- "_failed_count": "Variable storing the number of failed systems.",
- "_evolution_type_options": "List containing the evolution type options.",
- "_failed_prob": "Variable storing the total probability of all the failed systems",
- "_failed_systems_error_codes": "List storing the unique error codes raised by binary_c of the failed systems",
- "_grid_variables": "Dictionary storing the grid_variables. These contain properties which are accessed by the _generate_grid_code function",
- "_population_id": "Variable storing a unique 32-char hex string.",
- "_commandline_input": "String containing the arguments passed to the population object via the command line. Set and used by the population object.",
- "_system_generator": "Function object that contains the system generator function. This can be from a grid, or a source file, or a Monte Carlo grid.",
- "gridcode_filename": "Filename for the grid code. Set and used by the population object. TODO: allow the user to provide their own function, rather than only a generated function.",
- "log_args": "Boolean to log the arguments. Unused ", # TODO: fix the functionality for this and describe it properly
- "log_args_dir": "Directory to log the arguments to. Unused", # TODO: fix the functionality for this and describe it properly
- "log_file": "Log file for the population object. Unused", # TODO: fix the functionality for this and describe it properly
- "custom_logging_func_memaddr": "Memory address where the custom_logging_function is stored. Input: int",
- "_count": "Counter tracking which system the generator is on.",
- "_probtot": "Total probability of the population.", # TODO: check whether this is used properly throughout
- "_main_pid": "Main process ID of the master process. Used and set by the population object.",
- "_store_memaddr": "Memory address of the store object for binary_c.",
- "failed_systems_threshold": "Variable storing the maximum number of systems that are allowed to fail before logging their command line arguments to failed_systems log files",
- "parse_function": "Function that the user can provide to handle the output the binary_c. This function has to take the arguments (self, output). Its best not to return anything in this function, and just store stuff in the self.grid_results dictionary, or just output results to a file",
- "condor": "Int flag whether to use a condor type population evolution. Not implemented yet.", # TODO: describe this in more detail
- "slurm": "Int flag whether to use a Slurm type population evolution.", # TODO: describe this in more detail
- "weight": "Weight factor for each system. The calculated probability is multiplied by this. If the user wants each system to be repeated several times, then this variable should not be changed, rather change the _repeat variable instead, as that handles the reduction in probability per system. This is useful for systems that have a process with some random element in it.", # TODO: add more info here, regarding the evolution splitting.
- "repeat": "Factor of how many times a system should be repeated. Consider the evolution splitting binary_c argument for supernovae kick repeating.", # TODO: make sure this is used.
- "evolution_type": "Variable containing the type of evolution used of the grid. Multiprocessing, linear processing or possibly something else (e.g. for Slurm or Condor).",
- "combine_ensemble_with_thread_joining": "Boolean flag on whether to combine everything and return it to the user or if false: write it to data_dir/ensemble_output_{population_id}_{thread_id}.json",
- "log_runtime_systems": "Whether to log the runtime of the systems . Each systems run by the thread is logged to a file and is stored in the tmp_dir. (1 file per thread). Don't use this if you are planning to run a lot of systems. This is mostly for debugging and finding systems that take long to run. Integer, default = 0. if value is 1 then the systems are logged",
- "_total_mass_run": "To count the total mass that thread/process has ran",
- "_total_probability_weighted_mass_run": "To count the total mass * probability for each system that thread/process has ran",
- "_actually_evolve_system": "Whether to actually evolve the systems of just act as if. for testing. used in _process_run_population_grid",
- "max_queue_size": "Maximum size of the queue that is used to feed the processes. Don't make this too big! Default: 1000. Input: int",
- "_set_Moe2017_grid": "Internal flag whether the Moe and di Stefano (2017) grid has been loaded",
- "run_zero_probability_system": "Whether to run the zero probability systems. Default: True. Input: Boolean",
- "_zero_prob_stars_skipped": "Internal counter to track how many systems are skipped because they have 0 probability",
- "ensemble_factor_in_probability_weighted_mass": "Flag to multiply all the ensemble results with 1/probability_weighted_mass",
- "multiplicity_fraction_function": "Which multiplicity fraction function to use. 0: None, 1: Arenou 2010, 2: Rhagavan 2010, 3: Moe and di Stefano (2017) 2017",
- "m&s_options": "Internal variable that holds the Moe and di Stefano (2017) options. Don't write to this your self",
- "_loaded_Moe2017_data": "Internal variable storing whether the Moe and di Stefano (2017) data has been loaded into memory",
- "do_dry_run": "Whether to do a dry run to calculate the total probability for this run",
- "_Moe2017_JSON_data": "Location to store the loaded Moe&diStefano2017 dataset", # Stores the data
-}
-
-###
-#
-
-MIN_MASS_BINARY_C = float(
- return_binary_c_version_info(parsed=True)["macros"]["BINARY_C_MINIMUM_STELLAR_MASS"]
-)
-
-
-# Default options for the Moe & di Stefano grid
-moe_di_stefano_default_options = {
- # place holder for the JSON data to be used if a file
- # isn't specified
- "JSON": None,
- # resolution data
- "resolutions": {
- "M": [
- 20, # M1
- 20, # M2 (i.e. q)
- 0, # M3 currently unused
- 0, # M4 currently unused
- ],
- "logP": [
- 20, # P2 (binary period)
- 0, # P3 (triple period) currently unused
- 0, # P4 (quadruple period) currently unused
- ],
- "ecc": [
- 10, # e (binary eccentricity)
- 0, # e2 (triple eccentricity) currently unused
- 0, # e3 (quadruple eccentricity) currently unused
- ],
- },
- "samplerfuncs": {
- "M": [None, None, None, None],
- "logP": [None, None, None],
- "ecc": [None, None, None],
- },
- "ranges": {
- # stellar masses (Msun)
- "M": [
- MIN_MASS_BINARY_C
- * 1.05, # 0.08 is a tad bit above the minimum mass. Don't sample at 0.07, otherwise the first row of q values will have a phasevol of 0. Anything higher is fine.
- 80.0, # (rather arbitrary) upper mass cutoff
- ],
- "q": [
- None, # artificial qmin : set to None to use default
- None, # artificial qmax : set to None to use default
- ],
- "logP": [0.0, 8.0], # 0 = log10(1 day) # 8 = log10(10^8 days)
- "ecc": [0.0, 0.99],
- },
- # minimum stellar mass
- "Mmin": MIN_MASS_BINARY_C, # We take the value that binary_c has set as the default
- # multiplicity model (as a function of log10M1)
- #
- # You can use 'Poisson' which uses the system multiplicity
- # given by Moe and maps this to single/binary/triple/quad
- # fractions.
- #
- # Alternatively, 'data' takes the fractions directly
- # from the data, but then triples and quadruples are
- # combined (and there are NO quadruples).
- "multiplicity_model": "Poisson",
- # multiplicity modulator:
- # [single, binary, triple, quadruple]
- #
- # e.g. [1,0,0,0] for single stars only
- # [0,1,0,0] for binary stars only
- #
- # defaults to [1,1,0,0] i.e. all types
- #
- "multiplicity_modulator": [
- 1, # single
- 1, # binary
- 0, # triple
- 0, # quadruple
- ],
- # given a mix of multiplicities, you can either (noting that
- # here (S,B,T,Q) = appropriate modulator * model(S,B,T,Q) )
- #
- # 'norm' : normalise so the whole population is 1.0
- # after implementing the appropriate fractions
- # S/(S+B+T+Q), B/(S+B+T+Q), T/(S+B+T+Q), Q/(S+B+T+Q)
- #
- # 'raw' : stick to what is predicted, i.e.
- # S/(S+B+T+Q), B/(S+B+T+Q), T/(S+B+T+Q), Q/(S+B+T+Q)
- # without normalisation
- # (in which case the total probability < 1.0 unless
- # all you use single, binary, triple and quadruple)
- #
- # 'merge' : e.g. if you only have single and binary,
- # add the triples and quadruples to the binaries, so
- # binaries represent all multiple systems
- # ...
- # *** this is canonical binary population synthesis ***
- #
- # Note: if multiplicity_modulator == [1,1,1,1] this
- # option does nothing (equivalent to 'raw').
- #
- #
- # note: if you only set one multiplicity_modulator
- # to 1, and all the others to 0, then normalising
- # will mean that you effectively have the same number
- # of stars as single, binary, triple or quad (whichever
- # is non-zero) i.e. the multiplicity fraction is ignored.
- # This is probably not useful except for
- # testing purposes or comparing to old grids.
- "normalize_multiplicities": "merge",
- # q extrapolation (below 0.15 and above 0.9) method. We can choose from ['flat', 'linear', 'plaw2', 'nolowq']
- "q_low_extrapolation_method": "linear",
- "q_high_extrapolation_method": "linear",
-}
-
-moe_di_stefano_default_options_description = {
- "resolutions": "",
- "ranges": "",
- "Mmin": "Minimum stellar mass",
- "multiplicity_model": """
-multiplicity model (as a function of log10M1)
-
-You can use 'Poisson' which uses the system multiplicity
-given by Moe and maps this to single/binary/triple/quad
-fractions.
-
-Alternatively, 'data' takes the fractions directly
-from the data, but then triples and quadruples are
-combined (and there are NO quadruples).
-""",
- "multiplicity_modulator": """
-[single, binary, triple, quadruple]
-
-e.g. [1,0,0,0] for single stars only
- [0,1,0,0] for binary stars only
-
-defaults to [1,1,0,0] i.e. singles and binaries
-""",
- "normalize_multiplicities": """
-'norm': normalise so the whole population is 1.0
- after implementing the appropriate fractions
- S/(S+B+T+Q), B/(S+B+T+Q), T/(S+B+T+Q), Q/(S+B+T+Q)
- given a mix of multiplicities, you can either (noting that
- here (S,B,T,Q) = appropriate modulator * model(S,B,T,Q) )
- note: if you only set one multiplicity_modulator
- to 1, and all the others to 0, then normalising
- will mean that you effectively have the same number
- of stars as single, binary, triple or quad (whichever
- is non-zero) i.e. the multiplicity fraction is ignored.
- This is probably not useful except for
- testing purposes or comparing to old grids.
-
-'raw' : stick to what is predicted, i.e.
- S/(S+B+T+Q), B/(S+B+T+Q), T/(S+B+T+Q), Q/(S+B+T+Q)
- without normalisation
- (in which case the total probability < 1.0 unless
- all you use single, binary, triple and quadruple)
-
-'merge' : e.g. if you only have single and binary,
- add the triples and quadruples to the binaries, so
- binaries represent all multiple systems
- ...
- *** this is canonical binary population synthesis ***
-
- It only takes the maximum multiplicity into account,
- i.e. it doesn't multiply the resulting array by the multiplicity modulator again.
- This prevents the resulting array to always be 1 if only 1 multiplicity modulator element is nonzero
-
- Note: if multiplicity_modulator == [1,1,1,1]. this option does nothing (equivalent to 'raw').
-""",
- "q_low_extrapolation_method": """
-q extrapolation (below 0.15) method
- none
- flat
- linear2
- plaw2
- nolowq
-""",
- "q_high_extrapolation_method": "Same as q_low_extrapolation_method",
-}
-
-
-#################################
-# Grid options functions
-
-# Utility functions
-def grid_options_help(option: str) -> dict:
- """
- Function that prints out the description of a grid option. Useful function for the user.
-
- Args:
- option: which option you want to have the description of
-
- returns:
- dict containing the option, the description if its there, otherwise empty string. And if the key doesnt exist, the dict is empty
- """
-
- option_keys = grid_options_defaults_dict.keys()
- description_keys = grid_options_descriptions.keys()
-
- if not option in option_keys:
- print(
- "Error: This is an invalid entry. Option does not exist, please choose from the following options:\n\t{}".format(
- ", ".join(option_keys)
- )
- )
- return {}
-
- else:
- if not option in description_keys:
+secs_per_day = 86400 # probably needs to go somewhere more sensible
+
+class grid_options_defaults():
+ def __init__(self, **kwargs):
+
+ return
+
+ def grid_options_defaults_dict(self):
+
+ # Options dict
+ return {
+ ##########################
+ # general (or unordered..)
+ ##########################
+ "num_cores": 1, # total number of cores used to evolve the population
+ "num_cores_available": None, # set automatically, not by the user
+ "parse_function": None, # Function to parse the output with.
+ "multiplicity_fraction_function": 0, # Which multiplicity fraction function to use. 0: None, 1: Arenou 2010, 2: Rhagavan 2010, 3: Moe and di Stefano 2017
+ "tmp_dir": temp_dir(), # Setting the temp dir of the program
+ "cache_dir" : None, # Cache location
+ "status_dir" : None, #
+ "_main_pid": -1, # Placeholder for the main process id of the run.
+ "save_ensemble_chunks": True, # Force the ensemble chunk to be saved even if we are joining a thread (just in case the joining fails)
+ "combine_ensemble_with_thread_joining": True, # Flag on whether to combine everything and return it to the user or if false: write it to data_dir/ensemble_output_{population_id}_{thread_id}.json
+ "_commandline_input": "",
+ "log_runtime_systems": 0, # whether to log the runtime of the systems (1 file per thread. stored in the tmp_dir)
+ "_actually_evolve_system": True, # Whether to actually evolve the systems of just act as if. for testing. used in _process_run_population_grid
+ "max_queue_size": 0, # Maximum size of the system call queue. Set to 0 for this to be calculated automatically
+ "run_zero_probability_system": True, # Whether to run the zero probability systems
+ "_zero_prob_stars_skipped": 0,
+ "ensemble_factor_in_probability_weighted_mass": False, # Whether to multiply the ensemble results by 1/probability_weighted_mass
+ "do_dry_run": True, # Whether to do a dry run to calculate the total probability for this run
+ "custom_generator": None, # Place for the custom system generator
+ "exit_after_dry_run": False, # Exit after dry run?
+
+ #####################
+ # System information
+ #####################
+ "command_line": ' '.join(sys.argv),
+ "original_command_line": os.getenv('BINARY_C_PYTHON_ORIGINAL_CMD_LINE'),
+ "working_diretory": os.getcwd(),
+ "original_working_diretory": os.getenv('BINARY_C_PYTHON_ORIGINAL_WD'),
+ "start_time": datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"),
+ "original_submission_time" : os.getenv('BINARY_C_PYTHON_ORIGINAL_SUBMISSION_TIME'),
+
+ ##########################
+ # Execution log:
+ ##########################
+ "verbosity": 0, # Level of verbosity of the simulation
+ "log_file": os.path.join(
+ temp_dir(), "binary_c_python.log"
+ ), # Set to None to not log to file. The directory will be created
+ "log_dt": 5, # time between vb=1 logging outputs
+ "n_logging_stats": 50, # number of logging stats used to calculate time remaining (etc.) default = 50
+ ##########################
+ # binary_c files
+ ##########################
+ "_binary_c_executable": os.path.join(
+ os.environ["BINARY_C"], "binary_c"
+ ), # TODO: make this more robust
+ "_binary_c_shared_library": os.path.join(
+ os.environ["BINARY_C"], "src", "libbinary_c.so"
+ ), # TODO: make this more robust
+ "_binary_c_config_executable": os.path.join(
+ os.environ["BINARY_C"], "binary_c-config"
+ ), # TODO: make this more robust
+ "_binary_c_dir": os.environ["BINARY_C"],
+ ##########################
+ # Moe and di Stefano (2017) internal settings
+ ##########################
+ "_loaded_Moe2017_data": False, # Holds flag whether the Moe and di Stefano (2017) data is loaded into memory
+ "_set_Moe2017_grid": False, # Whether the Moe and di Stefano (2017) grid has been loaded
+ "Moe2017_options": None, # Holds the Moe and di Stefano (2017) options.
+ "_Moe2017_JSON_data": None, # Stores the data
+ ##########################
+ # Custom logging
+ ##########################
+ "C_auto_logging": None, # Should contain a dictionary where the keys are they headers
+ # and the values are lists of parameters that should be logged.
+ # This will get parsed by autogen_C_logging_code in custom_logging_functions.py
+ "C_logging_code": None, # Should contain a string which holds the logging code.
+ "custom_logging_func_memaddr": -1, # Contains the custom_logging functions memory address
+ "_custom_logging_shared_library_file": None, # file containing the .so file
+ ##########################
+ # Store pre-loading:
+ ##########################
+ "_store_memaddr": -1, # Contains the store object memory address, useful for pre loading.
+ # defaults to -1 and isn't used if that's the default then.
+ ##########################
+ # Log args: logging of arguments
+ ##########################
+ "log_args": 0, # unused
+ "log_args_dir": "/tmp/", # unused
+ ##########################
+ # Population evolution
+ ##########################
+ ## General
+ "evolution_type": "grid", # Flag for type of population evolution
+ "_evolution_type_options": [
+ "grid",
+ "custom_generator",
+ ], # available choices for type of population evolution. # TODO: fill later with Monte Carlo, source file
+ "_system_generator": None, # value that holds the function that generates the system
+ # (result of building the grid script)
+ "source_file_filename": None, # filename for the source
+ "_count": 0, # count of systems
+ "_total_starcount": 0, # Total count of systems in this generator
+ "_probtot": 0, # total probability
+ "weight": 1.0, # weighting for the probability
+ "repeat": 1, # number of times to repeat each system (probability is adjusted to be 1/repeat)
+ "_start_time_evolution": 0, # Start time of the grid
+ "_end_time_evolution": 0, # end time of the grid
+ "_errors_found": False, # Flag whether there are any errors from binary_c
+ "_errors_exceeded": False, # Flag whether the number of errors have exceeded the limit
+ "_failed_count": 0, # number of failed systems
+ "_failed_prob": 0, # Summed probability of failed systems
+ "failed_systems_threshold": 20, # Maximum failed systems per process allowed to fail before the process stops logging the failing systems.
+ "_failed_systems_error_codes": [], # List to store the unique error codes
+ "_population_id": 0, # Random id of this grid/population run, Unique code for the population. Should be set only once by the controller process.
+ "_total_mass_run": 0, # To count the total mass that thread/process has ran
+ "_total_probability_weighted_mass_run": 0, # To count the total mass * probability for each system that thread/process has ran
+ "modulo": 1, # run modulo n of the grid.
+ "start_at": 0, # start at the first model
+ ## Grid type evolution
+ "_grid_variables": {}, # grid variables
+ "gridcode_filename": None, # filename of gridcode
+ "symlink_latest_gridcode": True, # symlink to latest gridcode
+ "save_population_object" : None, # filename to which we should save a pickled grid object as the final thing we do
+ 'joinlist' : None,
+ 'do_analytics' : True, # if True, calculate analytics prior to return
+ 'save_snapshots' : True, # if True, save snapshots on SIGINT
+ 'restore_from_snapshot_file' : None, # file to restore from
+ 'restore_from_snapshot_dir' : None, # dir to restore from
+ 'exit_code' : 0, # return code
+ 'stop_queue' : False,
+ '_killed' : False,
+ '_queue_done' : False,
+ ## Monte carlo type evolution
+ # TODO: make MC options
+ ## Evolution from source file
+ # TODO: make run from sourcefile options.
+ ## Other no yet implemented parts for the population evolution part
+ # # start at this model number: handy during debugging
+ # # to skip large parts of the grid
+ # start_at => 0
+ # global_error_string => undef,
+ # monitor_files => [],
+ # nextlogtime => 0,
+ # nthreads => 1, # number of threads
+ # # start at model offset (0-based, so first model is zero)
+ # offset => 0,
+ # resolution=>{
+ # shift =>0,
+ # previous=>0,
+ # n =>{} # per-variable resolution
+ # },
+ # thread_q => undef,
+ # threads => undef, # array of threads objects
+ # tstart => [gettimeofday], # flexigrid start time
+ # __nvar => 0, # number of grid variables
+ # _varstub => undef,
+ # _lock => undef,
+ # _evcode_pids => [],
+ # };
+ ########################################
+ # Slurm stuff
+ ########################################
+ "slurm": 0, # dont use the slurm by default, 0 = no slurm, 1 = launch slurm jobs, 2 = run slurm jobs
+ "slurm_ntasks": 1, # CPUs required per array job: usually only need this to be 1
+ "slurm_dir": "", # working directory containing scripts output logs etc.
+ "slurm_njobs": 0, # number of scripts; set to 0 as default
+ "slurm_jobid": "", # slurm job id (%A)
+ "slurm_memory": '512MB', # memory required for the job
+ "slurm_warn_max_memory": '1024MB', # warn if we set it to more than this (usually by accident)
+ "slurm_postpone_join": 0, # if 1 do not join on slurm, join elsewhere. want to do it off the slurm grid (e.g. with more RAM)
+ "slurm_jobarrayindex": None, # slurm job array index (%a)
+ "slurm_jobname": "binary_grid", # default
+ "slurm_partition": None,
+ "slurm_time": 0, # total time. 0 = infinite time
+ "slurm_postpone_sbatch": 0, # if 1: don't submit, just make the script
+ "slurm_array": None, # override for --array, useful for rerunning jobs
+ "slurm_array_max_jobs" : None, # override for the max number of concurrent array jobs
+ "slurm_extra_settings": {}, # Dictionary of extra settings for Slurm to put in its launch script.
+ "slurm_sbatch": "sbatch", # sbatch command
+ "slurm_restart_dir" : None, # restart Slurm jobs from this directory
+ ########################################
+ # Condor stuff
+ ########################################
+ "condor": 0, # 1 to use condor, 0 otherwise
+ "condor_command": "", # condor command e.g. "evolve", "join"
+ "condor_dir": "", # working directory containing e.g. scripts, output, logs (e.g. should be NFS available to all)
+ "condor_njobs": "", # number of scripts/jobs that CONDOR will run in total
+ "condor_jobid": "", # condor job id
+ "condor_postpone_join": 0, # if 1, data is not joined, e.g. if you want to do it off the condor grid (e.g. with more RAM)
+ # "condor_join_machine": None, # if defined then this is the machine on which the join command should be launched (must be sshable and not postponed)
+ "condor_join_pwd": "", # directory the join should be in (defaults to $ENV{PWD} if undef)
+ "condor_memory": 1024, # in MB, the memory use (ImageSize) of the job
+ "condor_universe": "vanilla", # usually vanilla universe
+ "condor_extra_settings": {}, # Place to put extra configuration for the CONDOR submit file. The key and value of the dict will become the key and value of the line in te slurm batch file. Will be put in after all the other settings (and before the command). Take care not to overwrite something without really meaning to do so.
+ # snapshots and checkpoints
+ 'condor_snapshot_on_kill':0, # if 1 snapshot on SIGKILL before exit
+ 'condor_load_from_snapshot':0, # if 1 check for snapshot .sv file and load it if found
+ 'condor_checkpoint_interval':0, # checkpoint interval (seconds)
+ 'condor_checkpoint_stamp_times':0, # if 1 then files are given timestamped names
+ # (warning: lots of files!), otherwise just store the lates
+ 'condor_streams':0, # stream stderr/stdout by default (warning: might cause heavy network load)
+ 'condor_save_joined_file':0, # if 1 then results/joined contains the results
+ # (useful for debugging, otherwise a lot of work)
+ 'condor_requirements':'', # used?
+ # # resubmit options : if the status of a condor script is
+ # # either 'finished','submitted','running' or 'crashed',
+ # # decide whether to resubmit it.
+ # # NB Normally the status is empty, e.g. on the first run.
+ # # These are for restarting runs.
+ # condor_resubmit_finished:0,
+ 'condor_resubmit_submitted':0,
+ 'condor_resubmit_running':0,
+ 'condor_resubmit_crashed':0,
+ ##########################
+ # Unordered. Need to go through this. Copied from the perl implementation.
+ ##########################
+ ##
+ # return_array_refs:1, # quicker data parsing mode
+ # sort_args:1,
+ # save_args:1,
+ # nice:'nice -n +20', # nice command e.g. 'nice -n +10' or ''
+ # timeout:15, # seconds until timeout
+ # log_filename:"/scratch/davidh/results_simulations/tmp/log.txt",
+ # # current_log_filename:"/scratch/davidh/results_simulations/tmp/grid_errors.log",
+ ############################################################
+ # Set default grid properties (in %self->{_grid_options}}
+ # and %{$self->{_bse_options}})
+ # This is the first thing that should be called by the user!
+ ############################################################
+ # # set signal handlers for timeout
+ # $self->set_class_signal_handlers();
+ # # set operating system
+ # my $os = rob_misc::operating_system();
+ # %{$self->{_grid_options}}=(
+ # # save operating system
+ # operating_system:$os,
+ # # process name
+ # process_name : 'binary_grid'.$VERSION,
+ # grid_defaults_set:1, # so we know the grid_defaults function has been called
+ # # grid suspend files: assume binary_c by default
+ # suspend_files:[$tmp.'/force_binary_c_suspend',
+ # './force_binary_c_suspend'],
+ # snapshot_file:$tmp.'/binary_c-snapshot',
+ # ########################################
+ # # infomration about the running grid script
+ # ########################################
+ # working_directory:cwd(), # the starting directory
+ # perlscript:$0, # the name of the perlscript
+ # perlscript_arguments:join(' ',@ARGV), # arguments as a string
+ # perl_executable:$^X, # the perl executable
+ # command_line:join(' ',$0,@ARGV), # full command line
+ # process_ID:$$, # process ID of the main perl script
+ # ########################################
+ # # GRID
+ # ########################################
+ # # if undef, generate gridcode, otherwise load the gridcode
+ # # from this file. useful for debugging
+ # gridcode_from_file : undef,
+ # # assume binary_grid perl backend by default
+ # backend :
+ # $self->{_grid_options}->{backend} //
+ # $binary_grid2::backend //
+ # 'binary_grid::Perl',
+ # # custom C function for output : this automatically
+ # # binds if a function is available.
+ # C_logging_code : undef,
+ # C_auto_logging : undef,
+ # custom_output_C_function_pointer : binary_c_function_bind(),
+ # # control flow
+ 'rungrid' : 1, # usually run the grid, but can be 0
+ # # to skip it (e.g. for condor/slurm runs)
+ # merge_datafiles:'',
+ # merge_datafiles_filelist:'',
+ # # parameter space options
+ # binary:0, # set to 0 for single stars, 1 for binaries
+ # # if use_full_resolution is 1, then run a dummy grid to
+ # # calculate the resolution. this could be slow...
+ # use_full_resolution : 1,
+ # # the probability in any distribution must be within
+ # # this tolerance of 1.0, ignored if undef (if you want
+ # # to run *part* of the parameter space then this *must* be undef)
+ # probability_tolerance:undef,
+ # # how to deal with a failure of the probability tolerance:
+ # # 0 = nothing
+ # # 1 = warning
+ # # 2 = stop
+ # probability_tolerance_failmode:1,
+ # # add up and log system error count and probability
+ # add_up_system_errors:1,
+ # log_system_errors:1,
+ # # codes, paths, executables etc.
+ # # assume binary_c by default, and set its defaults
+ # code:'binary_c',
+ # arg_prefix:'--',
+ # prog:'binary_c', # executable
+ # nice:'nice -n +0', # nice command
+ # ionice:'',
+ # # compress output?
+ # binary_c_compression:0,
+ # # get output as array of pre-split array refs
+ # return_array_refs:1,
+ # # environment
+ # shell_environment:undef,
+ # libpath:undef, # for backwards compatibility
+ # # where is binary_c? need this to get the values of some counters
+ # rootpath:$self->okdir($ENV{BINARY_C_ROOTPATH}) //
+ # $self->okdir($ENV{HOME}.'/progs/stars/binary_c') //
+ # '.' , # last option is a fallback ... will fail if it doesn't exist
+ # srcpath:$self->okdir($ENV{BINARY_C_SRCPATH}) //
+ # $self->okdir($ENV{BINARY_C_ROOTPATH}.'/src') //
+ # $self->okdir($ENV{HOME}.'/progs/stars/binary_c/src') //
+ # './src' , # last option is fallback... will fail if it doesn't exist
+ # # stack size per thread in megabytes
+ # threads_stack_size:50,
+ # # thread sleep time between starting the evolution code and starting
+ # # the grid
+ # thread_presleep:0,
+ # # threads
+ # # Max time a thread can sit looping (with calls to tbse_line)
+ # # before a warning is issued : NB this does not catch real freezes,
+ # # just infinite loops (which still output)
+ # thread_max_freeze_time_before_warning:10,
+ # # run all models by default: modulo=1, offset=0
+ # modulo:1,
+ # offset:0,
+ # # max number of stars on the queue
+ # maxq_per_thread : 100,
+ # # data dump file : undef by default (do nothing)
+ # results_hash_dumpfile : '',
+ # # compress files with bzip2 by default
+ # compress_results_hash : 1,
+ # ########################################
+ # # CPU
+ # ########################################
+ # cpu_cap:0, # if 1, limits to one CPU
+ # cpu_affinity : 0, # do not bind to a CPU by default
+ # ########################################
+ # # Code, Timeouts, Signals
+ # ########################################
+ # binary_grid_code_filtering:1, # you want this, it's (MUCH!) faster
+ # pre_filter_file:undef, # dump pre filtered code to this file
+ # post_filter_file:undef, # dump post filtered code to this file
+ # timeout:30, # timeout in seconds
+ # timeout_vb:0, # no timeout logging
+ # tvb:0, # no thread logging
+ # nfs_sleep:1, # time to wait for NFS to catch up with file accesses
+ # # flexigrid checks the timeouts every
+ # # flexigrid_timeout_check_interval seconds
+ # flexigrid_timeout_check_interval:0.01,
+ # # this is set to 1 when the grid is finished
+ # flexigrid_finished:0,
+ # # allow signals by default
+ # 'no signals':0,
+ # # but perhaps disable specific signals?
+ # 'disable signal':{INT:0,ALRM:0,CONT:0,USR1:0,STOP:0},
+ # # dummy variables
+ # single_star_period:1e50, # orbital period of a single star
+ # #### timers : set timers to 0 (or empty list) to ignore,
+ # #### NB these must be given context (e.g. main::xyz)
+ # #### for functions not in binary_grid
+ # timers:0,
+ # timer_subroutines:[
+ # # this is a suggested default list
+ # 'flexigrid',
+ # 'set_next_alarm',
+ # 'vbout',
+ # 'vbout_fast',
+ # 'run_flexigrid_thread',
+ # 'thread_vb'
+ # ],
+ # ########################################
+ # # INPUT/OUTPUT
+ # ########################################
+ # blocking:undef, # not yet set
+ # # prepend command with stdbuf to stop buffering (if available)
+ # stdbuf_command:`stdbuf --version`=~/stdbuf \(GNU/ ? ' stdbuf -i0 -o0 -e0 ' : undef,
+ # vb:("@ARGV"=~/\Wvb=(\d+)\W/)[0] // 0, # set to 1 (or more) for verbose output to the screen
+ # log_dt_secs:1, # log output to stdout~every log_dt_secs seconds
+ # nmod:10, # every nmod models there is output to the screen,
+ # # if log_dt_secs has been exceeded also (ignored if 0)
+ # colour:1, # set to 1 to use the ANSIColor module for colour output
+ # log_args:0, # do not log args in files
+ # log_fins:0, # log end of runs too
+ # sort_args:0, # do not sort args
+ # save_args:0, # do not save args in a string
+ # log_args_dir:$tmp, # where to output the args files
+ # always_reopen_arg_files:0, # if 1 then arg files are always closed and reopened
+ # (may cause a lot of disk I/O)
+ # lazy_arg_sending:1, # if 1, the previous args are remembered and
+ # # only args that changed are sent (except M1, M2 etc. which always
+ # # need sending)
+ # # force output files to open on a local disk (not an NFS partion)
+ # # not sure how to do this on another OS
+ # force_local_hdd_use:($os eq 'unix'),
+ # # for verbose output, define the newline
+ # # For terminals use "\x0d", for files use "\n", in the
+ # # case of multiple threads this will be set to \n
+ # newline: "\x0d",
+ # # use reset_stars_defaults
+ # reset_stars_defaults:1,
+ # # set signal captures: argument determines behaviour when the code locks up
+ # # 0: exit
+ # # 1: reset and try the next star (does this work?!)
+ # alarm_procedure:1,
+ # # exit on eval failure?
+ # exit_on_eval_failure:1,
+ # ## functions: these should be set by perl lexical name
+ # ## (they are automatically converted to function pointers
+ # ## at runtime)
+ # # function to be called just before a thread is created
+ # thread_precreate_function:undef,
+ # thread_precreate_function_pointer:undef,
+ # # function to be called just after a thread is created
+ # # (from inside the thread just before *grid () call)
+ # threads_entry_function:undef,
+ # threads_entry_function_pointer:undef,
+ # # function to be called just after a thread is finished
+ # # (from inside the thread just after *grid () call)
+ # threads_flush_function:undef,
+ # threads_flush_function_pointer:undef,
+ # # function to be called just after a thread is created
+ # # (but external to the thread)
+ # thread_postrun_function:undef,
+ # thread_postrun_function_pointer:undef,
+ # # function to be called just before a thread join
+ # # (external to the thread)
+ # thread_prejoin_function:undef,
+ # thread_prejoin_function_pointer:undef,
+ # # default to using the internal join_flexigrid_thread function
+ # threads_join_function:'binary_grid2::join_flexigrid_thread',
+ # threads_join_function_pointer:sub{return $self->join_flexigrid_thread(@_)},
+ # # function to be called just after a thread join
+ # # (external to the thread)
+ # thread_postjoin_function:undef,
+ # thread_postjoin_function_pointer:undef,
+ # # usually, parse_bse in the main script is called
+ # parse_bse_function:'main::parse_bse',
+ # parse_bse_function_pointer:undef,
+ # # if starting_snapshot_file is defined, load initial
+ # # values for the grid from the snapshot file rather
+ # # than a normal initiation: this enables you to
+ # # stop and start a grid
+ # starting_sn apshot_file:undef,
+ }
+
+
+ def grid_options_descriptions(self):
+ # Grid containing the descriptions of the options # TODO: add input types for all of them
+ return {
+ "tmp_dir": "Directory where certain types of output are stored. The grid code is stored in that directory, as well as the custom logging libraries. Log files and other diagnostics will usually be written to this location, unless specified otherwise", # TODO: improve this
+ "status_dir" : "Directory where grid status is stored",
+ "_binary_c_dir": "Director where binary_c is stored. This options are not really used",
+ "_binary_c_config_executable": "Full path of the binary_c-config executable. This options is not used in the population object.",
+ "_binary_c_executable": "Full path to the binary_c executable. This options is not used in the population object.",
+ "_binary_c_shared_library": "Full path to the libbinary_c file. This options is not used in the population object",
+ "verbosity": "Verbosity of the population code. Default is 0, by which only errors will be printed. Higher values will show more output, which is good for debugging.",
+ # deprecated: "binary": "Set this to 1 if the population contains binaries. Input: int", # TODO: write what effect this has.
+ "num_cores": "The number of cores that the population grid will use. You can set this manually by entering an integer great than 0. When 0 uses all logical cores. When -1 uses all physical cores. Input: int",
+ "num_processes" : "Number of processes launched by multiprocessing. This should be set automatically by binary_c-python, not by the user.",
+ "_start_time_evolution": "Variable storing the start timestamp of the population evolution. Set by the object itself.", # TODO: make sure this is logged to a file
+ "_end_time_evolution": "Variable storing the end timestamp of the population evolution. Set by the object itself", # TODO: make sure this is logged to a file
+ "_total_starcount": "Variable storing the total number of systems in the generator. Used and set by the population object.",
+ "_custom_logging_shared_library_file": "filename for the custom_logging shared library. Used and set by the population object",
+ "_errors_found": "Variable storing a Boolean flag whether errors by binary_c are encountered.",
+ "_errors_exceeded": "Variable storing a Boolean flag whether the number of errors was higher than the set threshold (failed_systems_threshold). If True, then the command line arguments of the failing systems will not be stored in the failed_system_log files.",
+ "source_file_filename": "Variable containing the source file containing lines of binary_c command line calls. These all have to start with binary_c.", # TODO: Expand
+ "C_auto_logging": "Dictionary containing parameters to be logged by binary_c. The structure of this dictionary is as follows: the key is used as the headline which the user can then catch. The value at that key is a list of binary_c system parameters (like star[0].mass)",
+ "C_logging_code": "Variable to store the exact code that is used for the custom_logging. In this way the user can do more complex logging, as well as putting these logging strings in files.",
+ "_failed_count": "Variable storing the number of failed systems.",
+ "_evolution_type_options": "List containing the evolution type options.",
+ "_failed_prob": "Variable storing the total probability of all the failed systems",
+ "_failed_systems_error_codes": "List storing the unique error codes raised by binary_c of the failed systems",
+ "_grid_variables": "Dictionary storing the grid_variables. These contain properties which are accessed by the _generate_grid_code function",
+ "_population_id": "Variable storing a unique 32-char hex string.",
+ "_commandline_input": "String containing the arguments passed to the population object via the command line. Set and used by the population object.",
+ "_system_generator": "Function object that contains the system generator function. This can be from a grid, or a source file, or a Monte Carlo grid.",
+ "gridcode_filename": "Filename for the grid code. Set and used by the population object. TODO: allow the user to provide their own function, rather than only a generated function.",
+ "log_args": "Boolean to log the arguments. Unused ", # TODO: fix the functionality for this and describe it properly
+ "log_args_dir": "Directory to log the arguments to. Unused", # TODO: fix the functionality for this and describe it properly
+ "log_file": "Log file for the population object. Unused", # TODO: fix the functionality for this and describe it properly
+ "custom_logging_func_memaddr": "Memory address where the custom_logging_function is stored. Input: int",
+ "_count": "Counter tracking which system the generator is on.",
+ "_probtot": "Total probability of the population.", # TODO: check whether this is used properly throughout
+ "_main_pid": "Main process ID of the master process. Used and set by the population object.",
+ "_store_memaddr": "Memory address of the store object for binary_c.",
+ "failed_systems_threshold": "Variable storing the maximum number of systems that are allowed to fail before logging their command line arguments to failed_systems log files",
+ "parse_function": "Function that the user can provide to handle the output the binary_c. This function has to take the arguments (self, output). Its best not to return anything in this function, and just store stuff in the self.grid_results dictionary, or just output results to a file",
+ "condor": "Int flag whether to use a condor type population evolution. Not implemented yet.", # TODO: describe this in more detail
+ "slurm": "Int flag whether to use a Slurm type population evolution.", # TODO: describe this in more detail
+ "weight": "Weight factor for each system. The calculated probability is multiplied by this. If the user wants each system to be repeated several times, then this variable should not be changed, rather change the _repeat variable instead, as that handles the reduction in probability per system. This is useful for systems that have a process with some random element in it.", # TODO: add more info here, regarding the evolution splitting.
+ "repeat": "Factor of how many times a system should be repeated. Consider the evolution splitting binary_c argument for supernovae kick repeating.", # TODO: make sure this is used.
+ "evolution_type": "Variable containing the type of evolution used of the grid. Multiprocessing, linear processing or possibly something else (e.g. for Slurm or Condor).",
+ "combine_ensemble_with_thread_joining": "Boolean flag on whether to combine everything and return it to the user or if false: write it to data_dir/ensemble_output_{population_id}_{thread_id}.json",
+ "log_runtime_systems": "Whether to log the runtime of the systems . Each systems run by the thread is logged to a file and is stored in the tmp_dir. (1 file per thread). Don't use this if you are planning to run a lot of systems. This is mostly for debugging and finding systems that take long to run. Integer, default = 0. if value is 1 then the systems are logged",
+ "_total_mass_run": "To count the total mass that thread/process has ran",
+ "_total_probability_weighted_mass_run": "To count the total mass * probability for each system that thread/process has ran",
+ "_actually_evolve_system": "Whether to actually evolve the systems of just act as if. for testing. used in _process_run_population_grid",
+ "max_queue_size": "Maximum size of the queue that is used to feed the processes. Don't make this too big! Default: 1000. Input: int",
+ "_set_Moe2017_grid": "Internal flag whether the Moe and di Stefano (2017) grid has been loaded",
+ "run_zero_probability_system": "Whether to run the zero probability systems. Default: True. Input: Boolean",
+ "_zero_prob_stars_skipped": "Internal counter to track how many systems are skipped because they have 0 probability",
+ "ensemble_factor_in_probability_weighted_mass": "Flag to multiply all the ensemble results with 1/probability_weighted_mass",
+ "multiplicity_fraction_function": "Which multiplicity fraction function to use. 0: None, 1: Arenou 2010, 2: Rhagavan 2010, 3: Moe and di Stefano (2017) 2017",
+ "m&s_options": "Internal variable that holds the Moe and di Stefano (2017) options. Don't write to this your self",
+ "_loaded_Moe2017_data": "Internal variable storing whether the Moe and di Stefano (2017) data has been loaded into memory",
+ "do_dry_run": "Whether to do a dry run to calculate the total probability for this run",
+ "_Moe2017_JSON_data": "Location to store the loaded Moe&diStefano2017 dataset", # Stores the data
+ }
+
+
+ #################################
+ # Grid options functions
+
+ # Utility functions
+ def grid_options_help(option: str) -> dict:
+ """
+ Function that prints out the description of a grid option. Useful function for the user.
+
+ Args:
+ option: which option you want to have the description of
+
+ returns:
+ dict containing the option, the description if its there, otherwise empty string. And if the key doesnt exist, the dict is empty
+ """
+
+ option_keys = grid_options_defaults_dict.keys()
+ description_keys = grid_options_descriptions.keys()
+
+ if not option in option_keys:
print(
- "This option has not been described properly yet. Please contact on of the authors"
+ "Error: This is an invalid entry. Option does not exist, please choose from the following options:\n\t{}".format(
+ ", ".join(option_keys)
+ )
)
- return {option: ""}
+ return {}
+
else:
- print(grid_options_descriptions[option])
- return {option: grid_options_descriptions[option]}
+ if not option in description_keys:
+ print(
+ "This option has not been described properly yet. Please contact on of the authors"
+ )
+ return {option: ""}
+ else:
+ print(grid_options_descriptions[option])
+ return {option: grid_options_descriptions[option]}
-def grid_options_description_checker(print_info: bool = True) -> int:
- """
- Function that checks which descriptions are missing
+ def grid_options_description_checker(print_info: bool = True) -> int:
+ """
+ Function that checks which descriptions are missing
- Args:
- print_info: whether to print out information about which options contain proper descriptions and which do not
+ Args:
+ print_info: whether to print out information about which options contain proper descriptions and which do not
- Returns:
- the number of undescribed keys
- """
+ Returns:
+ the number of undescribed keys
+ """
- # Get the keys
- option_keys = grid_options_defaults_dict.keys()
- description_keys = grid_options_descriptions.keys()
+ # Get the keys
+ option_keys = grid_options_defaults_dict.keys()
+ description_keys = grid_options_descriptions.keys()
- #
- undescribed_keys = list(set(option_keys) - set(description_keys))
+ #
+ undescribed_keys = list(set(option_keys) - set(description_keys))
- if undescribed_keys:
- if print_info:
- print(
- "Warning: the following keys have no description yet:\n\t{}".format(
- ", ".join(sorted(undescribed_keys))
+ if undescribed_keys:
+ if print_info:
+ print(
+ "Warning: the following keys have no description yet:\n\t{}".format(
+ ", ".join(sorted(undescribed_keys))
+ )
)
- )
- print(
- "Total description progress: {:.2f}%%".format(
- 100 * len(description_keys) / len(option_keys)
+ print(
+ "Total description progress: {:.2f}%%".format(
+ 100 * len(description_keys) / len(option_keys)
+ )
)
- )
- return len(undescribed_keys)
-
-
-def write_grid_options_to_rst_file(output_file: str) -> None:
- """
- Function that writes the descriptions of the grid options to a rst file
-
- Tasks:
- TODO: separate things into private and public options
-
- Args:
- output_file: target file where the grid options descriptions are written to
- """
-
- # Get the options and the description
- options = grid_options_defaults_dict
- descriptions = grid_options_descriptions
-
- # Get those that do not have a description
- not_described_yet = list(set(options) - set(descriptions))
-
- # separate public and private options
- public_options = [key for key in options if not key.startswith("_")]
- private_options = [key for key in options if key.startswith("_")]
-
- # Check input
- if not output_file.endswith(".rst"):
- print("Filename doesn't end with .rst, please provide a proper filename")
- return None
-
- with open(output_file, "w") as f:
- print("Population grid code options", file=f)
- print("{}".format("=" * len("Population grid code options")), file=f)
- print(
- "The following chapter contains all grid code options, along with their descriptions",
- file=f,
- )
- print(
- "There are {} options that are not described yet.".format(
- len(not_described_yet)
- ),
- file=f,
- )
- print("\n", file=f)
+ return len(undescribed_keys)
- # Start public options part
- print_option_descriptions(
- f,
- public_options,
- descriptions,
- "Public options",
- "The following options are meant to be changed by the user.",
- )
-
- # Moe & di Stefano options:
- print_option_descriptions(
- f,
- moe_di_stefano_default_options,
- moe_di_stefano_default_options_description,
- "Moe & di Stefano sampler options",
- "The following options are meant to be changed by the user.",
- )
-
- # Start private options part
- print_option_descriptions(
- f,
- private_options,
- descriptions,
- "Private options",
- "The following options are not meant to be changed by the user, as these options are used and set internally by the object itself. The description still is provided, but just for documentation purposes.",
- )
-
-
-def print_option_descriptions(filehandle, options, descriptions, title, extra_text):
- # Start public options part
- print("{}".format(title), file=filehandle)
- print("{}".format("-" * len("{}".format(title))), file=filehandle)
- print("{}".format(extra_text), file=filehandle)
- print("\n", file=filehandle)
-
- for option in sorted(options):
- if option in descriptions:
+
+ def write_grid_options_to_rst_file(output_file: str) -> None:
+ """
+ Function that writes the descriptions of the grid options to a rst file
+
+ Tasks:
+ TODO: separate things into private and public options
+
+ Args:
+ output_file: target file where the grid options descriptions are written to
+ """
+
+ # Get the options and the description
+ options = grid_options_defaults_dict
+ descriptions = grid_options_descriptions
+
+ # Get those that do not have a description
+ not_described_yet = list(set(options) - set(descriptions))
+
+ # separate public and private options
+ public_options = [key for key in options if not key.startswith("_")]
+ private_options = [key for key in options if key.startswith("_")]
+
+ # Check input
+ if not output_file.endswith(".rst"):
+ print("Filename doesn't end with .rst, please provide a proper filename")
+ return None
+
+ with open(output_file, "w") as f:
+ print("Population grid code options", file=f)
+ print("{}".format("=" * len("Population grid code options")), file=f)
print(
- "| **{}**: {}".format(
- option, descriptions[option].replace("\n", "\n\t")
- ),
- file=filehandle,
+ "The following chapter contains all grid code options, along with their descriptions",
+ file=f,
)
- else:
print(
- "| **{}**: No description available yet".format(option),
- file=filehandle,
+ "There are {} options that are not described yet.".format(
+ len(not_described_yet)
+ ),
+ file=f,
+ )
+ print("\n", file=f)
+
+ # Start public options part
+ print_option_descriptions(
+ f,
+ public_options,
+ descriptions,
+ "Public options",
+ "The following options are meant to be changed by the user.",
+ )
+
+ # Moe & di Stefano options:
+ print_option_descriptions(
+ f,
+ moe_di_stefano_default_options,
+ moe_di_stefano_default_options_description,
+ "Moe & di Stefano sampler options",
+ "The following options are meant to be changed by the user.",
+ )
+
+ # Start private options part
+ print_option_descriptions(
+ f,
+ private_options,
+ descriptions,
+ "Private options",
+ "The following options are not meant to be changed by the user, as these options are used and set internally by the object itself. The description still is provided, but just for documentation purposes.",
)
- print("", file=filehandle)
+
+
+ def print_option_descriptions(filehandle, options, descriptions, title, extra_text):
+ # Start public options part
+ print("{}".format(title), file=filehandle)
+ print("{}".format("-" * len("{}".format(title))), file=filehandle)
+ print("{}".format(extra_text), file=filehandle)
+ print("\n", file=filehandle)
+
+ for option in sorted(options):
+ if option in descriptions:
+ print(
+ "| **{}**: {}".format(
+ option, descriptions[option].replace("\n", "\n\t")
+ ),
+ file=filehandle,
+ )
+ else:
+ print(
+ "| **{}**: No description available yet".format(option),
+ file=filehandle,
+ )
+ print("", file=filehandle)
diff --git a/binarycpython/utils/gridcode.py b/binarycpython/utils/gridcode.py
new file mode 100644
index 000000000..6ffacb040
--- /dev/null
+++ b/binarycpython/utils/gridcode.py
@@ -0,0 +1,997 @@
+"""
+ Binary_c-python's grid code generation functions.
+"""
+import datetime
+import importlib
+import json
+import os
+from typing import Union, Any
+
+
+_count = 0 # used for file symlinking (for testing only)
+
+class gridcode():
+
+ def __init__(self, **kwargs):
+ # don't do anything: we just inherit from this class
+ return
+
+ ###################################################
+ # Grid code functions
+ #
+ # Function below are used to run populations with
+ # a variable grid
+ ###################################################
+ def _gridcode_filename(self):
+ """
+ Returns a filename for the gridcode.
+ """
+ if self.grid_options['slurm'] > 0:
+ filename = os.path.join(
+ self.grid_options["tmp_dir"],
+ "binary_c_grid_{population_id}.{jobid}.{jobarrayindex}.py".format(
+ population_id=self.grid_options["_population_id"],
+ jobid=self.grid_options['slurm_jobid'],
+ jobarrayindex=self.grid_options['slurm_jobarrayindex'],
+ )
+ )
+ else:
+ filename = os.path.join(
+ self.grid_options["tmp_dir"],
+ "binary_c_grid_{population_id}.py".format(
+ population_id=self.grid_options["_population_id"]
+ ),
+ )
+ return filename
+
+ def _add_code(self, *args, indent=0):
+ """
+ Function to add code to the grid code string
+
+ add code to the code_string
+
+ indent (=0) is added once at the beginning
+ mindent (=0) is added for every line
+
+ don't use both!
+ """
+
+ indent_block = self._indent_block(indent)
+ for thing in args:
+ self.code_string += indent_block + thing
+
+ def _indent_block(self, n=0):
+ """
+ return an indent block, with n extra blocks in it
+ """
+ return (self.indent_depth + n) * self.indent_string
+
+ def _increment_indent_depth(self, delta):
+ """
+ increment the indent indent_depth by delta
+ """
+ self.indent_depth += delta
+
+ def _generate_grid_code(self, dry_run=False):
+ """
+ Function that generates the code from which the population will be made.
+
+ dry_run: when True, it will return the starcount at the end so that we know
+ what the total number of systems is.
+
+ The phasevol values are handled by generating a second array
+
+ # TODO: Add correct logging everywhere
+ # TODO: add part to handle separation if orbital_period is added. Idea. use default values
+ # for orbital parameters and possibly overwrite those or something.
+ # TODO: add sensible description to this function.
+ # TODO: Check whether all the probability and phasevol values are correct.
+ # TODO: import only the necessary packages/functions
+ # TODO: Put all the masses, eccentricities and periods in there already
+ # TODO: Put the certain blocks that are repeated in some sub functions
+ # TODO: make sure running systems with multiplicity 3+ is also possible.
+
+ Results in a generated file that contains a system_generator function.
+ """
+ self.verbose_print("Generating grid code", self.grid_options["verbosity"], 1)
+
+ total_grid_variables = len(self.grid_options["_grid_variables"])
+
+ self._add_code(
+ # Import packages
+ "import math\n",
+ "import numpy as np\n",
+ "from collections import OrderedDict\n",
+ "from binarycpython.utils.distribution_functions import *\n",
+ "from binarycpython.utils.spacing_functions import *\n",
+ "from binarycpython.utils.useful_funcs import *\n",
+ "\n\n",
+ # Make the function
+ "def grid_code(self, print_results=True):\n",
+ )
+
+ # Increase indent_depth
+ self._increment_indent_depth(+1)
+
+ self._add_code(
+ # Write some info in the function
+ "# Grid code generated on {}\n".format(datetime.datetime.now().isoformat()),
+ "# This function generates the systems that will be evolved with binary_c\n\n"
+ # Set some values in the generated code:
+ "# Setting initial values\n",
+ "_total_starcount = 0\n",
+ "starcounts = [0 for i in range({})]\n".format(total_grid_variables + 1),
+ "probabilities = {}\n",
+ "probabilities_list = [0 for i in range({})]\n".format(
+ total_grid_variables + 1
+ ),
+ "probabilities_sum = [0 for i in range({})]\n".format(
+ total_grid_variables + 1
+ ),
+ "parameter_dict = {}\n",
+ "phasevol = 1\n",
+ )
+
+ # Set up the system parameters
+ self._add_code(
+ "M_1 = None\n",
+ "M_2 = None\n",
+ "M_3 = None\n",
+ "M_4 = None\n",
+ "orbital_period = None\n",
+ "orbital_period_triple = None\n",
+ "orbital_period_quadruple = None\n",
+ "eccentricity = None\n",
+ "eccentricity2 = None\n",
+ "eccentricity3 = None\n",
+ "\n",
+ # Prepare the probability
+ "# setting probability lists\n",
+ )
+
+ for grid_variable_el in sorted(
+ self.grid_options["_grid_variables"].items(),
+ key=lambda x: x[1]["grid_variable_number"],
+ ):
+ # Make probabilities dict
+ grid_variable = grid_variable_el[1]
+ self._add_code('probabilities["{}"] = 0\n'.format(grid_variable["name"]))
+
+ #################################################################################
+ # Start of code generation
+ #################################################################################
+ self._add_code("\n")
+
+ # turn vb to True to have debugging output
+ vb = False
+
+ # Generate code
+ for loopnr, grid_variable_el in enumerate(
+ sorted(
+ self.grid_options["_grid_variables"].items(),
+ key=lambda x: x[1]["grid_variable_number"],
+ )
+ ):
+ self.verbose_print(
+ "Constructing/adding: {}".format(grid_variable_el[0]),
+ self.grid_options["verbosity"],
+ 2,
+ )
+ grid_variable = grid_variable_el[1]
+
+ ####################
+ # top code
+ if grid_variable["topcode"]:
+ self._add_code(grid_variable["topcode"])
+
+ #########################
+ # Setting up the for loop
+ # Add comment for for loop
+ self._add_code(
+ "# for loop for variable {name} gridtype {gridtype}".format(
+ name=grid_variable["name"],
+ gridtype=grid_variable["gridtype"],
+ ) + "\n",
+ "sampled_values_{} = {}".format(
+ grid_variable["name"], grid_variable["samplerfunc"]
+ )
+ + "\n")
+
+ if vb:
+ self._add_code(
+ "print('samples','{name}',':',sampled_values_{name})\n".format(
+ name=grid_variable["name"],
+ )
+ )
+
+ if vb:
+ self._add_code(
+ "print('sample {name} from',sampled_values_{name})".format(
+ name=grid_variable["name"]
+ )
+ + "\n"
+ )
+
+ # calculate number of values and starting location
+ #
+ # if we're sampling a continuous variable, we
+ # have one fewer grid point than the length of the
+ # sampled_values list
+ if ( grid_variable["gridtype"] in ["centred", "centre", "center", "edge", "left edge", "left", "right", "right edge"] ):
+ offset = -1
+ elif grid_variable["gridtype"] == "discrete":
+ # discrete variables sample all the points
+ offset = 0
+
+ start = 0
+
+ # for loop over the variable
+ if vb:
+ self._add_code(
+ "print(\"var {name} values \",sampled_values_{name},\" len \",len(sampled_values_{name})+{offset},\" gridtype {gridtype} offset {offset}\\n\")\n".format(
+ name=grid_variable["name"],
+ offset=offset,
+ gridtype=grid_variable['gridtype'],
+ )
+ )
+ self._add_code(
+ "for {name}_sample_number in range({start},len(sampled_values_{name})+{offset}):".format(
+ name=grid_variable["name"],
+ offset=offset,
+ start=start
+ )
+ + "\n"
+ )
+
+ self._increment_indent_depth(+1)
+
+ # {}_this_index is this grid point's index
+ # {}_prev_index and {}_next_index are the previous and next grid points,
+ # (which can be None if there is no previous or next, or if
+ # previous and next should not be used: this is deliberate)
+ #
+
+ if grid_variable["gridtype"] == "discrete":
+ # discrete grids only care about this,
+ # both prev and next should be None to
+ # force errors where they are used
+ self._add_code(
+ "{name}_this_index = {name}_sample_number ".format(
+ name=grid_variable["name"],
+ ),
+ )
+ self._add_code(
+ "\n",
+ "{name}_prev_index = None if {name}_this_index == 0 else ({name}_this_index - 1) ".format(
+ name=grid_variable["name"],
+ ),
+ "\n",
+ )
+ self._add_code(
+ "\n",
+ "{name}_next_index = None if {name}_this_index >= (len(sampled_values_{name})+{offset} - 1) else ({name}_this_index + 1)".format(
+ name=grid_variable["name"],
+ offset=offset
+ ),
+ "\n",
+ )
+
+ elif (grid_variable["gridtype"] in [ "centred","centre","center","edge","left","left edge" ] ):
+
+ # left and centred grids
+ self._add_code("if {}_sample_number == 0:\n".format(grid_variable["name"]))
+ self._add_code("{}_this_index = 0;\n".format(grid_variable["name"]), indent=1)
+ self._add_code("else:\n")
+ self._add_code("{name}_this_index = {name}_sample_number ".format(name=grid_variable["name"]),indent=1)
+ self._add_code("\n")
+ self._add_code("{name}_prev_index = ({name}_this_index - 1) if {name}_this_index > 0 else None ".format(name=grid_variable["name"]))
+ self._add_code("\n")
+ self._add_code("{name}_next_index = {name}_this_index + 1".format(name=grid_variable["name"]))
+ self._add_code("\n")
+
+ elif(grid_variable["gridtype"] in [ "right", "right edge" ] ):
+
+ # right edged grid
+ self._add_code("if {name}_sample_number == 0:\n".format(name=grid_variable["name"]))
+ self._add_code("{name}_this_index = 1;\n".format(name=grid_variable["name"]),indent=1)
+ self._add_code("else:\n")
+ self._add_code("{name}_this_index = {name}_sample_number + 1 ".format(name=grid_variable["name"],),indent=1)
+ self._add_code("\n")
+ self._add_code("{name}_prev_index = {name}_this_index - 1".format(name=grid_variable["name"]))
+ self._add_code("\n")
+ self._add_code("{name}_next_index = ({name}_this_index + 1) if {name}_this_index < len(sampled_values_{name}) else None".format(name=grid_variable["name"]))
+ self._add_code("\n")
+
+ # calculate phase volume
+ if(grid_variable["dphasevol"] == -1):
+ # no phase volume required so set it to 1.0
+ self._add_code("dphasevol_{name} = 1.0 # 666\n".format(name=grid_variable["name"]))
+
+ elif(grid_variable["gridtype"] in [ "right", "right edge" ] ):
+ # right edges always have this and prev defined
+ self._add_code(
+ "dphasevol_{name} = (sampled_values_{name}[{name}_this_index] - sampled_values_{name}[{name}_prev_index])".format(name=grid_variable["name"])
+ + "\n"
+ )
+ elif grid_variable["gridtype"] == "discrete":
+ # discrete might have next defined, use it if we can,
+ # otherwise use prev
+ self._add_code(
+ "dphasevol_{name} = (sampled_values_{name}[{name}_next_index] - sampled_values_{name}[{name}_this_index]) if {name}_next_index else (sampled_values_{name}[{name}_this_index] - sampled_values_{name}[{name}_prev_index])".format(name=grid_variable["name"])
+ + "\n"
+ )
+ else:
+ # left and centred always have this and next defined
+ self._add_code(
+ "dphasevol_{name} = (sampled_values_{name}[{name}_next_index] - sampled_values_{name}[{name}_this_index])".format(name=grid_variable["name"])
+ + "\n"
+ )
+
+
+ ##############
+ # Add phasevol check:
+ self._add_code("if dphasevol_{name} <= 0:\n".format(name=grid_variable["name"]))
+
+ # TODO: We might actually want to add the starcount and probability to the totals regardless.
+ # n that case we need another local variable which will prevent it from being run but will track those parameters
+ # Add phasevol check action:
+ self._add_code(
+ 'print("Grid generator: dphasevol_{name} <= 0! (this=",{name}_this_index,"=",sampled_values_{name}[{name}_this_index],", next=",{name}_next_index,"=",sampled_values_{name}[{name}_next_index],") Skipping current sample.")'.format(name=grid_variable["name"])
+ + "\n",
+ "continue\n",
+ indent=1,
+ )
+
+ if vb:
+ self._add_code(
+ "print('sample {name} from ',sampled_values_{name},' at this=',{name}_this_index,', next=',{name}_next_index)".format(name=grid_variable["name"])
+ + "\n"
+ )
+
+ # select sampled point location based on gridtype (left, centre or right)
+ if ( grid_variable["gridtype"] in ["edge", "left", "left edge", "right", "right edge", "discrete" ] ):
+ self._add_code(
+ "{name} = sampled_values_{name}[{name}_this_index]".format(
+ name=grid_variable["name"])
+ + "\n"
+ )
+ elif ( grid_variable["gridtype"] in ["centred", "centre", "center"] ):
+ self._add_code(
+ "{name} = 0.5 * (sampled_values_{name}[{name}_next_index] + sampled_values_{name}[{name}_this_index])".format(name=grid_variable["name"])
+ + "\n"
+ )
+ else:
+ msg = "Unknown gridtype value {type}.".format(type=grid_variable['gridtype'])
+ raise ValueError(msg)
+
+ if vb:
+ self._add_code(
+ "print('hence {name} = ',{name})\n".format(
+ name=grid_variable["name"]
+ )
+ )
+
+ #################################################################################
+ # Check condition and generate for loop
+
+ # If the grid variable has a condition, write the check and the action
+ if grid_variable["condition"]:
+ self._add_code(
+ # Add comment
+ "# Condition for {name}\n".format(name=grid_variable["name"]),
+
+ # Add condition check
+ "if not {condition}:\n".format(condition=grid_variable["condition"]),
+ indent=0,
+ )
+
+ # Add condition failed action:
+ if self.grid_options["verbosity"] >= 3:
+ self._add_code(
+ 'print("Grid generator: Condition for {name} not met!")'.format(
+ name=grid_variable["name"]
+ )
+ + "\n",
+ "continue" + "\n",
+ indent=1,
+ )
+ else:
+ self._add_code(
+ "continue" + "\n",
+ indent=1,
+ )
+ # Add some whitespace
+ self._add_code("\n")
+
+ # Add some whitespace
+ self._add_code("\n")
+
+ #########################
+ # Setting up pre-code and value in some cases
+ # Add pre-code
+ if grid_variable["precode"]:
+ self._add_code(
+ "{precode}".format(
+ precode=grid_variable["precode"].replace(
+ "\n", "\n" + self._indent_block(0)
+ )
+ )
+ + "\n"
+ )
+
+ # Set phasevol
+ self._add_code(
+ "phasevol *= dphasevol_{name}\n".format(
+ name=grid_variable["name"],
+ )
+ )
+
+ #######################
+ # Probabilities
+ # Calculate probability
+ self._add_code(
+ "\n",
+ "# Setting probabilities\n",
+ "d{name} = dphasevol_{name} * ({probdist})".format(
+ name=grid_variable["name"],
+ probdist=grid_variable["probdist"],
+ )
+ + "\n",
+ # Save probability sum
+ "probabilities_sum[{n}] += d{name}".format(
+ n=grid_variable["grid_variable_number"],
+ name=grid_variable["name"]
+ )
+ + "\n",
+ )
+
+ if grid_variable["grid_variable_number"] == 0:
+ self._add_code(
+ "probabilities_list[0] = d{name}".format(name=grid_variable["name"]) + "\n"
+ )
+ else:
+ self._add_code(
+ "probabilities_list[{this}] = probabilities_list[{prev}] * d{name}".format(
+ this=grid_variable["grid_variable_number"],
+ prev=grid_variable["grid_variable_number"] - 1,
+ name=grid_variable["name"],
+ )
+ + "\n"
+ )
+
+ ##############
+ # postcode
+ if grid_variable["postcode"]:
+ self._add_code(
+ "{postcode}".format(
+ postcode=grid_variable["postcode"].replace(
+ "\n", "\n" + self._indent_block(0)
+ )
+ )
+ + "\n"
+ )
+
+ #######################
+ # Increment starcount for this parameter
+ self._add_code(
+ "\n",
+ "# Increment starcount for {name}\n".format(name=grid_variable["name"]),
+ "starcounts[{n}] += 1".format(
+ n=grid_variable["grid_variable_number"],
+ )
+ + "\n",
+ # Add value to dict
+ 'parameter_dict["{name}"] = {name}'.format(
+ name=grid_variable["parameter_name"]
+ )
+ + "\n",
+ "\n",
+ )
+
+ self._increment_indent_depth(-1)
+
+ # The final parts of the code, where things are returned, are within the deepest loop,
+ # but in some cases code from a higher loop needs to go under it again
+ # SO I think its better to put an if statement here that checks
+ # whether this is the last loop.
+ if loopnr == len(self.grid_options["_grid_variables"]) - 1:
+ self._write_gridcode_system_call(
+ grid_variable,
+ dry_run,
+ grid_variable["branchpoint"],
+ grid_variable["branchcode"],
+ )
+
+ # increment indent_depth
+ self._increment_indent_depth(+1)
+
+ ####################
+ # bottom code
+ if grid_variable["bottomcode"]:
+ self._add_code(grid_variable["bottomcode"])
+
+ self._increment_indent_depth(-1)
+ self._add_code("\n")
+
+ # Write parts to write below the part that yield the results.
+ # this has to go in a reverse order:
+ # Here comes the stuff that is put after the deepest nested part that calls returns stuff.
+ # Here we will have a
+ reverse_sorted_grid_variables = sorted(
+ self.grid_options["_grid_variables"].items(),
+ key=lambda x: x[1]["grid_variable_number"],
+ reverse=True,
+ )
+ for loopnr, grid_variable_el in enumerate(reverse_sorted_grid_variables):
+ grid_variable = grid_variable_el[1]
+
+ self._increment_indent_depth(+1)
+ self._add_code(
+ "#" * 40 + "\n",
+ "# Code below is for finalising the handling of this iteration of the parameter {name}\n".format(
+ name=grid_variable["name"]
+ ),
+ )
+
+ # Set phasevol
+ # TODO: fix. this isn't supposed to be the value that we give it here. discuss
+ self._add_code("phasevol /= dphasevol_{name}\n\n".format(name=grid_variable["name"]))
+
+ self._increment_indent_depth(-2)
+
+ # Check the branchpoint part here. The branchpoint makes sure that we can construct
+ # a grid with several multiplicities and still can make the system calls for each
+ # multiplicity without reconstructing the grid each time
+ if grid_variable["branchpoint"] > 0:
+
+ self._increment_indent_depth(+1)
+
+ self._add_code(
+ # Add comment
+ "# Condition for branchpoint at {}".format(
+ reverse_sorted_grid_variables[loopnr + 1][1]["name"]
+ )
+ + "\n",
+ # # Add condition check
+ # "if not {}:".format(grid_variable["condition"])
+ # + "\n"
+ # Add branchpoint
+ "if multiplicity=={}:".format(grid_variable["branchpoint"]) + "\n",
+ )
+
+ self._write_gridcode_system_call(
+ reverse_sorted_grid_variables[loopnr + 1][1],
+ dry_run,
+ grid_variable["branchpoint"],
+ grid_variable["branchcode"],
+ )
+ self._increment_indent_depth(-1)
+ self._add_code("\n")
+
+ ###############################
+ # Finalising print statements
+ #
+ self._increment_indent_depth(+1)
+ self._add_code("\n", "#" * 40 + "\n", "if print_results:\n")
+ self._add_code(
+ "print('Grid has handled {starcount} stars with a total probability of {probtot:g}'.format(starcount=_total_starcount,probtot=self.grid_options['_probtot']))\n",
+ indent=1,
+ )
+
+ ################
+ # Finalising return statement for dry run.
+ #
+ if dry_run:
+ self._add_code("return _total_starcount\n")
+
+ self._increment_indent_depth(-1)
+ #################################################################################
+ # Stop of code generation. Here the code is saved and written
+
+ # Save the grid code to the grid_options
+ self.verbose_print(
+ "Saving grid code to grid_options", self.grid_options["verbosity"], 1
+ )
+
+ self.grid_options["code_string"] = self.code_string
+
+ # Write to file
+ gridcode_filename = self._gridcode_filename()
+
+ self.grid_options["gridcode_filename"] = gridcode_filename
+
+ self.verbose_print(
+ "{blue}Writing grid code to {file} [dry_run = {dry}]{reset}".format(
+ blue=self.ANSI_colours["blue"],
+ file=gridcode_filename,
+ dry=dry_run,
+ reset=self.ANSI_colours["reset"],
+ ),
+ self.grid_options["verbosity"],
+ 1,
+ )
+
+ with open(gridcode_filename, "w",encoding='utf-8') as file:
+ file.write(self.code_string)
+
+ # perhaps create symlink
+ if self.grid_options['slurm']==0 and \
+ self.grid_options["symlink_latest_gridcode"]:
+ global _count
+ symlink = os.path.join(
+ self.grid_options["tmp_dir"], "binary_c_grid-latest" + str(_count)
+ )
+ _count += 1
+ try:
+ os.unlink(symlink)
+ except:
+ pass
+
+ try:
+ os.symlink(gridcode_filename, symlink)
+ self.verbose_print(
+ "{blue}Symlinked grid code to {symlink} {reset}".format(
+ blue=self.ANSI_colours["blue"],
+ symlink=symlink,
+ reset=self.ANSI_colours["reset"]
+ ),
+ self.grid_options["verbosity"],
+ 1,
+ )
+ except OSError:
+ print("symlink failed")
+
+ def _write_gridcode_system_call(
+ self, grid_variable, dry_run, branchpoint, branchcode
+ ):
+ #################################################################################
+ # Here are the calls to the queuing or other solution. this part is for every system
+ # Add comment
+ self._increment_indent_depth(+1)
+ self._add_code("#" * 40 + "\n")
+
+ if branchcode:
+ self._add_code("# Branch code\nif {branchcode}:\n".format(branchcode=branchcode))
+
+ if branchpoint:
+ self._add_code(
+ "# Code below will get evaluated for every system at this level of multiplicity (last one of that being {name})\n".format(
+ name=grid_variable["name"]
+ )
+ )
+ else:
+ self._add_code(
+ "# Code below will get evaluated for every generated system\n"
+ )
+
+ # Factor in the custom weight input
+ self._add_code(
+ "\n",
+ "# Weigh the probability by a custom weighting factor\n",
+ 'probability = self.grid_options["weight"] * probabilities_list[{n}]'.format(
+ n=grid_variable["grid_variable_number"]
+ )
+ + "\n",
+ # Take into account the multiplicity fraction:
+ "\n",
+ "# Factor the multiplicity fraction into the probability\n",
+ "probability = probability * self._calculate_multiplicity_fraction(parameter_dict)"
+ + "\n",
+ # Add division by number of repeats
+ "\n",
+ "# Divide the probability by the number of repeats\n",
+ 'probability = probability / self.grid_options["repeat"]' + "\n",
+ # Now we yield the system self.grid_options["repeat"] times.
+ "\n",
+ "# Loop over the repeats\n",
+ 'for _ in range(self.grid_options["repeat"]):' + "\n",
+ )
+ self._add_code(
+ "_total_starcount += 1\n",
+ # set probability and phasevol values into the system dict
+ 'parameter_dict["{p}"] = {p}'.format(p="probability") + "\n",
+ 'parameter_dict["{v}"] = {v}'.format(v="phasevol") + "\n",
+ # Increment total probability
+ "self._increment_probtot(probability)\n",
+ indent=1,
+ )
+
+ if not dry_run:
+ # Handling of what is returned, or what is not.
+ self._add_code("yield(parameter_dict)\n", indent=1)
+
+ # If its a dry run, dont do anything with it
+ else:
+ self._add_code("pass\n", indent=1)
+
+ self._add_code("#" * 40 + "\n")
+
+ self._increment_indent_depth(-1)
+
+ return self.code_string
+
+ def _load_grid_function(self):
+ """
+ Function that loads the script containing the grid code.
+
+ TODO: Update this description
+ Test function to run grid stuff. mostly to test the import
+ """
+
+ # Code to load the
+ self.verbose_print(
+ message="Loading grid code function from {file}".format(
+ file=self.grid_options["gridcode_filename"]
+ ),
+ verbosity=self.grid_options["verbosity"],
+ minimal_verbosity=1,
+ )
+
+ spec = importlib.util.spec_from_file_location(
+ "binary_c_python_grid",
+ os.path.join(self.grid_options["gridcode_filename"]),
+ )
+ grid_file = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(grid_file)
+ generator = grid_file.grid_code
+
+ self.grid_options["_system_generator"] = generator
+
+ self.verbose_print("Grid code loaded", self.grid_options["verbosity"], 1)
+
+ def _last_grid_variable(self):
+ """
+ Function that returns the last grid variable
+ (i.e. the one with the highest grid_variable_number)
+ """
+
+ number = len(self.grid_options["_grid_variables"])
+ for grid_variable in self.grid_options["_grid_variables"]:
+ if (
+ self.grid_options["_grid_variables"][grid_variable][
+ "grid_variable_number"
+ ]
+ == number - 1
+ ):
+ return grid_variable
+
+ def update_grid_variable(self, name: str, **kwargs) -> None:
+ """
+ Function to update the values of a grid variable.
+
+ Args:
+ name:
+ name of the grid variable to be changed.
+ **kwargs:
+ key-value pairs to override the existing grid variable data. See add_grid_variable for these names.
+ """
+
+ grid_variable = None
+ try:
+ grid_variable = self.grid_options["_grid_variables"][name]
+ except KeyError:
+ msg = "Unknown grid variable {} - please create it with the add_grid_variable() method.".format(
+ name
+ )
+ raise KeyError(msg)
+
+ for key, value in kwargs.items():
+ grid_variable[key] = value
+ self.verbose_print(
+ "Updated grid variable: {}".format(json.dumps(grid_variable,
+ indent=4,
+ ensure_ascii=False)),
+ self.grid_options["verbosity"],
+ 1,
+ )
+
+ def delete_grid_variable(
+ self,
+ name: str,
+ ) -> None:
+ """
+ Function to delete a grid variable with the given name.
+
+ Args:
+ name:
+ name of the grid variable to be deleted.
+ """
+ try:
+ del self.grid_options["_grid_variables"][name]
+ self.verbose_print(
+ "Deleted grid variable: {}".format(name),
+ self.grid_options["verbosity"],
+ 1,
+ )
+ except:
+ msg = "Failed to remove grid variable {} : please check it exists.".format(
+ name
+ )
+ raise ValueError(msg)
+
+ def rename_grid_variable(self, oldname: str, newname: str) -> None:
+ """
+ Function to rename a grid variable.
+
+ note: this does NOT alter the order
+ of the self.grid_options["_grid_variables"] dictionary.
+
+ The order in which the grid variables are loaded into the grid is based on their
+ `grid_variable_number` property
+
+ Args:
+ oldname:
+ old name of the grid variable
+ newname:
+ new name of the grid variable
+ """
+
+ try:
+ self.grid_options["_grid_variables"][newname] = self.grid_options[
+ "_grid_variables"
+ ].pop(oldname)
+ self.grid_options["_grid_variables"][newname]["name"] = newname
+ self.verbose_print(
+ "Rename grid variable: {} to {}".format(oldname, newname),
+ self.grid_options["verbosity"],
+ 1,
+ )
+ except:
+ msg = "Failed to rename grid variable {} to {}.".format(oldname, newname)
+ raise ValueError(msg)
+
+ def add_grid_variable(
+ self,
+ name: str,
+ parameter_name: str,
+ longname: str,
+ valuerange: Union[list, str],
+ samplerfunc: str,
+ probdist: str,
+ dphasevol: Union[str, int] = -1,
+ gridtype: str = "centred",
+ branchpoint: int = 0,
+ branchcode: Union[str, None] = None,
+ precode: Union[str, None] = None,
+ postcode: Union[str, None] = None,
+ topcode: Union[str, None] = None,
+ bottomcode: Union[str, None] = None,
+ condition: Union[str, None] = None,
+ ) -> None:
+ """
+ Function to add grid variables to the grid_options.
+
+ The execution of the grid generation will be through a nested for loop.
+ Each of the grid variables will get create a deeper for loop.
+
+ The real function that generates the numbers will get written to a new file in the TMP_DIR,
+ and then loaded imported and evaluated.
+ beware that if you insert some destructive piece of code, it will be executed anyway.
+ Use at own risk.
+
+ Tasks:
+ - TODO: Fix this complex function.
+
+ Args:
+ name:
+ name of parameter used in the grid Python code.
+ This is evaluated as a parameter and you can use it throughout
+ the rest of the function
+
+ Examples:
+ name = 'lnm1'
+
+ parameter_name:
+ name of the parameter in binary_c
+
+ This name must correspond to a Python variable of the same name,
+ which is automatic if parameter_name == name.
+
+ Note: if parameter_name != name, you must set a
+ variable in "precode" or "postcode" to define a Python variable
+ called parameter_name
+
+ longname:
+ Long name of parameter
+
+ Examples:
+ longname = 'Primary mass'
+ range:
+ Range of values to take. Does not get used really, the samplerfunc is used to
+ get the values from
+
+ Examples:
+ range = [math.log(m_min), math.log(m_max)]
+ samplerfunc:
+ Function returning a list or numpy array of samples spaced appropriately.
+ You can either use a real function, or a string representation of a function call.
+
+ Examples:
+ samplerfunc = "const(math.log(m_min), math.log(m_max), {})".format(resolution['M_1'])
+
+ precode:
+ Extra room for some code. This code will be evaluated within the loop of the
+ sampling function (i.e. a value for lnm1 is chosen already)
+
+ Examples:
+ precode = 'M_1=math.exp(lnm1);'
+ postcode:
+ Code executed after the probability is calculated.
+ probdist:
+ Function determining the probability that gets assigned to the sampled parameter
+
+ Examples:
+ probdist = 'Kroupa2001(M_1)*M_1'
+ dphasevol:
+ part of the parameter space that the total probability is calculated with. Put to -1
+ if you want to ignore any dphasevol calculations and set the value to 1
+ Examples:
+ dphasevol = 'dlnm1'
+ condition:
+ condition that has to be met in order for the grid generation to continue
+ Examples:
+ condition = 'self.grid_options['binary']==1'
+ gridtype:
+ Method on how the value range is sampled. Can be either 'edge' (steps starting at
+ the lower edge of the value range) or 'centred'
+ (steps starting at lower edge + 0.5 * stepsize).
+
+ topcode:
+ Code added at the very top of the block.
+
+ bottomcode:
+ Code added at the very bottom of the block.
+ """
+
+ # check parameters
+ if False and dphasevol != -1.0 and gridtype == 'discrete':
+ print("Error making grid: you have set the phasevol to be not -1 and gridtype to discrete, but a discrete grid has no phasevol calculation. You should only set the gridtype to discrete and not set the phasevol in this case.")
+
+ self.exit(code=1)
+
+ # Add grid_variable
+ grid_variable = {
+ "name": name,
+ "parameter_name": parameter_name,
+ "longname": longname,
+ "valuerange": valuerange,
+ "samplerfunc": samplerfunc,
+ "precode": precode,
+ "postcode": postcode,
+ "probdist": probdist,
+ "dphasevol": dphasevol,
+ "condition": condition,
+ "gridtype": gridtype,
+ "branchpoint": branchpoint,
+ "branchcode": branchcode,
+ "topcode": topcode,
+ "bottomcode": bottomcode,
+ "grid_variable_number": len(self.grid_options["_grid_variables"]),
+ }
+
+ # Check for gridtype input
+ allowed_gridtypes = [
+ "edge",
+ "right",
+ "right edge",
+ "left",
+ "left edge",
+ "centred",
+ "centre",
+ "center",
+ "discrete"
+ ]
+ if not gridtype in allowed_gridtypes:
+ msg = "Unknown gridtype {gridtype}. Please choose one of: ".format(gridtype=gridtype) + ','.join(allowed_gridtypes)
+ raise ValueError(msg)
+
+ # Load it into the grid_options
+ self.grid_options["_grid_variables"][grid_variable["name"]] = grid_variable
+
+ self.verbose_print(
+ "Added grid variable: {}".format(json.dumps(grid_variable,
+ indent=4,
+ ensure_ascii=False)),
+ self.grid_options["verbosity"],
+ 2,
+ )
diff --git a/binarycpython/utils/metadata.py b/binarycpython/utils/metadata.py
new file mode 100644
index 000000000..afea81737
--- /dev/null
+++ b/binarycpython/utils/metadata.py
@@ -0,0 +1,115 @@
+"""
+Module containing binary_c-python's metadata functions
+"""
+
+import datetime
+import json
+import platform
+import time
+
+from binarycpython.utils.dicts import (
+ multiply_values_dict,
+)
+from binarycpython.utils.ensemble import (
+ binaryc_json_serializer,
+ )
+
+class metadata():
+ def __init__(self, **kwargs):
+ return
+
+ def add_system_metadata(self):
+ """
+ Add system's metadata to the grid_ensemble_results, and
+ add some system information to metadata.
+ """
+ # add metadata if it doesn't exist
+ if not "metadata" in self.grid_ensemble_results:
+ self.grid_ensemble_results["metadata"] = {}
+
+ # add date
+ self.grid_ensemble_results["metadata"]['date'] = datetime.datetime.now().strftime("%m/%d/%Y %H:%M:%S")
+
+ # add platform and build information
+ try:
+ self.grid_ensemble_results["metadata"]['platform'] = platform.platform()
+ self.grid_ensemble_results["metadata"]['platform_uname'] = list(platform.uname())
+ self.grid_ensemble_results["metadata"]['platform_machine'] = platform.machine()
+ self.grid_ensemble_results["metadata"]['platform_node'] = platform.node()
+ self.grid_ensemble_results["metadata"]['platform_release'] = platform.release()
+ self.grid_ensemble_results["metadata"]['platform_version'] = platform.version()
+ self.grid_ensemble_results["metadata"]['platform_processor'] = platform.processor()
+ self.grid_ensemble_results["metadata"]['platform_python_build'] = ' '.join(platform.python_build())
+ self.grid_ensemble_results["metadata"]['platform_python_version'] = platform.python_version()
+ except Exception as e:
+ print("platform call failed:",e)
+ pass
+
+ try:
+ self.grid_ensemble_results["metadata"]['hostname'] = platform.uname()[1]
+ except Exception as e:
+ print("platform call failed:",e)
+ pass
+
+ try:
+ self.grid_ensemble_results["metadata"]['duration'] = self.time_elapsed()
+ except Exception as e:
+ print("Failure to calculate time elapsed:",e)
+ pass
+
+ try:
+ self.grid_ensemble_results["metadata"]['CPU_time'] = self.CPU_time()
+ except Exception as e:
+ print("Failure to calculate CPU time consumed:",e)
+ pass
+ return
+
+
+ def add_ensemble_metadata(self,combined_output_dict):
+ """
+ Function to add metadata to the grid_ensemble_results and grid_options
+ """
+ self.grid_ensemble_results["metadata"] = {}
+ self.grid_ensemble_results["metadata"]["population_id"] = self.grid_options["_population_id"]
+ self.grid_ensemble_results["metadata"]["total_probability_weighted_mass"] = combined_output_dict["_total_probability_weighted_mass_run"]
+ self.grid_ensemble_results["metadata"]["factored_in_probability_weighted_mass"] = False
+ if self.grid_options["ensemble_factor_in_probability_weighted_mass"]:
+ multiply_values_dict(
+ self.grid_ensemble_results["ensemble"],
+ 1.0
+ / self.grid_ensemble_results["metadata"][
+ "total_probability_weighted_mass"
+ ],
+ )
+ self.grid_ensemble_results["metadata"]["factored_in_probability_weighted_mass"] = True
+ self.grid_ensemble_results["metadata"]["_killed"] = self.grid_options["_killed"]
+
+ # Add settings of the populations
+ all_info = self.return_all_info(
+ include_population_settings=True,
+ include_binary_c_defaults=True,
+ include_binary_c_version_info=True,
+ include_binary_c_help_all=True,
+ )
+ self.grid_ensemble_results["metadata"]["settings"] = json.loads(
+ json.dumps(all_info, default=binaryc_json_serializer, ensure_ascii=False)
+ )
+
+ ##############################
+ # Update grid options
+ for x in self._metadata_keylist():
+ self.grid_options[x] = combined_output_dict[x]
+ self.grid_options["_failed_systems_error_codes"] = list(set(combined_output_dict["_failed_systems_error_codes"]))
+
+
+ def _metadata_keylist(self):
+ return ["_failed_count",
+ "_failed_prob",
+ "_errors_exceeded",
+ "_errors_found",
+ "_probtot",
+ "_count",
+ "_total_mass_run",
+ "_total_probability_weighted_mass_run",
+ "_zero_prob_stars_skipped",
+ "_killed"]
diff --git a/binarycpython/utils/slurm.py b/binarycpython/utils/slurm.py
index e69de29bb..c46b7e092 100644
--- a/binarycpython/utils/slurm.py
+++ b/binarycpython/utils/slurm.py
@@ -0,0 +1,345 @@
+"""
+ Binary_c-python's slurm functions
+"""
+import os
+import datasize
+import lib_programname
+import multiprocessing
+import os
+import pathlib
+import signal
+import stat
+import subprocess
+import sys
+import time
+
+from binarycpython.utils.HPC import HPC
+
+class slurm(HPC):
+
+ def __init__(self, **kwargs):
+ # don't do anything: we just inherit from this class
+ return
+
+ def slurmpath(self,path):
+ """
+ Function to return the full slurm directory path.
+ """
+ return os.path.abspath(os.path.join(self.grid_options['slurm_dir'],path))
+
+ def slurm_status_file(self,
+ jobid=None,
+ jobarrayindex=None):
+ """
+ Return the slurm status file corresponding to the jobid and jobarrayindex, which default to grid_options slurm_jobid and slurm_jobarrayindex, respectively.
+ """
+ if jobid is None:
+ jobid= self.grid_options['slurm_jobid']
+ if jobarrayindex is None:
+ jobarrayindex = self.grid_options['slurm_jobarrayindex']
+ if jobid and jobarrayindex:
+ return os.path.join(self.slurmpath('status'),
+ self.grid_options['slurm_jobid'] + '.' + self.grid_options['slurm_jobarrayindex'])
+ else:
+ return None
+
+ def set_slurm_status(self,string):
+ """
+ Set the slurm status corresponing to the self object, which should have slurm_jobid and slurm_jobarrayindex set.
+ """
+ # save slurm jobid to file
+ idfile = os.path.join(self.grid_options["slurm_dir"],
+ "jobid")
+ if not os.path.exists(idfile):
+ with open(idfile,"w",encoding='utf-8') as fjobid:
+ fjobid.write("{jobid}\n".format(jobid=self.grid_options['slurm_jobid']))
+ fjobid.close()
+
+ # save slurm status
+ file = self.slurm_status_file()
+ if file:
+ with open(file,'w',encoding='utf-8') as f:
+ f.write(string)
+ f.close()
+ return
+
+ def get_slurm_status(self,
+ jobid=None,
+ jobarrayindex=None):
+ """
+ Get and return the slurm status corresponing to the self object, or jobid.jobarrayindex if they are passed in. If no status is found, returns None.
+ """
+ if jobid is None:
+ jobid = self.grid_options['slurm_jobid']
+ if jobarrayindex is None:
+ jobarrayindex = self.grid_options['slurm_jobarrayindex']
+
+ if jobid is None or jobarrayindex is None :
+ return None
+
+ path = pathlib.Path(self.slurm_status_file(jobid=jobid,
+ jobarrayindex=jobarrayindex))
+ if path:
+ return path.read_text().strip()
+ else:
+ return None
+
+ def slurm_outfile(self):
+ """
+ return a standard filename for the slurm chunk files
+ """
+ file = "{jobid}.{jobarrayindex}.gz".format(
+ jobid = self.grid_options['slurm_jobid'],
+ jobarrayindex = self.grid_options['slurm_jobarrayindex']
+ )
+ return os.path.abspath(os.path.join(self.grid_options['slurm_dir'],
+ 'results',
+ file))
+
+ def make_slurm_dirs(self):
+
+ # make the slurm directories
+ if not self.grid_options['slurm_dir']:
+ print("You must set self.grid_options['slurm_dir'] to a directory which we can use to set up binary_c-python's Slurm files. This should be unique to your set of grids.")
+ os.exit()
+
+ # make a list of directories, these contain the various slurm
+ # output, status files, etc.
+ dirs = []
+ for dir in ['stdout','stderr','results','status','snapshots']:
+ dirs.append(self.slurmpath(dir))
+
+ # make the directories: we do not allow these to already exist
+ # as the slurm directory should be a fresh location for each set of jobs
+ for dir in dirs:
+ try:
+ pathlib.Path(self.slurmpath(dir)).mkdir(exist_ok=False,
+ parents=True)
+ except:
+ print("Tried to make the directory {dir} but it already exists. When you launch a set of binary_c jobs on Slurm, you need to set your slurm_dir to be a fresh directory with no contents.".format(dir=dir))
+ self.exit(code=1)
+
+ # check that they have been made and exist: we need this
+ # because on network mounts (NFS) there's often a delay between the mkdir
+ # above and the actual directory being made. This shouldn't be too long...
+ fail = True
+ count = 0
+ count_warn = 10
+ while fail is True:
+ fail = False
+ count += 1
+ if count > count_warn:
+ print("Warning: Have been waiting about {} seconds for Slurm directories to be made, there seems to be significant delay...".format(count))
+ for dir in dirs:
+ if os.path.isdir(dir) is False:
+ fail = True
+ time.sleep(1)
+ break # break the "for dir in dirs:"
+
+ def slurm_grid(self):
+ """
+ function to be called when running grids when grid_options['slurm']>=1
+
+ if grid_options['slurm']==1, we set up the slurm script and launch the jobs, then exit.
+ if grid_options['slurm']==2, we are being called from the jobs to run the grids
+ """
+
+ # if slurm=1, we should have no evolution type, we just
+ # set up a load of slurm scripts and get them evolving
+ # in slurm array
+ if self.grid_options['slurm'] == 1:
+ self.grid_options['evolution_type'] = None
+
+ if self.grid_options['evolution_type'] == 'grid':
+ # run a grid of stars only, leaving the results
+ # in a file
+
+ # set output file
+ self.grid_options['save_population_object'] = slurm_outfile()
+
+ return self.evolve()
+ elif self.grid_options['evolution_type'] == 'join':
+ # should not happen!
+ return
+ else:
+ # setup and launch slurm jobs
+ self.make_slurm_dirs()
+
+ # check we're not using too much RAM
+ if datasize.DataSize(self.grid_options['slurm_memory']) > datasize.DataSize(self.grid_options['slurm_warn_max_memory']):
+ print("WARNING: you want to use {} MB of RAM : this is unlikely to be correct. If you believe it is, set slurm_warn_max_memory to something very large (it is currently {} MB)\n".format(
+ self.grid_options['slurm_memory'],
+ self.grid_options['slurm_warn_max_memory']))
+ self.exit(code=1)
+
+ # set up slurm_array
+ if not self.grid_options['slurm_array_max_jobs']:
+ self.grid_options['slurm_array_max_jobs'] = self.grid_options['slurm_njobs']
+ slurm_array = self.grid_options['slurm_array'] or "1-{njobs}\%{max_jobs}".format(
+ njobs=self.grid_options['slurm_njobs'],
+ max_jobs=self.grid_options['slurm_array_max_jobs'])
+
+ # get job id (might be passed in)
+ jobid = self.grid_options['slurm_jobid'] if self.grid_options['slurm_jobid'] != "" else '$SLURM_ARRAY_JOB_ID'
+
+ # get job array index
+ jobarrayindex = self.grid_options['slurm_jobarrayindex']
+ if jobarrayindex is None:
+ jobarrayindex = '$SLURM_ARRAY_TASK_ID'
+
+ if self.grid_options['slurm_njobs'] == 0:
+ print("binary_c-python Slurm : You must set grid_option slurm_njobs to be non-zero")
+ self.exit(code=1)
+
+ # build the grid command
+ grid_command = [
+ os.path.join("/usr","bin","env"),
+ sys.executable,
+ str(lib_programname.get_path_executed_script()),
+ ] + sys.argv[1:] + [
+ 'slurm=2',
+ 'start_at=' + str(jobarrayindex) + '-1', # do we need the -1?
+ 'modulo=' + str(self.grid_options['slurm_njobs']),
+ 'slurm_njobs=' + str(self.grid_options['slurm_njobs']),
+ 'slurm_dir=' + self.grid_options['slurm_dir'],
+ 'verbosity=' + str(self.grid_options['verbosity']),
+ 'num_cores=' + str(self.grid_options['num_processes'])
+ ]
+
+ grid_command = ' '.join(grid_command)
+
+ # make slurm script
+ scriptpath = self.slurmpath('slurm_script')
+ try:
+ script = open(scriptpath,'w',encoding='utf-8')
+ except IOError:
+ print("Could not open Slurm script at {path} for writing: please check you have set {slurm_dir} correctly (it is currently {slurm_dir} and can write to this directory.".format(path=scriptpath,
+ slurm_dir = self.grid_options['slurm_dir']))
+
+ # the joinfile contains the list of chunk files to be joined
+ joinfile = "{slurm_dir}/results/{jobid}.all".format(
+ slurm_dir=self.grid_options['slurm_dir'],
+ jobid=jobid
+ )
+
+ lines = [
+ "#!/bin/bash\n",
+ "# Slurm file for binary_grid2 and slurm\n",
+ "#SBATCH --error={slurm_dir}/stderr/\%A.\%a\n".format(slurm_dir=self.grid_options['slurm_dir']),
+ "#SBATCH --output={slurm_dir}/stdout/\%A.\%a\n".format(slurm_dir=self.grid_options['slurm_dir']),
+ "#SBATCH --job-name={slurm_jobname}\n".format(slurm_jobname=self.grid_options['slurm_jobname']),
+ "#SBATCH --partition={slurm_partition}\n".format(slurm_partition=self.grid_options['slurm_partition']),
+ "#SBATCH --time={slurm_time}\n".format(slurm_time=self.grid_options['slurm_time']),
+ "#SBATCH --mem={slurm_memory}\n".format(slurm_memory=self.grid_options['slurm_memory']),
+ "#SBATCH --ntasks={slurm_ntasks}\n".format(slurm_ntasks=self.grid_options['slurm_ntasks']),
+ "#SBATCH --array={slurm_array}\n".format(slurm_array=slurm_array),
+ "#SBATCH --cpus-per-task={ncpus}\n".format(ncpus=self.grid_options['num_processes'])
+ ]
+
+ for key in self.grid_options['slurm_extra_settings']:
+ lines += [ "#SBATCH --{key} = {value}\n".format(
+ key=key,
+ value=self.grid_options['slurm_extra_settings'][key]
+ )]
+
+ # save original command line, working directory, time
+ lines += [
+ "\nexport BINARY_C_PYTHON_ORIGINAL_CMD_LINE={cmdline}".format(cmdline=repr(self.grid_options['command_line'])),
+ "\nexport BINARY_C_PYTHON_ORIGINAL_WD=`pwd`",
+ "\nexport BINARY_C_PYTHON_ORIGINAL_SUBMISSION_TIME=`date`",
+ ]
+
+
+ lines += [
+ "\n# set status to \"running\"\n",
+ "echo \"running\" > {slurm_dir}/status/{jobid}.{jobarrayindex}\n\n".format(slurm_dir=self.grid_options['slurm_dir'],
+ jobid=jobid,
+ jobarrayindex=jobarrayindex),
+ "\n# make list of files\n",
+ "\necho {slurm_dir}/results/{jobid}.{jobarrayindex}.gz >> {slurm_dir}/results/{jobid}.all\n".format(slurm_dir=self.grid_options['slurm_dir'],
+ jobid=jobid,
+ jobarrayindex=jobarrayindex,
+ joinfile=joinfile),
+ "\n# run grid of stars and, if this returns 0, set status to finished\n",
+
+ # note: the next line ends in &&
+ "{grid_command} evolution_type=grid slurm_jobid={jobid} slurm_jobarrayindex={jobarrayindex} save_population_object={slurm_dir}/results/{jobid}.{jobarrayindex}.gz && echo -n \"finished\" > {slurm_dir}/status/{jobid}.{jobarrayindex} && \\\n".format(
+ slurm_dir=self.grid_options['slurm_dir'],
+ jobid=jobid,
+ jobarrayindex=jobarrayindex,
+ grid_command=grid_command),
+ ]
+
+ if not self.grid_options['slurm_postpone_join']:
+ lines += [
+ # the following line also ends in && so that if one fails, the rest
+ # also fail
+ "echo && echo \"Checking if we can join...\" && echo && \\\n",
+ "{grid_command} slurm=2 evolution_type=join joinlist={joinfile} slurm_jobid={jobid} slurm_jobarrayindex={jobarrayindex}\n\n".format(
+ grid_command=grid_command,
+ joinfile=joinfile,
+ jobid=jobid,
+ jobarrayindex=jobarrayindex
+ )]
+
+ # write to script, close it and make it executable by
+ # all (so the slurm user can pick it up)
+ script.writelines(lines)
+ script.close()
+ os.chmod(scriptpath,
+ stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC | \
+ stat.S_IRGRP | stat.S_IXGRP | \
+ stat.S_IROTH | stat.S_IXOTH)
+
+ if not self.grid_options['slurm_postpone_sbatch']:
+ # call sbatch to launch the jobs
+ cmd = [self.grid_options['slurm_sbatch'], scriptpath]
+ pipes = subprocess.Popen(cmd,
+ stdout = subprocess.PIPE,
+ stderr = subprocess.PIPE)
+ std_out, std_err = pipes.communicate()
+ if pipes.returncode != 0:
+ # an error happened!
+ err_msg = "{red}{err}\nReturn Code: {code}{reset}".format(err=std_err.strip(),
+ code=pipes.returncode,
+ red=self.ANSI_colours["red"],
+ reset=self.ANSI_colours["reset"],)
+ raise Exception(err_msg)
+
+ elif len(std_err):
+ print("{red}{err}{reset}".format(red=self.ANSI_colours["red"],
+ reset=self.ANSI_colours["reset"],
+ err=std_err.strip().decode('utf-8')))
+
+ print("{yellow}{out}{reset}".format(yellow=self.ANSI_colours["yellow"],
+ reset=self.ANSI_colours["reset"],
+ out=std_out.strip().decode('utf-8')))
+ else:
+ # just say we would have (use this for testing)
+ print("Slurm script is at {path} but has not been launched".format(path=scriptpath))
+
+
+ # some messages to the user, then return
+ if self.grid_options['slurm_postpone_sbatch'] == 1:
+ print("Slurm script written, but launching the jobs with sbatch was postponed.")
+ else:
+ print("Slurm jobs launched")
+ print("All done in slurm_grid().")
+ return
+
+ def slurm_jobid_from_dir(self,dir):
+ """
+ Return the Slurm jobid from a slurm directory, passed in
+ """
+ file = os.path.join(dir,'jobid')
+ f = open(file,"r",encoding='utf-8')
+ if not f:
+ print("Error: could not open {} to read the Slurm jobid of the directory {}".format(file,dir))
+ sys.exit(code=1)
+ oldjobid = f.read().strip()
+ f.close()
+ if not oldjobid:
+ print("Error: could not find jobid in {}".format(self.grid_options['slurm_restart_dir']))
+ self.exit(code=1)
+
+ return oldjobid
diff --git a/binarycpython/utils/version.py b/binarycpython/utils/version.py
new file mode 100644
index 000000000..52f63979a
--- /dev/null
+++ b/binarycpython/utils/version.py
@@ -0,0 +1,414 @@
+"""
+ Binary_c-python's version information
+"""
+
+import copy
+import os
+from binarycpython import _binary_c_bindings
+from typing import Union, Any
+
+from binarycpython.utils.functions import (
+ isfloat
+ )
+
+class version():
+
+ def __init__(self, **kwargs):
+ # don't do anything: we just inherit from this class
+ return
+
+
+ ########################################################
+ # version_info functions
+ ########################################################
+ def return_binary_c_version_info(self,
+ parsed: bool = True) -> Union[str, dict]:
+ """
+ Function that returns the version information of binary_c. This function calls the function
+ _binary_c_bindings.return_version_info()
+
+ Args:
+ parsed: Boolean flag whether to parse the version_info output of binary_c. default = False
+
+ Returns:
+ Either the raw string of binary_c or a parsed version of this in the form of a nested
+ dictionary
+ """
+
+ found_prev = False
+ if "BINARY_C_MACRO_HEADER" in os.environ:
+ # the env var is already present. lets save that and put that back later
+ found_prev = True
+ prev_value = os.environ["BINARY_C_MACRO_HEADER"]
+
+ #
+ os.environ["BINARY_C_MACRO_HEADER"] = "macroxyz"
+
+ # Get version_info
+ version_info = _binary_c_bindings.return_version_info().strip()
+
+ # parse if wanted
+ if parsed:
+ version_info = self.parse_binary_c_version_info(version_info)
+
+ # delete value
+ del os.environ["BINARY_C_MACRO_HEADER"]
+
+ # put stuff back if we found a previous one
+ if found_prev:
+ os.environ["BINARY_C_MACRO_HEADER"] = prev_value
+
+ return version_info
+
+
+ def parse_binary_c_version_info(self,
+ version_info_string: str) -> dict:
+ """
+ Function that parses the binary_c version info. Long function with a lot of branches
+
+ TODO: fix this function. stuff is missing: isotopes, macros, nucleosynthesis_sources
+
+ Args:
+ version_info_string: raw output of version_info call to binary_c
+
+ Returns:
+ Parsed version of the version info, which is a dictionary containing the keys: 'isotopes' for isotope info, 'argpairs' for argument pair info (TODO: explain), 'ensembles' for ensemble settings/info, 'macros' for macros, 'elements' for atomic element info, 'DTlimit' for (TODO: explain), 'nucleosynthesis_sources' for nucleosynthesis sources, and 'miscellaneous' for all those that were not caught by the previous groups. 'git_branch', 'git_build', 'revision' and 'email' are also keys, but its clear what those contain.
+ """
+
+ version_info_dict = {}
+
+ # Clean data and put in correct shape
+ splitted = version_info_string.strip().splitlines()
+ cleaned = {el.strip() for el in splitted if not el == ""}
+
+ ##########################
+ # Network:
+ # Split off all the networks and parse the info.
+
+ networks = {el for el in cleaned if el.startswith("Network ")}
+ cleaned = cleaned - networks
+
+ networks_dict = {}
+ for el in networks:
+ network_dict = {}
+ split_info = el.split("Network ")[-1].strip().split("==")
+
+ network_number = int(split_info[0])
+ network_dict["network_number"] = network_number
+
+ network_info_split = split_info[1].split(" is ")
+
+ shortname = network_info_split[0].strip()
+ network_dict["shortname"] = shortname
+
+ if not network_info_split[1].strip().startswith(":"):
+ network_split_info_extra = network_info_split[1].strip().split(":")
+
+ longname = network_split_info_extra[0].strip()
+ network_dict["longname"] = longname
+
+ implementation = (
+ network_split_info_extra[1].strip().replace("implemented in", "")
+ )
+ if implementation:
+ network_dict["implemented_in"] = [i.strip("()") for i in implementation.strip().split()]
+
+ networks_dict[network_number] = copy.deepcopy(network_dict)
+ version_info_dict["networks"] = networks_dict if networks_dict else None
+
+ ##########################
+ # Isotopes:
+ # Split off
+ isotopes = {el for el in cleaned if el.startswith("Isotope ")}
+ cleaned = cleaned - isotopes
+
+ isotope_dict = {}
+ for el in isotopes:
+ split_info = el.split("Isotope ")[-1].strip().split(" is ")
+
+ isotope_info = split_info[-1]
+ name = isotope_info.split(" ")[0].strip()
+
+ # Get details
+ mass_g = float(
+ isotope_info.split(",")[0].split("(")[1].split("=")[-1][:-2].strip()
+ )
+ mass_amu = float(
+ isotope_info.split(",")[0].split("(")[-1].split("=")[-1].strip()
+ )
+ mass_mev = float(
+ isotope_info.split(",")[-3].split("=")[-1].replace(")", "").strip()
+ )
+ A = int(isotope_info.split(",")[-1].strip().split("=")[-1].replace(")", ""))
+ Z = int(isotope_info.split(",")[-2].strip().split("=")[-1])
+
+ #
+ isotope_dict[int(split_info[0])] = {
+ "name": name,
+ "Z": Z,
+ "A": A,
+ "mass_mev": mass_mev,
+ "mass_g": mass_g,
+ "mass_amu": mass_amu,
+ }
+ version_info_dict["isotopes"] = isotope_dict if isotope_dict else None
+
+ ##########################
+ # Arg pairs:
+ # Split off
+ argpairs = set([el for el in cleaned if el.startswith("ArgPair")])
+ cleaned = cleaned - argpairs
+
+ argpair_dict = {}
+ for el in sorted(argpairs):
+ split_info = el.split("ArgPair ")[-1].split(" ")
+
+ if not argpair_dict.get(split_info[0], None):
+ argpair_dict[split_info[0]] = {split_info[1]: split_info[2]}
+ else:
+ argpair_dict[split_info[0]][split_info[1]] = split_info[2]
+
+ version_info_dict["argpairs"] = argpair_dict if argpair_dict else None
+
+ ##########################
+ # ensembles:
+ # Split off
+ ensembles = {el for el in cleaned if el.startswith("Ensemble")}
+ cleaned = cleaned - ensembles
+
+ ensemble_dict = {}
+ ensemble_filter_dict = {}
+ for el in ensembles:
+ split_info = el.split("Ensemble ")[-1].split(" is ")
+
+ if len(split_info) > 1:
+ if not split_info[0].startswith("filter"):
+ ensemble_dict[int(split_info[0])] = split_info[-1]
+ else:
+ filter_no = int(split_info[0].replace("filter ", ""))
+ ensemble_filter_dict[filter_no] = split_info[-1]
+
+ version_info_dict["ensembles"] = ensemble_dict if ensemble_dict else None
+ version_info_dict["ensemble_filters"] = (
+ ensemble_filter_dict if ensemble_filter_dict else None
+ )
+
+ ##########################
+ # macros:
+ # Split off
+ macros = {el for el in cleaned if el.startswith("macroxyz")}
+ cleaned = cleaned - macros
+
+ param_type_dict = {
+ "STRING": str,
+ "FLOAT": float,
+ "MACRO": str,
+ "INT": int,
+ "LONG_INT": int,
+ "UINT": int,
+ }
+
+ macros_dict = {}
+ for el in macros:
+ split_info = el.split("macroxyz ")[-1].split(" : ")
+ param_type = split_info[0]
+
+ new_split = "".join(split_info[1:]).split(" is ")
+ param_name = new_split[0].strip()
+ param_value = " is ".join(new_split[1:])
+ param_value = param_value.strip()
+
+ #print("macro ",param_name,"=",param_value," float?",isfloat(param_value)," int?",isint(param_value))
+
+ # If we're trying to set the value to "on", check that
+ # it doesn't already exist. If it does, do nothing, as the
+ # extra information is better than just "on"
+ if param_name in macros_dict:
+ #print("already exists (is ",macros_dict[param_name]," float? ",isfloat(macros_dict[param_name]),", int? ",isint(macros_dict[param_name]),") : check that we can improve it")
+ if macros_dict[param_name] == "on":
+ # update with better value
+ store = True
+ elif isfloat(macros_dict[param_name]) == False and isfloat(param_value) == True:
+ # store the number we now have to replace the non-number we had
+ store = True
+ else:
+ # don't override existing number
+ store = False
+
+ #if store:
+ # print("Found improved macro value of param",param_name,", was ",macros_dict[param_name],", is",param_value)
+ #else:
+ # print("Cannot improve: use old value")
+ else:
+ store = True
+
+ if store:
+ # Sometimes the macros have extra information behind it.
+ # Needs an update in outputting by binary_c (RGI: what does this mean David???)
+ try:
+ macros_dict[param_name] = param_type_dict[param_type](param_value)
+ except ValueError:
+ macros_dict[param_name] = str(param_value)
+
+ version_info_dict["macros"] = macros_dict if macros_dict else None
+
+ ##########################
+ # Elements:
+ # Split off:
+ elements = {el for el in cleaned if el.startswith("Element")}
+ cleaned = cleaned - elements
+
+ # Fill dict:
+ elements_dict = {}
+ for el in elements:
+ split_info = el.split("Element ")[-1].split(" : ")
+ name_info = split_info[0].split(" is ")
+
+ # get isotope info
+ isotopes = {}
+ if not split_info[-1][0] == "0":
+ isotope_string = split_info[-1].split(" = ")[-1]
+ isotopes = {
+ int(split_isotope.split("=")[0]): split_isotope.split("=")[1]
+ for split_isotope in isotope_string.split(" ")
+ }
+
+ elements_dict[int(name_info[0])] = {
+ "name": name_info[-1],
+ "atomic_number": int(name_info[0]),
+ "amt_isotopes": len(isotopes),
+ "isotopes": isotopes,
+ }
+ version_info_dict["elements"] = elements_dict if elements_dict else None
+
+ ##########################
+ # dt_limits:
+ # split off
+ dt_limits = {el for el in cleaned if el.startswith("DTlimit")}
+ cleaned = cleaned - dt_limits
+
+ # Fill dict
+ dt_limits_dict = {}
+ for el in dt_limits:
+ split_info = el.split("DTlimit ")[-1].split(" : ")
+ dt_limits_dict[split_info[1].strip()] = {
+ "index": int(split_info[0]),
+ "value": float(split_info[-1]),
+ }
+
+ version_info_dict["dt_limits"] = dt_limits_dict if dt_limits_dict else None
+
+ ##############################
+ # Units
+ units = {el for el in cleaned if el.startswith("Unit ")}
+ cleaned -= units
+ units_dict={}
+ for el in units:
+ split_info = el.split("Unit ")[-1].split(",")
+ s = split_info[0].split(" is ")
+
+ if len(s)==2:
+ long,short = [i.strip().strip("\"") for i in s]
+ elif len(s)==1:
+ long,short = None,s[0]
+ else:
+ print("Warning: Failed to split unit string {}".format(el))
+
+ to_cgs = (split_info[1].split())[3].strip().strip("\"")
+ code_units = split_info[2].split()
+ code_unit_type_num = int(code_units[3].strip().strip("\""))
+ code_unit_type = code_units[4].strip().strip("\"")
+ code_unit_cgs_value = code_units[9].strip().strip("\"").strip(")")
+ units_dict[long] = {
+ "long" : long,
+ "short" : short,
+ "to_cgs" : to_cgs,
+ "code_unit_type_num" : code_unit_type_num,
+ "code_unit_type" : code_unit_type,
+ "code_unit_cgs_value" : code_unit_cgs_value
+ }
+
+ units = {el for el in cleaned if el.startswith("Units: ")}
+ cleaned -= units
+ for el in units:
+ el = el[7:] # removes "Units: "
+ units_dict["units list"] = el.strip('Units:')
+
+ version_info_dict["units"] = units_dict
+
+ ##########################
+ # Nucleosynthesis sources:
+ # Split off
+ nucsyn_sources = {el for el in cleaned if el.startswith("Nucleosynthesis")}
+ cleaned -= nucsyn_sources
+
+ # Fill dict
+ nucsyn_sources_dict = {}
+ for el in nucsyn_sources:
+ split_info = el.split("Nucleosynthesis source")[-1].strip().split(" is ")
+ nucsyn_sources_dict[int(split_info[0])] = split_info[-1]
+
+ version_info_dict["nucleosynthesis_sources"] = (
+ nucsyn_sources_dict if nucsyn_sources_dict else None
+ )
+
+ ##########################
+ # miscellaneous:
+ # All those that I didn't catch with the above filters. Could try to get some more out though.
+ # TODO: filter a bit more.
+
+ misc_dict = {}
+
+ # Filter out git revision
+ git_revision = [el for el in cleaned if el.startswith("git revision")]
+ misc_dict["git_revision"] = (
+ git_revision[0].split("git revision ")[-1].replace('"', "")
+ )
+ cleaned = cleaned - set(git_revision)
+
+ # filter out git url
+ git_url = [el for el in cleaned if el.startswith("git URL")]
+ misc_dict["git_url"] = git_url[0].split("git URL ")[-1].replace('"', "")
+ cleaned = cleaned - set(git_url)
+
+ # filter out version
+ version = [el for el in cleaned if el.startswith("Version")]
+ misc_dict["version"] = str(version[0].split("Version ")[-1])
+ cleaned = cleaned - set(version)
+
+ git_branch = [el for el in cleaned if el.startswith("git branch")]
+ misc_dict["git_branch"] = git_branch[0].split("git branch ")[-1].replace('"', "")
+ cleaned = cleaned - set(git_branch)
+
+ build = [el for el in cleaned if el.startswith("Build")]
+ misc_dict["build"] = build[0].split("Build: ")[-1].replace('"', "")
+ cleaned = cleaned - set(build)
+
+ email = [el for el in cleaned if el.startswith("Email")]
+ misc_dict["email"] = email[0].split("Email ")[-1].split(",")
+ cleaned = cleaned - set(email)
+
+ other_items = set([el for el in cleaned if " is " in el])
+ cleaned = cleaned - other_items
+
+ for el in other_items:
+ split = el.split(" is ")
+ key = split[0].strip()
+ val = " is ".join(split[1:]).strip()
+ if key in misc_dict:
+ misc_dict[key + ' (alt)'] = val
+ else:
+ misc_dict[key] = val
+
+ misc_dict["uncaught"] = list(cleaned)
+
+ version_info_dict["miscellaneous"] = misc_dict if misc_dict else None
+ return version_info_dict
+
+ def minimum_stellar_mass(self):
+ """
+ Function to return the minimum stellar mass (in Msun) from binary_c.
+ """
+ if not self._minimum_stellar_mass:
+ self._minimum_stellar_mass = self.return_binary_c_version_info(parsed=True)["macros"]["BINARY_C_MINIMUM_STELLAR_MASS"]
+ return self._minimum_stellar_mass
--
GitLab