From 500273d47621fb7cb6814467ad3ecfb6cfb79ef6 Mon Sep 17 00:00:00 2001
From: dh00601 <dh00601@surrey.ac.uk>
Date: Sun, 9 Jan 2022 18:22:13 +0000
Subject: [PATCH] reformatted and making tests for hpc slurm and condor
---
badges/test_coverage.svg | 4 +-
binarycpython/utils/dicts.py | 23 +++++--
binarycpython/utils/functions.py | 21 +++++--
binarycpython/utils/grid.py | 18 ++++--
binarycpython/utils/plot_functions.py | 1 +
.../utils/population_extensions/HPC.py | 61 +++++++++++--------
.../Moe_di_Stefano_2017.py | 4 +-
.../utils/population_extensions/analytics.py | 1 +
.../utils/population_extensions/cache.py | 10 +--
.../utils/population_extensions/condor.py | 16 +++--
.../utils/population_extensions/dataIO.py | 34 +++++++----
.../distribution_functions.py | 6 +-
.../population_extensions/grid_logging.py | 12 ++--
.../grid_options_defaults.py | 14 +++--
.../utils/population_extensions/gridcode.py | 21 +++++--
.../utils/population_extensions/metadata.py | 12 +---
.../utils/population_extensions/slurm.py | 18 +++---
.../spacing_functions.py | 6 +-
docs/source/conf.py | 4 +-
19 files changed, 182 insertions(+), 104 deletions(-)
diff --git a/badges/test_coverage.svg b/badges/test_coverage.svg
index 6d68f472b..91a5e773d 100644
--- a/badges/test_coverage.svg
+++ b/badges/test_coverage.svg
@@ -15,7 +15,7 @@
<g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11">
<text x="31.5" y="15" fill="#010101" fill-opacity=".3">coverage</text>
<text x="31.5" y="14">coverage</text>
- <text x="80" y="15" fill="#010101" fill-opacity=".3">64%</text>
- <text x="80" y="14">64%</text>
+ <text x="80" y="15" fill="#010101" fill-opacity=".3">68%</text>
+ <text x="80" y="14">68%</text>
</g>
</svg>
diff --git a/binarycpython/utils/dicts.py b/binarycpython/utils/dicts.py
index 611af1606..68f7c9d33 100644
--- a/binarycpython/utils/dicts.py
+++ b/binarycpython/utils/dicts.py
@@ -13,13 +13,14 @@ import numpy as np
ALLOWED_NUMERICAL_TYPES = (int, float, complex, np.number)
UNION_ALLOWED_NUMERICAL_TYPES = Union[int, float, complex, np.number]
+
def keys_to_floats(input_dict: dict) -> dict:
"""
Function to convert all the keys of the dictionary to float to float
we need to convert keys to floats:
this is ~ a factor 10 faster than David's ``recursive_change_key_to_float`` routine, probably because this version only does the float conversion, nothing else.
-
+
Args:
input_dict: dict of which we want to turn all the keys to float types if possible
@@ -118,7 +119,9 @@ def recursive_change_key_to_string(input_dict: dict, custom_format: str = "{:g}"
# If dictionary type, call function again
if isinstance(input_dict[key], (dict, collections.OrderedDict)):
- new_dict[string_key] = recursive_change_key_to_string(input_dict[key], custom_format)
+ new_dict[string_key] = recursive_change_key_to_string(
+ input_dict[key], custom_format
+ )
else:
new_dict[string_key] = input_dict[key]
@@ -201,7 +204,9 @@ def multiply_float_values(input_dict, factor, ignore=None):
"""
path = []
- _recursive_normalize_floats(path, input_dict, factor, parent=input_dict, ignore=ignore)
+ _recursive_normalize_floats(
+ path, input_dict, factor, parent=input_dict, ignore=ignore
+ )
def subtract_dicts(dict_1: dict, dict_2: dict) -> dict:
@@ -507,7 +512,9 @@ def merge_dicts(dict_1: dict, dict_2: dict) -> dict:
try:
new_dict[key] = int(dict_1[key]) + int(dict_2[key])
except ValueError as e:
- msg = "{}: Failed to convert string (either '{}' or '{}') to an int".format(key, dict_1[key], dict_2[key])
+ msg = "{}: Failed to convert string (either '{}' or '{}') to an int".format(
+ key, dict_1[key], dict_2[key]
+ )
raise ValueError(msg) from e
# string-float clash : convert both to floats and save
@@ -520,7 +527,9 @@ def merge_dicts(dict_1: dict, dict_2: dict) -> dict:
try:
new_dict[key] = float(dict_1[key]) + float(dict_2[key])
except ValueError as e:
- msg = "{}: Failed to convert string (either '{}' or '{}') to an float".format(key, dict_1[key], dict_2[key])
+ msg = "{}: Failed to convert string (either '{}' or '{}') to an float".format(
+ key, dict_1[key], dict_2[key]
+ )
raise ValueError(msg) from e
# If the above cases have not dealt with it, then we should raise an error
@@ -637,7 +646,9 @@ def update_dicts(dict_1: dict, dict_2: dict) -> dict:
# See whether the types are actually the same
if not type(dict_1[key]) is type(dict_2[key]):
# Exceptions:
- if isinstance(dict_1[key], ALLOWED_NUMERICAL_TYPES) and isinstance(dict_2[key], ALLOWED_NUMERICAL_TYPES):
+ if isinstance(dict_1[key], ALLOWED_NUMERICAL_TYPES) and isinstance(
+ dict_2[key], ALLOWED_NUMERICAL_TYPES
+ ):
new_dict[key] = dict_2[key]
else:
diff --git a/binarycpython/utils/functions.py b/binarycpython/utils/functions.py
index 26c1ba91b..aec64d77c 100644
--- a/binarycpython/utils/functions.py
+++ b/binarycpython/utils/functions.py
@@ -149,7 +149,7 @@ def get_ANSI_colours():
def mem_use():
"""
- Return current process memory use in MB. (Takes no arguments)
+ Return current process memory use in MB. (Takes no arguments)
Note: this is per-thread only.
"""
@@ -271,6 +271,7 @@ def isfloat(x: Union[str, float, int]):
except ValueError:
return False
+
def isint(x: Union[str, float, int]):
"""
Function to return `True` if the "number" x, which could be a string, is an int, otherwise return `False`.
@@ -338,7 +339,7 @@ def pad_output_distribution(dist: dict, binwidth: float):
return dist
-class catchtime():
+class catchtime:
"""
Context manager to calculate time spent
"""
@@ -415,6 +416,7 @@ def call_binary_c_config(argument):
# utility functions
########################################################
+
def verbose_print(
message: str, verbosity: int, minimal_verbosity: int, newline: str = "\n"
) -> None:
@@ -1104,7 +1106,11 @@ def write_binary_c_parameter_descriptions_to_rst_file(output_file: str) -> None:
build_info = make_build_text()
if not output_file.endswith(".rst"):
- raise ValueError("Filename ({}) doesn't end with .rst, please provide a proper filename.".format(output_file))
+ raise ValueError(
+ "Filename ({}) doesn't end with .rst, please provide a proper filename.".format(
+ output_file
+ )
+ )
with open(output_file, "w", encoding="utf-8") as f:
@@ -1126,7 +1132,14 @@ def write_binary_c_parameter_descriptions_to_rst_file(output_file: str) -> None:
argdict = arguments_dict[el]["parameters"][arg]
print("| **Parameter**: {}".format(argdict["param_name"]), file=f)
- print("| **Description**: {}".format(argdict["description"].replace("|Rout/Rin-1|", "abs(Rout/Rin-1)")), file=f)
+ print(
+ "| **Description**: {}".format(
+ argdict["description"].replace(
+ "|Rout/Rin-1|", "abs(Rout/Rin-1)"
+ )
+ ),
+ file=f,
+ )
if "parameter_value_input_type" in argdict:
print(
"| **Parameter input type**: {}".format(
diff --git a/binarycpython/utils/grid.py b/binarycpython/utils/grid.py
index df3340849..c21dc4486 100644
--- a/binarycpython/utils/grid.py
+++ b/binarycpython/utils/grid.py
@@ -54,7 +54,7 @@ from binarycpython.utils.functions import (
get_help_all,
mem_use,
timedelta,
- now
+ now,
)
from binarycpython.utils.ensemble import (
binaryc_json_serializer,
@@ -70,14 +70,22 @@ from binarycpython.utils.dicts import (
from binarycpython.utils.population_extensions.analytics import analytics
from binarycpython.utils.population_extensions.cache import cache
from binarycpython.utils.population_extensions.dataIO import dataIO
-from binarycpython.utils.population_extensions.distribution_functions import distribution_functions
+from binarycpython.utils.population_extensions.distribution_functions import (
+ distribution_functions,
+)
from binarycpython.utils.population_extensions.grid_logging import grid_logging
-from binarycpython.utils.population_extensions.grid_options_defaults import grid_options_defaults
+from binarycpython.utils.population_extensions.grid_options_defaults import (
+ grid_options_defaults,
+)
from binarycpython.utils.population_extensions.gridcode import gridcode
from binarycpython.utils.population_extensions.HPC import HPC
from binarycpython.utils.population_extensions.metadata import metadata
-from binarycpython.utils.population_extensions.Moe_di_Stefano_2017 import Moe_di_Stefano_2017
-from binarycpython.utils.population_extensions.spacing_functions import spacing_functions
+from binarycpython.utils.population_extensions.Moe_di_Stefano_2017 import (
+ Moe_di_Stefano_2017,
+)
+from binarycpython.utils.population_extensions.spacing_functions import (
+ spacing_functions,
+)
from binarycpython.utils.population_extensions.version_info import version_info
from binarycpython import _binary_c_bindings
diff --git a/binarycpython/utils/plot_functions.py b/binarycpython/utils/plot_functions.py
index 0bc8e792e..4ee603a88 100644
--- a/binarycpython/utils/plot_functions.py
+++ b/binarycpython/utils/plot_functions.py
@@ -402,6 +402,7 @@ def dummy():
return None
+
def parse_function_hr_diagram(output: str):
"""
Parsing function for the HR plotting routine
diff --git a/binarycpython/utils/population_extensions/HPC.py b/binarycpython/utils/population_extensions/HPC.py
index 6d00fb104..fd55d76fc 100644
--- a/binarycpython/utils/population_extensions/HPC.py
+++ b/binarycpython/utils/population_extensions/HPC.py
@@ -52,7 +52,12 @@ class HPC(condor, slurm):
return int(n)
def HPC_make_joiningfile(
- self, hpc_jobid=None, hpc_dir=None, n=None, overwrite=False, error_on_overwrite=False
+ self,
+ hpc_jobid=None,
+ hpc_dir=None,
+ n=None,
+ overwrite=False,
+ error_on_overwrite=False,
):
"""
Function to make the joiningfile file that contains the filenames of results from each job. When all these exist, we can join.
@@ -87,7 +92,11 @@ class HPC(condor, slurm):
# this to remove any asynchronicity
lines = []
for i in range(0, n):
- lines += [os.path.join(prefix, "{hpc_jobid}.{i}.gz\n".format(hpc_jobid=hpc_jobid, i=i))]
+ lines += [
+ os.path.join(
+ prefix, "{hpc_jobid}.{i}.gz\n".format(hpc_jobid=hpc_jobid, i=i)
+ )
+ ]
string = "".join(lines)
# check the joiningfile doesn't exist
@@ -177,7 +186,11 @@ class HPC(condor, slurm):
)
)
except Exception as e:
- print("Failed to open joinlist at {joinlist} : {e}".format(joinlist=joinlist, e=e))
+ print(
+ "Failed to open joinlist at {joinlist} : {e}".format(
+ joinlist=joinlist, e=e
+ )
+ )
self.exit(code=1)
return joinlist
@@ -293,31 +306,33 @@ class HPC(condor, slurm):
"""
Function to return an HPC (Slurm or Condor) job id in the form x.y. Returns None if not an HPC job.
"""
+
if self.grid_options["slurm"] > 0:
- hpc_id = self.slurmID()
+ hpc_jobid = self.slurmID()
elif self.grid_options["condor"] > 0:
- hpc_id = self.condorID()
+ hpc_jobid = self.condorID()
else:
# not an HPC job
- hpc_id = None
- return hpc_id
+ hpc_jobid = None
+
+ return hpc_jobid
def HPC_jobID_tuple(self):
"""
Return the job ID as a tuple of ints, (x,y), or (None,None) on failure
"""
+
hpc_jobid = self.HPC_jobID()
- if hpc_jobid is None or id.startswith("None"):
- t = [None, None]
- elif self.HPC_job():
- print("JOBID", id)
- t = id.split(".")
- if not t[0]:
- t[0] = None
- if not t[1]:
- t[1] = None
- return tuple(t)
+ if hpc_jobid is None or hpc_jobid.startswith("None"):
+ split_hpc_jobid = [None, None]
+ else:
+ split_hpc_jobid = hpc_jobid.split(".")
+ if not split_hpc_jobid[0]:
+ split_hpc_jobid[0] = None
+ if not split_hpc_jobid[1]:
+ split_hpc_jobid[1] = None
+ return tuple(split_hpc_jobid)
def HPC_set_status(self, string):
"""
@@ -325,20 +340,14 @@ class HPC(condor, slurm):
Arguments:
string : the new contents of the status file
-
- Returns:
- True if the status was set, False otherwise.
- (As returned by either the appropriate Condor or Slurm function)
"""
if self.grid_options["slurm"] > 0:
- status = self.set_slurm_status(string)
+ self.set_slurm_status(string)
elif self.grid_options["condor"] > 0:
- status = self.set_condor_status(string)
+ self.set_condor_status(string)
else:
- status = None
-
- return status
+ pass
def HPC_get_status(self, job_id=None, job_index=None, hpc_dir=None):
"""
diff --git a/binarycpython/utils/population_extensions/Moe_di_Stefano_2017.py b/binarycpython/utils/population_extensions/Moe_di_Stefano_2017.py
index cffe9abdc..2988cbf3e 100644
--- a/binarycpython/utils/population_extensions/Moe_di_Stefano_2017.py
+++ b/binarycpython/utils/population_extensions/Moe_di_Stefano_2017.py
@@ -27,7 +27,6 @@ from binarycpython.utils.population_extensions.grid_options_defaults import (
from binarycpython.utils import moe_di_stefano_2017_data
-
class Moe_di_Stefano_2017:
"""
Extension to the population grid object that contains functionality to handle handle the Moe & distefano distributions
@@ -311,7 +310,8 @@ class Moe_di_Stefano_2017:
Moecache["period_distributions"].append(
[
float(logmass),
- float(logperiods[-1]) + 0.5 * dlog10P, # TODO: why this shift? to center it?
+ float(logperiods[-1])
+ + 0.5 * dlog10P, # TODO: why this shift? to center it?
json_data["log10M1"][logmass]["logP"][logperiods[-1]][
"normed_bin_frac_p_dist"
]
diff --git a/binarycpython/utils/population_extensions/analytics.py b/binarycpython/utils/population_extensions/analytics.py
index 4437c215f..b6fba1e3d 100644
--- a/binarycpython/utils/population_extensions/analytics.py
+++ b/binarycpython/utils/population_extensions/analytics.py
@@ -6,6 +6,7 @@ The class extension for the population object that contains analytics functional
import time
+
class analytics:
"""
Extension for the Population class containing the functions for analytics
diff --git a/binarycpython/utils/population_extensions/cache.py b/binarycpython/utils/population_extensions/cache.py
index b047f2f7f..ea744ee81 100644
--- a/binarycpython/utils/population_extensions/cache.py
+++ b/binarycpython/utils/population_extensions/cache.py
@@ -55,7 +55,7 @@ class cache:
def __init__(self, *args, **kwargs):
"""
Init function for the spacing_functions class
-
+
TODO: is this class necesarry to be defined *within* the cache class? can't it just be outside?
"""
@@ -148,7 +148,9 @@ class cache:
x = func.split(".")
modulename = "binarycpython.utils.population_extensions." + x[0]
module = importlib.import_module(modulename)
- _method = eval("module.{}.{}".format(x[0], x[1])) # TODO: we can do this differently with some .get call instead of eval
+ _method = eval(
+ "module.{}.{}".format(x[0], x[1])
+ ) # TODO: we can do this differently with some .get call instead of eval
_wrapped = getattr(_method, "__wrapped__", False)
# if function is wrapped...
@@ -250,10 +252,11 @@ class cache:
_method = eval("module.{}.{}".format(x[0], x[1]))
if testargs:
+
def _func_wrap(*args, **kwargs):
"""
wrap to return args and kwargs
-
+
TODO: i think this function can be defined elsewhere
"""
@@ -296,7 +299,6 @@ class cache:
)
)
-
"""
Cache speed test of function distribution_functions.powerlaw_constant
0 1 2 4 8 16 32 64 128 256
diff --git a/binarycpython/utils/population_extensions/condor.py b/binarycpython/utils/population_extensions/condor.py
index 5275b657e..ab3996aa7 100644
--- a/binarycpython/utils/population_extensions/condor.py
+++ b/binarycpython/utils/population_extensions/condor.py
@@ -18,6 +18,7 @@ import pathlib
import datasize
import lib_programname
+
class condor:
"""
Extension for the Population class containing the code for Condor grid runs
@@ -54,17 +55,21 @@ class condor:
Return the condor status file corresponding to the ClusterID and Process, which default to grid_options condor_ClusterID and condor_Process, respectively.
"""
return os.path.join(
- self.condorpath("status", condor_dir=condor_dir), self.condorID(ClusterID, Process)
+ self.condorpath("status", condor_dir=condor_dir),
+ self.condorID(ClusterID, Process),
)
def condor_check_requirements(self):
"""
Function to check whether the condor parameters in grid_options have been set appropriately.
"""
- if self.grid_options["condor"] > 0 and self.grid_options["condor_dir"] is None:
+ if self.grid_options["condor"] > 0 and (
+ self.grid_options["condor_dir"] is None
+ or not os.path.isdir(self.grid_options["condor_dir"])
+ ):
return (
False,
- "You have set condor={condor} but not set condor_dir (which is {condor_dir}). Please set it and try again.".format(
+ "You have set condor={condor} but not set condor_dir ({condor_dir}) correctly. Please set it and try again.".format(
condor=self.grid_options["condor"],
condor_dir=self.grid_options["condor_dir"],
),
@@ -124,7 +129,9 @@ class condor:
try:
path = pathlib.Path(
- self.condor_status_file(condor_dir=condor_dir, ClusterID=ClusterID, Process=Process)
+ self.condor_status_file(
+ condor_dir=condor_dir, ClusterID=ClusterID, Process=Process
+ )
)
# print("path={}".format(path))
# print("size={}".format(path.stat().st_size))
@@ -473,7 +480,6 @@ queue {njobs}
)
raise Exception(err_msg)
-
if len(std_err) > 0:
print(
"{red}{err}{reset}".format(
diff --git a/binarycpython/utils/population_extensions/dataIO.py b/binarycpython/utils/population_extensions/dataIO.py
index 0723da759..c7e4e94db 100644
--- a/binarycpython/utils/population_extensions/dataIO.py
+++ b/binarycpython/utils/population_extensions/dataIO.py
@@ -18,10 +18,7 @@ import msgpack
import flufl.lock
import compress_pickle
-from binarycpython.utils.ensemble import (
- ensemble_file_type,
- ensemble_compression
-)
+from binarycpython.utils.ensemble import ensemble_file_type, ensemble_compression
from binarycpython.utils.dicts import (
merge_dicts,
)
@@ -51,7 +48,11 @@ class dataIO:
return os.access(directory, os.F_OK) and os.access(directory, os.R_OK | os.W_OK)
def save_population_object(
- self, population_object=None, filename=None, confirmation=True, compression="gzip"
+ self,
+ population_object=None,
+ filename=None,
+ confirmation=True,
+ compression="gzip",
):
"""
Save pickled Population object to file at filename or, if filename is None, whatever is set at self.grid_options['save_population_object']
@@ -132,7 +133,9 @@ class dataIO:
# restore data
population_object.shared_memory = shared_memory
population_object.grid_options["_system_generator"] = system_generator
- del population_object.grid_ensemble_results["metadata"]["save_population_time"]
+ del population_object.grid_ensemble_results["metadata"][
+ "save_population_time"
+ ]
population_object.grid_options["store_memaddr"] = _store_memaddr
population_object.persistent_data_memory_dict = persistent_data_memory_dict
@@ -179,7 +182,9 @@ class dataIO:
# special cases
maxmem = 0
- if "max_memory_use" in refpop.grid_ensemble_results.get("metadata", {}) and "max_memory_use" in newpop.grid_ensemble_results.get("metadata", {}):
+ if "max_memory_use" in refpop.grid_ensemble_results.get(
+ "metadata", {}
+ ) and "max_memory_use" in newpop.grid_ensemble_results.get("metadata", {}):
maxmem = max(
refpop.grid_ensemble_results["metadata"]["max_memory_use"],
newpop.grid_ensemble_results["metadata"]["max_memory_use"],
@@ -245,11 +250,14 @@ class dataIO:
newpop = self.load_population_object(filename)
- if 'total_count' in newpop.grid_options:
+ if "total_count" in newpop.grid_options:
n = newpop.grid_options["total_count"]
elif "_count" in newpop.grid_options:
n = newpop.grid_options["_count"]
- elif 'metadata' in newpop.grid_ensemble_results and "_count" in newpop.grid_ensemble_results["metadata"]:
+ elif (
+ "metadata" in newpop.grid_ensemble_results
+ and "_count" in newpop.grid_ensemble_results["metadata"]
+ ):
n = newpop.grid_ensemble_results["metadata"]["_count"]
else:
n = -1
@@ -355,7 +363,9 @@ class dataIO:
if not file_type:
print(
- "Unable to determine file type from ensemble filename {} : it should be .json or .msgpack.".format(output_file)
+ "Unable to determine file type from ensemble filename {} : it should be .json or .msgpack.".format(
+ output_file
+ )
)
self.exit(code=1)
else:
@@ -371,7 +381,9 @@ class dataIO:
)
)
elif file_type == "msgpack":
- f = self.open(output_file, "w") # TODO: i think something is going wrong here. not sure but doing msgpack and .gz e.g gives an error about str input rather than bytes. i think this is because the self.open does not take into account that the msgpack stream requires different properties.
+ f = self.open(
+ output_file, "w"
+ ) # TODO: i think something is going wrong here. not sure but doing msgpack and .gz e.g gives an error about str input rather than bytes. i think this is because the self.open does not take into account that the msgpack stream requires different properties.
# msgpack output
msgpack.dump(data, f)
diff --git a/binarycpython/utils/population_extensions/distribution_functions.py b/binarycpython/utils/population_extensions/distribution_functions.py
index c26c929e6..774ae540c 100644
--- a/binarycpython/utils/population_extensions/distribution_functions.py
+++ b/binarycpython/utils/population_extensions/distribution_functions.py
@@ -140,7 +140,9 @@ class distribution_functions:
return powerlaw_const
@cachetools.cachedmethod(
- lambda self: self.caches["population_extensions._distribution_functions.powerlaw_constant"]
+ lambda self: self.caches[
+ "population_extensions._distribution_functions.powerlaw_constant"
+ ]
)
def powerlaw_constant(
self,
@@ -1221,7 +1223,7 @@ class distribution_functions:
)[0]
for q in np.arange(0.15, 0.950001, 0.1):
- if qmin<= q <= qmax:
+ if qmin <= q <= qmax:
qdata[q] = Moecache["rinterpolator_q"].interpolate(
[np.log10(options[m]), np.log10(options[p]), q]
)[0]
diff --git a/binarycpython/utils/population_extensions/grid_logging.py b/binarycpython/utils/population_extensions/grid_logging.py
index ce23420ab..19bf96cd1 100644
--- a/binarycpython/utils/population_extensions/grid_logging.py
+++ b/binarycpython/utils/population_extensions/grid_logging.py
@@ -16,15 +16,14 @@ from binarycpython.utils.functions import (
format_number,
trem,
remove_file,
- verbose_print
+ verbose_print,
)
from binarycpython.utils.population_extensions.grid_options_defaults import secs_per_day
from binarycpython.utils.custom_logging_functions import (
binary_c_log_code,
create_and_load_logging_function,
- autogen_C_logging_code
-
+ autogen_C_logging_code,
)
@@ -32,6 +31,7 @@ class grid_logging:
"""
The class extension for the population object that contains logging functionality
"""
+
def __init__(self, **kwargs):
"""
Init function for the grid_logging class
@@ -326,7 +326,7 @@ class grid_logging:
blue=self.ANSI_colours["blue"],
cmdline=cmdline_string,
reset=self.ANSI_colours["reset"],
- system_dict=system_dict
+ system_dict=system_dict,
)
)
@@ -346,7 +346,9 @@ class grid_logging:
# Pass the rest to the original verbose print
verbose_print(*args, **kwargs)
- def _boxed(self, *stringlist, colour="yellow on black", boxchar="*", separator="\n"):
+ def _boxed(
+ self, *stringlist, colour="yellow on black", boxchar="*", separator="\n"
+ ):
"""
Function to output a list of strings in a single box.
diff --git a/binarycpython/utils/population_extensions/grid_options_defaults.py b/binarycpython/utils/population_extensions/grid_options_defaults.py
index 6eac46ebe..388eaca63 100644
--- a/binarycpython/utils/population_extensions/grid_options_defaults.py
+++ b/binarycpython/utils/population_extensions/grid_options_defaults.py
@@ -104,9 +104,7 @@ class grid_options_defaults:
##########################
# binary_c files
##########################
- "_binary_c_executable": os.path.join(
- os.environ["BINARY_C"], "binary_c"
- ),
+ "_binary_c_executable": os.path.join(os.environ["BINARY_C"], "binary_c"),
"_binary_c_shared_library": os.path.join(
os.environ["BINARY_C"], "src", "libbinary_c.so"
),
@@ -536,12 +534,14 @@ class grid_options_defaults:
# Check input
if not output_file.endswith(".rst"):
- msg="Filename doesn't end with .rst, please provide a proper filename"
+ msg = "Filename doesn't end with .rst, please provide a proper filename"
raise ValueError(msg)
# M&S options
moe_di_stefano_default_options = self.get_Moe_di_Stefano_2017_default_options()
- moe_di_stefano_default_options_description = self.get_Moe_di_Stefano_2017_default_options_description()
+ moe_di_stefano_default_options_description = (
+ self.get_Moe_di_Stefano_2017_default_options_description()
+ )
with self.open(output_file, "w") as f:
print("Population grid code options", file=f)
@@ -585,7 +585,9 @@ class grid_options_defaults:
"The following options are not meant to be changed by the user, as these options are used and set internally by the object itself. The description still is provided, but just for documentation purposes.",
)
- def print_option_descriptions(self, filehandle, options, descriptions, title, extra_text):
+ def print_option_descriptions(
+ self, filehandle, options, descriptions, title, extra_text
+ ):
"""
Function to print the description of an option
"""
diff --git a/binarycpython/utils/population_extensions/gridcode.py b/binarycpython/utils/population_extensions/gridcode.py
index cfbc301cd..a795e55a7 100644
--- a/binarycpython/utils/population_extensions/gridcode.py
+++ b/binarycpython/utils/population_extensions/gridcode.py
@@ -96,7 +96,7 @@ class gridcode:
Results in a generated file that contains a system_generator function.
# TODO: make sure running systems with multiplicity 3+ is also possible.
- # TODO: there is a lot of things going on in this function. Make sure to describe clearly what happens here.
+ # TODO: there is a lot of things going on in this function. Make sure to describe clearly what happens here.
"""
self.verbose_print("Generating grid code", self.grid_options["verbosity"], 1)
@@ -822,7 +822,8 @@ class gridcode:
# run the hook function, only if given
if self.grid_options["dry_run_hook"]:
self._add_code(
- "self.grid_options['dry_run_hook'](self, parameter_dict)\n", indent=1
+ "self.grid_options['dry_run_hook'](self, parameter_dict)\n",
+ indent=1,
)
else:
# or pass
@@ -1001,6 +1002,7 @@ class gridcode:
the rest of the function
Examples::
+
name = 'lnM_1'
parameter_name:
@@ -1016,7 +1018,8 @@ class gridcode:
longname:
Long name of parameter
- Examples:
+ Examples::
+
longname = 'Primary mass'
range:
@@ -1024,6 +1027,7 @@ class gridcode:
get the values from
Examples::
+
range = [math.log(m_min), math.log(m_max)]
samplerfunc:
@@ -1031,6 +1035,7 @@ class gridcode:
You can either use a real function, or a string representation of a function call.
Examples::
+
samplerfunc = "self.const_linear(math.log(m_min), math.log(m_max), {})".format(resolution['M_1'])
precode:
@@ -1038,6 +1043,7 @@ class gridcode:
sampling function (i.e. a value for lnM_1 is chosen already)
Examples::
+
precode = 'M_1=math.exp(lnM_1);'
postcode:
@@ -1046,21 +1052,24 @@ class gridcode:
probdist:
Function determining the probability that gets assigned to the sampled parameter
- Examples:
+ Examples::
+
probdist = 'self.Kroupa2001(M_1)*M_1'
dphasevol:
part of the parameter space that the total probability is calculated with. Put to -1
if you want to ignore any dphasevol calculations and set the value to 1
- Examples::"
+ Examples::
+
dphasevol = 'dlnM_1'
condition:
condition that has to be met in order for the grid generation to continue
Examples::
- condition = 'self.grid_options['binary']==1'
+
+ condition = "self.grid_options['binary']==1"
gridtype:
Method on how the value range is sampled. Can be either 'edge' (steps starting at
diff --git a/binarycpython/utils/population_extensions/metadata.py b/binarycpython/utils/population_extensions/metadata.py
index fab614311..dd10a24d5 100644
--- a/binarycpython/utils/population_extensions/metadata.py
+++ b/binarycpython/utils/population_extensions/metadata.py
@@ -49,16 +49,10 @@ class metadata:
self.grid_ensemble_results["metadata"]["platform_uname"] = list(
platform.uname()
)
- self.grid_ensemble_results["metadata"][
- "platform_machine"
- ] = platform.machine()
+ self.grid_ensemble_results["metadata"]["platform_machine"] = platform.machine()
self.grid_ensemble_results["metadata"]["platform_node"] = platform.node()
- self.grid_ensemble_results["metadata"][
- "platform_release"
- ] = platform.release()
- self.grid_ensemble_results["metadata"][
- "platform_version"
- ] = platform.version()
+ self.grid_ensemble_results["metadata"]["platform_release"] = platform.release()
+ self.grid_ensemble_results["metadata"]["platform_version"] = platform.version()
self.grid_ensemble_results["metadata"][
"platform_processor"
] = platform.processor()
diff --git a/binarycpython/utils/population_extensions/slurm.py b/binarycpython/utils/population_extensions/slurm.py
index 65f772412..7e300ff97 100644
--- a/binarycpython/utils/population_extensions/slurm.py
+++ b/binarycpython/utils/population_extensions/slurm.py
@@ -63,10 +63,14 @@ class slurm:
"""
Function to check whether the slurm parameters in grid_options have been set appropriately.
"""
- if self.grid_options["slurm"] > 0 and self.grid_options["slurm_dir"] is None:
+
+ if self.grid_options["slurm"] > 0 and (
+ self.grid_options["slurm_dir"] is None
+ or not os.path.isdir(self.grid_options["slurm_dir"])
+ ):
return (
False,
- "You have set slurm={slurm} but not set slurm_dir (which is {slurm_dir}). Please set it and try again.".format(
+ "You have set slurm={slurm} but not set slurm_dir ({slurm_dir}) correctly. Please set it and try again.".format(
slurm=self.grid_options["slurm"],
slurm_dir=self.grid_options["slurm_dir"],
),
@@ -96,15 +100,15 @@ class slurm:
with self.open(idfile, "w", encoding="utf-8") as fjobid:
fjobid.write("{jobid}\n".format(jobid=self.grid_options["slurm_jobid"]))
fjobid.close()
- self.NFS_flush_hach(idfile)
+ self.NFS_flush_hack(idfile)
# save slurm status
- file = self.slurm_status_file(slurm_dir=slurm_dir)
- if file:
- with self.open(file, "w", encoding="utf-8") as f:
+ status_file = self.slurm_status_file(slurm_dir=slurm_dir)
+ if status_file:
+ with self.open(status_file, "w", encoding="utf-8") as f:
f.write(string)
f.close()
- self.NFS_fluch_hack(file)
+ self.NFS_flush_hack(status_file)
def get_slurm_status(self, jobid=None, jobarrayindex=None, slurm_dir=None):
"""
diff --git a/binarycpython/utils/population_extensions/spacing_functions.py b/binarycpython/utils/population_extensions/spacing_functions.py
index e2161e502..242fe7c0e 100644
--- a/binarycpython/utils/population_extensions/spacing_functions.py
+++ b/binarycpython/utils/population_extensions/spacing_functions.py
@@ -377,12 +377,12 @@ class spacing_functions:
):
"""
first thing to do is make a stellar lifetime table
-
+
we should use the bse_options_json passed in
so our lifetime_population uses the same physics
as the main grid
- TODO: Describe this function better with arguments and
+ TODO: Describe this function better with arguments and
"""
# convert bse_options to dict
@@ -522,7 +522,7 @@ class spacing_functions:
"""
Function to get a mass given a time, calculated by using the interpolator_time_mass
"""
-
+
return (
10.0
** interpolator_time_mass.interpolate([math.log10(linear_time)])[0]
diff --git a/docs/source/conf.py b/docs/source/conf.py
index a8a47d898..c9a2bfcb8 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -27,11 +27,12 @@ from git import Repo
from binarycpython.utils.functions import (
write_binary_c_parameter_descriptions_to_rst_file,
- call_binary_c_config
+ call_binary_c_config,
)
from binarycpython.utils.grid import Population
+
def generate_browser_url(ssh_url, branch=None):
"""
Function to generate the browser url for a git repo
@@ -95,6 +96,7 @@ Using binary_c with bit branch {binary_c_git_branch}: git revision: {binary_c_gi
with open("_templates/footer.html", "w") as outfile_filehandle:
outfile_filehandle.write(formatted_text)
+
#
def patched_m2r2_setup(app):
"""
--
GitLab