From f97399f873a8536d5f0a67ee7cafa321bc1620ff Mon Sep 17 00:00:00 2001
From: Robert Izzard <r.izzard@surrey.ac.uk>
Date: Mon, 8 Nov 2021 18:41:53 +0000
Subject: [PATCH] improved handling of directories, and provide more checks to
make sure they exist
---
binarycpython/utils/functions.py | 6 ++
binarycpython/utils/grid.py | 76 ++++++++++++++++----
binarycpython/utils/grid_options_defaults.py | 1 +
binarycpython/utils/spacing_functions.py | 2 +-
4 files changed, 69 insertions(+), 16 deletions(-)
diff --git a/binarycpython/utils/functions.py b/binarycpython/utils/functions.py
index 093ab1753..51058652f 100644
--- a/binarycpython/utils/functions.py
+++ b/binarycpython/utils/functions.py
@@ -1553,3 +1553,9 @@ def load_logfile(logfile: str) -> None:
event_list.append(" ".join(split_line[9:]))
print(event_list)
+
+def dir_ok(dir):
+ """
+ Function to test if we can read and write to a dir that must exist. Return True if all is ok, False otherwise.
+ """
+ return os.access(dir, os.F_OK) and os.access(dir, os.R_OK | os.W_OK)
diff --git a/binarycpython/utils/grid.py b/binarycpython/utils/grid.py
index b08016701..04fb95085 100644
--- a/binarycpython/utils/grid.py
+++ b/binarycpython/utils/grid.py
@@ -79,20 +79,21 @@ from binarycpython.utils.custom_logging_functions import (
)
from binarycpython.utils.functions import (
- get_defaults,
- remove_file,
+ check_if_in_shell,
+ conv_time_units,
+ dir_ok,
filter_arg_dict,
+ format_number,
+ get_ANSI_colours,
+ get_defaults,
get_help_all,
- return_binary_c_version_info,
- verbose_print,
get_moe_di_stefano_dataset,
- trem,
- conv_time_units,
mem_use,
- get_ANSI_colours,
- check_if_in_shell,
- format_number,
+ remove_file,
+ return_binary_c_version_info,
timedelta,
+ trem,
+ verbose_print,
)
from binarycpython.utils.ensemble import (
binaryc_json_serializer,
@@ -174,10 +175,6 @@ class Population:
# Set the options that are passed at creation of the object
self.set(**kwargs)
- # default status_dir to be tmp_dir if it doesn't exist
- if not self.grid_options['status_dir']:
- self.grid_options['status_dir'] = self.grid_options['tmp_dir']
-
# Load Moe and di Stefano options
self.grid_options["Moe2017_options"] = copy.deepcopy(
moe_di_stefano_default_options
@@ -1076,6 +1073,54 @@ class Population:
"d", [mem] * self.grid_options["num_processes"]
)
+ # check tmp_dir exists
+ if self.grid_options['tmp_dir'] is None \
+ or not os.path.isdir(self.grid_options['tmp_dir']):
+ print("grid_options['tmp_dir'] is not set or it is not a directory : this should point to a temporary directory location, preferably local to your CPUs")
+ sys.exit(1)
+
+
+ # set default directory locations
+
+ # default status_dir to be tmp_dir if it doesn't exist
+ if self.grid_options['status_dir'] is None:
+ self.grid_options['status_dir'] = os.path.join(self.grid_options['tmp_dir'],
+ 'status')
+
+ # default cache_dir based on tmp_dir
+ if self.grid_options['cache_dir'] is None:
+ self.grid_options['cache_dir'] = os.path.join(self.grid_options['tmp_dir'],
+ 'cache')
+
+ # check slurm_dir is set should this be required
+ if self.grid_options['slurm'] > 0 and \
+ self.grid_options['slurm_dir'] is None:
+ print("You have set slurm=",self.grid_options['slurm'],"but not set slurm_dir. Please set it and try again.")
+ sys.exit(1)
+
+ # make list of directories we want to use
+ dirs = ['tmp_dir','status_dir','cache_dir']
+ if self.grid_options['slurm']>0:
+ dirs += ['slurm_dir']
+ if self.grid_options['condor']>0:
+ dirs += ['condor_dir']
+
+ # try to make directories if they don't exist
+ for dir in dirs:
+ path = self.grid_options[dir]
+ if path != None:
+ pathlib.Path(path).mkdir(exist_ok=True,
+ parents=True)
+
+ # check directories exist and can be written to
+ for dir in dirs:
+ path = self.grid_options[dir]
+ if path != None and dir_ok(path) == False:
+ print("Directory",dir,", currently set to",path,", cannot be written to. Please check that this directory is correct and you have write access.")
+ sys.exit(1)
+
+ return
+
def clean(self) -> None:
"""
Clean the contents of the population object so it can be reused.
@@ -1164,7 +1209,8 @@ class Population:
# if we're running a slurm grid, exit here
# unless we're joining
- if self.grid_options["slurm"] >= 1 and self.grid_options['evolution_type'] != 'join':
+ if self.grid_options["slurm"] >= 1 and \
+ self.grid_options['evolution_type'] != 'join':
sys.exit()
##
@@ -5048,7 +5094,7 @@ eccentricity3=0
# build the grid command
grid_command = [
- "/usr/bin/env",
+ os.path.join("/usr","bin","env"),
sys.executable,
str(lib_programname.get_path_executed_script()),
] + sys.argv[1:] + [
diff --git a/binarycpython/utils/grid_options_defaults.py b/binarycpython/utils/grid_options_defaults.py
index cfd9460ad..aa4be941d 100644
--- a/binarycpython/utils/grid_options_defaults.py
+++ b/binarycpython/utils/grid_options_defaults.py
@@ -34,6 +34,7 @@ grid_options_defaults_dict = {
"parse_function": None, # Function to parse the output with.
"multiplicity_fraction_function": 0, # Which multiplicity fraction function to use. 0: None, 1: Arenou 2010, 2: Rhagavan 2010, 3: Moe and di Stefano 2017
"tmp_dir": temp_dir(), # Setting the temp dir of the program
+ "cache_dir" : None, # Cache location
"status_dir" : None, #
"_main_pid": -1, # Placeholder for the main process id of the run.
"save_ensemble_chunks": True, # Force the ensemble chunk to be saved even if we are joining a thread (just in case the joining fails)
diff --git a/binarycpython/utils/spacing_functions.py b/binarycpython/utils/spacing_functions.py
index b76af0d8b..daec4c41f 100644
--- a/binarycpython/utils/spacing_functions.py
+++ b/binarycpython/utils/spacing_functions.py
@@ -200,7 +200,7 @@ def const_dt(self,cachedir=None,usecache=True,**kwargs):
"""
if cachedir == None:
- cachedir = self.grid_options['tmp_dir'] + '/const_dt_cache'
+ cachedir = self.grid_options['cache_dir'] + '/const_dt_cache'
cache = Cache(cachedir)
@cache.memoize()
def _const_dt(cachedir,
--
GitLab