From 479e17f297461826309e0eeb23b8678164bfc73f Mon Sep 17 00:00:00 2001
From: Robert Izzard <r.izzard@surrey.ac.uk>
Date: Mon, 8 Nov 2021 18:59:06 +0000
Subject: [PATCH] improve sub-dir handling
---
binarycpython/utils/grid.py | 41 +++++++++++++++----------------------
1 file changed, 17 insertions(+), 24 deletions(-)
diff --git a/binarycpython/utils/grid.py b/binarycpython/utils/grid.py
index 56de13f4f..0d5dca6d8 100644
--- a/binarycpython/utils/grid.py
+++ b/binarycpython/utils/grid.py
@@ -404,7 +404,6 @@ class Population:
def set_status(self,
string,
- dir="process_status",
format_statment="process_{}.txt",
ID=None):
"""
@@ -418,7 +417,6 @@ class Population:
with open(
os.path.join(
self.grid_options["status_dir"],
- dir,
format_statment.format(ID),
),
"w",
@@ -1031,7 +1029,7 @@ class Population:
# manually specify number of cores made available
self.grid_options['num_processes'] = self.grid_options['num_cores']
- def _pre_run_cleanup(self) -> None:
+ def _pre_run_setup(self) -> None:
"""
Function to clean up some stuff in the grid before a run (like results, ensemble results etc)
"""
@@ -1100,6 +1098,8 @@ class Population:
# make list of directories we want to use
dirs = ['tmp_dir','status_dir','cache_dir']
+
+
if self.grid_options['slurm']>0:
dirs += ['slurm_dir']
if self.grid_options['condor']>0:
@@ -1109,8 +1109,7 @@ class Population:
for dir in dirs:
path = self.grid_options[dir]
if path != None:
- pathlib.Path(path).mkdir(exist_ok=True,
- parents=True)
+ os.makedirs(path,exist_ok=True)
# check directories exist and can be written to
for dir in dirs:
@@ -1119,18 +1118,28 @@ class Population:
print("Directory {dir} currently set to {path} cannot be written to. Please check that this directory is correct and you have write access.".format(dir=dir,path=path))
sys.exit(1)
+ # Make sure the subdirs of the tmp dir exist
+ subdirs = ['failed_systems','current_system','process_summary','runtime_systems']
+ for subdir in subdirs:
+ path = os.path.join(self.grid_options["tmp_dir"], subdir)
+ os.makedirs(path,exist_ok=True)
+ if dir_ok(path) == False:
+ print("Sub-Directory {subdir} (in tmp_dir) currently set to {path} cannot be written to. Please check that this directory is correct and you have write access.".format(subdir=subdir,path=path))
+ sys.exit(1)
+
+
return
def clean(self) -> None:
"""
Clean the contents of the population object so it can be reused.
- Calling _pre_run_cleanup()
+ Calling _pre_run_setup()
TODO: decide to deprecate this function
"""
- self._pre_run_cleanup()
+ self._pre_run_setup()
def evolve(self) -> None:
"""
@@ -1148,7 +1157,7 @@ class Population:
"""
# Just to make sure we don't have stuff from a previous run hanging around
- self._pre_run_cleanup()
+ self._pre_run_setup()
if self.grid_options["slurm"]>=1:
self.grid_options["symlink_latest_gridcode"] = False
@@ -2137,22 +2146,6 @@ class Population:
function
"""
- # Make sure the subdirs of the tmp dir exist:
- os.makedirs(
- os.path.join(self.grid_options["tmp_dir"], "failed_systems"), exist_ok=True
- )
- os.makedirs(
- os.path.join(self.grid_options["tmp_dir"], "current_system"), exist_ok=True
- )
- os.makedirs(
- os.path.join(self.grid_options["tmp_dir"], "process_status"), exist_ok=True
- )
- os.makedirs(
- os.path.join(self.grid_options["tmp_dir"], "process_summary"), exist_ok=True
- )
- os.makedirs(
- os.path.join(self.grid_options["tmp_dir"], "runtime_systems"), exist_ok=True
- )
# Check for parse function
if not self.grid_options["parse_function"]:
--
GitLab