From 8b25626435e6c65c090a6bde3843fbaafbc8e62a Mon Sep 17 00:00:00 2001
From: Robert Izzard <r.izzard@surrey.ac.uk>
Date: Tue, 16 Nov 2021 20:17:54 +0000
Subject: [PATCH] add option HPC_rebuild_joinlist to build the joinlist
automatically
---
binarycpython/utils/HPC.py | 36 +++++++++++++++++++++++++++++-------
1 file changed, 29 insertions(+), 7 deletions(-)
diff --git a/binarycpython/utils/HPC.py b/binarycpython/utils/HPC.py
index 8e13ea576..ada6b0cc0 100644
--- a/binarycpython/utils/HPC.py
+++ b/binarycpython/utils/HPC.py
@@ -24,13 +24,26 @@ class HPC(condor,slurm):
def HPC_joinfiles(self,joinlist=None):
"""
Function to load in the joinlist to a list and return it.
+
+ If grid_options['HPC_rebuild_joinlist'] is True, we rebuild it.
"""
+ prefix = os.path.join(self.HPC_dir(),
+ 'results')
+ if self.grid_options['HPC_rebuild_joinlist'] == 1:
+ # we should rebuild the joinlist from the
+ # files we find at the prefix directory
+ list = glob.glob(str(prefix) + '/*.gz')
+ return list
+
if joinlist is None:
joinlist = self.grid_options['joinlist']
try:
f = open(joinlist,'r',encoding='utf-8')
list = f.read().splitlines()
f.close()
+
+ if self.grid_options['HPC_prepend_dir_to_joinlist'] = True:
+ list = [os.path.join(prefix,file) for file in list]
except:
print("Failed to open joinlist at {list}".format(list=joinlist))
self.exit(code=1)
@@ -92,7 +105,8 @@ class HPC(condor,slurm):
"""
Function to return True if we're running an HPC (Slurm or Condor) job, False otherwise.
"""
- if self.grid_options['slurm'] > 0 or self.grid_options['condor'] > 0:
+ if self.grid_options['slurm'] > 0 or \
+ self.grid_options['condor'] > 0:
x = True
else:
x = False
@@ -354,14 +368,22 @@ class HPC(condor,slurm):
"""
Function to return an HPC job's snapshot filename.
"""
- if self.grid_options['slurm'] > 0:
- file = os.path.join(self.grid_options['slurm_dir'],
- 'snapshots',
- self.HPCjobID() + '.gz')
- elif self.grid_options['condor'] > 0:
- file = os.path.join(self.grid_options['condor_dir'],
+ if self.HPCjob():
+ file = os.path.join(self.HPC_dir,
'snapshots',
self.HPCjobID() + '.gz')
else:
file = None
return file
+
+ def HPC_dir(self):
+ """
+ Function to return an HPC job's directory.
+ """
+ if self.grid_options['slurm'] > 0:
+ d = self.grid_options['slurm_dir']
+ elif self.grid_options['condor'] > 0:
+ d = self.grid_options['condor_dir']
+ else:
+ d = None
+ return d
--
GitLab