add option HPC_rebuild_joinlist to build the joinlist automatically

binarycpython/utils/HPC.py

@@ -24,13 +24,26 @@ class HPC(condor,slurm):
     def HPC_joinfiles(self,joinlist=None):
         """
         Function to load in the joinlist to a list and return it.
+
+        If grid_options['HPC_rebuild_joinlist'] is True, we rebuild it.
         """
+        prefix = os.path.join(self.HPC_dir(),
+                              'results')
+        if self.grid_options['HPC_rebuild_joinlist'] == 1:
+            # we should rebuild the joinlist from the
+            # files we find at the prefix directory
+            list = glob.glob(str(prefix) + '/*.gz')
+            return list
+
         if joinlist is None:
             joinlist = self.grid_options['joinlist']
         try:
             f = open(joinlist,'r',encoding='utf-8')
             list = f.read().splitlines()
             f.close()
+
+            if self.grid_options['HPC_prepend_dir_to_joinlist'] = True:
+                list = [os.path.join(prefix,file) for file in list]
         except:
             print("Failed to open joinlist at {list}".format(list=joinlist))
             self.exit(code=1)
@@ -92,7 +105,8 @@ class HPC(condor,slurm):
         """
         Function to return True if we're running an HPC (Slurm or Condor) job, False otherwise.
         """
-        if self.grid_options['slurm'] > 0 or self.grid_options['condor'] > 0:
+        if self.grid_options['slurm'] > 0 or \
+           self.grid_options['condor'] > 0:
             x = True
         else:
             x = False
@@ -354,14 +368,22 @@ class HPC(condor,slurm):
         """
         Function to return an HPC job's snapshot filename.
         """
-        if self.grid_options['slurm'] > 0:
-            file = os.path.join(self.grid_options['slurm_dir'],
-                                'snapshots',
-                                self.HPCjobID() + '.gz')
-        elif self.grid_options['condor'] > 0:
-            file = os.path.join(self.grid_options['condor_dir'],
+        if self.HPCjob():
+            file = os.path.join(self.HPC_dir,
                                 'snapshots',
                                 self.HPCjobID() + '.gz')
         else:
             file = None
         return file
+
+    def HPC_dir(self):
+        """
+        Function to return an HPC job's directory.
+        """
+        if self.grid_options['slurm'] > 0:
+            d = self.grid_options['slurm_dir']
+        elif self.grid_options['condor'] > 0:
+            d = self.grid_options['condor_dir']
+        else:
+            d = None
+        return d