backport changes to use locked_close() etc.

binarycpython/utils/dataIO.py

@@ -93,7 +93,7 @@ class dataIO():
                 object.grid_ensemble_results["metadata"] = {}
 
             # add datestamp
-            object.grid_ensemble_results["metadata"]['save_population_time'] = self.now() 
+            object.grid_ensemble_results["metadata"]['save_population_time'] = self.now()
 
             # add extra metadata
             object.add_system_metadata()
@@ -252,7 +252,7 @@ class dataIO():
 
         # set the start position for new stars
         self.grid_options['start_at'] = newpop.grid_options['start_at']
-        
+
         print("Loaded from snapshot at {file} : {nstars} stars, start at star {nstart}".format(
             file=file,
             nstars=0,#self.grid_options[''],
@@ -489,6 +489,25 @@ class dataIO():
         if self.HPC_job():
             self.HPC_set_status(string)
 
+    def locked_close(self,
+                     file,
+                     lock):
+        """
+        Partner function to locked_open()
+
+        Closes and unlocks the file
+        """
+        print("locked_close:\nFile now = {}".format(file))
+        print("Lock now = {}".format(lock))
+        if file:
+            print("Close file {}".format(file))
+            file.close()
+        if lock:
+            print("Unlock {}".format(lock))
+            lock.unlock()
+        print("File now = {}".format(file))
+        print("Lock now = {}".format(lock))
+
     def locked_open(self,
                     file,
                     mode="r",
@@ -504,11 +523,11 @@ class dataIO():
          We check whether the file exists, in which case just return
          (None,None), and if we cannot get the lock we return
          (None,None).
-        
+
         If the file does not exist, we keep trying to lock until it does.
 
          To do the locking, we use flufl.lock which is NFS safe.
-        
+
         Args:
             lock_lifetime: (passed to flufl.lock.Lock()) default 60 seconds.
                            It should take less than this time to write the file.
@@ -518,57 +537,70 @@ class dataIO():
 
         Returns:
             (file_object, lock_object) tuple.
-            If the file was not opened, returns (None,None). 
+            If the file was not opened, returns (None,None).
         """
-        
+
         # set the lockfile path: this should be the same
         # for all processes, so it's just the original file
         # plus the lock_suffix
         lockfile = file + lock_suffix
+        print("lockfile={}".format(lockfile))
 
         while True:
             # if the file exists, just return
-            if os.path.isfile(lockfile):
+            if os.path.isfile(file):
+                print("file {} already exists".format(file))
                 return (None,None)
-        
+
             # make the lock object by opening the lockfile
             lock = flufl.lock.Lock(lockfile,
                                    default_timeout=lock_timeout)
+            print("post-lock: {}",format(lock))
 
             if lock:
                 # we have the lockfile, so set the lifetime and try to lock it
                 lock.lifetime = datetime.timedelta(seconds=lock_lifetime)
                 try:
+                    print("try to lock",format(lock))
                     lock.lock()
+                    print("locked {}".format(lock))
                 except:
                     pass
-                
+
                 # if we acquired the lock, try to open the file
                 if lock.is_locked:
-                    
+                    print("is locked {}",lock)
+
                     # check it hasn't been opened in the meantime
                     if os.path.isfile(file):
+                        print("in the meantime, {} has been opened".format(file))
                         lock.unlock()
+                        print("unlocked {} : return None,None".format(lock))
                         return (None,None)
 
                     # All is apparently ok: file is locked and does not
                     # exist so open the file
                     try:
+                        print("Try to open {}",file)
                         f = open(file,
                                  mode=mode,
                                  encoding=encoding,
                                  **kwargs)
+                        print("Return locked file {}, {}".format(f,lock))
                         return (f,lock)
-                    
+
                     # error on open should be fatal
                     except Exception as e:
                         print("Error in locked_open() : {}",e)
                         if fatal_open_errors:
+                            print("fatal EXIT")
                             self.exit(1)
                         else:
+                            print("unlock {}".format(lock))
                             lock.unlock()
+                            print("unlocked {} return None,None".format(lock))
                             return (None,None)
-            
+
             # failed to lock this time, keep trying
             # (we shouldn't lock up the CPU because the timeout is non-zero)
             continue

binarycpython/utils/grid.py

@@ -613,21 +613,20 @@ class Population(analytics,
                 # open locked settings file, then output if we get the lock
                 (f,lock) = self.locked_open(settings_fullname,
                                             mode="w")
-                if lock:
+                if lock and f:
                     self.verbose_print(
                         "Writing settings to {}".format(settings_fullname),
                         self.grid_options["verbosity"],
                         1,
                     )
-                    if f:
-                        json.dump(
-                            all_info_cleaned,
-                            f,
-                            indent=indent,
-                            default=binaryc_json_serializer,
-                            ensure_ascii=ensure_ascii
-                        )
-                    lock.unlock()
+                    json.dump(
+                        all_info_cleaned,
+                        f,
+                        indent=indent,
+                        default=binaryc_json_serializer,
+                        ensure_ascii=ensure_ascii
+                    )
+                self.locked_close(f,lock)
                 return settings_fullname
             else:
                 msg = "Exporting all info without passing a value for `outfile` requires custom_options['data_dir'] to be present. That is not the cause. Either set the `data_dir` or pass a value for `outfile` "

binarycpython/utils/gridcode.py

@@ -385,7 +385,7 @@ class gridcode():
                 )
 
                 # Add condition failed action:
-                if self.grid_options["verbosity"] >= 3:
+                if self.grid_options["verbosity"] >= 4:
                     self._add_code(
                         'print("Grid generator: Condition for {name} not met!")'.format(
                             name=grid_variable["name"]