From f12cc475c95e09d9d5c532f585ad9c61b37d1ebd Mon Sep 17 00:00:00 2001
From: Robert Izzard <r.izzard@surrey.ac.uk>
Date: Mon, 22 Nov 2021 11:02:02 +0000
Subject: [PATCH] backported more changes from the working HPC/condor machine
---
binarycpython/utils/HPC.py | 34 ++++++++++++++---------------
binarycpython/utils/dataIO.py | 41 +++++++++++++++--------------------
2 files changed, 35 insertions(+), 40 deletions(-)
diff --git a/binarycpython/utils/HPC.py b/binarycpython/utils/HPC.py
index c335811bb..ad91a04e1 100644
--- a/binarycpython/utils/HPC.py
+++ b/binarycpython/utils/HPC.py
@@ -35,7 +35,7 @@ class HPC(condor,slurm):
else:
n = None
return int(n)
-
+
def HPC_make_joiningfile(self,
id=None,
dir=None,
@@ -56,7 +56,7 @@ class HPC(condor,slurm):
Returns:
True if the file is made, False otherwise.
-
+
"""
# defaults
@@ -86,16 +86,16 @@ class HPC(condor,slurm):
encoding="utf-8")
# write to it if we are first to obtain unique access
- x = False
- if lock:
- if f:
- for i in range(0,n):
- f.write(os.path.join(prefix,
- "{i}.gz\n".format(i=i)))
- x = True
- lock.unlock()
+ if lock and f:
+ for i in range(0,n):
+ f.write(os.path.join(prefix,
+ "{i}.gz\n".format(i=i)))
+ x = True
+ else:
+ x = False
+ self.locked_close(f,lock)
return x
-
+
def HPC_joinfiles(self,joinlist=None):
"""
Function to load in the joinlist to a list and return it.
@@ -192,10 +192,10 @@ class HPC(condor,slurm):
else:
x = False
return x
-
+
def HPC_job_task(self):
"""
- Function to return the HPC task number, which is 1 when setting
+ Function to return the HPC task number, which is 1 when setting
up and running the scripts, 2 when joining data.
"""
if self.grid_options['slurm'] > 0:
@@ -205,7 +205,7 @@ class HPC(condor,slurm):
else:
x = 0
return x
-
+
def HPC_job_type(self):
"""
Function to return a string telling us the type of an HPC job, i.e.
@@ -296,7 +296,7 @@ class HPC(condor,slurm):
dirs = []
return dirs
-
+
def HPC_grid(self,makejoiningfile=True):
"""
Function to call the appropriate HPC grid function
@@ -313,7 +313,7 @@ class HPC(condor,slurm):
id is not None:
print("Making joiningfile from HPC_grid().")
self.HPC_make_joiningfile()
-
+
if self.grid_options['slurm'] > 0:
x = self.slurm_grid()
elif self.grid_options['condor'] > 0:
@@ -322,7 +322,7 @@ class HPC(condor,slurm):
x =None # should not happen
return x
-
+
def HPC_check_requirements(self):
"""
Function to check HPC option requirements have been met. Returns a tuple: (True,"") if all is ok, (False,<warning string>) otherwise.
diff --git a/binarycpython/utils/dataIO.py b/binarycpython/utils/dataIO.py
index ccc0c4a66..a609960fc 100644
--- a/binarycpython/utils/dataIO.py
+++ b/binarycpython/utils/dataIO.py
@@ -509,7 +509,7 @@ class dataIO():
print("Lock now = {}".format(lock))
def locked_open(self,
- file,
+ filename,
mode="r",
encoding="utf-8",
lock_suffix='.lock',
@@ -518,13 +518,13 @@ class dataIO():
fatal_open_errors=True,
**kwargs):
"""
- Wrapper for Python's open(file)
+ Wrapper for Python's open(filename)
- We check whether the file exists, in which case just return
- (None,None), and if we cannot get the lock we return
- (None,None).
+ We check whether the file's lockfile already exists, in which
+ case just return (None,None), and if we cannot obtain a
+ lock on the file we also return (None,None).
- If the file does not exist, we keep trying to lock until it does.
+ If the file does not exist, we keep trying to lock until it does.
To do the locking, we use flufl.lock which is NFS safe.
@@ -543,17 +543,19 @@ class dataIO():
# set the lockfile path: this should be the same
# for all processes, so it's just the original file
# plus the lock_suffix
- lockfile = file + lock_suffix
- print("lockfile={}".format(lockfile))
+ lockfilename = filename + lock_suffix
+ print("lockfile={}".format(lockfilename))
while True:
# if the file exists, just return
- if os.path.isfile(file):
- print("file {} already exists".format(file))
+ if os.path.isfile(lockfilename):
+ print("lockfile at {} already exists (corresponding to file at {})".format(
+ lockfilename,
+ filename))
return (None,None)
# make the lock object by opening the lockfile
- lock = flufl.lock.Lock(lockfile,
+ lock = flufl.lock.Lock(lockfilename,
default_timeout=lock_timeout)
print("post-lock: {}",format(lock))
@@ -569,20 +571,13 @@ class dataIO():
# if we acquired the lock, try to open the file
if lock.is_locked:
- print("is locked {}",lock)
+ print("{} is locked by {} to {}",
+ filename,lock,lockfilename)
- # check it hasn't been opened in the meantime
- if os.path.isfile(file):
- print("in the meantime, {} has been opened".format(file))
- lock.unlock()
- print("unlocked {} : return None,None".format(lock))
- return (None,None)
-
- # All is apparently ok: file is locked and does not
- # exist so open the file
+ # All is apparently ok: file is locked
try:
- print("Try to open {}",file)
- f = open(file,
+ print("Try to open file at {}",filename)
+ f = open(filename,
mode=mode,
encoding=encoding,
**kwargs)
--
GitLab