From 911134679c18aeaeb08cc58d78997292421206c9 Mon Sep 17 00:00:00 2001
From: Izzard <ri0005@orca.eps.surrey.ac.uk>
Date: Fri, 19 Nov 2021 11:28:44 +0000
Subject: [PATCH] added NFS file locking with flufl.lock()
---
binarycpython/utils/HPC.py | 39 +++++++++++++++++++++++++++-----------
1 file changed, 28 insertions(+), 11 deletions(-)
diff --git a/binarycpython/utils/HPC.py b/binarycpython/utils/HPC.py
index 7b543fb5c..33343107a 100644
--- a/binarycpython/utils/HPC.py
+++ b/binarycpython/utils/HPC.py
@@ -9,6 +9,7 @@
Condor API.
"""
import datetime
+import flufl.lock
import glob
import os
import pathlib
@@ -79,17 +80,33 @@ class HPC(condor,slurm):
self.exit(code=1)
x = False
else:
- # write the joiningfile
- print("Making joiningfile at {file} with range 0 to {n}".format(
- file=file,
- n=n
- ))
- with open(file,"w",encoding="utf-8") as f:
- for i in range(0,n):
- f.write(os.path.join(prefix,
- "{i}.gz\n".format(i=i)))
- f.close()
- x = True
+ # lock the joiningfile
+ lock = flufl.lock.Lock(file)
+
+ # writing this file should not take > 1 hour
+ lock.lifetime = datetime.timedelta(seconds=3600)
+
+ if os.path.isfile(file):
+ print("Cannot make joiningfile at {file} because it already exists.".format(file=file))
+ x = False
+ else:
+ if lock.is_locked:
+ # write the joiningfile
+ print("Making joiningfile at {file} with range 0 to {n}".format(
+ file=file,
+ n=n
+ ))
+ with open(file,"w",encoding="utf-8") as f:
+ for i in range(0,n):
+ f.write(os.path.join(prefix,
+ "{i}.gz\n".format(i=i)))
+ x = True
+ else:
+ print("Could not lock joiningfile at {file}".format(file=file))
+ x = False
+
+ # unlock the file
+ lock.unlock()
return x
def HPC_joinfiles(self,joinlist=None):
--
GitLab