diff --git a/badges/test_coverage.svg b/badges/test_coverage.svg
index 91a5e773dbb94d146314a63950271bd5af8fad14..012a8497e3c9a2a79933b6a7062337837907a8d7 100644
--- a/badges/test_coverage.svg
+++ b/badges/test_coverage.svg
@@ -15,7 +15,7 @@
<g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11">
<text x="31.5" y="15" fill="#010101" fill-opacity=".3">coverage</text>
<text x="31.5" y="14">coverage</text>
- <text x="80" y="15" fill="#010101" fill-opacity=".3">68%</text>
- <text x="80" y="14">68%</text>
+ <text x="80" y="15" fill="#010101" fill-opacity=".3">70%</text>
+ <text x="80" y="14">70%</text>
</g>
</svg>
diff --git a/binarycpython/tests/main.py b/binarycpython/tests/main.py
index 84866198d6c56f762ebbad458fe1f1fd03644692..a5d43085c7b52e905c25c0f9fdcd8f619db012a3 100755
--- a/binarycpython/tests/main.py
+++ b/binarycpython/tests/main.py
@@ -135,6 +135,9 @@ from binarycpython.tests.tests_population_extensions.test_HPC import (
test_HPC_check_requirements,
test_HPC_set_status,
test_HPC_get_status,
+ test_HPC_job_task,
+ test_HPC_dir,
+ test_HPC_id_from_dir,
)
from binarycpython.tests.tests_population_extensions.test_condor import (
test_condorID,
@@ -142,6 +145,8 @@ from binarycpython.tests.tests_population_extensions.test_condor import (
test_condor_check_requirements,
test_set_condor_status,
test_get_condor_status,
+ test_condor_outfile,
+ test_make_condor_dirs,
)
from binarycpython.tests.tests_population_extensions.test_slurm import (
test_slurmID,
@@ -149,6 +154,8 @@ from binarycpython.tests.tests_population_extensions.test_slurm import (
test_slurm_check_requirements,
test_set_slurm_status,
test_get_slurm_status,
+ test_slurm_outfile,
+ test_make_slurm_dirs,
)
from binarycpython.tests.test_stellar_types import *
diff --git a/binarycpython/tests/tests_population_extensions/test_HPC.py b/binarycpython/tests/tests_population_extensions/test_HPC.py
index ccfc1a7e2ee8d242a6d055b2af708975de8f2076..7bb35a2f4f2e4a4fbc784c1dca9b9f8f25447640 100644
--- a/binarycpython/tests/tests_population_extensions/test_HPC.py
+++ b/binarycpython/tests/tests_population_extensions/test_HPC.py
@@ -26,7 +26,7 @@ import unittest
from binarycpython.utils.functions import Capturing, temp_dir
from binarycpython.utils.grid import Population
-TMP_DIR = temp_dir("tests", "test_condor")
+TMP_DIR = temp_dir("tests", "test_HPC")
shutil.rmtree(TMP_DIR)
os.makedirs(TMP_DIR, exist_ok=True)
@@ -392,13 +392,13 @@ class test_HPC_check_requirements(unittest.TestCase):
slurm_pop.grid_options["slurm"] = 1
# First the False test
- result_1 = slurm_pop.slurm_check_requirements()
+ result_1 = slurm_pop.HPC_check_requirements()
self.assertFalse(result_1[0])
self.assertTrue(len(result_1[1]) > 0)
# First the True test
slurm_pop.grid_options["slurm_dir"] = TMP_DIR
- result_2 = slurm_pop.slurm_check_requirements()
+ result_2 = slurm_pop.HPC_check_requirements()
self.assertTrue(result_2[0])
self.assertTrue(len(result_2[1]) == 0)
@@ -412,7 +412,7 @@ class test_HPC_check_requirements(unittest.TestCase):
"""
none_pop = Population(tmp_dir=TMP_DIR)
- result_none = none_pop.slurm_check_requirements()
+ result_none = none_pop.HPC_check_requirements()
self.assertTrue(result_none[0])
self.assertTrue(len(result_none[1]) == 0)
@@ -586,5 +586,153 @@ class test_HPC_get_status(unittest.TestCase):
self.assertEqual(status, "test_set_slurm_status")
+class test_HPC_job_task(unittest.TestCase):
+ """
+ Unittests for function HPC_get_status
+ """
+
+ def test_condor(self):
+ with Capturing() as output:
+ self._test_condor()
+
+ def _test_condor(self):
+ """
+ Unit test for HPC_job_task for condor
+ """
+
+ condor_pop = Population(tmp_dir=TMP_DIR)
+ condor_pop.grid_options["condor"] = 1
+ condor_pop.grid_options["condor_ClusterID"] = 2
+ condor_pop.grid_options["condor_Process"] = 3
+ condor_pop.grid_options["condor_dir"] = TMP_DIR
+
+ #
+ job_task = condor_pop.HPC_job_task()
+ self.assertEqual(job_task, 1)
+
+
+ def test_slurm(self):
+ with Capturing() as output:
+ self._test_slurm()
+
+ def _test_slurm(self):
+ """
+ Unit test for HPC_set_status for slurm
+ """
+
+ slurm_pop = Population(tmp_dir=TMP_DIR)
+ slurm_pop.grid_options["slurm"] = 1
+ slurm_pop.grid_options["slurm_jobid"] = 4
+ slurm_pop.grid_options["slurm_jobarrayindex"] = 5
+ slurm_pop.grid_options["slurm_dir"] = TMP_DIR
+
+ #
+ job_task = slurm_pop.HPC_job_task()
+ self.assertEqual(job_task, 1)
+
+
+class test_HPC_dir(unittest.TestCase):
+ """
+ Unittests for function HPC_dir
+ """
+
+ def test_condor(self):
+ with Capturing() as output:
+ self._test_condor()
+
+ def _test_condor(self):
+ """
+ Unit test for HPC_dir for condor
+ """
+
+ condor_pop = Population(tmp_dir=TMP_DIR)
+ condor_pop.grid_options["condor"] = 1
+ condor_pop.grid_options["condor_ClusterID"] = 2
+ condor_pop.grid_options["condor_Process"] = 3
+ condor_pop.grid_options["condor_dir"] = os.path.join(TMP_DIR, "condor")
+
+ #
+ self.assertEqual(condor_pop.HPC_dir(), os.path.join(TMP_DIR, "condor"))
+
+
+ def test_slurm(self):
+ with Capturing() as output:
+ self._test_slurm()
+
+ def _test_slurm(self):
+ """
+ Unit test for HPC_dir for slurm
+ """
+
+ slurm_pop = Population(tmp_dir=TMP_DIR)
+ slurm_pop.grid_options["slurm"] = 1
+ slurm_pop.grid_options["slurm_jobid"] = 4
+ slurm_pop.grid_options["slurm_jobarrayindex"] = 5
+ slurm_pop.grid_options["slurm_dir"] = os.path.join(TMP_DIR, "slurm")
+
+ #
+ self.assertEqual(slurm_pop.HPC_dir(), os.path.join(TMP_DIR, "slurm"))
+
+ def test_none(self):
+ with Capturing() as output:
+ self._test_slurm()
+
+ def _test_none(self):
+ """
+ Unit test for HPC_dir for none
+ """
+
+ none_pop = Population(tmp_dir=TMP_DIR)
+
+ #
+ self.assertEqual(none_pop.HPC_dir(), None)
+
+
+class test_HPC_id_from_dir(unittest.TestCase):
+ """
+ Unittests for function HPC_id_from_dir
+ """
+
+ def test_condor(self):
+ with Capturing() as output:
+ self._test_condor()
+
+ def _test_condor(self):
+ """
+ Unit test for HPC_id_from_dir for condor
+ """
+
+ condor_pop = Population(tmp_dir=TMP_DIR)
+ condor_pop.grid_options["condor"] = 1
+ condor_pop.grid_options["condor_ClusterID"] = 2
+ condor_pop.grid_options["condor_Process"] = 3
+ condor_pop.grid_options["condor_dir"] = TMP_DIR
+
+ # id
+ id_from_dir = condor_pop.HPC_id_from_dir(condor_pop.HPC_dir())
+
+ self.assertEqual(id_from_dir.strip(), str(2))
+
+ def test_slurm(self):
+ with Capturing() as output:
+ self._test_slurm()
+
+ def _test_slurm(self):
+ """
+ Unit test for HPC_id_from_dir for slurm
+ """
+
+ slurm_pop = Population(tmp_dir=TMP_DIR)
+ slurm_pop.grid_options["slurm"] = 1
+ slurm_pop.grid_options["slurm_jobid"] = 4
+ slurm_pop.grid_options["slurm_jobarrayindex"] = 5
+ slurm_pop.grid_options["slurm_dir"] = os.path.join(TMP_DIR)
+
+ # id
+ id_from_dir = slurm_pop.HPC_id_from_dir(slurm_pop.HPC_dir())
+
+ self.assertEqual(id_from_dir.strip(), str(4))
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/binarycpython/tests/tests_population_extensions/test_condor.py b/binarycpython/tests/tests_population_extensions/test_condor.py
index 7a07e38ed7626effa2034a59ccdfd6040095a8c9..a3a85a3fd118d4e80462b1a9993e18c07d56aaee 100644
--- a/binarycpython/tests/tests_population_extensions/test_condor.py
+++ b/binarycpython/tests/tests_population_extensions/test_condor.py
@@ -3,8 +3,6 @@ Unit classes for the _condor module population extension
TODO: condorpath
TODO: condor_status_file
-TODO: condor_outfile
-TODO: make_condor_dirs
TODO: condor_grid
TODO: condor_queue_stats
"""
@@ -188,5 +186,64 @@ class test_get_condor_status(unittest.TestCase):
self.assertEqual(status, "test_get_condor_status")
+class test_condor_outfile(unittest.TestCase):
+ """
+ Unittests for function condor_outfile
+ """
+
+ def test_condor_outfile(self):
+ with Capturing() as output:
+ self._test_condor_outfile()
+
+ def _test_condor_outfile(self):
+ """
+ Unit test for condor_outfile for condor
+ """
+
+ condor_pop = Population(tmp_dir=TMP_DIR)
+ condor_pop.grid_options["condor"] = 1
+ condor_pop.grid_options["condor_ClusterID"] = 2
+ condor_pop.grid_options["condor_Process"] = 3
+ condor_pop.grid_options["condor_dir"] = TMP_DIR
+
+ outfile = condor_pop.condor_outfile()
+ condor_id = condor_pop.condorID()
+ self.assertEqual(outfile, os.path.abspath(os.path.join(condor_pop.grid_options["condor_dir"], "results", "{}.gz".format(condor_id))))
+
+
+class test_make_condor_dirs(unittest.TestCase):
+ """
+ Unittests for function make_condor_dirs
+ """
+
+ def test_make_condor_dirs(self):
+ with Capturing() as output:
+ self._test_make_condor_dirs()
+
+ def _test_make_condor_dirs(self):
+ """
+ Unit test for make_condor_dirs for condor
+ """
+
+ condor_pop = Population(tmp_dir=TMP_DIR)
+ condor_pop.grid_options["condor"] = 1
+ condor_pop.grid_options["condor_ClusterID"] = 2
+ condor_pop.grid_options["condor_Process"] = 3
+ condor_pop.grid_options["condor_dir"] = TMP_DIR
+
+ shutil.rmtree(TMP_DIR)
+ os.makedirs(TMP_DIR)
+
+ #
+ condor_pop.make_condor_dirs()
+
+ self.assertTrue(os.path.isdir(os.path.join(TMP_DIR, "stdout")))
+ self.assertTrue(os.path.isdir(os.path.join(TMP_DIR, "stderr")))
+ self.assertTrue(os.path.isdir(os.path.join(TMP_DIR, "log")))
+ self.assertTrue(os.path.isdir(os.path.join(TMP_DIR, "results")))
+ self.assertTrue(os.path.isdir(os.path.join(TMP_DIR, "status")))
+ self.assertTrue(os.path.isdir(os.path.join(TMP_DIR, "snapshots")))
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/binarycpython/tests/tests_population_extensions/test_slurm.py b/binarycpython/tests/tests_population_extensions/test_slurm.py
index 1a78d33a7486aa40e2b8480b0266f06d50c1a038..e3e2b1d9dba2762a90efb455626e5d8109a7e901 100644
--- a/binarycpython/tests/tests_population_extensions/test_slurm.py
+++ b/binarycpython/tests/tests_population_extensions/test_slurm.py
@@ -3,8 +3,6 @@ Unittests for slurm module
TODO: slurmpath
TODO: slurm_status_file
-TODO: slurm_outfile
-TODO: make_slurm_dirs
TODO: slurm_grid
TODO: slurm_queue_stats
"""
@@ -185,5 +183,62 @@ class test_get_slurm_status(unittest.TestCase):
self.assertEqual(status, "test_set_slurm_status")
+class test_slurm_outfile(unittest.TestCase):
+ """
+ Unittests for function slurm_outfile
+ """
+
+ def test_slurm_outfile(self):
+ with Capturing() as output:
+ self._test_slurm_outfile()
+
+ def _test_slurm_outfile(self):
+ """
+ Unit test for slurm_outfile for slurm
+ """
+
+ slurm_pop = Population(tmp_dir=TMP_DIR)
+ slurm_pop.grid_options["slurm"] = 1
+ slurm_pop.grid_options["slurm_jobid"] = 4
+ slurm_pop.grid_options["slurm_jobarrayindex"] = 5
+ slurm_pop.grid_options["slurm_dir"] = TMP_DIR
+
+ outfile = slurm_pop.slurm_outfile()
+ slurm_id = slurm_pop.slurmID()
+ self.assertEqual(outfile, os.path.abspath(os.path.join(slurm_pop.grid_options["slurm_dir"], "results", "{}.gz".format(slurm_id))))
+
+
+class test_make_slurm_dirs(unittest.TestCase):
+ """
+ Unittests for function slurm_outfile
+ """
+
+ def test_make_slurm_dirs(self):
+ with Capturing() as output:
+ self._test_make_slurm_dirs()
+
+ def _test_make_slurm_dirs(self):
+ """
+ Unit test for slurm_outfile for slurm
+ """
+
+ slurm_pop = Population(tmp_dir=TMP_DIR)
+ slurm_pop.grid_options["slurm"] = 1
+ slurm_pop.grid_options["slurm_jobid"] = 4
+ slurm_pop.grid_options["slurm_jobarrayindex"] = 5
+ slurm_pop.grid_options["slurm_dir"] = TMP_DIR
+
+ shutil.rmtree(TMP_DIR)
+ os.makedirs(TMP_DIR)
+
+ #
+ slurm_pop.make_slurm_dirs()
+
+ self.assertTrue(os.path.isdir(os.path.join(TMP_DIR, "stdout")))
+ self.assertTrue(os.path.isdir(os.path.join(TMP_DIR, "stderr")))
+ self.assertTrue(os.path.isdir(os.path.join(TMP_DIR, "results")))
+ self.assertTrue(os.path.isdir(os.path.join(TMP_DIR, "status")))
+ self.assertTrue(os.path.isdir(os.path.join(TMP_DIR, "snapshots")))
+
if __name__ == "__main__":
unittest.main()
diff --git a/binarycpython/utils/ensemble.py b/binarycpython/utils/ensemble.py
index 1808ed7c31595b021a4c43fb5d1b377af3d26674..d2414c4f67aa9f90eb8f881694678d6b21daa5e4 100644
--- a/binarycpython/utils/ensemble.py
+++ b/binarycpython/utils/ensemble.py
@@ -168,6 +168,7 @@ def load_ensemble(
# simplejson is faster than standard json and "just works"
# on the big Moe set in 37s
data = simplejson.load(file_object, object_hook=_hook)
+ file_object.close()
# standard json module
# on the big Moe set takes 42s
@@ -175,6 +176,7 @@ def load_ensemble(
# object_hook=_hook)
elif filetype == "msgpack":
data = msgpack.load(file_object, object_hook=_hook)
+ file_object.close()
if timing:
print(