now killing the child or parent process kills them both

Slurm restarts work, just take your old (unfinished) Slurm dir and set it with slurm_restart_dir=<whatever>

binarycpython/utils/functions.py

@@ -113,6 +113,7 @@ def get_ANSI_colours():
         "magenta": Fore.MAGENTA,
         "white": Fore.WHITE,
         "black": Fore.BLACK,
+        "bold": Style.BRIGHT
     }
 
     background_colours = {

binarycpython/utils/grid_options_defaults.py

@@ -44,7 +44,7 @@ grid_options_defaults_dict = {
     "_commandline_input": "",
     "log_runtime_systems": 0,  # whether to log the runtime of the systems (1 file per thread. stored in the tmp_dir)
     "_actually_evolve_system": True,  # Whether to actually evolve the systems of just act as if. for testing. used in _process_run_population_grid
-    "max_queue_size": 1000,  # Maximum size of the system call queue.
+    "max_queue_size": 0,  # Maximum size of the system call queue. Set to 0 for this to be calculated automatically
     "run_zero_probability_system": True,  # Whether to run the zero probability systems
     "_zero_prob_stars_skipped": 0,
     "ensemble_factor_in_probability_weighted_mass": False,  # Whether to multiply the ensemble results by 1/probability_weighted_mass
@@ -150,6 +150,10 @@ grid_options_defaults_dict = {
     'save_snapshots' : True, # if True, save snapshots on SIGINT
     'restore_from_snapshot_file' : None, # file to restore from
     'restore_from_snapshot_dir' : None, # dir to restore from
+    'exit_code' : 0, # return code
+    'stop_queue' : False,
+    '_killed' : False,
+    '_queue_done' : False,
     ## Monte carlo type evolution
     # TODO: make MC options
     ## Evolution from source file
@@ -196,6 +200,7 @@ grid_options_defaults_dict = {
     "slurm_array": None,  # override for --array, useful for rerunning jobs
     "slurm_extra_settings": {},  # Dictionary of extra settings for Slurm to put in its launch script.
     "slurm_sbatch": "sbatch", # sbatch command
+    "slurm_restart_dir" : None, # restart Slurm jobs from this directory
     ########################################
     # Condor stuff
     ########################################