From d95a587da6fca3d73440da741de598127d991ebb Mon Sep 17 00:00:00 2001
From: David Hendriks <davidhendriks93@gmail.com>
Date: Wed, 23 Sep 2020 20:06:55 +0100
Subject: [PATCH] repaired bugs, fixed the free_store_memaddr, worked on slum &
condor
---
binarycpython/utils/grid.py | 420 ++++++++++++-------
binarycpython/utils/grid_options_defaults.py | 72 ++--
include/binary_c_python.h | 2 +-
src/binary_c_python.c | 9 +-
src/binary_c_python_api.c | 2 +
tests/core/test_persistent_data.py | 5 +-
tests/core/test_return_store_memaddr.py | 12 +-
7 files changed, 325 insertions(+), 197 deletions(-)
diff --git a/binarycpython/utils/grid.py b/binarycpython/utils/grid.py
index 719c51ba0..1d3439d3e 100644
--- a/binarycpython/utils/grid.py
+++ b/binarycpython/utils/grid.py
@@ -15,6 +15,7 @@ import datetime
import time
import logging
import argparse
+import subprocess
import importlib.util
from pathos.helpers import mp as pathos_multiprocess
@@ -80,7 +81,7 @@ class Population:
"""
self.defaults = get_defaults()
- self.cleaned_up_defaults = self.cleanup_defaults()
+ self.cleaned_up_defaults = self._cleanup_defaults()
# Different sections of options
self.bse_options = {} # bse_options is just empty.
@@ -215,6 +216,7 @@ class Population:
# Grab the input and split them up, while accepting only non-empty entries
cmdline_args = args.cmdline
+ self.grid_options['commandline_input'] = cmdline_args
split_args = [
cmdline_arg
for cmdline_arg in cmdline_args.split(" ")
@@ -234,7 +236,7 @@ class Population:
# unpack the dictionary into the setting function that handles where the values are set
self.set(**cmdline_dict)
- def return_argline(self, parameter_dict=None):
+ def _return_argline(self, parameter_dict=None):
"""
Function to create the string for the arg line from a parameter dict
"""
@@ -250,17 +252,17 @@ class Population:
return argline
def add_grid_variable(
- self,
- name,
- longname,
- valuerange,
- resolution,
- spacingfunc,
- probdist,
- dphasevol,
- parameter_name,
- precode=None,
- condition=None,
+ self,
+ name,
+ longname,
+ valuerange,
+ resolution,
+ spacingfunc,
+ probdist,
+ dphasevol,
+ parameter_name,
+ precode=None,
+ condition=None,
):
"""
Function to add grid variables to the grid_options.
@@ -343,7 +345,7 @@ class Population:
return options
- def return_binary_c_version_info(self, parsed=False):
+ def _return_binary_c_version_info(self, parsed=False):
"""
Function that returns the version information of binary_c
"""
@@ -355,7 +357,7 @@ class Population:
return version_info
- def return_binary_c_defaults(self):
+ def _return_binary_c_defaults(self):
"""
Function that returns the defaults of the binary_c version that is used.
"""
@@ -363,12 +365,12 @@ class Population:
return self.defaults
def return_all_info(
- self,
- include_population_settings=True,
- include_binary_c_defaults=True,
- include_binary_c_version_info=True,
- include_binary_c_help_all=True,
- ):
+ self,
+ include_population_settings=True,
+ include_binary_c_defaults=True,
+ include_binary_c_version_info=True,
+ include_binary_c_help_all=True,
+ ):
"""
Function that returns all the information about the population and binary_c
"""
@@ -383,7 +385,7 @@ class Population:
#
if include_binary_c_defaults:
- binary_c_defaults = self.return_binary_c_defaults()
+ binary_c_defaults = self._return_binary_c_defaults()
all_info["binary_c_defaults"] = binary_c_defaults
if include_binary_c_version_info:
@@ -397,14 +399,14 @@ class Population:
return all_info
def export_all_info(
- self,
- use_datadir=True,
- outfile=None,
- include_population_settings=True,
- include_binary_c_defaults=True,
- include_binary_c_version_info=True,
- include_binary_c_help_all=True,
- ):
+ self,
+ use_datadir=True,
+ outfile=None,
+ include_population_settings=True,
+ include_binary_c_defaults=True,
+ include_binary_c_version_info=True,
+ include_binary_c_help_all=True,
+ ):
"""
Function that exports the all_info to a json file
@@ -473,7 +475,7 @@ class Population:
)
)
- def set_custom_logging(self):
+ def _set_custom_logging(self):
"""
Function/routine to set all the custom logging so that the function memory pointer
is known to the grid.
@@ -524,7 +526,7 @@ class Population:
# Ensemble functions
###################################################
- def load_persistent_data_memory_dict(self):
+ def _load_persistent_data_memory_dict(self):
"""
Function that loads a set amount (amt_cores) of persistent data memory adresses to
pass to binary_c.
@@ -543,7 +545,7 @@ class Population:
1,
)
- def free_persistent_data_memory_and_combine_results_and_output(self):
+ def _free_persistent_data_memory_and_combine_results_and_output(self):
"""
Function that loads a set amount of persisten data memory adresses to
pass to binary_c.
@@ -598,22 +600,21 @@ class Population:
def evolve(self):
"""
Entrypoint function of the whole object. From here, based on the settings,
- we set up a SLURM or CONDOR grid, or if no setting is given we go straight
+ we set up a SLURM or CONDOR grid, or if no setting is given we go straight
to evolving the population
"""
# Check which type:
if self.grid_options['slurm'] == 1:
# Execute slurm subroutines
- self.slurm_grid()
+ self._slurm_grid()
elif self.grid_options['condor'] == 1:
# Execute condor subroutines
- self.condor_grid()
-
- else:
+ self._condor_grid()
+ else:
# Execute population evolution subroutines
- self.evolve_population()
+ self._evolve_population()
def _evolve_population(self):
"""
@@ -623,21 +624,21 @@ class Population:
Choices here are:
- to evolve a population via multiprocessing or linearly on 1 core.
- to evolve a population via a variable grid, a source file or MC
-
+
TODO: include options for different ways of generating a population here.
"""
##
# Prepare code/initialise grid.
# set custom logging, set up store_memaddr, build grid code. dry run grid code.
- self.setup()
+ self._setup()
##
# Evolve systems: via grid_options one can choose to do this linearly, or
# multiprocessing method.
if (
- self.grid_options["evolution_type"]
- in self.grid_options["evolution_type_options"]
+ self.grid_options["evolution_type"]
+ in self.grid_options["evolution_type_options"]
):
if self.grid_options["evolution_type"] == "mp":
self._evolve_population_mp()
@@ -653,7 +654,7 @@ class Population:
##
# Clean up code: remove files, unset values.
- self.cleanup()
+ self._cleanup()
def _evolve_population_mp(self):
"""
@@ -680,7 +681,7 @@ class Population:
# TODO: calculate the chunksize value based on: total starcount and cores used.
_ = list(
pool.imap_unordered(
- self.evolve_system_mp, self.yield_system_mp(), chunksize=20
+ self.evolve_system_mp, self._yield_system_mp(), chunksize=20
)
)
@@ -698,7 +699,7 @@ class Population:
full_system_dict = self.bse_options.copy()
full_system_dict.update(system)
- binary_cmdline_string = self.return_argline(full_system_dict)
+ binary_cmdline_string = self._return_argline(full_system_dict)
out = binary_c_python_api.run_system(
argstring=binary_cmdline_string,
custom_logging_func_memaddr=self.grid_options[
@@ -707,14 +708,14 @@ class Population:
store_memaddr=self.grid_options["store_memaddr"],
population=1,
)
- self.print_info(
+ self._print_info(
i + 1, self.grid_options["total_starcount"], full_system_dict
)
if self.grid_options["parse_function"]:
self.grid_options["parse_function"](self, out)
- def _evolve_system_mp(self, binary_cmdline_string):
+ def evolve_system_mp(self, binary_cmdline_string):
"""
Function that the multiprocessing evolution method calls to evolve a system
"""
@@ -740,9 +741,9 @@ class Population:
full_system_dict = self.bse_options.copy()
full_system_dict.update(system)
- binary_cmdline_string = self.return_argline(full_system_dict)
+ binary_cmdline_string = self._return_argline(full_system_dict)
- self.print_info(
+ self._print_info(
i + 1, self.grid_options["total_starcount"], full_system_dict
)
yield binary_cmdline_string
@@ -758,10 +759,10 @@ class Population:
"""
### Custom logging code:
- self.set_custom_logging()
+ self._set_custom_logging()
# Get argument line
- argline = self.return_argline(self.bse_options)
+ argline = self._return_argline(self.bse_options)
verbose_print("Running {}".format(argline), self.grid_options["verbosity"], 1)
# Run system
@@ -779,7 +780,7 @@ class Population:
# TODO: make a switch to turn this off
if clean_up_custom_logging_files:
- self.clean_up_custom_logging(evol_type="single")
+ self._clean_up_custom_logging(evol_type="single")
# Parse
if self.grid_options["parse_function"]:
@@ -807,18 +808,14 @@ class Population:
#######################
### Custom logging code:
- self.set_custom_logging()
+ self._set_custom_logging()
### Load store
- self.grid_options["store_memaddr"] = binary_c_python_api.return_store("")
+ self.grid_options["store_memaddr"] = binary_c_python_api.return_store_memaddr()
### ensemble:
- ## Load persistent_data_memaddr if necessary:
- if self.bse_options["ensemble"] == 1:
- self.load_persistent_data_memory_dict()
-
## check the settings:
- if self.bse_options["ensemble"] == 1:
+ if self.bse_options.get("ensemble", None):
if not self.bse_options["ensemble_defer"] == 1:
verbose_print(
"Error, if you want to run an ensemble in a population, the output needs to be deferred",
@@ -827,8 +824,11 @@ class Population:
)
raise ValueError
+ ## Load persistent_data_memaddr if necessary:
+ self._load_persistent_data_memory_dict()
+
# Check which type of population generation
- if grid_options["population_type_options"] == "grid":
+ if self.grid_options["population_type"] == "grid":
#######################
# Dry run and getting starcount
self.grid_options["probtot"] = 0
@@ -839,13 +839,13 @@ class Population:
raise ValueError
# Set up the grid code with a dry run option to see total probability
- self.generate_grid_code(dry_run=True)
+ self._generate_grid_code(dry_run=True)
# Load the grid code
- self.load_grid_function()
+ self._load_grid_function()
# Do a dry run
- self.dry_run()
+ self._dry_run()
print(
"Total starcount for this run will be: {}".format(
@@ -863,22 +863,23 @@ class Population:
] = time.time() # Setting start time of grid
#
- self.generate_grid_code(dry_run=False)
+ self._generate_grid_code(dry_run=False)
#
- self.load_grid_function()
+ self._load_grid_function()
# Source file
- elif grid_options["population_type_options"] == "source_file":
+ elif self.grid_options["population_type"] == "source_file":
#######################
# Dry run and getting starcount
self.grid_options["probtot"] = 0
# Load the grid code
- self.load_source_file_function()
+ # TODO: fix this function
+ # self._load_source_file_function()
# Do a dry run
- self.dry_run_source_file()
+ self._dry_run_source_file()
print(
"Total starcount for this run will be: {}".format(
@@ -896,10 +897,12 @@ class Population:
] = time.time() # Setting start time of grid
#
- self.load_source_file_function(dry_run=False)
+ # TODO: fix this function
+ # self._load_source_file_function()
+ # self._load_source_file_function(dry_run=False)
#
- self.load_grid_function()
+ self._load_grid_function()
#######
def _cleanup(self):
@@ -914,8 +917,8 @@ class Population:
"""
# Output the ensemble if necessary:
- if self.bse_options["ensemble"] == 1:
- self.free_persistent_data_memory_and_combine_results_and_output()
+ if self.bse_options.get("ensemble", None):
+ self._free_persistent_data_memory_and_combine_results_and_output()
# Reset values
self.grid_options["count"] = 0
@@ -927,7 +930,7 @@ class Population:
# Unload functions
# Unload store
- binary_c_python_api.binary_c_free_store_memaddr(
+ binary_c_python_api.free_store_memaddr(
self.grid_options["store_memaddr"]
)
@@ -938,7 +941,7 @@ class Population:
# a variable grid
###################################################
- def generate_grid_code(self, dry_run=False):
+ def _generate_grid_code(self, dry_run=False):
"""
Function that generates the code from which the population will be made.
@@ -1015,8 +1018,8 @@ class Population:
code_string += indent * depth + "# setting probability lists\n"
# Prepare the probability
for grid_variable_el in sorted(
- self.grid_options["grid_variables"].items(),
- key=lambda x: x[1]["grid_variable_number"],
+ self.grid_options["grid_variables"].items(),
+ key=lambda x: x[1]["grid_variable_number"],
):
# Make probabilities dict
grid_variable = grid_variable_el[1]
@@ -1089,6 +1092,7 @@ class Population:
)
# TODO: Make clear that the phasevol only works good
+ # TODO: add option to ignore this phasevol calculation and set it to 1
# if you sample linearly in that thing.
code_string += (
indent * depth
@@ -1250,7 +1254,7 @@ class Population:
# Increment total probability
code_string += (
- indent * (depth + 1) + "self.increment_probtot(probability)\n"
+ indent * (depth + 1) + "self._increment_probtot(probability)\n"
)
if not dry_run:
@@ -1341,7 +1345,7 @@ class Population:
with open(gridcode_filename, "w") as file:
file.write(code_string)
- def load_grid_function(self):
+ def _load_grid_function(self):
"""
TODO: Update this description
Test function to run grid stuff. mostly to test the import
@@ -1368,7 +1372,7 @@ class Population:
verbose_print("Grid code loaded", self.grid_options["verbosity"], 1)
- def dry_run(self):
+ def _dry_run(self):
"""
Function to dry run the grid and know how many stars it will run
@@ -1379,7 +1383,7 @@ class Population:
total_starcount = system_generator(self)
self.grid_options["total_starcount"] = total_starcount
- def print_info(self, run_number, total_systems, full_system_dict):
+ def _print_info(self, run_number, total_systems, full_system_dict):
"""
Function to print info about the current system and the progress of the grid.
@@ -1399,7 +1403,7 @@ class Population:
# time_passed = time.time() - self.grid_options["start_time_evolution"]
if run_number % print_freq == 0:
- binary_cmdline_string = self.return_argline(full_system_dict)
+ binary_cmdline_string = self._return_argline(full_system_dict)
info_string = "{color_part_1} \
{text_part_1}{end_part_1}{color_part_2} \
{text_part_2}{end_part_2}".format(
@@ -1426,7 +1430,7 @@ class Population:
# a file containing binary_c calls
###################################################
- def dry_run_source_file(self):
+ def _dry_run_source_file(self):
"""
Function to go through the source_file and count the amount of lines and the total probability
"""
@@ -1444,7 +1448,7 @@ class Population:
total_starcount = system_generator(self)
self.grid_options["total_starcount"] = total_starcount
- def load_source_file(self, check=False):
+ def _load_source_file(self, check=False):
"""
Function that loads the source_file that contains a binary_c calls
"""
@@ -1477,9 +1481,9 @@ class Population:
verbose_print("Source file loaded", self.grid_options["verbosity"], 1)
- def dict_from_line_source_file(self):
+ def _dict_from_line_source_file(self):
"""
- Function that creates a dict from a binary_c argline
+ Function that creates a dict from a binary_c argline
"""
if line.startswith("binary_c "):
@@ -1491,20 +1495,18 @@ class Population:
for i in range(0, len(split_line), 2):
if "." in split_line[i+1]:
arg_dict[split_line[i]] = float(split_line[i + 1])
- else:
+ else:
arg_dict[split_line[i]] = int(split_line[i + 1])
return arg_dict
-
-
###################################################
# SLURM functions
#
# subroutines to run SLURM grids
###################################################
- def slurm_grid(self):
+ def _slurm_grid(self):
"""
Main function that manages the SLURM setup.
@@ -1516,7 +1518,7 @@ class Population:
Which stage is used is determined by the value of grid_options['slurm_command']:
- <empty>: the function will know its the user that executed the script and
+ <empty>: the function will know its the user that executed the script and
it will set up the necessary condor stuff
'evolve': evolve_population is called to evolve the population of stars
@@ -1524,28 +1526,30 @@ class Population:
'join': We will attempt to join the output
"""
+ # Check version
# TODO: Put in function
slurm_version = get_slurm_version()
if not slurm_version:
- verbose_print("SLURM: Error: No installation of slurm found", self.grid_options['verbosity'], 0)
+ verbose_print("SLURM: Error: No installation of slurm found",
+ self.grid_options['verbosity'], 0)
else:
major_version = int(slurm_version.split(".")[0])
minor_version = int(slurm_version.split(".")[1])
- # if (major_version == 8) and (minor_version > 4):
- # verbose_print("SLURM: Found version {} which is new enough".format(slurm_version), self.grid_options['verbosity'], 0)
if (major_version > 17):
- verbose_print("SLURM: Found version {} which is new enough".format(slurm_version), self.grid_options['verbosity'], 0)
+ verbose_print("SLURM: Found version {} which is new enough".format(slurm_version),
+ self.grid_options['verbosity'], 1)
else:
- verbose_print("SLURM: Found version {} which is too old (we require 17+)".format(slurm_version), self.grid_options['verbosity'], 0)
+ verbose_print("SLURM: Found version {} which is too old (we require 17+)".format(slurm_version),
+ self.grid_options['verbosity'], 0)
+
+ verbose_print("SLURM: Running slurm grid. command={}".format(self.grid_options['slurm_command']),
+ self.grid_options['verbosity'], 1)
- verbose_print("SLURM: Running slurm grid. command={}".format(self.grid_options['slurm_command']), self.grid_options['verbosity'], 1)
if not self.grid_options['slurm_command']:
# Setting up
verbose_print("SLURM: Main controller script. Setting up", self.grid_options['verbosity'], 1)
- # Check settings:
- # TODO: check settings
# Set up working directories:
verbose_print("SLURM: creating working directories", self.grid_options['verbosity'], 1)
@@ -1554,10 +1558,9 @@ class Population:
# Create command
python_details = get_python_details()
scriptname = path_of_calling_script()
-
- command = "".join([
- "{}".format(python_details['executable']),
- "{}".format(scriptname),
+ command = "{} {}".format(python_details['executable'], scriptname)
+ command += " --cmdline \"{}\"".format(" ".join([
+ "{}".format(self.grid_options['commandline_input']),
"offset=$jobarrayindex",
"modulo={}".format(self.grid_options['slurm_njobs']),
"vb={}".format(self.grid_options['verbosity']),
@@ -1565,29 +1568,81 @@ class Population:
"slurm_jobarrayindex=$jobarrayindex",
"slurm_jobname='binary_grid_'$jobid'.'$jobarrayindex",
"slurm_njobs={}".format(self.grid_options['slurm_njobs']),
- "slurm_dir={}".format(self.grid_options['slurm_dir'])
- ])
+ "slurm_dir={}".format(self.grid_options['slurm_dir']),
+ "rungrid=1",
+ "slurm_command=evolve"
+ ]).strip())
+ # Construct dict with settings for the script while checking the settings at the same time
+ # Check settings:
+ # TODO: check settings
# Create SLURM_DIR script:
- # TODO: create the condor script.
slurm_script_options = {}
slurm_script_options['n'] = self.grid_options['slurm_njobs']
slurm_script_options['njobs'] = self.grid_options['slurm_njobs']
slurm_script_options['dir'] = self.grid_options['slurm_dir']
slurm_script_options['memory'] = self.grid_options['slurm_memory']
- slurm_script_options['working_dir'] = self.grid_options['slurm_working_dir']
- slurm_script_options['command'] = self.grid_options['command']
- slurm_script_options['streams'] = self.grid_options['streams']
-
- print(slurm_script_options)
+ slurm_script_options['working_dir'] = self.grid_options['slurm_dir'] #TODO: check this
+ slurm_script_options['command'] = command
+ # slurm_script_options['streams'] = self.grid_options['streams']
+
+ # Construct the script
+ slurm_script_contents = ""
+ slurm_script_contents += "#!/bin/bash\n"
+ slurm_script_contents += "# Slurm file for binary_grid and slurm\n"
+ slurm_script_contents += "#SBATCH --error={}/stderr/%A.%a\n".format(self.grid_options['slurm_dir'])
+ slurm_script_contents += "#SBATCH --output={}/stdout/%A.%a\n".format(self.grid_options['slurm_dir'])
+ slurm_script_contents += "#SBATCH --job-name={}\n".format(self.grid_options['slurm_jobname'])
+ slurm_script_contents += "#SBATCH --partition={}\n".format(self.grid_options['slurm_partition'])
+ slurm_script_contents += "#SBATCH --time={}\n".format(self.grid_options['slurm_time'])
+ slurm_script_contents += "#SBATCH --mem={}\n".format(self.grid_options['slurm_memory'])
+ slurm_script_contents += "#SBATCH --ntasks={}\n".format(self.grid_options['slurm_ntasks'])
+ slurm_script_contents += "#SBATCH --array={}\n".format(self.grid_options['slurm_array'])
+ slurm_script_contents += "\n"
+
+ if self.grid_options['slurm_extra_settings']:
+ slurm_script_contents += "# Extra settings by user:"
+ slurm_script_contents += "\n".join(["--{}={}".format(key, self.grid_options['slurm_extra_settings'][key]) for key in self.grid_options['slurm_extra_settings']])
+
+ slurm_script_contents += "# set status to \"running\"\n"
+ slurm_script_contents += "echo \"running\" > {}/status/$jobid.$jobarrayindex\n\n".format(self.grid_options['slurm_dir'])
+ slurm_script_contents += "# run grid of stars\n"
+ slurm_script_contents += "{}\n\n".format(command)
+ slurm_script_contents += "# set status to \"finished\"\n"
+ slurm_script_contents += "echo \"finished\" > {}/status/$jobid.$jobarrayindex\n".format(self.grid_options['slurm_dir'])
+ slurm_script_contents += "\n"
+
+ if self.grid_options['slurm_postpone_join']:
+ slurm_script_contents += "{} rungrid=0 results_hash_dumpfile={}/results/$jobid.all slurm_command=join\n".format(command, self.grid_options['slurm_dir'])
+
+ # Write script to file
+ slurm_script_filename = os.path.join(self.grid_options['slurm_dir'], 'slurm_script')
+ with open(slurm_script_filename, 'w') as slurm_script_file:
+ slurm_script_file.write(slurm_script_contents)
+
+ # Execute or postpone
+ if self.grid_options['slurm_postpone_sbatch']:
+ # Execute or postpone the real call to sbatch
+ sbatch_command = "sbatch {}".format(slurm_script_filename)
+ verbose_print("running slurm script {}".format(slurm_script_filename),
+ self.grid_options['verbosity'], 0)
+ # subprocess.Popen(sbatch_command, close_fds=True)
+ # subprocess.Popen(sbatch_command, creationflags=subprocess.DETACHED_PROCESS)
+ verbose_print("Submitted scripts.",
+ self.grid_options['verbosity'], 0)
+ else:
+ verbose_print("Slurm script is in {} but hasnt been executed".format(slurm_script_filename),
+ self.grid_options['verbosity'], 0)
+ verbose_print("all done!", self.grid_options['verbosity'], 0)
+ exit()
elif self.grid_options['slurm_command'] == 'evolve':
- # Part to evolve the population.
+ # Part to evolve the population.
# TODO: decide how many CPUs
verbose_print("SLURM: Evolving population", self.grid_options['verbosity'], 1)
- #
+ #
self._evolve_population()
elif self.grid_options['slurm_command'] == 'join':
@@ -1600,7 +1655,7 @@ class Population:
# subroutines to run CONDOR grids
###################################################
- def condor_grid(self):
+ def _condor_grid(self):
"""
Main function that manages the CONDOR setup.
@@ -1641,17 +1696,14 @@ class Population:
# Setting up
verbose_print("CONDOR: Main controller script. Setting up", self.grid_options['verbosity'], 1)
- # Check settings:
- # TODO: check settings
-
# Set up working directories:
- verbose_print("CONDOR: creating working directories", verbosity, minimal_verbosity)
+ verbose_print("CONDOR: creating working directories", self.grid_options['verbosity'], 1)
create_directories_hpc(self.grid_options['condor_dir'])
# Create command
+ current_workingdir = os.getcwd()
python_details = get_python_details()
scriptname = path_of_calling_script()
-
# command = "".join([
# "{}".python_details['executable'],
# "{}".scriptname,
@@ -1659,10 +1711,6 @@ class Population:
# "modulo={}".format(self.grid_options['condor_njobs']),
# "vb={}".format(self.grid_options['verbosity'])
-
-
-
-
# "results_hash_dumpfile=$self->{_grid_options}{slurm_dir}/results/$jobid.$jobarrayindex",
# 'slurm_jobid='.$jobid,
# 'slurm_jobarrayindex='.$jobarrayindex,
@@ -1671,11 +1719,10 @@ class Population:
# "slurm_dir=$self->{_grid_options}{slurm_dir}",
# );
-
- # Create CONDOR script:
- # TODO: create the condor script.
+ # Create directory with info for the condor script. By creating this directory we also check whether all the values are set correctly
+ # TODO: create the condor script.
condor_script_options = {}
- # condor_script_options['n'] =
+ # condor_script_options['n'] =
condor_script_options['njobs'] = self.grid_options['condor_njobs']
condor_script_options['dir'] = self.grid_options['condor_dir']
condor_script_options['memory'] = self.grid_options['condor_memory']
@@ -1683,20 +1730,100 @@ class Population:
condor_script_options['command'] = self.grid_options['command']
condor_script_options['streams'] = self.grid_options['streams']
+ # TODO: condor works with running an executable.
+
+ # Create script contents
+ condor_script_contents = ""
+ condor_script_contents += """
+#################################################
+#
+# Condor script to run a binary_grid via python
+#
+#################################################
+"""
+ condor_script_contents += "Executable\t= {}".format(executable)
+ condor_script_contents += "arguments\t= {}".format(arguments)
+ condor_script_contents += "environment\t= {}".format(environment)
+ condor_script_contents += "universe\t= {}".format(self.grid_options['condor_universe'])
+ condor_script_contents += "\n"
+ condor_script_contents += "output\t= {}/stdout/$id\n".format(self.grid_options['condor_dir'])
+ condor_script_contents += "error\t={}/sterr/$id".format(self.grid_options['condor_dir'])
+ condor_script_contents += "log\t={}\n".format(self.grid_options['condor_dir'])
+ condor_script_contents += "initialdir\t={}\n".format(current_workingdir)
+ condor_script_contents += "remote_initialdir\t={}\n".format(current_workingdir)
+ condor_script_contents += "\n"
+ condor_script_contents += "steam_output\t={}".format(stream)
+ condor_script_contents += "steam_error\t={}".format(stream)
+ condor_script_contents += "+WantCheckpoint = False"
+ condor_script_contents += "\n"
+ condor_script_contents += "request_memory\t={}".format(self.grid_options['condor_memory'])
+ condor_script_contents += "ImageSize\t={}".format(self.grid_options['condor_memory'])
+ condor_script_contents += "\n"
+
+
+ if self.grid_options['condor_extra_settings']:
+ slurm_script_contents += "# Extra settings by user:"
+ slurm_script_contents += "\n".join(["{}\t={}".format(key, self.grid_options['condor_extra_settings'][key]) for key in self.grid_options['condor_extra_settings']])
+
+ condor_script_contents += "\n"
+
+ # request_memory = $_[0]{memory}
+ # ImageSize = $_[0]{memory}
+
+ # Requirements = (1) \&\& (".
+ # $self->{_grid_options}{condor_requirements}.")\n";
+ #
+ # file name: my_program.condor
+ # Condor submit description file for my_program
+ # Executable = my_program
+ # Universe = vanilla
+ # Error = logs/err.$(cluster)
+ # Output = logs/out.$(cluster)
+ # Log = logs/log.$(cluster)
+
+ # should_transfer_files = YES
+ # when_to_transfer_output = ON_EXIT
+ # transfer_input_files = files/in1,files/in2
+
+ # Arguments = files/in1 files/in2 files/out1
+ # Queue
+
+ # Write script contents to file
+ if self.grid_options['condor_postpone_join']:
+ condor_script_contents += "{} rungrid=0 results_hash_dumpfile={}/results/$jobid.all condor_command=join\n".format(command, self.grid_options['condor_dir'])
+
+ condor_script_filename = os.path.join(self.grid_options['condor_dir'], 'condor_script')
+ with open(condor_script_filename, 'w') as condor_script_file:
+ condor_script_file.write(condor_script_contents)
+
+ if self.grid_options['condor_postpone_sbatch']:
+ # Execute or postpone the real call to sbatch
+ submit_command = "condor_submit {}".format(condor_script_filename)
+ verbose_print("running condor script {}".format(condor_script_filename),
+ self.grid_options['verbosity'], 0)
+ # subprocess.Popen(sbatch_command, close_fds=True)
+ # subprocess.Popen(sbatch_command, creationflags=subprocess.DETACHED_PROCESS)
+ verbose_print("Submitted scripts.",
+ self.grid_options['verbosity'], 0)
+ else:
+ verbose_print("Condor script is in {} but hasnt been executed".format(condor_script_filename),
+ self.grid_options['verbosity'], 0)
+ verbose_print("all done!", self.grid_options['verbosity'], 0)
+ exit()
-
elif self.grid_options['condor_command'] == 'evolve':
- # Part to evolve the population.
+ # TODO: write this function
+ # Part to evolve the population.
# TODO: decide how many CPUs
verbose_print("CONDOR: Evolving population", self.grid_options['verbosity'], 1)
-
- #
+ #
self._evolve_population()
elif self.grid_options['condor_command'] == 'join':
+ # TODO: write this function
# Joining the output.
verbose_print("CONDOR: Joining results", self.grid_options['verbosity'], 1)
@@ -1719,19 +1846,8 @@ class Population:
pass
-
-
-
-
-
-
-
-
-
-
-
def write_binary_c_calls_to_file(
- self, output_dir=None, output_filename=None, include_defaults=False
+ self, output_dir=None, output_filename=None, include_defaults=False
):
"""
Function that loops over the gridcode and writes the generated parameters to a file.
@@ -1767,10 +1883,10 @@ class Population:
raise ValueError
#
- self.generate_grid_code(dry_run=False)
+ self._generate_grid_code(dry_run=False)
#
- self.load_grid_function()
+ self._load_grid_function()
if self.grid_options["system_generator"]:
# Check if there is an output dir configured
@@ -1812,13 +1928,13 @@ class Population:
# update values with current system values
full_system_dict.update(system)
- binary_cmdline_string = self.return_argline(full_system_dict)
+ binary_cmdline_string = self._return_argline(full_system_dict)
file.write(binary_cmdline_string + "\n")
else:
print("Error. No grid function found!")
raise ValueError
- def cleanup_defaults(self):
+ def _cleanup_defaults(self):
"""
Function to clean up the default values:
@@ -1832,12 +1948,12 @@ class Population:
TODO: Rethink this functionality. seems a bit double, could also be just outside of the class
"""
- binary_c_defaults = self.return_binary_c_defaults().copy()
+ binary_c_defaults = self._return_binary_c_defaults().copy()
cleaned_dict = filter_arg_dict(binary_c_defaults)
return cleaned_dict
- def clean_up_custom_logging(self, evol_type):
+ def _clean_up_custom_logging(self, evol_type):
"""
Function to clean up the custom logging.
Has two types:
@@ -1883,20 +1999,20 @@ class Population:
if evol_type == "MC":
pass
- def increment_probtot(self, prob):
+ def _increment_probtot(self, prob):
"""
Function to add to the total probability
"""
self.grid_options["probtot"] += prob
- def increment_count(self):
+ def _increment_count(self):
"""
Function to add to the total amount of stars
"""
self.grid_options["count"] += 1
- def set_loggers(self):
+ def _set_loggers(self):
"""
Function to set the loggers for the execution of the grid
"""
diff --git a/binarycpython/utils/grid_options_defaults.py b/binarycpython/utils/grid_options_defaults.py
index 0d694fcc6..18c001bab 100644
--- a/binarycpython/utils/grid_options_defaults.py
+++ b/binarycpython/utils/grid_options_defaults.py
@@ -16,6 +16,8 @@ grid_options_defaults_dict = {
"tmp_dir": temp_dir(), # Setting the temp dir of the program
"main_pid": -1, # Placeholder for the main process id of the run.
# "output_dir":
+ "commandline_input": "",
+
##########################
# Execution log:
##########################
@@ -118,55 +120,48 @@ grid_options_defaults_dict = {
########################################
# Slurm stuff
########################################
- # slurm_ntasks=>1, # 1 CPU required per job
- # slurm_partition=>'all', # MUST be defined
- # slurm_jobname=>'binary_grid', # not required but useful
- # slurm_use_all_node_CPUs=>0, # if given nodes, set to 1
- # # if given CPUs, set to 0
-
"slurm": 0, # dont use the slurm by default. 1 = use slurm
- "slurm_command": "", # Command that slurm runs (e.g. run_flexigrid or join_datafiles)
- "slurm_dir": "", # working directory containin scripts output logs etc.
+ "slurm_ntasks": 1, # CPUs required per array job: usually only need this
+ "slurm_command": "", # Command that slurm runs (e.g. evolve or join_datafiles)
+ "slurm_dir": "", # working directory containing scripts output logs etc.
"slurm_njobs": 0, # number of scripts; set to 0 as default
"slurm_jobid": '', # slurm job id (%A)
"slurm_memory": 512, # in MB, the memory use of the job
"slurm_warn_max_memory": 1024, # in MB : warn if mem req. > this
"slurm_use_all_node_CPUs": 0, # 1 = use all of a node's CPUs. 0 = use a given amount of CPUs
- "slurm_postpone_join": 0, # if 1 do not join on slurm, join elsewhere
-
- # slurm_jobarrayindex=>'', # slurm job array index (%a)
- # slurm_jobname=>'binary_grid', # set to binary_grid
- # slurm_postpone_join=>0, # if 1, data is not joined, e.g. if you
- # # want to do it off the slurm grid (e.g. with more RAM)
- # slurm_postpone_sbatch=>0, # if 1, don't submit, just make the script
- # # (defaults to $ENV{PWD} if undef)
- # slurm_partition=>undef,
- # slurm_ntasks=>1, # 1 CPU required per array job: usually only need this
- # slurm_time=>0, # 0 = infinite time
- # # you will want to use this if your Slurm SelectType is e.g. linear
- # # which means it allocates all the CPUs in a node to the job
- # slurm_control_CPUs=>0, # if so, leave this many for Perl control (0)
- # slurm_array=>undef,# override for --array, useful for rerunning jobs
+ "slurm_postpone_join": 0, # if 1 do not join on slurm, join elsewhere. want to do it off the slurm grid (e.g. with more RAM)
+ "slurm_jobarrayindex": '', # slurm job array index (%a)
+ "slurm_jobname": 'binary_grid', # default
+ "slurm_partition": None,
+ "slurm_time": 0, # total time. 0 = infinite time
+ "slurm_postpone_sbatch": 0, # if 1: don't submit, just make the script
+ "slurm_array": None,# override for --array, useful for rerunning jobs
+ "slurm_use_all_node_CPUs": 0, # if given nodes, set to 1
+ # if given CPUs, set to 0
+ # you will want to use this if your Slurm SelectType is e.g. linear
+ # which means it allocates all the CPUs in a node to the job
+ "slurm_control_CPUs": 0, # if so, leave this many for Pythons control (0)
+ "slurm_array": None, # override for --array, useful for rerunning jobs
+ "slurm_partition": None, # MUST be defined
+ "slurm_extra_settings": {}, # Place to put extra configuration for the SLURM batch file. The key and value of the dict will become the key and value of the line in te slurm batch file. Will be put in after all the other settings (and before the command). Take care not to overwrite something without really meaning to do so.
########################################
# Condor stuff
########################################
- # condor=>0, # 1 to use condor, 0 otherwise
- # condor_command=>'',# condor command e.g. "run_flexigrid",
- # # "join_datafiles"
- # condor_dir=>'', # working directory containing e.g.
- # # scripts, output, logs (e.g. should be NFS available to all)
- # condor_njobs=>'', # number of scripts
- # condor_jobid=>'', # condor job id
- # condor_postpone_join=>0, # if 1, data is not joined, e.g. if you
- # # want to do it off the condor grid (e.g. with more RAM)
- # condor_join_machine=>undef, # if defined then this is the machine on which the join command
- # should be launched (must be sshable and not postponed)
- # condor_join_pwd=>undef, # directory the join should be in
- # # (defaults to $ENV{PWD} if undef)
- # condor_memory=>1024, # in MB, the memory use (ImageSize) of the job
- # condor_universe=>'vanilla', # usually vanilla universe
+ "condor": 0, # 1 to use condor, 0 otherwise
+ "condor_command": '', # condor command e.g. "evolve", "join"
+ "condor_dir": '', # working directory containing e.g. scripts, output, logs (e.g. should be NFS available to all)
+ "condor_njobs": '', # number of scripts/jobs that CONDOR will run in total
+ "condor_jobid": '', # condor job id
+ "condor_postpone_join": 0, # if 1, data is not joined, e.g. if you want to do it off the condor grid (e.g. with more RAM)
+ # "condor_join_machine": None, # if defined then this is the machine on which the join command should be launched (must be sshable and not postponed)
+ "condor_join_pwd": '', # directory the join should be in (defaults to $ENV{PWD} if undef)
+ "condor_memory": 1024, # in MB, the memory use (ImageSize) of the job
+ "condor_universe": 'vanilla', # usually vanilla universe
+ "condor_extra_settings": {}, # Place to put extra configuration for the CONDOR submit file. The key and value of the dict will become the key and value of the line in te slurm batch file. Will be put in after all the other settings (and before the command). Take care not to overwrite something without really meaning to do so.
+
+ # snapshots and checkpoints
# condor_snapshot_on_kill=>0, # if 1 snapshot on SIGKILL before exit
# condor_load_from_snapshot=>0, # if 1 check for snapshot .sv file and load it if found
# condor_checkpoint_interval=>0, # checkpoint interval (seconds)
@@ -185,6 +180,7 @@ grid_options_defaults_dict = {
# condor_resubmit_submitted=>0,
# condor_resubmit_running=>0,
# condor_resubmit_crashed=>0,
+
##########################
# Unordered. Need to go through this. Copied from the perl implementation.
##########################
diff --git a/include/binary_c_python.h b/include/binary_c_python.h
index 8373b511a..f539ca9f3 100644
--- a/include/binary_c_python.h
+++ b/include/binary_c_python.h
@@ -71,7 +71,7 @@ int free_store_memaddr(long int store_memaddr,
#define NO_OUTPUT
#ifdef BINARY_C_PYTHON_DEBUG
- #define debug_printf(fmt, ...) printf(fmt, __VA_ARGS__);
+ #define debug_printf(fmt, ...) printf(fmt, ##__VA_ARGS__);
#else
#define debug_printf(fmt, ...) /* Do nothing */
#endif
diff --git a/src/binary_c_python.c b/src/binary_c_python.c
index 1bcafb324..e6634e959 100644
--- a/src/binary_c_python.c
+++ b/src/binary_c_python.c
@@ -506,12 +506,14 @@ static PyObject* binary_c_free_persistent_data_memaddr_and_return_json_output(Py
static PyObject* binary_c_free_store_memaddr(PyObject *self, PyObject *args)
{
/* Python binding that calls the c function that free's the store memory */
-
long int store_memaddr = -1;
/* Parse the input tuple */
if(!PyArg_ParseTuple(args, "l", &store_memaddr))
{
+ // printf("Error: got a bad input\n");
+ fprintf(stderr,
+ "Error (in function: binary_c_free_store_memaddr): Got a bad input\n");
return NULL;
}
@@ -530,13 +532,14 @@ static PyObject* binary_c_free_store_memaddr(PyObject *self, PyObject *args)
if(error_buffer != NULL && strlen(error_buffer)>0)
{
+ // printf("Error (in function: binary_c_free_store_memaddr): %s", error_buffer);
fprintf(stderr,
- "Error (in function: binary_c_free_store): %s\n",
+ "Error (in function: binary_c_free_store_memaddr): %s\n",
error_buffer);
}
Safe_free(buffer);
Safe_free(error_buffer);
- return 0;
+ return Py_BuildValue("");
}
diff --git a/src/binary_c_python_api.c b/src/binary_c_python_api.c
index 9d8484931..974d35471 100644
--- a/src/binary_c_python_api.c
+++ b/src/binary_c_python_api.c
@@ -522,6 +522,7 @@ int free_store_memaddr(long int store_memaddr,
{
// load the store from the integer that has been passed
store = (void*)store_memaddr;
+ debug_printf("Took long int store_memaddr %ld and loaded it to %p\n", store_memaddr, (void*)&store);
}
else
{
@@ -538,6 +539,7 @@ int free_store_memaddr(long int store_memaddr,
-1 // argc
);
+ printf("freed store memaddr\n");
/* output to strings */
stardata->preferences->internal_buffering = INTERNAL_BUFFERING_STORE;
stardata->preferences->batchmode = BATCHMODE_LIBRARY;
diff --git a/tests/core/test_persistent_data.py b/tests/core/test_persistent_data.py
index e5b4e251e..6b27bbaa5 100644
--- a/tests/core/test_persistent_data.py
+++ b/tests/core/test_persistent_data.py
@@ -372,7 +372,8 @@ if __name__ == "__main__":
# test_return_persistent_data_memaddr()
# test_passing_persistent_data_to_run_system()
# test_full_ensemble_output()
- test_adding_ensemble_output()
- # test_free_and_json_output()
+ # test_adding_ensemble_output()
+ test_free_and_json_output()
# test_combine_with_empty_json()
+ # test_all()
print("Done")
diff --git a/tests/core/test_return_store_memaddr.py b/tests/core/test_return_store_memaddr.py
index 1e9083336..678077c0f 100644
--- a/tests/core/test_return_store_memaddr.py
+++ b/tests/core/test_return_store_memaddr.py
@@ -3,13 +3,23 @@ import textwrap
def test_return_store_memaddr():
- output = binary_c_python_api.return_store_memaddr("")
+ output = binary_c_python_api.return_store_memaddr()
print("function: test_return_store")
print("store memory adress:")
print(textwrap.indent(str(output), "\t"))
+def test_unload_store_memaddr():
+ output = binary_c_python_api.return_store_memaddr()
+
+ print("function: test_return_store")
+ print("store memory adress:")
+ print(textwrap.indent(str(output), "\t"))
+ print(type(output))
+ _ = binary_c_python_api.free_store_memaddr(output)
+ print("freed store memaddr")
####
if __name__ == "__main__":
test_return_store_memaddr()
+ test_unload_store_memaddr()
--
GitLab