From 056386bffcf3115fb4c4c0029f67fd2ad4c094fe Mon Sep 17 00:00:00 2001
From: David Hendriks <davidhendriks93@gmail.com>
Date: Mon, 9 Mar 2020 17:46:34 +0000
Subject: [PATCH] updated code, removed some old files, reformatted everything
---
binarycpython/utils/functions.py | 14 +-
binarycpython/utils/grid.py | 149 ++++++++++--------
binarycpython/utils/grid_options_defaults.py | 59 +++++--
binarycpython/utils/spacing_functions.py | 1 +
examples/example_population.py | 45 +++---
tests/population/grid_tests.py | 24 ++-
tests/population/profiling/readout_profile.py | 5 +-
...pulation_comparing_with_multiprocessing.py | 37 +++--
.../scaling/generate_scaling_script.py | 51 ------
.../scaling/generate_test_script.py | 37 ++---
tests/population/scaling/plot_amdahl.py | 12 +-
tests/population/scaling/plot_scaling.py | 82 +++++++---
tests/population/scaling/plot_scaling_new.py | 54 +++----
tests/population/scaling/scaling_script.py | 57 ++++---
14 files changed, 334 insertions(+), 293 deletions(-)
delete mode 100644 tests/population/scaling/generate_scaling_script.py
diff --git a/binarycpython/utils/functions.py b/binarycpython/utils/functions.py
index 675441579..688ee8c44 100644
--- a/binarycpython/utils/functions.py
+++ b/binarycpython/utils/functions.py
@@ -10,12 +10,14 @@ from binarycpython.utils.custom_logging_functions import (
create_and_load_logging_function,
)
+
def output_lines(output):
"""
Function that outputs the lines that were recieved from the binary_c run.
"""
return output.splitlines()
+
def parse_binary_c_version_info(version_info_string):
version_info_dict = {}
@@ -117,9 +119,7 @@ def create_hdf5(data_dir, name):
# Make HDF5:
# Create the file
- hdf5_filename = os.path.join(
- data_dir, "{}".format(name)
- )
+ hdf5_filename = os.path.join(data_dir, "{}".format(name))
print("Creating {}".format(hdf5_filename))
f = h5py.File(hdf5_filename, "w")
@@ -397,11 +397,15 @@ def get_help(param_name="", print_help=True, return_dict=False, fail_silently=Fa
return_dict: wether to return the help info dictionary
"""
-
+
available_arg_keys = get_arg_keys()
if not param_name:
- print("Please set the param_name to any of the following:\n {}".format(sorted(available_arg_keys)))
+ print(
+ "Please set the param_name to any of the following:\n {}".format(
+ sorted(available_arg_keys)
+ )
+ )
return None
else:
if param_name in available_arg_keys:
diff --git a/binarycpython/utils/grid.py b/binarycpython/utils/grid.py
index fdab898bc..0b916daa5 100644
--- a/binarycpython/utils/grid.py
+++ b/binarycpython/utils/grid.py
@@ -22,8 +22,11 @@ from binarycpython.utils.custom_logging_functions import (
create_and_load_logging_function,
temp_dir,
)
-from binarycpython.utils.functions import get_defaults, parse_binary_c_version_info, output_lines
-
+from binarycpython.utils.functions import (
+ get_defaults,
+ parse_binary_c_version_info,
+ output_lines,
+)
# Todo-list
@@ -354,12 +357,18 @@ class Population(object):
# Copy dict
all_info_cleaned = copy.deepcopy(all_info)
-
# Clean the all_info_dict: (i.e. transform the function objects to strings)
- if all_info_cleaned.get('population_settings', None):
- if all_info_cleaned['population_settings']['grid_options']['parse_function']:
- all_info_cleaned['population_settings']['grid_options']['parse_function'] = str(all_info_cleaned['population_settings']['grid_options']['parse_function'])
-
+ if all_info_cleaned.get("population_settings", None):
+ if all_info_cleaned["population_settings"]["grid_options"][
+ "parse_function"
+ ]:
+ all_info_cleaned["population_settings"]["grid_options"][
+ "parse_function"
+ ] = str(
+ all_info_cleaned["population_settings"]["grid_options"][
+ "parse_function"
+ ]
+ )
if use_datadir:
base_name = os.path.splitext(self.custom_options["base_filename"])[0]
@@ -451,7 +460,7 @@ class Population(object):
#######################
# Dry run and getting starcount
- self.grid_options['probtot'] = 0
+ self.grid_options["probtot"] = 0
self.generate_grid_code(dry_run=True)
@@ -459,20 +468,27 @@ class Population(object):
self.dry_run()
- print("Total starcount for this run will be: {}".format(self.grid_options['total_starcount']))
+ print(
+ "Total starcount for this run will be: {}".format(
+ self.grid_options["total_starcount"]
+ )
+ )
#######################
- # Reset values and prepare the grid function
- self.grid_options['probtot'] = 0 # To make sure that the values are reset. TODO: fix this in a cleaner way
- self.grid_options['start_time_grid'] = time.time() # Setting start time of grid
+ # Reset values and prepare the grid function
+ self.grid_options[
+ "probtot"
+ ] = 0 # To make sure that the values are reset. TODO: fix this in a cleaner way
+ self.grid_options[
+ "start_time_evolution"
+ ] = time.time() # Setting start time of grid
- #
+ #
self.generate_grid_code(dry_run=False)
- #
+ #
self.load_grid_function()
-
def cleanup(self):
"""
Function that handles all the cleaning up after the grid has been generated and/or run
@@ -485,14 +501,13 @@ class Population(object):
"""
# Reset values
- self.grid_options['count'] = 0
- self.grid_options['probtot'] = 0
- self.grid_options['system_generator'] = None
+ self.grid_options["count"] = 0
+ self.grid_options["probtot"] = 0
+ self.grid_options["system_generator"] = None
# Remove files
# Unload functions
-
def evolve_system_mp(self, binary_cmdline_string):
"""
Function that the multiprocessing evolution method calls to evolve a system
@@ -503,9 +518,8 @@ class Population(object):
self.grid_options["custom_logging_func_memaddr"],
self.grid_options["store_memaddr"],
)
- if self.grid_options['parse_function']:
- self.grid_options['parse_function'](self, out)
-
+ if self.grid_options["parse_function"]:
+ self.grid_options["parse_function"](self, out)
def yield_system_mp(self):
"""
@@ -518,12 +532,13 @@ class Population(object):
binary_cmdline_string = self.return_argline(full_system_dict)
- self.print_info(i+1, self.grid_options['total_starcount'], full_system_dict)
+ self.print_info(
+ i + 1, self.grid_options["total_starcount"], full_system_dict
+ )
yield binary_cmdline_string
print("generator done")
-
def evolve_single(self, parse_function=None, clean_up_custom_logging_files=True):
"""
Function to run a single system
@@ -536,7 +551,7 @@ class Population(object):
# Get argument line
argline = self.return_argline(self.bse_options)
- print('Running {}'.format(argline))
+ print("Running {}".format(argline))
# Run system
out = binary_c_python_api.run_system(
argline,
@@ -555,13 +570,13 @@ class Population(object):
else:
return out
-
def evolve_population_mp(self):
"""
Function to evolve the population with multiprocessing approach. Using pathos to be able to include class-owned functions.
"""
import multiprocessing as mp
+
# from pathos.multiprocessing import ProcessingPool as Pool
from pathos.pools import _ProcessPool as Pool
@@ -569,14 +584,17 @@ class Population(object):
p = Pool(processes=self.grid_options["amt_cores"])
# Execute
- # TODO: calculate the chunksize value based on: total starcount and cores used.
- r = list(p.imap_unordered(self.evolve_system_mp, self.yield_system_mp(), chunksize=20))
+ # TODO: calculate the chunksize value based on: total starcount and cores used.
+ r = list(
+ p.imap_unordered(
+ self.evolve_system_mp, self.yield_system_mp(), chunksize=20
+ )
+ )
# Handle clean termination of the whole multiprocessing (making sure there are no zombie processes (https://en.wikipedia.org/wiki/Zombie_process))
p.close()
p.join()
-
def evolve_population_lin(self):
"""
Function to evolve the population linearly (i.e. 1 core, no multiprocessing)
@@ -592,8 +610,9 @@ class Population(object):
self.grid_options["custom_logging_func_memaddr"],
self.grid_options["store_memaddr"],
)
- self.print_info(i+1, self.grid_options['total_starcount'], full_system_dict)
-
+ self.print_info(
+ i + 1, self.grid_options["total_starcount"], full_system_dict
+ )
def evolve_population(self):
"""
@@ -602,24 +621,30 @@ class Population(object):
##
# Prepare code/initialise grid.
- # set custom logging, set up store_memaddr, build grid code. dry run grid code.
+ # set custom logging, set up store_memaddr, build grid code. dry run grid code.
self.setup()
##
# Evolve systems: via grid_options one can choose to do this linearly, or multiprocessing method.
- if self.grid_options['evolution_type'] in self.grid_options['evolution_type_options']:
- if self.grid_options['evolution_type'] == 'mp':
+ if (
+ self.grid_options["evolution_type"]
+ in self.grid_options["evolution_type_options"]
+ ):
+ if self.grid_options["evolution_type"] == "mp":
self.evolve_population_mp()
- elif self.grid_options['evolution_type'] == 'linear':
+ elif self.grid_options["evolution_type"] == "linear":
self.evolve_population_lin()
else:
- print("Warning. you chose a wrong option for the grid evolution types. Please choose from the following: {}.".format(self.grid_options['evolution_type_options']))
+ print(
+ "Warning. you chose a wrong option for the grid evolution types. Please choose from the following: {}.".format(
+ self.grid_options["evolution_type_options"]
+ )
+ )
##
# Clean up code: remove files, unset values.
self.cleanup()
-
###################################################
# Function to test evolution algorithms
###################################################
@@ -673,7 +698,6 @@ class Population(object):
print("Error while deleting file {}".format(file))
raise FileNotFoundError
-
def clean_up_custom_logging(self, evol_type):
"""
Function to clean up the custom logging.
@@ -694,7 +718,6 @@ class Population(object):
# TODO: Unset function memory adress
# print(self.grid_options["custom_logging_func_memaddr"])
-
# remove shared library files
if self.grid_options["custom_logging_shared_library_file"]:
self.remove_file(
@@ -707,7 +730,6 @@ class Population(object):
print("Cleaning up the custom logging stuffs. type: population")
# TODO: make sure that these also work. not fully sure if necessary tho. whether its a single file, or a dict of files/memaddresses
-
def increment_probtot(self, prob):
"""
Function to add to the total probability
@@ -715,7 +737,6 @@ class Population(object):
self.grid_options["probtot"] += prob
-
def increment_count(self):
"""
Function to add to the total amount of stars
@@ -876,14 +897,18 @@ class Population(object):
code_string += (
indent * depth
- + "sampled_values_{} = {}".format(grid_variable["name"], grid_variable["spacingfunc"])
+ + "sampled_values_{} = {}".format(
+ grid_variable["name"], grid_variable["spacingfunc"]
+ )
+ "\n"
)
- # TODO: Make clear that the phasevol only works good if you sample linearly in that thing.
+ # TODO: Make clear that the phasevol only works good if you sample linearly in that thing.
code_string += (
indent * depth
- + "phasevol_{} = sampled_values_{}[1]-sampled_values_{}[0]".format(grid_variable["name"], grid_variable["name"], grid_variable["name"])
+ + "phasevol_{} = sampled_values_{}[1]-sampled_values_{}[0]".format(
+ grid_variable["name"], grid_variable["name"], grid_variable["name"]
+ )
+ "\n"
)
@@ -917,7 +942,7 @@ class Population(object):
# Set phasevol
code_string += indent * (depth + 1) + "phasevol *= phasevol_{}\n".format(
- grid_variable["name"],
+ grid_variable["name"],
)
#######################
@@ -927,7 +952,11 @@ class Population(object):
code_string += indent * (depth + 1) + "# Setting probabilities\n"
code_string += (
indent * (depth + 1)
- + "d{} = phasevol_{} * {}".format(grid_variable["name"], grid_variable["name"], grid_variable["probdist"])
+ + "d{} = phasevol_{} * {}".format(
+ grid_variable["name"],
+ grid_variable["name"],
+ grid_variable["probdist"],
+ )
+ "\n"
)
@@ -1079,7 +1108,6 @@ class Population(object):
)
code_string += indent * (depth + 1) + "\n"
-
depth -= 1
################
@@ -1092,14 +1120,12 @@ class Population(object):
+ "print('Grid has handled {} stars'.format(total_starcount))\n"
)
code_string += (
- indent * (depth + 1) + "print('with a total probability of {}'.format(self.grid_options['probtot']))\n"
+ indent * (depth + 1)
+ + "print('with a total probability of {}'.format(self.grid_options['probtot']))\n"
)
if dry_run:
- code_string += (
- indent * (depth + 1) + "return total_starcount\n"
- )
-
+ code_string += indent * (depth + 1) + "return total_starcount\n"
#################################################################################
# Stop of code generation. Here the code is saved and written
@@ -1122,7 +1148,6 @@ class Population(object):
with open(gridcode_filename, "w") as f:
f.write(code_string)
-
def load_grid_function(self):
"""
Test function to run grid stuff. mostly to test the import
@@ -1151,7 +1176,6 @@ class Population(object):
if self.grid_options["verbose"] > 0:
print("Grid code loaded")
-
def dry_run(self):
"""
Function to dry run the grid and know how many stars it will run
@@ -1161,8 +1185,7 @@ class Population(object):
system_generator = self.grid_options["system_generator"]
total_starcount = system_generator(self)
- self.grid_options['total_starcount'] = total_starcount
-
+ self.grid_options["total_starcount"] = total_starcount
def write_binary_c_calls_to_file(self, output_dir=None, output_filename=None):
"""
@@ -1205,7 +1228,6 @@ class Population(object):
print("Error. No grid function found!")
raise KeyError
-
def print_info(self, run_number, total_systems, full_system_dict):
"""
Function to print info about the current system and the progress of the grid.
@@ -1215,30 +1237,27 @@ class Population(object):
"""
# Define frequency
- if self.grid_options['verbose'] == 1:
+ if self.grid_options["verbose"] == 1:
print_freq = 1
else:
print_freq = 10
-
# Calculate amount of time left
-
# calculate amount of time passed
- time_passed = time.time() - self.grid_options['start_time_grid']
+ time_passed = time.time() - self.grid_options["start_time_evolution"]
- if run_number%print_freq==0:
+ if run_number % print_freq == 0:
binary_cmdline_string = self.return_argline(full_system_dict)
info_string = "{color_part_1} {text_part_1}{end_part_1}{color_part_2} {text_part_2}{end_part_2}".format(
color_part_1="\033[1;32;41m",
text_part_1="{}/{}".format(run_number, total_systems),
end_part_1="\033[0m",
-
color_part_2="\033[1;32;42m",
text_part_2="{}".format(binary_cmdline_string),
end_part_2="\033[0m",
-
- )
+ )
print(info_string)
+
################################################################################################
diff --git a/binarycpython/utils/grid_options_defaults.py b/binarycpython/utils/grid_options_defaults.py
index 529e9e23d..78f160eae 100644
--- a/binarycpython/utils/grid_options_defaults.py
+++ b/binarycpython/utils/grid_options_defaults.py
@@ -4,12 +4,18 @@ import sys
from binarycpython.utils.custom_logging_functions import temp_dir
grid_options_defaults_dict = {
- # general
+ ##########################
+ # general (or unordered..)
+ ##########################
"amt_cores": 1, # total amount of cores used to evolve the population
"verbose": 0, # Level of verbosity of the simulation
- "evolution_type": 'mp', # Flag for type of population evolution
- "evolution_type_options": ['mp', 'linear'], # available choices for type of population evolution
+ "binary": 0, # FLag on whether the systems are binary systems or single systems.
+ "parse_function": None, # FUnction to parse the output with.
+ "tmp_dir": temp_dir(), # Setting the temp dir of the program
+ # "output_dir":
+ ##########################
# binary_c files
+ ##########################
"binary_c_executable": os.path.join(
os.environ["BINARY_C"], "binary_c-config"
), # TODO: make this more robust
@@ -20,34 +26,55 @@ grid_options_defaults_dict = {
os.environ["BINARY_C"], "binary_c-config"
), # TODO: make this more robust
"binary_c_dir": os.environ["BINARYC_DIR"],
- # Output dir
- # "output_dir":
+ ##########################
# Custom logging
+ ##########################
"C_auto_logging": None, # Should contain a dictionary where the kes are they headers and the values are lists of parameters that should be logged. This will get parsed by autogen_C_logging_code in custom_loggion_functions.py
"C_logging_code": None, # Should contain a string which holds the logging code.
"custom_logging_func_memaddr": -1, # Contains the custom_logging functions memory address
"custom_logging_shared_library_file": None,
+ ##########################
# Store pre-loading:
+ ##########################
"store_memaddr": -1, # Contains the store object memory adress, useful for preloading. defaults to -1 and isnt used if thats the default then.
+ ##########################
# Log args: logging of arguments
+ ##########################
"log_args": 0, #
"log_args_dir": "/tmp/",
- # Grid variables: instructions to generate the values of the parameters
+ ##########################
+ # Population evolution
+ ##########################
+ ## General
+ "evolution_type": "mp", # Flag for type of population evolution
+ "evolution_type_options": [
+ "mp",
+ "linear",
+ ], # available choices for type of population evolution
+ "system_generator": None, # value that holds the function that generates the system (result of building the grid script)
+ "population_type": "grid", #
+ "population_type_options": [
+ "grid"
+ ], # TODO: fill later with monte carlo, source file etc
+ "count": 0, # total count of systems
+ "probtot": 0, # total probability
+ "weight": 1.0, # weighting for the probability
+ "repeat": 1.0, # number of times to repeat each system (probability is adjusted to be 1/repeat)
+ "start_time_evolution": 0, # Start time of the grid
+ ## Grid type evolution
"grid_variables": {}, # grid variables
"grid_code": None, # literal grid code
"gridcode_filename": None, # filename of gridcode
- "count": 0, # total count of systems
- "probtot": 0, # total probabilit
- "system_generator": None, # value that holds the function that generates the system (result of building the grid script)
- # binary
- "binary": 0,
+ ## Monte carlo type evolution
+ # TODO: make MC options
+ ## Evolution from source file
+ # TODO: make run from sourcefile options.
+ # Grid variables: instructions to generate the values of the parameters
# Locations:
- "tmp_dir": temp_dir(), # Setting the temp dir of the program
# Probability:
- "weight": 1.0, # weighting for the probability
- "repeat": 1.0, # number of times to repeat each system (probability is adjusted to be 1/repeat)
- "parse_function": None, #
- "start_time_grid": 0, # Start time of the grid
+ ##########################
+ # Unordered. Need to go through this. Copied from the perl implementation.
+ ##########################
##
# return_array_refs=>1, # quicker data parsing mode
# sort_args=>1,
diff --git a/binarycpython/utils/spacing_functions.py b/binarycpython/utils/spacing_functions.py
index fbd3d35c7..84aeecdf0 100644
--- a/binarycpython/utils/spacing_functions.py
+++ b/binarycpython/utils/spacing_functions.py
@@ -1,4 +1,5 @@
import numpy as np
+
def const(min, max, steps):
return np.linspace(min, max, steps)
diff --git a/examples/example_population.py b/examples/example_population.py
index 63f88c62f..922d01056 100644
--- a/examples/example_population.py
+++ b/examples/example_population.py
@@ -21,19 +21,20 @@ def output_lines(output):
"""
return output.splitlines()
+
def parse_function(self, output):
# extract info from the population instance
# TODO: think about whether this is smart. Passing around this object might be an overkill
- # Get some information from the
- data_dir = self.custom_options['data_dir']
- base_filename = self.custom_options['base_filename']
+ # Get some information from the
+ data_dir = self.custom_options["data_dir"]
+ base_filename = self.custom_options["base_filename"]
# Check directory, make if necessary
os.makedirs(data_dir, exist_ok=True)
- seperator = ' '
-
+ seperator = " "
+
# Create filename
outfilename = os.path.join(data_dir, base_filename)
@@ -42,16 +43,17 @@ def parse_function(self, output):
headerline = el.split()[0]
# CHeck the header and act accordingly
- if (headerline=='MY_STELLAR_DATA'):
- parameters = ['time', 'mass', 'zams_mass', 'probability', 'radius']
+ if headerline == "MY_STELLAR_DATA":
+ parameters = ["time", "mass", "zams_mass", "probability", "radius"]
values = el.split()[1:]
if not os.path.exists(outfilename):
- with open(outfilename, 'w') as f:
- f.write(seperator.join(parameters)+'\n')
+ with open(outfilename, "w") as f:
+ f.write(seperator.join(parameters) + "\n")
+
+ with open(outfilename, "a") as f:
+ f.write(seperator.join(values) + "\n")
- with open(outfilename, 'a') as f:
- f.write(seperator.join(values)+'\n')
# Create population object
example_pop = Population()
@@ -70,12 +72,10 @@ example_pop.set(
orbital_period=45000000080, # bse_options
max_evolution_time=15000, # bse_options
eccentricity=0.02, # bse_options
-
# grid_options
amt_cores=1, # grid_options
- verbose=1, # verbosity. Not fully configured correctly yet but having it value of 1 prints alot of stuff
- parse_function=parse_function, # Setting the parse function thats used in the evolve_population
-
+ verbose=1, # verbosity. Not fully configured correctly yet but having it value of 1 prints alot of stuff
+ parse_function=parse_function, # Setting the parse function thats used in the evolve_population
# Custom options # TODO: need to be set in grid_options probably
data_dir=os.path.join(
temp_dir(), "example_python_population_result"
@@ -85,7 +85,7 @@ example_pop.set(
### Custom logging
-## Below example requires changing the parse function
+## Below example requires changing the parse function
## very simple example of custom logging. Will work but need to change the parse function to handle that nicely.
# example_pop.set(
# C_auto_logging={
@@ -113,23 +113,24 @@ if(stardata->star[0].stellar_type >= 2)
/* Kill the simulation to save time */
stardata->model.max_evolution_time = stardata->model.time - stardata->model.dtm;
};
-""")
+"""
+)
# Add grid variables
-resolution = {'M_1': 100}
+resolution = {"M_1": 100}
# Mass
example_pop.add_grid_variable(
name="lnm1",
longname="Primary mass",
valuerange=[2, 150],
- resolution="{}".format(resolution['M_1']),
- spacingfunc="const(math.log(2), math.log(150), {})".format(resolution['M_1']),
+ resolution="{}".format(resolution["M_1"]),
+ spacingfunc="const(math.log(2), math.log(150), {})".format(resolution["M_1"]),
precode="M_1=math.exp(lnm1)",
probdist="three_part_powerlaw(M_1, 0.1, 0.5, 1.0, 150, -1.3, -2.3, -2.3)*M_1",
dphasevol="dlnm1",
parameter_name="M_1",
- condition="", # Impose a condition on this grid variable. Mostly for a check for yourself
+ condition="", # Impose a condition on this grid variable. Mostly for a check for yourself
)
# Exporting of all the settings can be done with .export_all_info()
@@ -156,4 +157,4 @@ print(output)
# Wrapping up the results to an hdf5 file can be done by using the create_hdf5(<directory containing data and settings>)
# This function takes the settings file (ending in _settings.json) and the data files (ending in .dat) from the data_dir
# and packing them into an hdf5 file, which is then written into the same data_dir directory
-create_hdf5(data_dir=example_pop.custom_options["data_dir"], name='example_pop.hdf5')
\ No newline at end of file
+create_hdf5(data_dir=example_pop.custom_options["data_dir"], name="example_pop.hdf5")
diff --git a/tests/population/grid_tests.py b/tests/population/grid_tests.py
index 44c3e1f20..57b211596 100644
--- a/tests/population/grid_tests.py
+++ b/tests/population/grid_tests.py
@@ -289,21 +289,19 @@ test_pop.set(
# ###
# testing population:
-test_pop.set(verbose=1,
- amt_cores=2,
- binary=0,
+test_pop.set(
+ verbose=1, amt_cores=2, binary=0,
)
-
-resolution = {'M_1': 10, 'per': 10}
+resolution = {"M_1": 10, "per": 10}
test_pop.add_grid_variable(
name="M_1",
longname="log primary mass",
valuerange=[10, 100],
- resolution="{}".format(resolution['M_1']),
- spacingfunc="const(10, 100, {})".format(resolution['M_1']),
+ resolution="{}".format(resolution["M_1"]),
+ spacingfunc="const(10, 100, {})".format(resolution["M_1"]),
# precode="M_1=math.exp(lnm1)",
probdist="flat(M_1)",
# probdist='self.custom_options["extra_prob_function"](M_1)',
@@ -316,8 +314,8 @@ test_pop.add_grid_variable(
name="period",
longname="period",
valuerange=["M_1", 20],
- resolution="{}".format(resolution['per']),
- spacingfunc="np.linspace(1, 10, {})".format(resolution['per']),
+ resolution="{}".format(resolution["per"]),
+ spacingfunc="np.linspace(1, 10, {})".format(resolution["per"]),
precode="orbital_period = period**2",
probdist="flat(orbital_period)",
parameter_name="orbital_period",
@@ -325,13 +323,9 @@ test_pop.add_grid_variable(
condition='self.grid_options["binary"]==0',
)
-test_pop.set(verbose=1,
- amt_cores=2,
- binary=0,
- evolution_type='linear'
-)
+test_pop.set(verbose=1, amt_cores=2, binary=0, evolution_type="linear")
# test_pop.test_evolve_population_mp()
# test_pop.evolve_population_mp_chunks()
-test_pop.evolve_population()
\ No newline at end of file
+test_pop.evolve_population()
diff --git a/tests/population/profiling/readout_profile.py b/tests/population/profiling/readout_profile.py
index 5975c324f..654f07514 100644
--- a/tests/population/profiling/readout_profile.py
+++ b/tests/population/profiling/readout_profile.py
@@ -1,5 +1,4 @@
import pstats
-p = pstats.Stats('profile_run.txt')
-p.sort_stats('tottime').print_stats(10)
-
+p = pstats.Stats("profile_run.txt")
+p.sort_stats("tottime").print_stats(10)
diff --git a/tests/population/scaling/evolve_population_comparing_with_multiprocessing.py b/tests/population/scaling/evolve_population_comparing_with_multiprocessing.py
index 78840fbea..f41769e35 100644
--- a/tests/population/scaling/evolve_population_comparing_with_multiprocessing.py
+++ b/tests/population/scaling/evolve_population_comparing_with_multiprocessing.py
@@ -79,25 +79,24 @@ def parse_function(self, output):
f.write(seperator.join(values) + "\n")
-resolution = {'M_1': res_m_1, 'per': res_per}
+resolution = {"M_1": res_m_1, "per": res_per}
total_systems = np.prod([el for el in resolution.values()])
-result_dir = 'scaling_results'
+result_dir = "scaling_results"
# AMT_CORES = int(amt_cores)
test_pop = Population()
-test_pop.set(verbose=1,
- amt_cores=AMT_CORES,
- binary=1,
+test_pop.set(
+ verbose=1, amt_cores=AMT_CORES, binary=1,
)
test_pop.add_grid_variable(
name="M_1",
longname="log primary mass",
valuerange=[1, 100],
- resolution="{}".format(resolution['M_1']),
- spacingfunc="const(1, 100, {})".format(resolution['M_1']),
+ resolution="{}".format(resolution["M_1"]),
+ spacingfunc="const(1, 100, {})".format(resolution["M_1"]),
precode="",
# precode="M_1=math.exp(lnm1)",
probdist="Kroupa2001(M_1)",
@@ -112,8 +111,8 @@ test_pop.add_grid_variable(
name="period",
longname="period",
valuerange=["M_1", 20],
- resolution="{}".format(resolution['per']),
- spacingfunc="np.linspace(1, 10, {})".format(resolution['per']),
+ resolution="{}".format(resolution["per"]),
+ spacingfunc="np.linspace(1, 10, {})".format(resolution["per"]),
precode="orbital_period = period**2",
probdist="flat(orbital_period)",
parameter_name="orbital_period",
@@ -125,14 +124,18 @@ test_pop.add_grid_variable(
# MP
total_mp_start = time.time()
-#evolve_mp_time = test_pop.test_evolve_population_mp()
+# evolve_mp_time = test_pop.test_evolve_population_mp()
evolve_mp_time = test_pop.test_evolve_population_mp_chunks()
# evolve_mp_time = test_pop.test_evolve_population_mp_chunks_better()
total_mp_stop = time.time()
total_mp = total_mp_stop - total_mp_start
-print("MP ({} nodes) run with {} systems: {} of which {} spent on evolving the systems".format(AMT_CORES, total_systems, total_mp, evolve_mp_time))
+print(
+ "MP ({} nodes) run with {} systems: {} of which {} spent on evolving the systems".format(
+ AMT_CORES, total_systems, total_mp, evolve_mp_time
+ )
+)
# Lin
total_lin_start = time.time()
@@ -143,15 +146,19 @@ total_lin_stop = time.time()
total_lin = total_lin_stop - total_lin_start
-print("linear run with {} systems: {} of which {} spent on evolving the systems".format(total_systems, total_lin, evolve_lin_time))
+print(
+ "linear run with {} systems: {} of which {} spent on evolving the systems".format(
+ total_systems, total_lin, evolve_lin_time
+ )
+)
####
-speed_up = total_lin/total_mp
-print("The speed up by using MP is: {}".format(total_lin/total_mp))
+speed_up = total_lin / total_mp
+print("The speed up by using MP is: {}".format(total_lin / total_mp))
# Write to file:
# amt_cores, amt_systems, total_time_lin, total_time_mp, speedup
-#with open(os.path.join(result_dir, "comparison_result_{}.dat".format(name_testcase)), "a") as f:
+# with open(os.path.join(result_dir, "comparison_result_{}.dat".format(name_testcase)), "a") as f:
# res = (AMT_CORES, total_systems, total_lin, total_mp, speed_up)
# f.write(str(res) + "\n")
diff --git a/tests/population/scaling/generate_scaling_script.py b/tests/population/scaling/generate_scaling_script.py
deleted file mode 100644
index 5286de381..000000000
--- a/tests/population/scaling/generate_scaling_script.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# Script to generate the test script :P
-import socket, psutil
-import numpy as np
-
-amount_of_cores = psutil.cpu_count(logical=False)
-amount_of_cpus = psutil.cpu_count()
-hostname = socket.gethostname()
-
-if amount_of_cpus <= 4:
- stepsize = 1
-elif 4 < amount_of_cpus <= 24:
- stepsize = 2
-elif 24 < amount_of_cpus <= 48:
- stepsize = 4
-
-
-amt_repeat = 3
-
-with open('run_scaling_{}.sh', .format(hostname), 'w') as f:
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-# Generate the lines to run
-with open('run_tests_{}.sh'.format(hostname), 'w') as f:
- command = ""
- for cpu_count in np.arange(stepsize, amount_of_cpus + stepsize, stepsize):
-
- command += "python3 evolve_population_comparing_with_multiprocessing.py 10 10 {} {}\n".format(cpu_count, hostname)
- command += "python3 evolve_population_comparing_with_multiprocessing.py 20 20 {} {}\n".format(cpu_count, hostname)
- command += "python3 evolve_population_comparing_with_multiprocessing.py 25 25 {} {}\n".format(cpu_count, hostname)
- command += "python3 evolve_population_comparing_with_multiprocessing.py 35 35 {} {}\n".format(cpu_count, hostname)
- command += "python3 evolve_population_comparing_with_multiprocessing.py 50 50 {} {}\n".format(cpu_count, hostname)
-
- command += "\n"
-
- f.write(command)
- command = ""
diff --git a/tests/population/scaling/generate_test_script.py b/tests/population/scaling/generate_test_script.py
index 5eec6a1ee..a9c4f1edd 100644
--- a/tests/population/scaling/generate_test_script.py
+++ b/tests/population/scaling/generate_test_script.py
@@ -1,4 +1,4 @@
-# Script to generate the test script :P
+# Script to generate the test script :P
import socket, psutil
import numpy as np
@@ -14,29 +14,26 @@ elif 24 < amount_of_cpus <= 48:
stepsize = 4
-
-
-
-
-
-
-
-
-
-
-
-
-
# Generate the lines to run
-with open('run_tests_{}.sh'.format(hostname), 'w') as f:
+with open("run_tests_{}.sh".format(hostname), "w") as f:
command = ""
for cpu_count in np.arange(stepsize, amount_of_cpus + stepsize, stepsize):
- command += "python3 evolve_population_comparing_with_multiprocessing.py 10 10 {} {}\n".format(cpu_count, hostname)
- command += "python3 evolve_population_comparing_with_multiprocessing.py 20 20 {} {}\n".format(cpu_count, hostname)
- command += "python3 evolve_population_comparing_with_multiprocessing.py 25 25 {} {}\n".format(cpu_count, hostname)
- command += "python3 evolve_population_comparing_with_multiprocessing.py 35 35 {} {}\n".format(cpu_count, hostname)
- command += "python3 evolve_population_comparing_with_multiprocessing.py 50 50 {} {}\n".format(cpu_count, hostname)
+ command += "python3 evolve_population_comparing_with_multiprocessing.py 10 10 {} {}\n".format(
+ cpu_count, hostname
+ )
+ command += "python3 evolve_population_comparing_with_multiprocessing.py 20 20 {} {}\n".format(
+ cpu_count, hostname
+ )
+ command += "python3 evolve_population_comparing_with_multiprocessing.py 25 25 {} {}\n".format(
+ cpu_count, hostname
+ )
+ command += "python3 evolve_population_comparing_with_multiprocessing.py 35 35 {} {}\n".format(
+ cpu_count, hostname
+ )
+ command += "python3 evolve_population_comparing_with_multiprocessing.py 50 50 {} {}\n".format(
+ cpu_count, hostname
+ )
command += "\n"
diff --git a/tests/population/scaling/plot_amdahl.py b/tests/population/scaling/plot_amdahl.py
index 240d7fa1c..53cd235ac 100644
--- a/tests/population/scaling/plot_amdahl.py
+++ b/tests/population/scaling/plot_amdahl.py
@@ -1,11 +1,13 @@
import matplotlib.pyplot as plt
import numpy as np
-def amdahl(f,n):
- return 1.0/((1-f) + (f/n))
+
+def amdahl(f, n):
+ return 1.0 / ((1 - f) + (f / n))
+
cores = np.arange(1, 10, 0.1)
-values_list = []
+values_list = []
par_vals = np.arange(0, 1.1, 0.1)
@@ -15,5 +17,5 @@ for par_val in par_vals:
for values in values_list:
- plt.plot(cores, values, 'b-')
-plt.show()
\ No newline at end of file
+ plt.plot(cores, values, "b-")
+plt.show()
diff --git a/tests/population/scaling/plot_scaling.py b/tests/population/scaling/plot_scaling.py
index f0061147f..575b210da 100644
--- a/tests/population/scaling/plot_scaling.py
+++ b/tests/population/scaling/plot_scaling.py
@@ -8,11 +8,12 @@ import numpy as np
def calc_mean_and_std(arr):
return np.mean(arr), np.std(arr)
+
def calculate_results(results, unique_amt_cores, unique_amt_systems):
"""
Function to calculate the numbers
"""
-
+
calculated_results = []
for i in unique_amt_cores:
for j in unique_amt_systems:
@@ -59,7 +60,10 @@ def calculate_results(results, unique_amt_cores, unique_amt_systems):
return calculated_results
-def plot_speedup_and_efficiency(calculated_results, unique_amt_cores, unique_amt_systems):
+
+def plot_speedup_and_efficiency(
+ calculated_results, unique_amt_cores, unique_amt_systems
+):
x_position_shift = 0
y_position_shift = -0.05
max_speedup = 0
@@ -100,7 +104,7 @@ def plot_speedup_and_efficiency(calculated_results, unique_amt_cores, unique_amt
marker="^",
label="Speed up & efficiency of {} systems".format(amt_systems),
)
-
+
ax2.plot(cores, efficiency, alpha=0.5)
x_position_shift += 0.1
@@ -118,16 +122,23 @@ def plot_speedup_and_efficiency(calculated_results, unique_amt_cores, unique_amt
ax1.grid()
ax1.legend(loc=4)
- fig.savefig(os.path.join(img_dir, "speedup_scaling_{}.{}".format(name_testcase, "png")))
- fig.savefig(os.path.join(img_dir, "speedup_scaling_{}.{}".format(name_testcase, "pdf")))
- fig.savefig(os.path.join(img_dir, "speedup_scaling_{}.{}".format(name_testcase, "eps")))
+ fig.savefig(
+ os.path.join(img_dir, "speedup_scaling_{}.{}".format(name_testcase, "png"))
+ )
+ fig.savefig(
+ os.path.join(img_dir, "speedup_scaling_{}.{}".format(name_testcase, "pdf"))
+ )
+ fig.savefig(
+ os.path.join(img_dir, "speedup_scaling_{}.{}".format(name_testcase, "eps"))
+ )
plt.show()
+
def plot_runtime(calculated_results, unique_amt_cores, unique_amt_systems):
fig = plt.figure()
ax = fig.add_subplot(111)
- color_options = ['blue', 'green', 'red', 'black', 'yellow', 'magenta']
+ color_options = ["blue", "green", "red", "black", "yellow", "magenta"]
x_position_shift = 0
y_position_shift = -0.05
@@ -143,40 +154,65 @@ def plot_runtime(calculated_results, unique_amt_cores, unique_amt_systems):
std_mp = []
for el in calculated_results:
- if el['systems']==amt_systems:
- cores.append(el['cores'] + x_position_shift)
+ if el["systems"] == amt_systems:
+ cores.append(el["cores"] + x_position_shift)
- mean_times_linear.append(el['mean_time_sequential'])
- std_linear.append(el['std_sequential'])
+ mean_times_linear.append(el["mean_time_sequential"])
+ std_linear.append(el["std_sequential"])
- mean_times_multiprocessing.append(el['mean_time_multiprocessing'])
- std_mp.append(el['std_multiprocessing'])
+ mean_times_multiprocessing.append(el["mean_time_multiprocessing"])
+ std_mp.append(el["std_multiprocessing"])
# # add number of runs its based on
# plt.text(el['cores'] + x_position_shift+0.01, el['mean_ratio']+y_position_shift, el['total_runs'])
- ax.errorbar(cores, mean_times_linear, std_linear, color=colors[i], linewidth=1, marker='P', label='{} systems linear run'.format(amt_systems))
- ax.errorbar(cores, mean_times_multiprocessing, std_mp, linewidth=1, color=colors[i], marker='d', label='{} systems MP run'.format(amt_systems))
+ ax.errorbar(
+ cores,
+ mean_times_linear,
+ std_linear,
+ color=colors[i],
+ linewidth=1,
+ marker="P",
+ label="{} systems linear run".format(amt_systems),
+ )
+ ax.errorbar(
+ cores,
+ mean_times_multiprocessing,
+ std_mp,
+ linewidth=1,
+ color=colors[i],
+ marker="d",
+ label="{} systems MP run".format(amt_systems),
+ )
x_position_shift += 0.1
- ax.set_title("Total time for sequential and MP for different amounts of systems on {}".format(name_testcase))
+ ax.set_title(
+ "Total time for sequential and MP for different amounts of systems on {}".format(
+ name_testcase
+ )
+ )
ax.set_xlabel("Amount of cores used")
ax.set_ylabel("Total time taken (s)")
ax.set_xlim(0, max(unique_amt_cores) + 20)
- ax.set_yscale('log')
+ ax.set_yscale("log")
ax.grid()
ax.legend()
- fig.savefig(os.path.join(img_dir, 'total_time_scaling_{}.{}'.format(name_testcase, 'png')))
- fig.savefig(os.path.join(img_dir, 'total_time_scaling_{}.{}'.format(name_testcase, 'pdf')))
- fig.savefig(os.path.join(img_dir, 'total_time_scaling_{}.{}'.format(name_testcase, 'eps')))
+ fig.savefig(
+ os.path.join(img_dir, "total_time_scaling_{}.{}".format(name_testcase, "png"))
+ )
+ fig.savefig(
+ os.path.join(img_dir, "total_time_scaling_{}.{}".format(name_testcase, "pdf"))
+ )
+ fig.savefig(
+ os.path.join(img_dir, "total_time_scaling_{}.{}".format(name_testcase, "eps"))
+ )
plt.show()
-
-#####################
+#####################
# Configure
result_file = "comparison_result_laptop.dat"
@@ -218,4 +254,4 @@ plot_speedup_and_efficiency(calculated_results, unique_amt_cores, unique_amt_sys
########################################################################################
# Plot run time
-plot_runtime(calculated_results, unique_amt_cores, unique_amt_systems)
\ No newline at end of file
+plot_runtime(calculated_results, unique_amt_cores, unique_amt_systems)
diff --git a/tests/population/scaling/plot_scaling_new.py b/tests/population/scaling/plot_scaling_new.py
index e739b3323..9125e2409 100644
--- a/tests/population/scaling/plot_scaling_new.py
+++ b/tests/population/scaling/plot_scaling_new.py
@@ -6,15 +6,17 @@ import numpy as np
import json
import math
-def amdahl(f,n):
- return 1.0/((1-f) + (f/n))
+
+def amdahl(f, n):
+ return 1.0 / ((1 - f) + (f / n))
+
#################################
# Files
-scaling_result_dir = 'scaling_results'
+scaling_result_dir = "scaling_results"
filenames = [
- 'astro2_2500_systems.json',
- 'astro2_3000_systems.json',
+ "astro2_2500_systems.json",
+ "astro2_3000_systems.json",
]
result_jsons = []
for filename in filenames:
@@ -27,26 +29,29 @@ ax2 = ax1.twinx()
for jsonfile in result_jsons:
print(jsonfile)
- with open(jsonfile, 'r') as f:
+ with open(jsonfile, "r") as f:
result_data = json.loads(f.read())
# Get linear data
- linear_data = result_data['linear']
+ linear_data = result_data["linear"]
linear_mean = np.mean(linear_data)
linear_stdev = np.std(linear_data)
cpus, speedups, efficiencies, stddev_speedups = [], [], [], []
- for amt_cpus in result_data['mp']:
+ for amt_cpus in result_data["mp"]:
# Get mp data
- mp_data = result_data['mp'][amt_cpus]
+ mp_data = result_data["mp"][amt_cpus]
mp_mean = np.mean(mp_data)
mp_stdev = np.std(mp_data)
- # Calc
+ # Calc
amt_cpus = int(amt_cpus)
- speedup = linear_mean/mp_mean
- stddev_speedup = math.sqrt((linear_stdev/linear_mean)**2 + (mp_stdev/mp_mean)**2) * speedup
- efficiency = speedup/int(amt_cpus)
+ speedup = linear_mean / mp_mean
+ stddev_speedup = (
+ math.sqrt((linear_stdev / linear_mean) ** 2 + (mp_stdev / mp_mean) ** 2)
+ * speedup
+ )
+ efficiency = speedup / int(amt_cpus)
# Add to lists
cpus.append(amt_cpus)
@@ -61,7 +66,9 @@ for jsonfile in result_jsons:
stddev_speedups,
linestyle="None",
marker="^",
- label="Speed up & efficiency of {} systems".format(result_data['amt_systems']),
+ label="Speed up & efficiency of {} systems".format(
+ result_data["amt_systems"]
+ ),
)
ax2.plot(cpus, efficiencies, alpha=0.5)
@@ -71,7 +78,7 @@ for jsonfile in result_jsons:
# Do Amdahls law fitting
# cores = np.arange(1, 48, 0.1)
-# values_list = []
+# values_list = []
# par_step = 0.005
# par_vals = np.arange(.95, 1, par_step)
@@ -84,12 +91,11 @@ for jsonfile in result_jsons:
# ax1.plot(cores, values, label="par_val={}".format(par_vals[i]))
-
#################################
# Adding plot make up
ax1.set_title(
"Speed up ratio vs amount of cores for different amounts of systems on {}".format(
- 'name_testcase'
+ "name_testcase"
)
)
@@ -102,22 +108,12 @@ ax1.set_ylabel("Speed up ratio (time_linear/time_parallel)")
# ax2.set_ylim(0, 1)
-
-
-
-
-
-
-
-
ax1.grid()
ax1.legend(loc=4)
-ax1.set_xscale('log')
-ax2.set_xscale('log')
+ax1.set_xscale("log")
+ax2.set_xscale("log")
# fig.savefig(os.path.join(img_dir, "speedup_scaling_{}.{}".format(name_testcase, "png")))
# fig.savefig(os.path.join(img_dir, "speedup_scaling_{}.{}".format(name_testcase, "pdf")))
# fig.savefig(os.path.join(img_dir, "speedup_scaling_{}.{}".format(name_testcase, "eps")))
plt.show()
-
-
diff --git a/tests/population/scaling/scaling_script.py b/tests/population/scaling/scaling_script.py
index 24d87a7a4..e7bc30e11 100644
--- a/tests/population/scaling/scaling_script.py
+++ b/tests/population/scaling/scaling_script.py
@@ -1,4 +1,4 @@
-# Script to generate the test script :P
+# Script to generate the test script :P
import socket, psutil
import numpy as np
import time
@@ -24,44 +24,43 @@ elif 4 < amount_of_cpus:
# stepsize = 4
cpu_list = [1]
-for i in range(1, int(amount_of_cpus/stepsize) + 1 ):
- cpu_amt = i * stepsize
+for i in range(1, int(amount_of_cpus / stepsize) + 1):
+ cpu_amt = i * stepsize
if not cpu_amt in cpu_list:
cpu_list.append(i * stepsize)
# set some info
amt_repeats = 5
-resolution = {'M_1': 50, 'per': 60}
+resolution = {"M_1": 50, "per": 60}
total_systems = int(np.prod([el for el in resolution.values()]))
-result_dir = 'scaling_results'
-testcase = 'linear vs MP batched'
+result_dir = "scaling_results"
+testcase = "linear vs MP batched"
# Create dictionairy in which to store all the results:
result_dict = {}
#
-result_dict['amt_systems'] = total_systems
-result_dict['hostname'] = hostname
-result_dict['amt_logical_cores'] = amount_of_cpus
-result_dict['amt_of_physical_cores'] = amount_of_cores
-result_dict['testcase'] = testcase
+result_dict["amt_systems"] = total_systems
+result_dict["hostname"] = hostname
+result_dict["amt_logical_cores"] = amount_of_cpus
+result_dict["amt_of_physical_cores"] = amount_of_cores
+result_dict["testcase"] = testcase
-#################
+#################
# Configuring population
test_pop = Population()
test_pop.set(
- verbose=1,
- binary=1,
+ verbose=1, binary=1,
)
test_pop.add_grid_variable(
name="M_1",
longname="log primary mass",
valuerange=[1, 100],
- resolution="{}".format(resolution['M_1']),
- spacingfunc="const(1, 100, {})".format(resolution['M_1']),
+ resolution="{}".format(resolution["M_1"]),
+ spacingfunc="const(1, 100, {})".format(resolution["M_1"]),
probdist="Kroupa2001(M_1)",
# probdist='self.custom_options["extra_prob_function"](M_1)',
dphasevol="dlnm1",
@@ -73,8 +72,8 @@ test_pop.add_grid_variable(
name="period",
longname="period",
valuerange=["M_1", 20],
- resolution="{}".format(resolution['per']),
- spacingfunc="np.linspace(1, 10, {})".format(resolution['per']),
+ resolution="{}".format(resolution["per"]),
+ spacingfunc="np.linspace(1, 10, {})".format(resolution["per"]),
precode="orbital_period = period**2",
probdist="flat(orbital_period)",
parameter_name="orbital_period",
@@ -92,11 +91,15 @@ for repeat in range(amt_repeats):
evolve_lin_time = test_pop.test_evolve_population_lin()
total_lin = time.time() - total_lin_start
- print("linear run with {} systems: {} of which {} spent on evolving the systems".format(total_systems, total_lin, evolve_lin_time))
+ print(
+ "linear run with {} systems: {} of which {} spent on evolving the systems".format(
+ total_systems, total_lin, evolve_lin_time
+ )
+ )
linear_times.append(total_lin)
-result_dict['linear'] = linear_times
+result_dict["linear"] = linear_times
#######################################################################################
# MP runs
@@ -107,19 +110,25 @@ for cpu_amt in cpu_list:
test_pop.set(amt_cores=cpu_amt)
- #
+ #
for repeat in range(amt_repeats):
total_mp_start = time.time()
evolve_mp_time = test_pop.evolve_population_mp_chunks()
total_mp = time.time() - total_mp_start
- print("MP ({} nodes) run with {} systems: {} of which {} spent on evolving the systems".format(cpu_amt, total_systems, total_mp, evolve_mp_time))
+ print(
+ "MP ({} nodes) run with {} systems: {} of which {} spent on evolving the systems".format(
+ cpu_amt, total_systems, total_mp, evolve_mp_time
+ )
+ )
mp_times.append(total_mp)
mp_dict[cpu_amt] = mp_times
-result_dict['mp'] = mp_dict
+result_dict["mp"] = mp_dict
# Write to file
-with open('scaling_results/{}_{}_systems.json'.format(hostname, total_systems), 'w') as f:
+with open(
+ "scaling_results/{}_{}_systems.json".format(hostname, total_systems), "w"
+) as f:
f.write(json.dumps(result_dict))
--
GitLab