From 5255a155584c4b4a940b71a1b7cd6c17543eba69 Mon Sep 17 00:00:00 2001
From: David Hendriks <davidhendriks93@gmail.com>
Date: Sun, 8 Mar 2020 21:58:04 +0000
Subject: [PATCH] updated grid code to be more clean, and have dedicated setup
and cleanup functionalities. hopefully code will stay cleaner
---
binarycpython/utils/grid.py | 459 +++++--------------
binarycpython/utils/grid_options_defaults.py | 2 +
examples/example_population.py | 6 +-
src/binary_c_python_api.c | 2 +-
tests/population/grid_tests.py | 10 +-
5 files changed, 124 insertions(+), 355 deletions(-)
diff --git a/binarycpython/utils/grid.py b/binarycpython/utils/grid.py
index 3c3a9c416..fdab898bc 100644
--- a/binarycpython/utils/grid.py
+++ b/binarycpython/utils/grid.py
@@ -433,105 +433,14 @@ class Population(object):
# Evolution functions
###################################################
- def evolve_single(self, parse_function=None, clean_up_custom_logging_files=True):
- """
- Function to run a single system
-
- The output of the run gets returned, unless a parse function is given to this function.
+ def setup(self):
"""
+ Function to set up the necessary stuff for the population evolution:
+ # TODO: Make other kinds of populations possible. i.e, read out type of grid, and set up accordingly
- ### Custom logging code:
- self.set_custom_logging()
-
- # Get argument line
- argline = self.return_argline(self.bse_options)
- print('Running {}'.format(argline))
- # Run system
- out = binary_c_python_api.run_system(
- argline,
- self.grid_options["custom_logging_func_memaddr"],
- self.grid_options["store_memaddr"],
- ) # Todo: change this to run_binary again but then build in checks in binary
+ # TODO: make this function more general. Have it explicitly set the system_generator function
- # TODO: add call to function that cleans up the temp customlogging dir, and unloads the loaded libraries.
- # TODO: make a switch to turn this off
- if clean_up_custom_logging_files:
- self.clean_up_custom_logging(evol_type="single")
-
- # Parse
- if parse_function:
- parse_function(self, out)
- else:
- return out
-
- def evolve_population(self, parse_function, custom_arg_file=None):
"""
- The function that will evolve the population. This function contains many steps
-
- TODO: fix the verbosity in this function when this function is finished
- """
-
- ### Custom logging code:
- self.set_custom_logging()
-
- ### Load store
- if self.grid_options["verbose"] > 0:
- print("loading binary_c store information")
- self.grid_options["store_memaddr"] = binary_c_python_api.return_store("")
-
- # Execute.
- # TODO: CHange this part alot. This is not finished whatsoever
- out = binary_c_python_api.run_population(
- self.return_argline(),
- self.grid_options["custom_logging_func_memaddr"],
- self.grid_options["store_memaddr"],
- )
-
- print(out)
-
- ### Arguments
- # If user inputs a file containing arg lines then use that
- if custom_arg_file:
- # check if file exists
- if os.path.isfile(custom_arg_file):
- # load file
- with open(custom_arg_file) as f:
- # Load lines into list
- temp = f.read().splitlines()
-
- # Filter out all the lines that dont start with binary_c
- population_arglines = [
- line for line in temp if line.startswith("binary_c")
- ]
-
- else:
- # generate population from options
-
- pass
-
- quit()
-
- #######
- # Do stuff
- for line in population_arglines:
- print(line)
-
- pass
-
- # TODO: add call to function that cleans up the temp customlogging dir, and unloads the loaded libraries.
-
- ###################################################
- # Testing functions
- ###################################################
-
- def test_evolve_population_lin(self):
- """
- Test function to evolve a population in a linear way.
-
- returns total time spent on the actual interfacing with binaryc
- """
-
- import time
#######################
### Custom logging code:
@@ -550,304 +459,170 @@ class Population(object):
self.dry_run()
- total_starcount_run = self.grid_options['total_starcount']
- print("Total starcount for this run will be: {}".format(total_starcount_run))
+ print("Total starcount for this run will be: {}".format(self.grid_options['total_starcount']))
#######################
- # Linear run
- start_lin = time.time()
-
+ # Reset values and prepare the grid function
self.grid_options['probtot'] = 0 # To make sure that the values are reset. TODO: fix this in a cleaner way
-
+ self.grid_options['start_time_grid'] = time.time() # Setting start time of grid
+
+ #
self.generate_grid_code(dry_run=False)
+ #
self.load_grid_function()
- for i, system in enumerate(self.grid_options["system_generator"](self)):
- full_system_dict = self.bse_options.copy()
- full_system_dict.update(system)
-
- binary_cmdline_string = self.return_argline(full_system_dict)
- out = binary_c_python_api.run_population(
- binary_cmdline_string,
- self.grid_options["custom_logging_func_memaddr"],
- self.grid_options["store_memaddr"],
- )
- print("{}/{}".format(i+1, total_starcount_run), binary_cmdline_string)
-
- stop_lin = time.time()
- print(
- "Without mp: {} systems took {}s".format(total_starcount_run, stop_lin-start_lin))
-
- return stop_lin-start_lin
-
-
- def test_evolve_population_mp(self):
+ def cleanup(self):
"""
- Test function to evolve a population in a parallel way.
-
- returns total time spent on the actual interfacing with binaryc
+ Function that handles all the cleaning up after the grid has been generated and/or run
+
+ - reset values to 0
+ - remove grid file
+ - unload grid function/module
+ - remove dry grid file
+ - unload dry grid function/module
"""
- import time
- import multiprocessing as mp
- from pathos.multiprocessing import ProcessingPool as Pool
-
- #######################
- ### Custom logging code:
- self.set_custom_logging()
-
- ### Load store
- self.grid_options["store_memaddr"] = binary_c_python_api.return_store("")
-
- #######################
- # Dry run and getting starcount
+ # Reset values
+ self.grid_options['count'] = 0
self.grid_options['probtot'] = 0
+ self.grid_options['system_generator'] = None
- self.generate_grid_code(dry_run=True)
+ # Remove files
+ # Unload functions
- self.load_grid_function()
- self.dry_run()
-
- total_starcount_run = self.grid_options['total_starcount']
- print("Total starcount for this run will be: {}".format(total_starcount_run))
-
- #######################
- # MP run
- self.grid_options['probtot'] = 0 # To make sure that the values are reset. TODO: fix this in a cleaner way
-
- start_mp = time.time()
-
- self.generate_grid_code(dry_run=False)
-
- self.load_grid_function()
+ def evolve_system_mp(self, binary_cmdline_string):
+ """
+ Function that the multiprocessing evolution method calls to evolve a system
+ """
- def evolve_system(binary_cmdline_string):
- # print(binary_cmdline_string)
- # pass
- # print('next')
- # self.set_bse_option("M_1", mass)
- # out = binary_c_python_api.run_population(
- # binary_cmdline_string,
- # self.grid_options["custom_logging_func_memaddr"],
- # self.grid_options["store_memaddr"],
- # )
- pass
- # # parse_function(self, out)
+ out = binary_c_python_api.run_population(
+ binary_cmdline_string,
+ self.grid_options["custom_logging_func_memaddr"],
+ self.grid_options["store_memaddr"],
+ )
+ if self.grid_options['parse_function']:
+ self.grid_options['parse_function'](self, out)
- def yield_system():
- for i, system in enumerate(self.grid_options["system_generator"](self)):
- full_system_dict = self.bse_options.copy()
- full_system_dict.update(system)
- binary_cmdline_string = self.return_argline(full_system_dict)
- # print("{}/{}".format(i+1, total_starcount_run), binary_cmdline_string)
- yield binary_cmdline_string
- # yield i
- print("generator done")
+ def yield_system_mp(self):
+ """
+ Function that the multiprocessing evolution method calls to yield systems
+ """
- # Create pool
- p = Pool(nodes=self.grid_options["amt_cores"])
+ for i, system in enumerate(self.grid_options["system_generator"](self)):
+ full_system_dict = self.bse_options.copy()
+ full_system_dict.update(system)
- # Execute
- r = list(p.imap(evolve_system, yield_system()))
+ binary_cmdline_string = self.return_argline(full_system_dict)
- stop_mp = time.time()
+ self.print_info(i+1, self.grid_options['total_starcount'], full_system_dict)
+ yield binary_cmdline_string
- # Give feedback
- print(
- "with mp: {} systems took {}s using {} cores".format(
- self.grid_options['total_starcount'],
- stop_mp - start_mp,
- self.grid_options["amt_cores"],
- )
- )
+ print("generator done")
- return stop_mp - start_mp
- def test_evolve_population_mp_chunks(self):
+ def evolve_single(self, parse_function=None, clean_up_custom_logging_files=True):
"""
- Test function to evolve a population in a parallel way.
-
- returns total time spent on the actual interfacing with binaryc
+ Function to run a single system
+
+ The output of the run gets returned, unless a parse function is given to this function.
"""
- import time
- import multiprocessing as mp
- # from pathos.multiprocessing import ProcessingPool as Pool
- from pathos.pools import _ProcessPool as Pool
- #######################
### Custom logging code:
self.set_custom_logging()
- ### Load store
- self.grid_options["store_memaddr"] = binary_c_python_api.return_store("")
-
- #######################
- # Dry run and getting starcount
- self.grid_options['probtot'] = 0
-
- self.generate_grid_code(dry_run=True)
-
- self.load_grid_function()
-
- self.dry_run()
-
- total_starcount_run = self.grid_options['total_starcount']
- print("Total starcount for this run will be: {}".format(total_starcount_run))
-
- #######################
- # MP run
- self.grid_options['probtot'] = 0 # To make sure that the values are reset. TODO: fix this in a cleaner way
-
- start_mp = time.time()
-
- self.generate_grid_code(dry_run=False)
-
- self.load_grid_function()
-
- def evolve_system(binary_cmdline_string):
+ # Get argument line
+ argline = self.return_argline(self.bse_options)
+ print('Running {}'.format(argline))
+ # Run system
+ out = binary_c_python_api.run_system(
+ argline,
+ self.grid_options["custom_logging_func_memaddr"],
+ self.grid_options["store_memaddr"],
+ ) # Todo: change this to run_binary again but then build in checks in binary
+ # TODO: add call to function that cleans up the temp customlogging dir, and unloads the loaded libraries.
+ # TODO: make a switch to turn this off
+ if clean_up_custom_logging_files:
+ self.clean_up_custom_logging(evol_type="single")
- # print(binary_cmdline_string)
- # pass
- # print('next')
- # self.set_bse_option("M_1", mass)
- out = binary_c_python_api.run_population(
- binary_cmdline_string,
- self.grid_options["custom_logging_func_memaddr"],
- self.grid_options["store_memaddr"],
- )
+ # Parse
+ if parse_function:
+ return parse_function(self, out)
+ else:
+ return out
- parse_function(self, out)
- # pass
- def yield_system():
- for i, system in enumerate(self.grid_options["system_generator"](self)):
- full_system_dict = self.bse_options.copy()
- full_system_dict.update(system)
+ def evolve_population_mp(self):
+ """
+ Function to evolve the population with multiprocessing approach. Using pathos to be able to include class-owned functions.
+ """
- binary_cmdline_string = self.return_argline(full_system_dict)
- # print("{}/{}".format(i+1, total_starcount_run), binary_cmdline_string)
- yield binary_cmdline_string
- # yield i
- print("generator done")
+ import multiprocessing as mp
+ # from pathos.multiprocessing import ProcessingPool as Pool
+ from pathos.pools import _ProcessPool as Pool
# Create pool
p = Pool(processes=self.grid_options["amt_cores"])
# Execute
# TODO: calculate the chunksize value based on: total starcount and cores used.
- r = list(p.imap_unordered(evolve_system, yield_system(), chunksize=1000))
+ r = list(p.imap_unordered(self.evolve_system_mp, self.yield_system_mp(), chunksize=20))
- stop_mp = time.time()
-
- # Give feedback
- print(
- "with mp: {} systems took {}s using {} cores".format(
- self.grid_options['total_starcount'],
- stop_mp - start_mp,
- self.grid_options["amt_cores"],
- )
- )
+ # Handle clean termination of the whole multiprocessing (making sure there are no zombie processes (https://en.wikipedia.org/wiki/Zombie_process))
+ p.close()
+ p.join()
- return stop_mp - start_mp
- def evolve_population_mp_chunks(self):
+ def evolve_population_lin(self):
"""
- Test function to evolve a population in a parallel way.
-
- returns total time spent on the actual interfacing with binaryc
+ Function to evolve the population linearly (i.e. 1 core, no multiprocessing)
"""
- import time
- import multiprocessing as mp
- # from pathos.multiprocessing import ProcessingPool as Pool
- from pathos.pools import _ProcessPool as Pool
- #######################
- ### Custom logging code:
- self.set_custom_logging()
-
- ### Load store
- self.grid_options["store_memaddr"] = binary_c_python_api.return_store("")
-
- #######################
- # Dry run and getting starcount
- self.grid_options['probtot'] = 0
-
- self.generate_grid_code(dry_run=True)
-
- self.load_grid_function()
-
- self.dry_run()
-
- total_starcount_run = self.grid_options['total_starcount']
- print("Total starcount for this run will be: {}".format(total_starcount_run))
-
- #######################
- # MP run
- self.grid_options['probtot'] = 0 # To make sure that the values are reset. TODO: fix this in a cleaner way
- self.grid_options['start_time_grid'] = time.time() # Setting start time of grid
-
-
- start_mp = time.time()
-
- self.generate_grid_code(dry_run=False)
-
- self.load_grid_function()
+ for i, system in enumerate(self.grid_options["system_generator"](self)):
+ full_system_dict = self.bse_options.copy()
+ full_system_dict.update(system)
- # def evolve_system(binary_cmdline_string):
- def evolve_system(binary_cmdline_string):
+ binary_cmdline_string = self.return_argline(full_system_dict)
out = binary_c_python_api.run_population(
binary_cmdline_string,
self.grid_options["custom_logging_func_memaddr"],
self.grid_options["store_memaddr"],
)
- if self.grid_options['parse_function']:
- self.grid_options['parse_function'](self, out)
+ self.print_info(i+1, self.grid_options['total_starcount'], full_system_dict)
- def yield_system():
- for i, system in enumerate(self.grid_options["system_generator"](self)):
- full_system_dict = self.bse_options.copy()
- full_system_dict.update(system)
-
- binary_cmdline_string = self.return_argline(full_system_dict)
-
- self.print_info(i+1, total_starcount_run, full_system_dict)
- # print("{}/{}".format(i+1, total_starcount_run), binary_cmdline_string)
- yield binary_cmdline_string
-
- print("generator done")
-
- # Create pool
- p = Pool(processes=self.grid_options["amt_cores"])
-
- # Execute
- # TODO: calculate the chunksize value based on: total starcount and cores used.
- r = list(p.imap_unordered(evolve_system, yield_system(), chunksize=20))
-
- # Handle clean termination of the whole multiprocessing (making sure there are no zombie processes (https://en.wikipedia.org/wiki/Zombie_process))
- p.close()
- p.join()
-
- stop_mp = time.time()
- self.grid_options['start_time_grid'] = 0
+ def evolve_population(self):
+ """
+ Function to evolve populations. This is the main function. Handles the setting up, evolving and cleaning up of a population of stars.
+ """
- # Give feedback
- print(
- "with mp: {} systems took {}s using {} cores".format(
- self.grid_options['total_starcount'],
- stop_mp - start_mp,
- self.grid_options["amt_cores"],
- )
- )
+ ##
+ # Prepare code/initialise grid.
+ # set custom logging, set up store_memaddr, build grid code. dry run grid code.
+ self.setup()
+
+ ##
+ # Evolve systems: via grid_options one can choose to do this linearly, or multiprocessing method.
+ if self.grid_options['evolution_type'] in self.grid_options['evolution_type_options']:
+ if self.grid_options['evolution_type'] == 'mp':
+ self.evolve_population_mp()
+ elif self.grid_options['evolution_type'] == 'linear':
+ self.evolve_population_lin()
+ else:
+ print("Warning. you chose a wrong option for the grid evolution types. Please choose from the following: {}.".format(self.grid_options['evolution_type_options']))
- return stop_mp - start_mp
+ ##
+ # Clean up code: remove files, unset values.
+ self.cleanup()
+ ###################################################
+ # Function to test evolution algorithms
+ ###################################################
def test_evolve_single(self):
"""
@@ -898,6 +673,7 @@ class Population(object):
print("Error while deleting file {}".format(file))
raise FileNotFoundError
+
def clean_up_custom_logging(self, evol_type):
"""
Function to clean up the custom logging.
@@ -932,7 +708,6 @@ class Population(object):
# TODO: make sure that these also work. not fully sure if necessary tho. whether its a single file, or a dict of files/memaddresses
-
def increment_probtot(self, prob):
"""
Function to add to the total probability
@@ -940,6 +715,7 @@ class Population(object):
self.grid_options["probtot"] += prob
+
def increment_count(self):
"""
Function to add to the total amount of stars
@@ -1346,18 +1122,6 @@ class Population(object):
with open(gridcode_filename, "w") as f:
f.write(code_string)
- def cleanup_grid(self):
- """
- Function that handles all the cleaning up after the grid has been generated and/or run
-
- - reset values to 0
- - remove grid file
- - unload grid function/module
- - remove dry grid file
- - unload dry grid function/module
- """
-
- pass
def load_grid_function(self):
"""
@@ -1387,6 +1151,7 @@ class Population(object):
if self.grid_options["verbose"] > 0:
print("Grid code loaded")
+
def dry_run(self):
"""
Function to dry run the grid and know how many stars it will run
diff --git a/binarycpython/utils/grid_options_defaults.py b/binarycpython/utils/grid_options_defaults.py
index d37d6a1bd..529e9e23d 100644
--- a/binarycpython/utils/grid_options_defaults.py
+++ b/binarycpython/utils/grid_options_defaults.py
@@ -7,6 +7,8 @@ grid_options_defaults_dict = {
# general
"amt_cores": 1, # total amount of cores used to evolve the population
"verbose": 0, # Level of verbosity of the simulation
+ "evolution_type": 'mp', # Flag for type of population evolution
+ "evolution_type_options": ['mp', 'linear'], # available choices for type of population evolution
# binary_c files
"binary_c_executable": os.path.join(
os.environ["BINARY_C"], "binary_c-config"
diff --git a/examples/example_population.py b/examples/example_population.py
index e3a9cab26..63f88c62f 100644
--- a/examples/example_population.py
+++ b/examples/example_population.py
@@ -146,12 +146,12 @@ example_pop.export_all_info()
## Executing a single system
## This uses the M_1 orbital period etc set with the set function
-# output = example_pop.evolve_single()
-# print(output)
+output = example_pop.evolve_single()
+print(output)
## Executing a population
## This uses the values generated by the grid_variables
-example_pop.evolve_population_mp_chunks() # TODO: update this function call
+# example_pop.evolve_population_mp_chunks() # TODO: update this function call
# Wrapping up the results to an hdf5 file can be done by using the create_hdf5(<directory containing data and settings>)
# This function takes the settings file (ending in _settings.json) and the data files (ending in .dat) from the data_dir
diff --git a/src/binary_c_python_api.c b/src/binary_c_python_api.c
index 903e60257..68be4e74c 100644
--- a/src/binary_c_python_api.c
+++ b/src/binary_c_python_api.c
@@ -395,7 +395,7 @@ int return_arglines(char ** const buffer,
stardata->preferences->batchmode = BATCHMODE_LIBRARY;
/* List available arguments */
- // binary_c_list_args(stardata);
+ binary_c_list_args(stardata);
/* get buffer pointer */
binary_c_buffer_info(stardata,buffer,nbytes);
diff --git a/tests/population/grid_tests.py b/tests/population/grid_tests.py
index e7e63f37c..44c3e1f20 100644
--- a/tests/population/grid_tests.py
+++ b/tests/population/grid_tests.py
@@ -290,7 +290,7 @@ test_pop.set(
# ###
# testing population:
test_pop.set(verbose=1,
- amt_cores=1,
+ amt_cores=2,
binary=0,
)
@@ -326,10 +326,12 @@ test_pop.add_grid_variable(
)
test_pop.set(verbose=1,
- amt_cores=1,
+ amt_cores=2,
binary=0,
+ evolution_type='linear'
)
-
# test_pop.test_evolve_population_mp()
-test_pop.evolve_population_mp_chunks()
+# test_pop.evolve_population_mp_chunks()
+
+test_pop.evolve_population()
\ No newline at end of file
--
GitLab