From 8f52d55744e9531986ee10f7b3fc589975469e91 Mon Sep 17 00:00:00 2001
From: David Hendriks <davidhendriks93@gmail.com>
Date: Sat, 24 Jul 2021 21:45:16 +0100
Subject: [PATCH] updated code to not explicity destroy the interpolators
---
binarycpython/utils/distribution_functions.py | 6 ------
binarycpython/utils/functions.py | 3 ++-
binarycpython/utils/grid.py | 16 +++------------
binarycpython/utils/grid_options_defaults.py | 20 +++++++++----------
4 files changed, 14 insertions(+), 31 deletions(-)
diff --git a/binarycpython/utils/distribution_functions.py b/binarycpython/utils/distribution_functions.py
index 0b0e488c2..e7a789116 100644
--- a/binarycpython/utils/distribution_functions.py
+++ b/binarycpython/utils/distribution_functions.py
@@ -946,15 +946,11 @@ def cosmic_SFH_madau_dickinson2014(z):
# known value if we try to interpolate outside of the tables.
# There are still some open tasks and improvements that can be made:
#
-# TODO: check all the raise ValueErrors to make them more appropriate
-# TODO: Put the json checking stuff in a different function
-# TODO: Clean up the logging
# TODO: Solve the memory issues that are present.
# Are the interpolators not cleaned?
# TODO: Parallellize the setting up of the interpolators
# TODO: Generalize the code such that we can input other/newer tables
-
########################################################################
import py_rinterpolate
@@ -1265,8 +1261,6 @@ def build_q_table(options, m, p, verbosity=0):
verbosity,
_MS_VERBOSITY_LEVEL,
)
- Moecache["rinterpolator_q_given_{}_log10{}".format(m, p)].destroy()
- gc.collect()
#
if not incache:
diff --git a/binarycpython/utils/functions.py b/binarycpython/utils/functions.py
index 039d36c23..310ff4bb6 100644
--- a/binarycpython/utils/functions.py
+++ b/binarycpython/utils/functions.py
@@ -14,6 +14,7 @@ import tempfile
import copy
import inspect
import sys
+import time
from io import StringIO
from typing import Union, Any
@@ -1893,7 +1894,7 @@ def binaryc_json_serializer(obj: Any) -> Any:
Function objects will be turned into str representations of themselves
Args:
- obj: obj being process
+ obj: The object that might not be serializable
Returns:
Either string representation of object if the object is a function, or the object itself
diff --git a/binarycpython/utils/grid.py b/binarycpython/utils/grid.py
index 95f049785..454f7e6e0 100644
--- a/binarycpython/utils/grid.py
+++ b/binarycpython/utils/grid.py
@@ -1482,15 +1482,9 @@ class Population:
stream_logger.debug(f"Process-{self.process_ID} is finished.")
# Clean up the interpolators if they exist
+
# TODO: make a cleanup function for the individual threads
# TODO: make sure this is necessary. Actually its probably not, because we have a centralized queue
- verbose_print(
- "Process {}: Cleaning up interpolators".format(ID),
- self.grid_options["verbosity"],
- 1,
- )
- self._clean_interpolators()
- gc.collect()
return
@@ -1731,9 +1725,6 @@ class Population:
# Unload/free custom_logging_code
# TODO: cleanup custom logging code.
- # Clean up the interpolator functions
- self._clean_interpolators()
-
# Also remove the rest of the contents
keys_moecache = list(Moecache.keys())
for key in keys_moecache:
@@ -3973,8 +3964,7 @@ eccentricity3=0
# Second: is the moecache filled.
if not Moecache:
- msg = "Moecache is empty. It needs to be filled with the data for the interpolators"
- raise ValueError(msg)
+ print("Moecache is empty. It needs to be filled with the data for the interpolators. Loading the data now")
# Load the data
self._load_moe_distefano_data()
@@ -3988,7 +3978,7 @@ eccentricity3=0
# Calculate the multiplicity fraction
ding = Moe_de_Stefano_2017_multiplicity_fractions(self.grid_options['m&s_options'], self.grid_options["verbosity"])
print(ding)
-
+
# Set the prev value back
self.grid_options['m&s_options'][prev_M1_value_ms]
diff --git a/binarycpython/utils/grid_options_defaults.py b/binarycpython/utils/grid_options_defaults.py
index 52e8c7cf6..02e993a01 100644
--- a/binarycpython/utils/grid_options_defaults.py
+++ b/binarycpython/utils/grid_options_defaults.py
@@ -11,8 +11,6 @@ There are several other functions in this module, mostly to generate help texts
With this its also possible to automatically generate a document containing all the setting names + descriptions.
All the options starting with _ should not be changed by the user except when you really know what you're doing (which is probably hacking the code :P)
-
-# TODO: add moe distefano
"""
import os
@@ -24,30 +22,24 @@ _MS_VERBOSITY_LEVEL = 5
_MS_VERBOSITY_INTERPOLATOR_LEVEL = 6
_MS_VERBOSITY_INTERPOLATOR_EXTRA_LEVEL = 7
-
# Options dict
grid_options_defaults_dict = {
##########################
# general (or unordered..)
##########################
"amt_cores": 1, # total amount of cores used to evolve the population
- "binary": 0, # FLag on whether the systems are binary systems or single systems.
"parse_function": None, # FUnction to parse the output with.
+ "multiplicity_fraction_function": 0, # Which multiplicity fraction function to use. 0: None, 1: Arenou 2010, 2: Rhagavan 2010, 3: M&S 2017
"tmp_dir": temp_dir(), # Setting the temp dir of the program
"_main_pid": -1, # Placeholder for the main process id of the run.
"combine_ensemble_with_thread_joining": True, # Flag on whether to combine everything and return it to the user or if false: write it to data_dir/ensemble_output_{popuation_id}_{thread_id}.json
- # "output_dir":
"_commandline_input": "",
"log_runtime_systems": 0, # whether to log the runtime of the systems (1 file per thread. stored in the tmp_dir)
"_actually_evolve_system": True, # Whether to actually evolve the systems of just act as if. for testing. used in _process_run_population_grid
- "max_queue_size": 1000, # Maximum size of the system call queue. Can't be too big!
- "_set_ms_grid": False, # Whether the M&S grid has been loaded
+ "max_queue_size": 1000, # Maximum size of the system call queue.
"run_zero_probability_system": True, # Whether to run the zero probability systems
"_zero_prob_stars_skipped": 0,
"ensemble_factor_in_probability_weighted_mass": False, # Whether to multiply the ensemble results by 1/probability_weighted_mass
- "multiplicity_fraction_function": 0, # Which multiplicity fraction function to use. 0: None, 1: Arenou 2010, 2: Rhagavan 2010, 3: M&S 2017
- "_m&s_options": None, # Holds the M&S options.
- "_loaded_ms_data": False, # Holds flag whether the M&S data is loaded into memory
##########################
# Execution log:
##########################
@@ -69,6 +61,12 @@ grid_options_defaults_dict = {
), # TODO: make this more robust
"_binary_c_dir": os.environ["BINARY_C"],
##########################
+ # M&S internal settings
+ ##########################
+ "_loaded_ms_data": False, # Holds flag whether the M&S data is loaded into memory
+ "_set_ms_grid": False, # Whether the M&S grid has been loaded
+ "m&s_options": None, # Holds the M&S options.
+ ##########################
# Custom logging
##########################
"C_auto_logging": None, # Should contain a dictionary where the kes are they headers
@@ -491,7 +489,7 @@ grid_options_descriptions = {
"_zero_prob_stars_skipped": "Internal counter to track how many systems are skipped because they have 0 probability",
"ensemble_factor_in_probability_weighted_mass": "Flag to multiply all the ensemble results with 1/probability_weighted_mass",
"multiplicity_fraction_function": "Which multiplicity fraction function to use. 0: None, 1: Arenou 2010, 2: Rhagavan 2010, 3: M&S 2017",
- "_m&s_options": "Internal variable that holds the M&S options. Don't write to this your self",
+ "m&s_options": "Internal variable that holds the M&S options. Don't write to this your self",
"_loaded_ms_data": "Internal variable storing whether the M&S data has been loaded into memory",
}
--
GitLab