From 3f98ca519e23e2971ff68900d7e30fb48100b8ad Mon Sep 17 00:00:00 2001
From: David Hendriks <davidhendriks93@gmail.com>
Date: Tue, 18 May 2021 12:28:19 +0100
Subject: [PATCH] working on M&S
---
binarycpython/utils/distribution_functions.py | 38 +++++++++++---
binarycpython/utils/functions.py | 13 +++--
binarycpython/utils/grid.py | 49 +++++++++++++++++--
binarycpython/utils/grid_options_defaults.py | 2 +
4 files changed, 89 insertions(+), 13 deletions(-)
diff --git a/binarycpython/utils/distribution_functions.py b/binarycpython/utils/distribution_functions.py
index 79e0b3c89..63459057a 100644
--- a/binarycpython/utils/distribution_functions.py
+++ b/binarycpython/utils/distribution_functions.py
@@ -1113,7 +1113,7 @@ def Moe_de_Stefano_2017_multiplicity_fractions(options):
# return array reference
return result
-
+# @profile
def build_q_table(options, m, p):
############################################################
#
@@ -1138,12 +1138,13 @@ def build_q_table(options, m, p):
# We should ignore those parts of the table and renormalize. If we are below the lowest value of qmin in the table we need to extrapolate the data
# We can check if we have a cached value for this already:
- # TODO: fix this cache check
+ # TODO: fix this cache check.
incache = False
if Moecache.get("rinterpolator_q_metadata", None):
- if Moecache["rinterpolator_q_metadata"][m] == options[m]:
- if Moecache["rinterpolator_q_metadata"] == options[p]:
+ if (Moecache["rinterpolator_q_metadata"][m]) and (Moecache["rinterpolator_q_metadata"][p]):
+ if (Moecache["rinterpolator_q_metadata"][m] == options[m]) and (Moecache["rinterpolator_q_metadata"][p] == options[p]):
incache = True
+ # print("INCACHE: {}".format(incache))
#
if not incache:
@@ -1167,7 +1168,7 @@ def build_q_table(options, m, p):
qeps = 1e-8 # small number but such that qeps+1 != 1
if qeps + 1 == 1.0:
- printf("qeps (= {}) +1 == 1. Make qeps larger".format(qeps))
+ print("qeps (= {}) +1 == 1. Make qeps larger".format(qeps))
if qmin >= qmax:
# there may be NO binaries in this part of the parameter space:
@@ -1335,6 +1336,7 @@ def build_q_table(options, m, p):
else:
print("No other methods available")
raise ValueError
+
# TODO: consider implementing this
# elsif($method =~ /^(log)?poly(\d+)/)
# {
@@ -1393,10 +1395,30 @@ def build_q_table(options, m, p):
for q in sorted(qdata.keys()):
tmp_table.append([q, qdata[q]])
+ if Moecache.get("rinterpolator_q_given_{}_log10{}".format(m, p), None):
+ print("Present interpolator: {}".format(Moecache["rinterpolator_q_given_{}_log10{}".format(m, p)]))
+ print("Destroying present interpolator:")
+ interpolator = Moecache["rinterpolator_q_given_{}_log10{}".format(m, p)]
+ print(interpolator)
+ print(type(interpolator))
+ print(dir(interpolator))
+ x=Moecache["rinterpolator_q_given_{}_log10{}".format(m, p)].interpolate([0.5])
+ print("Interpolated a value q=0.5: {}".format(x))
+ Moecache["rinterpolator_q_given_{}_log10{}".format(m, p)].destroy()
+ print(interpolator)
+ print(type(interpolator))
+ print(dir(interpolator))
+ print("Present interpolator: {}".format(Moecache["rinterpolator_q_given_{}_log10{}".format(m, p)]))
+ x=Moecache["rinterpolator_q_given_{}_log10{}".format(m, p)].interpolate([0.5])
+ print("Interpolated a value q=0.5: {}".format(x))
+ # del Moecache["rinterpolator_q_given_{}_log10{}".format(m, p)]
+
+ print("CREATING A NEW TABLE Q table")
# Make an interpolation table to contain our modified data
q_interpolator = py_rinterpolate.Rinterpolate(
table=tmp_table, nparams=1, ndata=1 # Contains the table of data # q #
)
+ print("CREATed A NEW TABLE Q table")
# TODO: build a check in here to see if the interpolator build was successful
# print("Can renormalize?: {}".format(can_renormalize))
@@ -1434,7 +1456,10 @@ def build_q_table(options, m, p):
print("Error: > 1e-6 in q probability integral: {}".format(I))
# set this new table in the cache
+ print("STORING Q INTERPOLATOR AS {}".format("rinterpolator_q_given_{}_log10{}".format(m, p)))
Moecache["rinterpolator_q_given_{}_log10{}".format(m, p)] = q_interpolator
+ print("STORed Q INTERPOLATOR AS {}".format("rinterpolator_q_given_{}_log10{}".format(m, p)))
+
if not Moecache.get("rinterpolator_q_metadata", None):
Moecache["rinterpolator_q_metadata"] = {}
Moecache["rinterpolator_q_metadata"][m] = options[m]
@@ -1756,7 +1781,8 @@ def Moe_de_Stefano_2017_pdf(options):
print_info = 1
if print_info:
- print("Probability density")
+ # print("Probability density")
+
if multiplicity == 1:
print(
"M1={} q=N/A log10P=N/A ({}): {} -> {}\n".format(
diff --git a/binarycpython/utils/functions.py b/binarycpython/utils/functions.py
index 7bf7e1b81..3ded7b1d9 100644
--- a/binarycpython/utils/functions.py
+++ b/binarycpython/utils/functions.py
@@ -580,9 +580,9 @@ def parse_binary_c_version_info(version_info_string: str) -> dict:
new_split = "".join(split_info[1:]).split(" is ")
param_name = new_split[0]
param_value = " is ".join(new_split[1:])
+
# Sometimes the macros have extra information behind it. Needs an update in outputting by binary_c
try:
- print(param_name, param_type, param_value)
macros_dict[param_name] = param_type_dict[param_type](param_value)
except ValueError:
macros_dict[param_name] = str(param_value)
@@ -1588,16 +1588,21 @@ def extract_ensemble_json_from_string(binary_c_output: str) -> dict:
json dictionary with the parsed ENSEMBLE_JSON data
"""
- json = None
+ json_dict = None
try:
+ # If there is no output just return an empty dict:
+ if not binary_c_output:
+ json_dict = {}
+ return json_dict
+
ensemble_jsons_strings = [
line
for line in binary_c_output.splitlines()
if line.startswith("ENSEMBLE_JSON")
]
- json = handle_ensemble_string_to_json(
+ json_dict = handle_ensemble_string_to_json(
ensemble_jsons_strings[0][len("ENSEMBLE_JSON ") :]
)
@@ -1614,7 +1619,7 @@ def extract_ensemble_json_from_string(binary_c_output: str) -> dict:
0,
)
- return json
+ return json_dict
class binarycDecoder(json.JSONDecoder):
diff --git a/binarycpython/utils/grid.py b/binarycpython/utils/grid.py
index 345f88fe0..00faa9021 100644
--- a/binarycpython/utils/grid.py
+++ b/binarycpython/utils/grid.py
@@ -692,6 +692,9 @@ class Population:
If neither of the above is set, we continue without using HPC routines
(that doesn't mean this cannot be run on a server with many cores)
+
+ Returns an dictionary containing the analytics of the run
+ TODO: change the way this is done. Slurm & CONDOR should probably do this different
"""
# Just to make sure we don't have stuff from a previous run hanging around
@@ -728,6 +731,9 @@ class Population:
"total_probability_weighted_mass_run": self.grid_options["_total_probability_weighted_mass_run"],
}
+ # print(Moecache)
+ print(Moecache.keys())
+
##
# Clean up code: remove files, unset values. This is placed in the general evolve function,
# because that makes for easier control
@@ -876,6 +882,10 @@ class Population:
pool.close()
pool.join()
+ print("OUTSIDE THREAD")
+ print(Moecache.keys())
+ print("OUTSIDE THREAD")
+
# Handle the results by merging all the dictionaries. How that merging happens exactly is
# described in the merge_dicts description.
combined_output_dict = {}
@@ -945,6 +955,15 @@ class Population:
This function is called by _evolve_population_grid
"""
+
+
+
+ import tracemalloc
+ tracemalloc.start()
+ # ... start your application ...
+
+ snapshot1 = tracemalloc.take_snapshot()
+
# set start timer
start_process_time = datetime.datetime.now()
@@ -1055,6 +1074,7 @@ class Population:
# they match the keys known to binary_c.
# Won't do that every system cause that is a bit of a waste of computing time.
if localcounter == 0:
+ # TODO: Put this someplace else and wrap in a functioncall
for key in full_system_dict.keys():
if not key in self.available_keys:
# Deal with special keys
@@ -1075,9 +1095,10 @@ class Population:
raise ValueError(msg)
start_runtime_binary_c = time.time()
- # TODO: build flag to actually evolve the system
+
# Evolve the system
- self._evolve_system_mp(full_system_dict)
+ if self.grid_options['_actually_evolve_system']:
+ self._evolve_system_mp(full_system_dict)
end_runtime_binary_c = time.time()
@@ -1216,6 +1237,24 @@ class Population:
0,
)
+ print("INSIDE THREAD")
+ print(Moecache.keys())
+ for key in Moecache.keys():
+ print(key, Moecache[key])
+ print("")
+ print("INSIDE THREAD")
+
+
+ # ... call the function leaking memory ...
+ snapshot2 = tracemalloc.take_snapshot()
+
+ top_stats = snapshot2.compare_to(snapshot1, 'lineno')
+
+ print("[ Top 10 differences ]")
+ for stat in top_stats[:10]:
+ print(stat)
+
+
return output_dict
# Single system
@@ -1945,6 +1984,7 @@ class Population:
# code_string += indent * (depth + 2) + "print('phasevol_lnm1: ',phasevol_lnm1); print('phasevol_multiplicity: ',phasevol_multiplicity);\n"
# code_string += indent * (depth + 2) + "print(probabilities_list)\n"
# code_string += indent * (depth + 2) + "print(parameter_dict)\n"
+ code_string += indent * (depth + 2) + "print('YOO IK GA LEKKER NOG EEN RONDJE')\n"
code_string += indent * (depth + 2) + "yield(parameter_dict)\n"
# If its a dry run, dont do anything with it
@@ -2981,6 +3021,7 @@ class Population:
]
)
+ print("Size multiplicity table: {}", len(Moecache["multiplicity_table"]))
############################################################
# a small log10period which we can shift just outside the
# table to force integration out there to zero
@@ -3045,6 +3086,7 @@ class Population:
/ dlog10P,
]
)
+ print("Size period_distributions table: {}", len(Moecache["period_distributions"]))
############################################################
# distributions as a function of mass, period, q
@@ -3123,6 +3165,7 @@ class Population:
0.0,
]
)
+ print("Size period_distributions table: {}".format(len(Moecache["period_distributions"])))
# Write to logfile
with open("/tmp/moecache.json", "w") as cache_filehandle:
@@ -3243,7 +3286,7 @@ class Population:
resolution=options["resolutions"]["logP"][0],
probdist=1.0,
condition='(self.grid_options["multiplicity"] >= 2)',
- branchpoint=1,
+ # branchpoint=1,
gridtype="centred",
dphasevol="({} * dlog10per)".format(LOG_LN_CONVERTER),
valuerange=[options["ranges"]["logP"][0], options["ranges"]["logP"][1]],
diff --git a/binarycpython/utils/grid_options_defaults.py b/binarycpython/utils/grid_options_defaults.py
index 45c4e017a..b40b674dd 100644
--- a/binarycpython/utils/grid_options_defaults.py
+++ b/binarycpython/utils/grid_options_defaults.py
@@ -33,6 +33,7 @@ grid_options_defaults_dict = {
# "output_dir":
"_commandline_input": "",
"log_runtime_systems": 0, # whether to log the runtime of the systems (1 file per thread. stored in the tmp_dir)
+ "_actually_evolve_system": True, # Whether to actually evolve the systems of just act as if. for testing. used in _process_run_population_grid
##########################
# Execution log:
##########################
@@ -469,6 +470,7 @@ grid_options_descriptions = {
"log_runtime_systems": "Whether to log the runtime of the systems . Each systems run by the thread is logged to a file and is stored in the tmp_dir. (1 file per thread). Don't use this if you are planning to run alot of systems. This is mostly for debugging and finding systems that take long to run. Integer, default = 0. if value is 1 then the systems are logged",
"_total_mass_run": "To count the total mass that thread/process has ran",
"_total_probability_weighted_mass_run": "To count the total mass * probability for each system that thread/process has ran",
+ "_actually_evolve_system": "Whether to actually evolve the systems of just act as if. for testing. used in _process_run_population_grid"
}
###
--
GitLab