updated scaling plots and function to compare the grid runs. i need to keep in...

updated scaling plots and function to compare the grid runs. i need to keep in mind that these calculations are very probably limited the the physical amount of cores the machine has

binarycpython/utils/grid.py

@@ -794,40 +794,147 @@ class Population(object):
             )
         )
 
-        # quit()
-        # ########################
-
-        # # evolve with mp
-        # start_mp = time.time()
-
-        # def evolve_mp(mass):
-        #     self.set_bse_option("M_1", mass)
-        #     out = binary_c_python_api.run_population(
-        #         self.return_argline(),
-        #         self.grid_options["custom_logging_func_memaddr"],
-        #         self.grid_options["store_memaddr"],
-        #     )
-        #     parse_function(self, out)
-
-        # p = Pool(nodes=self.grid_options["amt_cores"])
-
-        # def g(mass_distribution):
-        #     masses = mass_distribution
-        #     for mass in masses:
-        #         yield mass
-        #     print("generator done")
-
-        # r = list(p.imap(evolve_mp, g(mass_distribution)))
-        # stop_mp = time.time()
-
-        # print(
-        #     "with mp: {} systems took {}s using {} cores".format(
-        #         len(mass_distribution),
-        #         stop_mp - start_mp,
-        #         self.grid_options["amt_cores"],
-        #     )
-        # )
+    def test_evolve_population_lin(self):
+        """
+        Test function to evolve a population in a linear way.
+
+        returns total time spent on the actual interfacing with binaryc 
+        """
+
+        import time
+
+        #######################
+        ### Custom logging code:
+        self.set_custom_logging()
+
+        ### Load store
+        self.grid_options["store_memaddr"] = binary_c_python_api.return_store("")
+
+        #######################
+        # Dry run and getting starcount
+        self.grid_options['probtot'] = 0
+
+        self.generate_grid_code(dry_run=True)
+
+        self.load_grid_function()
+
+        self.dry_run()
+
+        total_starcount_run = self.grid_options['total_starcount']
+        print("Total starcount for this run will be: {}".format(total_starcount_run))
+
+        #######################
+        # Linear run
+        start_lin = time.time()
+
+        self.grid_options['probtot'] = 0 # To make sure that the values are reset. TODO: fix this in a cleaner way
+ 
+        self.generate_grid_code(dry_run=False)
+
+        self.load_grid_function()
+
+        for i, system in enumerate(self.grid_options["system_generator"](self)):
+            full_system_dict = self.bse_options.copy()
+            full_system_dict.update(system)
+
+            binary_cmdline_string = self.return_argline(full_system_dict)
+            out = binary_c_python_api.run_population(
+                binary_cmdline_string,
+                self.grid_options["custom_logging_func_memaddr"],
+                self.grid_options["store_memaddr"],
+            )
+            print("{}/{}".format(i+1, total_starcount_run), binary_cmdline_string)
+
+        stop_lin = time.time()
+
+        print(
+            "Without mp: {} systems took {}s".format(total_starcount_run, stop_lin-start_lin))
+
+        return stop_lin-start_lin
+
+
+    def test_evolve_population_mp(self):
+        """
+        Test function to evolve a population in a parallel way.
+
+        returns total time spent on the actual interfacing with binaryc 
+        """
+
+        import time
+        import multiprocessing as mp
+        from pathos.multiprocessing import ProcessingPool as Pool
+
+        #######################
+        ### Custom logging code:
+        self.set_custom_logging()
+
+        ### Load store
+        self.grid_options["store_memaddr"] = binary_c_python_api.return_store("")
+
+        #######################
+        # Dry run and getting starcount
+        self.grid_options['probtot'] = 0
+
+        self.generate_grid_code(dry_run=True)
+
+        self.load_grid_function()
+
+        self.dry_run()
+
+        total_starcount_run = self.grid_options['total_starcount']
+        print("Total starcount for this run will be: {}".format(total_starcount_run))
+
+        #######################
+        # MP run
+        self.grid_options['probtot'] = 0 # To make sure that the values are reset. TODO: fix this in a cleaner way
+
+        start_mp = time.time()
+
+        self.generate_grid_code(dry_run=False)
+
+        self.load_grid_function()
+
+        def evolve_system(binary_cmdline_string):
+            # print(binary_cmdline_string)
+            # pass
+            # print('next')
+            # self.set_bse_option("M_1", mass)
+            out = binary_c_python_api.run_population(
+                binary_cmdline_string,
+                self.grid_options["custom_logging_func_memaddr"],
+                self.grid_options["store_memaddr"],
+            )
+            # # parse_function(self, out)
+
+        def yield_system():
+            for i, system in enumerate(self.grid_options["system_generator"](self)):
+                full_system_dict = self.bse_options.copy()
+                full_system_dict.update(system)
+
+                binary_cmdline_string = self.return_argline(full_system_dict)
+                print("{}/{}".format(i+1, total_starcount_run), binary_cmdline_string)
+
+                yield binary_cmdline_string
+            print("generator done")
+
+        # Create pool
+        p = Pool(nodes=self.grid_options["amt_cores"])
+
+        # Execute
+        r = list(p.imap(evolve_system, yield_system()))
+
+        stop_mp = time.time()
+
+        # Give feedback
+        print(
+            "with mp: {} systems took {}s using {} cores".format(
+                self.grid_options['total_starcount'],
+                stop_mp - start_mp,
+                self.grid_options["amt_cores"],
+            )
+        )
 
+        return stop_mp - start_mp
 
     def test_evolve_single(self):
         """

tests/population/grid_tests.py

@@ -289,7 +289,6 @@ test_pop.set(
 
 # ###
 # testing population:
-
 test_pop.set(verbose=1, 
     amt_cores=1,
     binary=0,

tests/population/scaling/evolve_population_comparing_with_multiprocessing.py

+import os
+import json
+import time
+import pickle
+import sys
+import numpy as np
+
+from binarycpython.utils.grid import Population
+from binarycpython.utils.functions import get_help_all, get_help, create_hdf5
+
+
+# import argparse
+
+
+# parser = argparse.ArgumentParser()
+# parser.add_argument(
+#     "amt_systems", help="the amount of systems",
+# )
+
+# parser.add_argument(
+#     "amt_nodes", help="the amount of nodes that are used for the multiprocessing",
+# )
+
+# parser.add_argument(
+#     "name_testcase", help="The name of the testcase (e.g. laptop, cluster etc)",
+# )
+
+
+# args = parser.parse_args()
+
+# amt_systems = args.amt_systems
+# amt_nodes = args.amt_nodes
+# name_testcase = args.name_testcase
+
+
+## Quick script to get some output about which stars go supernova when.
+def output_lines(output):
+    """
+    Function that outputs the lines that were recieved from the binary_c run. 
+    """
+    return output.splitlines()
+
+
+def parse_function(self, output):
+    # extract info from the population instance
+    # TODO: think about whether this is smart. Passing around this object might be an overkill
+
+    # Get some information from the
+    data_dir = self.custom_options["data_dir"]
+    base_filename = self.custom_options["base_filename"]
+
+    # Check directory, make if necessary
+    os.makedirs(data_dir, exist_ok=True)
+
+    # Create filename
+    outfilename = os.path.join(data_dir, base_filename)
+
+    # Go over the output.
+    for el in output_lines(output):
+        headerline = el.split()[0]
+
+        # CHeck the header and act accordingly
+        if headerline == "DAVID_SN":
+            parameters = ["time", "mass_1", "prev_mass_1", "zams_mass_1", "SN_type"]
+            values = el.split()[1:]
+            seperator = "\t"
+
+            if not os.path.exists(outfilename):
+                with open(outfilename, "w") as f:
+                    f.write(seperator.join(parameters) + "\n")
+
+            with open(outfilename, "a") as f:
+                f.write(seperator.join(values) + "\n")
+
+test_pop = Population()
+
+test_pop.set(verbose=1, 
+    amt_cores=4,
+    binary=1,
+)
+
+
+resolution = {'M_1': 10, 'per': 10}
+
+test_pop.add_grid_variable(
+    name="M_1",
+    longname="log primary mass",
+    valuerange=[10, 100],
+    resolution="{}".format(resolution['M_1']),
+    spacingfunc="const(10, 100, {})".format(resolution['M_1']),
+    precode="",
+    # precode="M_1=math.exp(lnm1)",
+    probdist="flat(M_1)",
+    # probdist='self.custom_options["extra_prob_function"](M_1)',
+    dphasevol="dlnm1",
+    parameter_name="M_1",
+    condition="",
+)
+
+### Grid generating test.
+test_pop.add_grid_variable(
+    name="period",
+    longname="period",
+    valuerange=["M_1", 20],
+    resolution="{}".format(resolution['per']),
+    spacingfunc="np.linspace(1, 10, {})".format(resolution['per']),
+    precode="orbital_period = period**2",
+    probdist="flat(orbital_period)",
+    parameter_name="orbital_period",
+    dphasevol="dper",
+    condition='self.grid_options["binary"]==1',
+)
+
+test_pop.set(verbose=1, 
+    amt_cores=2,
+    binary=1,
+)
+
+total_systems = np.prod([el for el in resolution.values()])
+AMT_CORES = 2
+
+# Lin
+total_lin_start = time.time()
+
+evolve_lin_time = test_pop.test_evolve_population_lin()
+
+total_lin_stop = time.time()
+
+total_lin = total_lin_stop - total_lin_start
+
+print("linear run with {} systems: {} of which {} spent on evolving the systems".format(total_systems, total_lin, evolve_lin_time))
+
+# MP
+total_mp_start = time.time()
+
+evolve_mp_time = test_pop.test_evolve_population_mp()
+
+total_mp_stop = time.time()
+
+total_mp = total_mp_stop - total_mp_start
+print("MP ({} nodes) run with {} systems: {} of which {} spent on evolving the systems".format(AMT_CORES, total_systems, total_mp, evolve_mp_time))
+
+print("The speed up by using MP is: {}".format(total_lin/total_mp))
\ No newline at end of file

tests/population/scaling/plot_scaling.py

@@ -179,9 +179,9 @@ def plot_runtime(calculated_results, unique_amt_cores, unique_amt_systems):
 ##################### 
 
 # Configure
-result_file = "comparison_result.dat"
-result_file = "comparison_result_astro1.dat"
-name_testcase = "Astro1"
+result_file = "comparison_result_laptop.dat"
+# result_file = "comparison_result_astro1.dat"
+name_testcase = "laptop"
 img_dir = "scaling_plots"
 
 # Readout file