updating scaling plots. added pool closing and joining at the end of the MP...

updating scaling plots. added pool closing and joining at the end of the MP part because on astro2 I suddenly got the error of having too many open files

binarycpython/utils/grid.py

@@ -812,6 +812,10 @@ class Population(object):
         # TODO: calculate the chunksize value based on: total starcount and cores used. 
         r = list(p.imap_unordered(evolve_system, yield_system(), chunksize=20))
 
+        # Handle clean termination of the whole multiprocessing
+        p.close()
+        p.join()
+
         stop_mp = time.time()
 
         # Give feedback

tests/population/scaling/plot_scaling_new.py

@@ -78,10 +78,9 @@ ax1.set_xlim(0, max(cpus) + 4)
 
 ax1.grid()
 ax1.legend(loc=4)
+# fig.savefig(os.path.join(img_dir, "speedup_scaling_{}.{}".format(name_testcase, "png")))
+# fig.savefig(os.path.join(img_dir, "speedup_scaling_{}.{}".format(name_testcase, "pdf")))
+# fig.savefig(os.path.join(img_dir, "speedup_scaling_{}.{}".format(name_testcase, "eps")))
 plt.show()
 
 
-
-    # fig.savefig(os.path.join(img_dir, "speedup_scaling_{}.{}".format(name_testcase, "png")))
-    # fig.savefig(os.path.join(img_dir, "speedup_scaling_{}.{}".format(name_testcase, "pdf")))
-    # fig.savefig(os.path.join(img_dir, "speedup_scaling_{}.{}".format(name_testcase, "eps")))
\ No newline at end of file

tests/population/scaling/scaling_script.py

@@ -10,10 +10,12 @@ from binarycpython.utils.functions import get_help_all, get_help, create_hdf5
 
 import argparse
 
+# Get some info
 amount_of_cores = psutil.cpu_count(logical=False)
 amount_of_cpus = psutil.cpu_count()
 hostname = socket.gethostname()
 
+# Generate list of cpu amounts to use
 if amount_of_cpus <= 4:
     stepsize = 1
 elif 4 < amount_of_cpus <= 24:
@@ -28,13 +30,25 @@ for i in range(1, int(amount_of_cpus/stepsize) + 1 ):
         cpu_list.append(i * stepsize)
 
 
+# set some info
 amt_repeats = 5
-
-resolution = {'M_1': 50, 'per': 50}
+resolution = {'M_1': 5, 'per': 5}
 total_systems = int(np.prod([el for el in resolution.values()]))
 result_dir = 'scaling_results'
+testcase = 'linear vs MP batched'
+
+# Create dictionairy in which to store all the results:
+result_dict = {}
 
+#
+result_dict['amt_systems'] = total_systems
+result_dict['hostname'] = hostname
+result_dict['amt_logical_cores'] = amount_of_cpus
+result_dict['amt_of_physical_cores'] = amount_of_cores
+result_dict['testcase'] = testcase
 
+################# 
+# Configuring population
 test_pop = Population()
 
 test_pop.set(
@@ -48,8 +62,6 @@ test_pop.add_grid_variable(
     valuerange=[1, 100],
     resolution="{}".format(resolution['M_1']),
     spacingfunc="const(1, 100, {})".format(resolution['M_1']),
-    precode="",
-    # precode="M_1=math.exp(lnm1)",
     probdist="Kroupa2001(M_1)",
     # probdist='self.custom_options["extra_prob_function"](M_1)',
     dphasevol="dlnm1",
@@ -71,24 +83,14 @@ test_pop.add_grid_variable(
 )
 
 #######################################################################################
-
-# Scaling dict:
-result_dict = {}
-
-#
-result_dict['amt_systems'] = total_systems
-
+# Execute grids
 
 # Linear runs
 linear_times = []
 for repeat in range(amt_repeats):
     total_lin_start = time.time()
-
     evolve_lin_time = test_pop.test_evolve_population_lin()
-
-    total_lin_stop = time.time()
-
-    total_lin = total_lin_stop - total_lin_start
+    total_lin = time.time() - total_lin_start
 
     print("linear run with {} systems: {} of which {} spent on evolving the systems".format(total_systems, total_lin, evolve_lin_time))
     linear_times.append(total_lin)
@@ -96,31 +98,28 @@ for repeat in range(amt_repeats):
 
 result_dict['linear'] = linear_times
 
-#######################################################################################
-# MP runs
-mp_dict = {}
-for cpu_amt in cpu_list:
-
-    mp_times = []
-
-    test_pop.set(amt_cores=cpu_amt)
+# #######################################################################################
+# # MP runs
+# mp_dict = {}
+# for cpu_amt in cpu_list:
 
-    # 
-    for repeat in range(amt_repeats):
-        # MP
-        total_mp_start = time.time()
+#     mp_times = []
 
-        evolve_mp_time = test_pop.evolve_population_mp_chunks()
-        total_mp_stop = time.time()
+#     test_pop.set(amt_cores=cpu_amt)
 
-        total_mp = total_mp_stop - total_mp_start
-        mp_times.append(total_mp)
-        print("MP ({} nodes) run with {} systems: {} of which {} spent on evolving the systems".format(cpu_amt, total_systems, total_mp, evolve_mp_time))
+#     # 
+#     for repeat in range(amt_repeats):
+#         total_mp_start = time.time()
+#         evolve_mp_time = test_pop.evolve_population_mp_chunks()
+#         total_mp = time.time() - total_mp_start
 
-    mp_dict[cpu_amt] = mp_times
+#         print("MP ({} nodes) run with {} systems: {} of which {} spent on evolving the systems".format(cpu_amt, total_systems, total_mp, evolve_mp_time))
+#         mp_times.append(total_mp)
 
-result_dict['mp'] = mp_dict
+#     mp_dict[cpu_amt] = mp_times
 
+# result_dict['mp'] = mp_dict
 
+# Write to file
 with open('scaling_results/{}_{}_systems.json'.format(hostname, total_systems), 'w') as f:
     f.write(json.dumps(result_dict))
\ No newline at end of file