From c6e3f007017c68ea9884638c2f607101cf2f4683 Mon Sep 17 00:00:00 2001
From: David Hendriks <davidhendriks93@gmail.com>
Date: Mon, 24 Oct 2022 14:39:29 +0100
Subject: [PATCH] changing the start method of multiprocessing
---
binarycpython/utils/grid.py | 5 +-
examples/notebook_population.ipynb | 236 +++++++++--------------------
2 files changed, 78 insertions(+), 163 deletions(-)
diff --git a/binarycpython/utils/grid.py b/binarycpython/utils/grid.py
index 26b5520c3..a0a290c1d 100644
--- a/binarycpython/utils/grid.py
+++ b/binarycpython/utils/grid.py
@@ -23,6 +23,10 @@ import functools
import gc
import json
import multiprocessing
+
+# Set method to fork.
+multiprocessing.set_start_method('fork')
+
import os
import psutil
import queue
@@ -1260,7 +1264,6 @@ class Population(
# Set process name
setproctitle.setproctitle("binarycpython parent process")
-
# if max_queue_size is zero, calculate automatically
# to be double the number of processes - you don't want to
# make the queue too large because when it's killed you
diff --git a/examples/notebook_population.ipynb b/examples/notebook_population.ipynb
index ebc19cff0..267daec0a 100644
--- a/examples/notebook_population.ipynb
+++ b/examples/notebook_population.ipynb
@@ -17,7 +17,7 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 2,
"id": "bf6b8673-a2b5-4b50-ad1b-e90671f57470",
"metadata": {},
"outputs": [],
@@ -32,6 +32,16 @@
"# help(Population) # Uncomment to see the public functions of this object"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "id": "cf687b99-04f7-4d33-9f74-3d9d66d0b530",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from binarycpython import Population"
+ ]
+ },
{
"cell_type": "markdown",
"id": "f268eff3-4e08-4f6b-8b59-f22dba4d2074",
@@ -50,7 +60,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 3,
"id": "79ab50b7-591f-4883-af09-116d1835a751",
"metadata": {},
"outputs": [
@@ -127,7 +137,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 4,
"id": "b9c2471a-a5b0-48b7-a50b-2f0d22100926",
"metadata": {},
"outputs": [
@@ -136,10 +146,15 @@
"output_type": "stream",
"text": [
"ok\n",
- "File at /tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result/example_pop_settings.json.gz already exists: cannot write to it\n",
+ "lockfile=/tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result/example_pop_settings.json.gz.lock\n",
+ "post-lock: <Lock /tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result/example_pop_settings.json.gz.lock [unlocked: 0:00:15] pid=147908 at 0x7f5278ecd970x>\n",
+ "try to lock <Lock /tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result/example_pop_settings.json.gz.lock [unlocked: 0:01:00] pid=147908 at 0x7f5278ecd970x>\n",
+ "locked <Lock /tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result/example_pop_settings.json.gz.lock [locked: 0:01:00] pid=147908 at 0x7f5278ecd970x>\n",
+ "/tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result/example_pop_settings.json.gz is locked by <Lock /tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result/example_pop_settings.json.gz.lock [locked: 0:01:00] pid=147908 at 0x7f5278ecd970x> to /tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result/example_pop_settings.json.gz.lock\n",
+ "Try to open file at /tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result/example_pop_settings.json.gz\n",
+ "Return locked file <_io.TextIOWrapper name='/tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result/example_pop_settings.json.gz' encoding='utf-8'>, <Lock /tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result/example_pop_settings.json.gz.lock [locked: 0:01:00] pid=147908 at 0x7f5278ecd970x>\n",
"ok\n",
- "ok pre\n",
- "ok ret\n"
+ "Writing settings to /tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result/example_pop_settings.json.gz\n"
]
},
{
@@ -148,7 +163,7 @@
"'/tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result/example_pop_settings.json.gz'"
]
},
- "execution_count": 3,
+ "execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
@@ -177,7 +192,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 5,
"id": "68c84521-9ae8-4020-af7a-5334173db969",
"metadata": {},
"outputs": [
@@ -305,7 +320,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 6,
"id": "048db541-3e92-4c5d-a25c-9c5a34b9c857",
"metadata": {
"scrolled": true,
@@ -327,7 +342,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 7,
"id": "47979841-2c26-4b26-8945-603d013dc93a",
"metadata": {},
"outputs": [],
@@ -392,7 +407,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 8,
"id": "0c986215-93b1-4e30-ad79-f7c397e9ff7d",
"metadata": {},
"outputs": [
@@ -457,7 +472,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 9,
"id": "fd197154-a8ce-4865-8929-008d3483101a",
"metadata": {},
"outputs": [
@@ -465,7 +480,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "adding: parse_function=<function parse_function at 0x7f2b6ca163a0> to grid_options\n"
+ "adding: parse_function=<function parse_function at 0x7f523208e280> to grid_options\n"
]
}
],
@@ -530,7 +545,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 10,
"id": "8ea376c1-1e92-45af-8cab-9d7fdca564eb",
"metadata": {
"tags": []
@@ -550,27 +565,27 @@
"* Total probability is 0.0443872 *\n",
"**********************************\n",
"\n",
- "EXAMPLE_COMPACT_OBJECT 3.598268106227e+01 1.30592 8.75988 0.00193614 13\n",
- "EXAMPLE_COMPACT_OBJECT 2.436983545111e+01 1.35842 10.9948 0.00144093 13\n",
- "EXAMPLE_COMPACT_OBJECT 1.690157944401e+01 1.43124 13.7998 0.00107238 13\n",
- "EXAMPLE_COMPACT_OBJECT 1.242397939068e+01 1.52416 17.3205 0.000798096 13\n",
- "EXAMPLE_COMPACT_OBJECT 9.756794139032e+00 1.66914 21.7394 0.000593966 13\n",
- "EXAMPLE_COMPACT_OBJECT 8.401414766976e+00 1.73729 27.2857 0.000442046 13\n",
- "EXAMPLE_COMPACT_OBJECT 7.536373523810e+00 1.80677 34.247 0.000328983 13\n",
- "EXAMPLE_COMPACT_OBJECT 7.393982410080e+00 1.82164 42.9844 0.000244839 13\n",
- "EXAMPLE_COMPACT_OBJECT 7.396470605248e+00 1.82129 53.9508 0.000182216 13\n",
+ "EXAMPLE_COMPACT_OBJECT 3.598268106261e+01 1.30592 8.75988 0.00193614 13\n",
+ "EXAMPLE_COMPACT_OBJECT 2.436983545179e+01 1.35842 10.9948 0.00144093 13\n",
+ "EXAMPLE_COMPACT_OBJECT 1.690157943854e+01 1.43124 13.7998 0.00107238 13\n",
+ "EXAMPLE_COMPACT_OBJECT 1.242397939734e+01 1.52416 17.3205 0.000798096 13\n",
+ "EXAMPLE_COMPACT_OBJECT 9.758088010684e+00 1.6691 21.7394 0.000593966 13\n",
+ "EXAMPLE_COMPACT_OBJECT 8.484238931453e+00 1.66351 27.2857 0.000442046 13\n",
+ "EXAMPLE_COMPACT_OBJECT 7.699271848690e+00 1.64817 34.247 0.000328983 13\n",
+ "EXAMPLE_COMPACT_OBJECT 7.568274926111e+00 1.64805 42.9844 0.000244839 13\n",
+ "EXAMPLE_COMPACT_OBJECT 7.569952912843e+00 1.64805 53.9508 0.000182216 13\n",
"Do join of subprocesses ...\n",
- "EXAMPLE_COMPACT_OBJECT 7.399005684057e+00 1.82041 67.7151 0.00013561 13\n",
- "EXAMPLE_COMPACT_OBJECT 7.443375325717e+00 1.81645 84.9909 0.000100925 13\n",
- "EXAMPLE_COMPACT_OBJECT 7.451195752942e+00 1.81559 106.674 7.51114e-05 13\n",
- "EXAMPLE_COMPACT_OBJECT 7.452661646076e+00 1.81543 133.89 5.59e-05 13\n",
- "Joined subprocesses.\n",
+ "EXAMPLE_COMPACT_OBJECT 7.571923249967e+00 1.64787 67.7151 0.00013561 13\n",
+ "EXAMPLE_COMPACT_OBJECT 7.612944004713e+00 1.64836 84.9909 0.000100925 13\n",
+ "EXAMPLE_COMPACT_OBJECT 7.620339349962e+00 1.64833 106.674 7.51114e-05 13\n",
+ "EXAMPLE_COMPACT_OBJECT 7.622200536471e+00 1.64833 133.89 5.59e-05 13\n",
+ "Joined all subprocesses.\n",
"**********************************************************\n",
- "* Population-ce756bb317f64099a459bf8b55a746ac finished! *\n",
+ "* Population-0339fb6d69f146188239ec3595eb47d5 finished! *\n",
"* The total probability is 0.0443872. *\n",
- "* It took a total of 0.73s to run 19 systems on 2 cores *\n",
- "* = 1.46s of CPU time. *\n",
- "* Maximum memory use 293.406 MB *\n",
+ "* It took a total of 2.76s to run 19 systems on 2 cores *\n",
+ "* = 5.53s of CPU time. *\n",
+ "* Maximum memory use 488.414 MB *\n",
"**********************************************************\n",
"\n",
"No failed systems were found in this run.\n",
@@ -598,18 +613,10 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": null,
"id": "e1f0464b-0424-4022-b34b-5b744bc2c59d",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "{'population_id': 'ce756bb317f64099a459bf8b55a746ac', 'evolution_type': 'grid', 'failed_count': 0, 'failed_prob': 0, 'failed_systems_error_codes': [], 'errors_exceeded': False, 'errors_found': False, 'total_probability': 0.044387171445641534, 'total_count': 19, 'start_timestamp': 1646563001.7193637, 'end_timestamp': 1646563002.4480088, 'time_elapsed': 0.7286450862884521, 'total_mass_run': 649.905447944397, 'total_probability_weighted_mass_run': 0.28133908148630704, 'zero_prob_stars_skipped': 0}\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"print(analytics)"
]
@@ -626,69 +633,20 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": null,
"id": "83f8e519-4f7c-474a-ad95-f175a34fae81",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Help on method write_binary_c_calls_to_file in module binarycpython.utils.population_extensions.dataIO:\n",
- "\n",
- "write_binary_c_calls_to_file(output_dir: Optional[str] = None, output_filename: Optional[str] = None, include_defaults: bool = False, encoding='utf-8') -> None method of binarycpython.utils.grid.Population instance\n",
- " Function that loops over the grid code and writes the generated parameters to a file.\n",
- " In the form of a command line call\n",
- " \n",
- " Only useful when you have a variable grid as system_generator. MC wouldn't be that useful\n",
- " \n",
- " Also, make sure that in this export there are the basic parameters\n",
- " like m1,m2,sep, orb-per, ecc, probability etc.\n",
- " \n",
- " On default this will write to the datadir, if it exists\n",
- " \n",
- " Args:\n",
- " output_dir: (optional, default = None) directory where to write the file to. If custom_options['data_dir'] is present, then that one will be used first, and then the output_dir\n",
- " output_filename: (optional, default = None) filename of the output. If not set it will be called \"binary_c_calls.txt\"\n",
- " include_defaults: (optional, default = None) whether to include the defaults of binary_c in the lines that are written. Beware that this will result in very long lines, and it might be better to just export the binary_c defaults and keep them in a separate file.\n",
- " \n",
- " Returns:\n",
- " filename: filename that was used to write the calls to\n",
- "\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"help(example_pop.write_binary_c_calls_to_file)"
]
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": null,
"id": "dacfed75-dfe3-4afd-a0ff-a4be17746021",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Generating grid code\n",
- "Save grid code to grid_options\n",
- "Write grid code to /tmp/binary_c_python-david/notebooks/notebook_population/binary_c_grid_ce756bb317f64099a459bf8b55a746ac.py [dry_run = False]\n",
- "Symlinked grid code to /tmp/binary_c_python-david/notebooks/notebook_population/binary_c_grid-latest2 \n",
- "Load grid code function from /tmp/binary_c_python-david/notebooks/notebook_population/binary_c_grid_ce756bb317f64099a459bf8b55a746ac.py\n",
- "Grid code loaded\n",
- "Writing binary_c calls to /tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result/binary_c_calls.txt\n",
- "Grid has handled 19 stars with a total probability of 0.0443872\n",
- "/tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result/binary_c_calls.txt\n",
- "binary_c M_1 2.2406484012210224 eccentricity 0.02 max_evolution_time 15000 orbital_period 45000000080 phasevol 0.22723621650191106 probability 0.011394572976608001\n",
- "binary_c M_1 2.812296769855663 eccentricity 0.02 max_evolution_time 15000 orbital_period 45000000080 phasevol 0.22723621650191117 probability 0.008480166685456411\n",
- "binary_c M_1 3.5297876799548944 eccentricity 0.02 max_evolution_time 15000 orbital_period 45000000080 phasevol 0.22723621650191106 probability 0.006311182276049824\n",
- "binary_c M_1 4.430329401616038 eccentricity 0.02 max_evolution_time 15000 orbital_period 45000000080 phasevol 0.22723621650191106 probability 0.004696962123378559\n",
- "(abridged)\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"example_pop.set(verbosity=1)\n",
"calls_filename = example_pop.write_binary_c_calls_to_file()\n",
@@ -718,7 +676,7 @@
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": 12,
"id": "7212b6be-9787-4122-a7f1-86538cf38179",
"metadata": {},
"outputs": [
@@ -731,8 +689,6 @@
"ok\n",
"File at /tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result/example_pop_settings.json.gz already exists: cannot write to it\n",
"ok\n",
- "ok pre\n",
- "ok ret\n",
"Do dry run? True\n",
"Doing dry run to calculate total starcount and probability\n",
"Grid has handled 19 stars with a total probability of 0.0443872\n",
@@ -743,13 +699,13 @@
"**********************************\n",
"\n",
"Do join of subprocesses ...\n",
- "Joined subprocesses.\n",
+ "Joined all subprocesses.\n",
"**********************************************************\n",
- "* Population-0fa4c2b8707741a5ab41d209ef95a3a4 finished! *\n",
+ "* Population-19273c7d83674b89902b735e67ae3ae3 finished! *\n",
"* The total probability is 0.0443872. *\n",
- "* It took a total of 0.61s to run 19 systems on 2 cores *\n",
- "* = 1.23s of CPU time. *\n",
- "* Maximum memory use 299.531 MB *\n",
+ "* It took a total of 4.23s to run 19 systems on 2 cores *\n",
+ "* = 8.46s of CPU time. *\n",
+ "* Maximum memory use 493.797 MB *\n",
"**********************************************************\n",
"\n",
"No failed systems were found in this run.\n",
@@ -758,19 +714,19 @@
"\n",
"\n",
"time mass zams_mass probability radius stellar_type\n",
- "3.598268106227e+01 1.30592 8.75988 0.00193614 1.72498e-05 13\n",
- "2.436983545111e+01 1.35842 10.9948 0.00144093 1.72498e-05 13\n",
- "1.690157944401e+01 1.43124 13.7998 0.00107238 1.72498e-05 13\n",
- "1.242397939068e+01 1.52416 17.3205 0.000798096 1.72498e-05 13\n",
- "9.756794139032e+00 1.66914 21.7394 0.000593966 1.72498e-05 13\n",
- "8.401414766976e+00 1.73729 27.2857 0.000442046 1.72498e-05 13\n",
- "7.536373523810e+00 1.80677 34.247 0.000328983 1.72498e-05 13\n",
- "7.393982410080e+00 1.82164 42.9844 0.000244839 1.72498e-05 13\n",
- "7.396470605248e+00 1.82129 53.9508 0.000182216 1.72498e-05 13\n",
- "7.399005684057e+00 1.82041 67.7151 0.00013561 1.72498e-05 13\n",
- "7.451195752942e+00 1.81559 106.674 7.51114e-05 1.72498e-05 13\n",
- "7.443375325717e+00 1.81645 84.9909 0.000100925 1.72498e-05 13\n",
- "7.452661646076e+00 1.81543 133.89 5.59e-05 1.72498e-05 13\n",
+ "2.436983545179e+01 1.35842 10.9948 0.00144093 1.72498e-05 13\n",
+ "3.598268106261e+01 1.30592 8.75988 0.00193614 1.72498e-05 13\n",
+ "1.690157943854e+01 1.43124 13.7998 0.00107238 1.72498e-05 13\n",
+ "1.242397939734e+01 1.52416 17.3205 0.000798096 1.72498e-05 13\n",
+ "9.758088010684e+00 1.6691 21.7394 0.000593966 1.72498e-05 13\n",
+ "8.484238931453e+00 1.66351 27.2857 0.000442046 1.72498e-05 13\n",
+ "7.699271848690e+00 1.64817 34.247 0.000328983 1.72498e-05 13\n",
+ "7.568274926111e+00 1.64805 42.9844 0.000244839 1.72498e-05 13\n",
+ "7.569952912843e+00 1.64805 53.9508 0.000182216 1.72498e-05 13\n",
+ "7.571923249967e+00 1.64787 67.7151 0.00013561 1.72498e-05 13\n",
+ "7.612944004713e+00 1.64836 84.9909 0.000100925 1.72498e-05 13\n",
+ "7.620339349962e+00 1.64833 106.674 7.51114e-05 1.72498e-05 13\n",
+ "7.622200536471e+00 1.64833 133.89 5.59e-05 1.72498e-05 13\n",
"\n"
]
}
@@ -919,54 +875,10 @@
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": null,
"id": "79acdbb2-7dd6-45c4-9609-80994f03619a",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "<<<< Warning: Key does not match previously known parameter: adding: data_dir=/tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result to custom_options >>>>\n",
- "<<<< Warning: Key does not match previously known parameter: adding: base_filename=example_pop.dat to custom_options >>>>\n",
- "ok\n",
- "File at /tmp/binary_c_python-david/notebooks/notebook_population/example_python_population_result/example_pop_settings.json.gz already exists: cannot write to it\n",
- "ok\n",
- "ok pre\n",
- "ok ret\n",
- "Do dry run? True\n",
- "Doing dry run to calculate total starcount and probability\n",
- "Grid has handled 8 stars with a total probability of 0.0211592\n",
- "**********************************\n",
- "* Dry run *\n",
- "* Total starcount is 8 *\n",
- "* Total probability is 0.0211592 *\n",
- "**********************************\n",
- "\n",
- "Do join of subprocesses ...\n",
- "Joined subprocesses.\n",
- "**********************************************************\n",
- "* Population-0eb5c0c9abd34607a6ee060b26a7e32f finished! *\n",
- "* The total probability is 0.0211592. *\n",
- "* It took a total of 0.84s to run 8 systems on 2 cores *\n",
- "* = 1.68s of CPU time. *\n",
- "* Maximum memory use 300.125 MB *\n",
- "**********************************************************\n",
- "\n",
- "No failed systems were found in this run.\n",
- "Do analytics\n",
- "Added analytics to metadata\n",
- "\n",
- "\n",
- "time mass_1 zams_mass_1 mass_2 zams_mass_2 stellar_type_1 prev_stellar_type_1 stellar_type_2 prev_stellar_type_2 metallicity probability\n",
- "1.378266748188e+01 1.66293 50.9713 1.78767 12.8178 13 13 13 8 0.02 0.000339963\n",
- "1.817608462595e+01 1.82104 50.9713 1.41436 12.8178 13 13 13 5 0.02 0.000193036\n",
- "7.422997711686e+00 1.82479 50.9713 1.82171 38.2535 13 13 13 8 0.02 0.000193036\n",
- "1.205711924468e+01 1.73765 50.9713 0 38.2535 13 13 15 8 0.02 0.000339963\n",
- "\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"def parse_function(self, output):\n",
" \"\"\"\n",
--
GitLab