From fdaf033ef12887b48a1ae4a003e2cacaf9b909a6 Mon Sep 17 00:00:00 2001
From: David Hendriks <davidhendriks93@gmail.com>
Date: Thu, 3 Dec 2020 15:29:50 +0000
Subject: [PATCH] Fixed ensemble issues
---
.../tests/c_bindings/test_ensemble.py | 170 +++++-------------
binarycpython/utils/functions.py | 18 +-
2 files changed, 52 insertions(+), 136 deletions(-)
diff --git a/binarycpython/tests/c_bindings/test_ensemble.py b/binarycpython/tests/c_bindings/test_ensemble.py
index e57b321a1..cf05dd1a8 100644
--- a/binarycpython/tests/c_bindings/test_ensemble.py
+++ b/binarycpython/tests/c_bindings/test_ensemble.py
@@ -140,93 +140,6 @@ def pickle_n_checksize(obj):
print("size: {}mb".format(file_stats.st_size / (1024 * 1024)))
os.remove(name)
-
-def test_full_ensemble_output_new():
- """
- Function to just output the whole ensemble
- """
-
- argstring_1 = return_argstring(defer_ensemble=0, ensemble_filters_off=0)
- print(argstring_1)
- quit()
- output_1 = _binary_c_bindings.run_system(argstring=argstring_1)
- # print(output_1)
- ensemble_jsons_1 = [
- line for line in output_1.splitlines() if line.startswith("ENSEMBLE_JSON")
- ]
-
- print("start")
- start = time.time()
- json_1 = eval(ensemble_jsons_1[0][len("ENSEMBLE_JSON ") :])
-
- # json_1 = eval(
- # [line for line in _binary_c_bindings.run_system(argstring=argstring_1).splitlines() if line.startswith("ENSEMBLE_JSON")][0][len("ENSEMBLE_JSON ") :]
- # )
- stop = time.time()
- print("stop")
-
- pickle_n_checksize(json_1)
-
- print("took {}s to decode".format(stop - start))
- # print("Size of the json in memory: {}".format(sys.getsizeof(json_1)))
-
- print(json_1.keys())
- # assert statements:
- assert "number_counts" in json_1.keys()
- assert "HRD" in json_1.keys()
- assert "HRD(t)" in json_1.keys()
- assert "Xyield" in json_1.keys()
- assert "distributions" in json_1.keys()
- assert "scalars" in json_1.keys()
-
-
-def test_full_ensemble_output():
- """
- Function to just output the whole ensemble
- """
-
- argstring_1 = return_argstring(defer_ensemble=0, ensemble_filters_off=1)
-
- # print(argstring_1)
- # quit()
- argstring_1 += " ensemble_filter_MERGED 1 "
- argstring_1 += " ensemble_filter_ORBIT 1 "
- argstring_1 += " ensemble_filter_SCALARS 1 "
- argstring_1 += " ensemble_filter_CHEMICALLY_PECULIAR 1 "
- argstring_1 += " ensemble_filter_SPECTRAL_TYPES 1 "
- argstring_1 += " ensemble_filter_HRD 1 "
-
- output_1 = _binary_c_bindings.run_system(argstring=argstring_1)
- pickle_n_checksize(output_1)
- ensemble_jsons_1 = [
- line for line in output_1.splitlines() if line.startswith("ENSEMBLE_JSON")
- ]
-
- pickle_n_checksize(ensemble_jsons_1[0][len("ENSEMBLE_JSON ") :])
-
- print("start")
- start = time.time()
- json_1 = handle_ensemble_string_to_json(
- ensemble_jsons_1[0][len("ENSEMBLE_JSON ") :]
- )
- stop = time.time()
- print("stop")
-
- pickle_n_checksize(json_1)
-
- print("took {}s to decode".format(stop - start))
- # print("Size of the json in memory: {}".format(sys.getsizeof(json_1)))
-
- # print(json_1)
- # assert statements:
- assert "number_counts" in json_1.keys()
- assert "HRD" in json_1.keys()
- assert "HRD(t)" in json_1.keys()
- assert "Xyield" in json_1.keys()
- assert "distributions" in json_1.keys()
- assert "scalars" in json_1.keys()
-
-
def test_adding_ensemble_output():
"""
Function that adds the output of 2 ensembles and compares it to the output that we get by deferring the first output
@@ -377,27 +290,6 @@ def test_adding_ensemble_output():
), assert_message_1
# assert inspect_dict(test_1_merged_dict, print_structure=False) == inspect_dict(test_3_json, print_structure=False), assert_message_2
-
-def test_combine_with_empty_json():
- """
- Test for merging with an empty dict
- """
-
- argstring_1 = return_argstring(defer_ensemble=0)
- output_1 = _binary_c_bindings.run_system(argstring=argstring_1)
- ensemble_jsons_1 = [
- line for line in output_1.splitlines() if line.startswith("ENSEMBLE_JSON")
- ]
- json_1 = handle_ensemble_string_to_json(
- ensemble_jsons_1[0][len("ENSEMBLE_JSON ") :]
- )
-
- assert_message = (
- "combining output json with empty dict should give same result as initial json"
- )
- assert merge_dicts(json_1, {}) == json_1, assert_message
-
-
def test_free_and_json_output():
"""
Function that tests the freeing of the memory adress and the output of the json
@@ -415,37 +307,69 @@ def test_free_and_json_output():
persistent_data_memaddr = _binary_c_bindings.return_persistent_data_memaddr()
# Evolve and defer output
- print("evolving")
output_1_deferred = _binary_c_bindings.run_system(
argstring=argstring_1, persistent_data_memaddr=persistent_data_memaddr
)
- print("Evolved")
- print("Output:")
- print(textwrap.indent(str(output_1_deferred), "\t"))
# Free memory adress
- print("freeing")
json_output_by_freeing = _binary_c_bindings.free_persistent_data_memaddr_and_return_json_output(
persistent_data_memaddr
)
- print("Freed")
- print("Output:")
- print(textwrap.indent(str(json_output_by_freeing), "\t"))
+ # print(textwrap.indent(str(json_output_by_freeing), "\t"))
parsed_json = handle_ensemble_string_to_json(
json_output_by_freeing.splitlines()[0][len("ENSEMBLE_JSON ") :],
)
- print(parsed_json)
- # ensemble_jsons_1 = [line for line in output_1.splitlines() if line.startswith("ENSEMBLE_JSON")]
- # json_1 = json.loads(ensemble_jsons_1[0][len("ENSEMBLE_JSON "):], cls=binarycDecoder)
+ assert "number_counts" in parsed_json.keys(), "Output not correct"
+
+def test_combine_with_empty_json():
+ """
+ Test for merging with an empty dict
+ """
+
+ argstring_1 = return_argstring(defer_ensemble=0)
+ output_1 = _binary_c_bindings.run_system(argstring=argstring_1)
+ ensemble_jsons_1 = [
+ line for line in output_1.splitlines() if line.startswith("ENSEMBLE_JSON")
+ ]
+ json_1 = handle_ensemble_string_to_json(
+ ensemble_jsons_1[0][len("ENSEMBLE_JSON ") :]
+ )
+ assert_message = (
+ "combining output json with empty dict should give same result as initial json"
+ )
+ assert merge_dicts(json_1, {}) == json_1, assert_message
+
+#############
+def test_full_ensemble_output():
+ """
+ Function to just output the whole ensemble
+ """
+
+ argstring_1 = return_argstring(defer_ensemble=0, ensemble_filters_off=0)
+ # print(argstring_1)
+
+ output_1 = _binary_c_bindings.run_system(argstring=argstring_1)
+ ensemble_jsons_1 = [
+ line for line in output_1.splitlines() if line.startswith("ENSEMBLE_JSON")
+ ]
+ json_1 = handle_ensemble_string_to_json(ensemble_jsons_1[0][len("ENSEMBLE_JSON ") :])
+
+ # assert statements:
+ assert "number_counts" in json_1.keys()
+ assert "HRD" in json_1.keys()
+ assert "HRD(t)" in json_1.keys()
+ # assert "Xyield" in json_1.keys()
+ assert "distributions" in json_1.keys()
+ assert "scalars" in json_1.keys()
+
def all():
- # test_return_persistent_data_memaddr()
- # test_passing_persistent_data_to_run_system()
- test_full_ensemble_output_new()
- # test_full_ensemble_output()
+ test_return_persistent_data_memaddr()
+ test_passing_persistent_data_to_run_system()
+ test_full_ensemble_output()
test_adding_ensemble_output()
test_free_and_json_output()
test_combine_with_empty_json()
diff --git a/binarycpython/utils/functions.py b/binarycpython/utils/functions.py
index 4effffa2f..fa79c8ef0 100644
--- a/binarycpython/utils/functions.py
+++ b/binarycpython/utils/functions.py
@@ -972,19 +972,11 @@ def binaryc_json_serializer(obj):
def handle_ensemble_string_to_json(raw_output):
"""
- Function that deals with the raw output of the ensemble and
- creates a working JSON dictionary out of it.
+ Function that deals with the raw output of the ensemble and
+ creates a working JSON dictionary out of it.
- It seems that JSON does not like strings that contain dictionaries which have a number as a key, but more specifically it is the fact that binary-c outputs as 5e2 instead of
-
- e.g. with python 3.6.4: json.loads("{5e2: 10}") leads to a JSONDecodeError. To fix this, I do the following:
-
- with ensemble_output being the literal string of output, and json_output the correct jsonified version of that.
-
- eval_dict = ast.literal_eval(ensemble_output)
- json_output = json.loads(json.dumps(eval_dict), cls=binarycDecoder)
-
- Note: this does pose somewhat of a risk, since eval'ing something might lead to unexpected behaviour.
+ Having this wrapper makes it easy to
"""
- return json.loads(json.dumps(ast.literal_eval(raw_output)), cls=binarycDecoder)
+ # return json.loads(json.dumps(ast.literal_eval(raw_output)), cls=binarycDecoder)
+ return json.loads(raw_output, cls=binarycDecoder)
--
GitLab