autogen_C_logging_code() (in module binarycpython.utils.custom_logging_functions)

binary_c_log_code() (in module binarycpython.utils.custom_logging_functions)

binary_c_write_log_code() (in module binarycpython.utils.custom_logging_functions)

compile_shared_lib() (in module binarycpython.utils.custom_logging_functions)

create_and_load_logging_function() (in module binarycpython.utils.custom_logging_functions)

from_binary_c_config() (in module binarycpython.utils.custom_logging_functions)

return_compilation_dict() (in module binarycpython.utils.custom_logging_functions)

Arenou2010_binary_fraction() (in module binarycpython.utils.distribution_functions)

Izzard2012_period_distribution() (in module binarycpython.utils.distribution_functions)

Kroupa2001() (in module binarycpython.utils.distribution_functions)

calculate_constants_three_part_powerlaw() (in module binarycpython.utils.distribution_functions)

cosmic_SFH_madau_dickinson2014() (in module binarycpython.utils.distribution_functions)

duquennoy1991() (in module binarycpython.utils.distribution_functions)

flatsections() (in module binarycpython.utils.distribution_functions)

gaussian() (in module binarycpython.utils.distribution_functions)

gaussian_func() (in module binarycpython.utils.distribution_functions)

gaussian_normalizing_const() (in module binarycpython.utils.distribution_functions)

imf_chabrier2003() (in module binarycpython.utils.distribution_functions)

imf_scalo1986() (in module binarycpython.utils.distribution_functions)

imf_scalo1998() (in module binarycpython.utils.distribution_functions)

imf_tinsley1980() (in module binarycpython.utils.distribution_functions)

interpolate_in_mass_izzard2012() (in module binarycpython.utils.distribution_functions)

powerlaw() (in module binarycpython.utils.distribution_functions)

powerlaw_constant() (in module binarycpython.utils.distribution_functions)

prepare_dict() (in module binarycpython.utils.distribution_functions)

raghavan2010_binary_fraction() (in module binarycpython.utils.distribution_functions)

set_opts() (in module binarycpython.utils.distribution_functions)

three_part_powerlaw() (in module binarycpython.utils.distribution_functions)

binaryc_json_serializer() (in module binarycpython.utils.functions)

example_parse_output() (in module binarycpython.utils.functions)

extract_ensemble_json_from_string() (in module binarycpython.utils.functions)

handle_ensemble_string_to_json() (in module binarycpython.utils.functions)

parse_binary_c_version_info() (in module binarycpython.utils.functions)

return_binary_c_version_info() (in module binarycpython.utils.functions)

write_binary_c_parameter_descriptions_to_rst_file() (in module binarycpython.utils.functions)

add_grid_variable() (binarycpython.utils.grid.Population method)

evolve_population() (binarycpython.utils.grid.Population method)

return_population_settings() (binarycpython.utils.grid.Population method)

write_binary_c_calls_to_file() (binarycpython.utils.grid.Population method)

grid_options_description_checker() (in module binarycpython.utils.grid_options_defaults)

grid_options_help() (in module binarycpython.utils.grid_options_defaults)

write_grid_options_to_rst_file() (in module binarycpython.utils.grid_options_defaults)

parse_function_hr_diagram() (in module binarycpython.utils.plot_functions)

parse_function_masses() (in module binarycpython.utils.plot_functions)

parse_function_orbit() (in module binarycpython.utils.plot_functions)

plot_HR_diagram() (in module binarycpython.utils.plot_functions)

calc_period_from_sep() (in module binarycpython.utils.useful_funcs)

calc_sep_from_period() (in module binarycpython.utils.useful_funcs)

minimum_period_for_RLOF() (in module binarycpython.utils.useful_funcs)

minimum_separation_for_RLOF() (in module binarycpython.utils.useful_funcs)

"""
Module containing functions for the custom logging functionality.
The functions here make it possible for the user to define binaryc output logs on runtime
"""

import os
import textwrap
import subprocess
import socket
import ctypes
import uuid
from typing import Union, Tuple, Optional
from binarycpython.utils.functions import temp_dir, remove_file, verbose_print

def autogen_C_logging_code(logging_dict: dict, verbose: int = 0) -> Optional[str]:
    """
    Function that autogenerates PRINTF statements for binaryc.
    Input is a dictionary where the key is the header of that logging line
    and items which are lists of parameters that will be put in that logging line

    The list elements are all appended to 'stardata->' in the autogenerated code.

    Example:
        Input dictionary should look like this::

            {'MY_STELLAR_DATA':
                [
                    'model.time',
                    'star[0].mass',
                    'model.probability',
                    'model.dt'
                ]
            }

    Args:
        logging_dict: Dictionary containing lists of parameters that binary_c has to output. The keys are used by binary_c as start of the sentence.
        verbose: Level of verbosity. Defaults to zero if not set explicilty.

    Returns:
        string containing C printf statement built to output the parameters given as input.
    """

    # Check if the input is of the correct form
    if not isinstance(logging_dict, dict):
        print("Error: please use a dictionary as input")
        return None

    code = ""

    # Loop over dict keys
    for key in logging_dict:
        if verbose > 0:
            print(
                "Generating Print statement for custom logging code with {} as a header".format(
                    key
                )
            )
        logging_dict_entry = logging_dict[key]

        # Check if item is of correct type:
        if isinstance(logging_dict_entry, list):

            # Construct print statement
            code += 'Printf("{}'.format(key)
            code += " {}".format("%g " * len(logging_dict_entry))
            code = code.strip()
            code += '\\n"'

            # Add format keys
            for param in logging_dict_entry:
                code += ",((double)stardata->{})".format(param)
            code += ");\n"

        else:
            print(
                "Error: please use a list for the list of parameters that you want to have logged"
            )
            return None
    code = code.strip()

    return code

####################################################################################
def binary_c_log_code(code: str, verbose: int = 0) -> str:
    """
    Function to construct the code to construct the custom logging function

    Example:
        Code to log and terminate evolution when the primary star becomes a NS::

            if(stardata->star[0].stellar_type>=NS)
            {
                if (stardata->model.time < stardata->model.max_evolution_time)
                {
                    Printf("EXAMPLE_LOG_CO %30.12e %g %g %g %g %d %d\\n",
                        //
                        stardata->model.time, // 1

                        stardata->star[0].mass, //2
                        stardata->previous_stardata->star[0].mass, //3

                        stardata->star[0].radius, //4
                        stardata->previous_stardata->star[0].radius, //5

                        stardata->star[0].stellar_type, //6
                        stardata->previous_stardata->star[0].stellar_type //7
                  );
                };
                /* Kill the simulation to save time */
                stardata->model.max_evolution_time = stardata->model.time - stardata->model.dtm;
            };

    Args:
        code: Exact c-statement to output information in binary_c. Can be wrapped in logical statements.

    Returns:
        string containing the custom logging code. This includes all the includes and other definitions. This code will be used as the shared library
    """

    verbose_print(
        "Creating the code for the shared library for the custom logging", verbose, 0
    )

    if not "Printf" in code:
        print(
            "Error: There has to be at least a printf statement in the provided code. Aborting"
        )

    # Create code
    custom_logging_function_string = """\
#pragma push_macro(\"MAX\")
#pragma push_macro(\"MIN\")
#undef MAX
#undef MIN
#include \"binary_c.h\"
#include \"RLOF/RLOF_prototypes.h\"

// add visibility __attribute__ ((visibility ("default"))) to it 
void binary_c_API_function custom_output_function(struct stardata_t * stardata);
void binary_c_API_function custom_output_function(struct stardata_t * stardata)
{{
    // struct stardata_t * stardata = (struct stardata_t *)x;
    {};
}}

#undef MAX 
#undef MIN
#pragma pop_macro(\"MIN\")
#pragma pop_macro(\"MAX\")\
    """.format(
        code
    )

    # print(repr(textwrap.dedent(custom_logging_function_string)))
    return textwrap.dedent(custom_logging_function_string)

def binary_c_write_log_code(code: str, filename: str, verbose: int = 0) -> None:
    """
    Function to write the generated logging code to a file

    Args:
        code: string containing the custom logging code to write to a file.
        filename: target filename.
    """

    # TODO: change this. I don't like the cwd
    cwd = os.getcwd()
    filePath = os.path.join(cwd, filename)

    # Remove if it exists
    remove_file(filePath, verbose)

    if verbose > 0:
        print("Writing the custom logging code to {}".format(filePath))

    # Write again
    with open(filePath, "w") as file:
        file.write(code)

def from_binary_c_config(config_file: str, flag: str) -> str:
    """
    Function to run the binaryc_config command with flags

    Args:
        config_file: binary_c-config filepath TODO: change the name of this
        flag: flag used in the binary_c-config call.

    Returns:
        returns the result of <binary_c-config> <flag>
    """

    res = subprocess.check_output(
        "{config_file} {flag}".format(config_file=config_file, flag=flag),
        shell=True,
        stderr=subprocess.STDOUT,
    )

    # convert and chop off newline
    res = res.decode("utf-8").rstrip()

    return res

def return_compilation_dict(verbose: int = 0) -> dict:
    """
    Function to build the compile command for the shared library

    Inspired by binary_c_inline_config command in perl

    TODO: this function still has some cleaning up to do wrt default values for the compile command
    # https://developers.redhat.com/blog/2018/03/21/compiler-and-linker-flags-gcc/

    Args:

    Returns:
        string containing the command to build the shared library
    """

        print(
            "Calling the binary_c config code to get the info to build the shared library"
        )
    # use binary_c-config to get necessary flags
    BINARY_C_DIR = os.getenv("BINARY_C")
    if BINARY_C_DIR:
        BINARY_C_CONFIG = os.path.join(BINARY_C_DIR, "binary_c-config")
        BINARY_C_SRC_DIR = os.path.join(BINARY_C_DIR, "src")
    else:
        raise NameError("Envvar BINARY_C doesnt exist")

    # TODO: make more options for the compiling
    cc = from_binary_c_config(BINARY_C_CONFIG, "cc")

    # Check for binary_c
    BINARY_C_EXE = os.path.join(BINARY_C_DIR, "binary_c")
    if not os.path.isfile(BINARY_C_EXE):
        print("We require  binary_c executable; have you built binary_c?")
        raise NameError("BINARY_C executable doesnt exist")

    # TODO: debug
    libbinary_c = "-lbinary_c"
    binclibs = from_binary_c_config(BINARY_C_CONFIG, "libs")
    libdirs = "{} -L{}".format(
        from_binary_c_config(BINARY_C_CONFIG, "libdirs"), BINARY_C_SRC_DIR
    )
    bincflags = from_binary_c_config(BINARY_C_CONFIG, "cflags")
    bincincdirs = from_binary_c_config(BINARY_C_CONFIG, "incdirs")

    # combine
    binclibs = " {} {} {}".format(libdirs, libbinary_c, binclibs)

    # setup defaults:
    defaults = {
        "cc": "gcc",  # default compiler
        "ccflags": bincflags,
        "ld": "ld",  # 'ld': $Config{ld}, # default linker
        "debug": 0,
        "inc": "{} -I{}".format(bincincdirs, BINARY_C_SRC_DIR),
        # inc => ' '.($Config{inc}//' ').' '.$bincincdirs." -I$srcdir ",
        #   include the defaults plus # GSL and binary_c
        # 'libname': libname, # libname is usually just binary_c corresponding to libbinary_c.so
        "libs": binclibs,
    }

    # set values with defaults. TODO: make other input possile.
    ld = defaults["ld"]
    # debug = defaults["debug"]
    inc = defaults[
        "inc"
    ]  # = ($ENV{BINARY_GRID2_INC} // $defaults{inc}).' '.($ENV{BINARY_GRID2_EXTRAINC} // '');
    libs = defaults[
        "libs"
    ]  # = ($ENV{BINARY_GRID2_LIBS} // $defaults{libs}).' '.($ENV{BINARY_GRID2_EXTRALIBS}//'');
    ccflags = defaults["ccflags"]  #  = $ENV{BINARY_GRID2_CCFLAGS}
    # // ($defaults{ccflags}) . ($ENV{BINARY_GRID2_EXTRACCFLAGS} // '');

    # you must define _SEARCH_H to prevent it being loaded twice
    ccflags += " -shared -D_SEARCH_H"

    # remove the visibility=hidden for this compilation
    ccflags = ccflags.replace("-fvisibility=hidden", "")

    # ensure library paths to the front of the libs:
    libs_content = libs.split(" ")
    library_paths = [el for el in libs_content if el.startswith("-L")]
    non_library_paths = [
        el for el in libs_content if (not el.startswith("-L") and not el == "")
    ]
    libs = "{} {}".format(" ".join(library_paths), " ".join(non_library_paths))

        print(
            "Got options to compile:\n\tcc = {cc}\n\tccflags = {ccflags}\n\tld = {ld}\n\tlibs = {libs}\n\tinc = {inc}\n\n".format(
                cc=cc, ccflags=ccflags, ld=ld, libs=libs, inc=inc
            )
        )

    return {"cc": cc, "ld": ld, "ccflags": ccflags, "libs": libs, "inc": inc}

def compile_shared_lib(
    code: str, sourcefile_name: str, outfile_name: str, verbose: int = 0
) -> None:
    """
    Function to write the custom logging code to a file and then compile it.

    TODO: nicely put in the -fPIC
    TODO: consider returning a status

    Args:
        code: string containing the custom logging code
        sourcefile_name: name of the file that will contain the code
        outfile_name: name of the file that will be the shared library
    """

    # Write code to file
    binary_c_write_log_code(code, sourcefile_name, verbose)

    # Remove the library if present:
    remove_file(outfile_name, verbose)

    # create compilation command
    compilation_dict = return_compilation_dict(verbose)

    # Construct full command
    command = (
        "{cc} -fPIC {ccflags} {libs} -o {outfile_name} {sourcefile_name} {inc}".format(
            cc=compilation_dict["cc"],
            ccflags=compilation_dict["ccflags"],
            libs=compilation_dict["libs"],
            outfile_name=outfile_name,
            sourcefile_name=sourcefile_name,
            inc=compilation_dict["inc"],
        )
    )

    # remove extra whitespaces:
    command = " ".join(command.split())

    # Execute compilation and create the library
        print(
            "Building shared library for custom logging with (binary_c.h) on {}\n".format(
                socket.gethostname()
            )
        )
        print(
            "Executing following command to compile the shared library:\n{command}".format(
                command=command
            )
        )
    res = subprocess.check_output("{command}".format(command=command), shell=True)

        if res:
            print("Output of compilation command:\n{}".format(res))

def create_and_load_logging_function(
    custom_logging_code: str, verbose: int = 0
) -> Tuple[int, str]:
    """
    Function to automatically compile the shared library with the given
    custom logging code and load it with ctypes.

    This function is more or less the main function of this module and unless you know what you're doing with the other functions
    I recommend using this in function in combination with a function that generates the exact code (like :meth:`~binarycpython.utils.custom_logging_code.binary_c_log_code`)

    Args:
        custom_logging_code: string containing the custom logging code

    Returns:
        memory adress of the custom logging function in a int type.
    """

    #

    library_name = os.path.join(
        temp_dir(), "libcustom_logging_{}.so".format(uuid.uuid4().hex)
    )

    compile_shared_lib(
        custom_logging_code,
        sourcefile_name=os.path.join(temp_dir(), "custom_logging.c"),
        outfile_name=library_name,
        verbose=verbose,
    )

        print("loading shared library for custom logging")

    # Loading library
    _ = ctypes.CDLL("libgslcblas.so", mode=ctypes.RTLD_GLOBAL)
    _ = ctypes.CDLL("libgsl.so", mode=ctypes.RTLD_GLOBAL)
    _ = ctypes.CDLL("libbinary_c.so", mode=ctypes.RTLD_GLOBAL)

    libcustom_logging = ctypes.CDLL(
        library_name,
        mode=ctypes.RTLD_GLOBAL,
    )  # loads the shared library

    # Get memory adress of function. mimicking a pointer
    func_memaddr = ctypes.cast(
        libcustom_logging.custom_output_function, ctypes.c_void_p
    ).value

    if not isinstance(func_memaddr, int):
        print(
            "Something went wrong. The memory adress returned by the ctypes.cast is not an integer. It has the value {}".format(
                func_memaddr
            )
        )
        raise ValueError

        print(
            "loaded shared library for custom logging. \
            custom_output_function is loaded in memory at {}".format(
            )
        )

    return func_memaddr, library_name

"""
Module containing the predefined distribution functions

The user can use any of these distribution functions to
generate probability distributions for sampling populations

There are distributions for the following parameters:
    - mass
    - period
    - mass ratio
    - binary fraction

Tasks:
    - TODO: make some things globally present? rob does this in his module..i guess it saves calculations but not sure if im gonna do that now
    - TODO: make global constants stuff
    - TODO: add eccentricity distribution: thermal
    - TODO: Add SFH distributions depending on redshift
    - TODO: Add metallicity distributions depending on redshift
    - TODO: Add initial rotational velocity distributions
"""

import math
import numpy as np
from typing import Optional, Union
from binarycpython.utils.useful_funcs import calc_period_from_sep

###
# File containing probability distributions
# Mostly copied from the perl modules
LOG_LN_CONVERTER = 1.0 / math.log(10.0)
distribution_constants = {}  # To store the constants in

def prepare_dict(global_dict: dict, list_of_sub_keys: list) -> None:
    """
    Function that makes sure that the global dict is prepared to have a value set there. This dictionary will store values and factors for the distribution functions, so that they dont have to be calculated each time.

    Args:
        global_dict: globablly acessible dictionary where factors are stored in
        list_of_sub_keys: List of keys that must become be(come) present in the global_dict
    """

    internal_dict_value = global_dict

    # This loop almost mimics a recursive loop into the dictionary.
    # It checks whether the first key of the list is present, if not; set it with an empty dict.
    # Then it overrides itself to be that (new) item, and goes on to do that again, until the list exhausted
    for k in list_of_sub_keys:
        # If the sub key doesnt exist then make an empty dict
        if not internal_dict_value.get(k, None):
            internal_dict_value[k] = {}
        internal_dict_value = internal_dict_value[k]

def set_opts(opts: dict, newopts: dict) -> dict:
    """
    Function to take a default dict and override it with newer values.

    # TODO: consider changing this to just a dict.update

    Args:
        opts: dictionary with default values
        newopts: dictionary with new values

    Returns:
        returns an updated dictionary
    """

    if newopts:
        for opt in newopts.keys():
            if opt in opts.keys():
                opts[opt] = newopts[opt]

    return opts

def flat() -> float:
    """
    Dummy distribution function that returns 1

    Returns:
        a flat uniform distribution: 1
    """

    return 1.0

def number(value: Union[int, float]) -> Union[int, float]:
    """
    Dummy distribution function that returns the input

    Args:
        value: the value that will be returned by this function.

    Returns:
        the value that was provided
    """

    return value

def const(
    min_bound: Union[int, float], max_bound: Union[int, float], val: float = None
) -> Union[int, float]:
    """
    a constant distribution function between min=min_bound and max=max_bound.

    Args:
        min_bound: lower bound of the range
        max_bound: upper bound of the range

    Returns:
            returns the value of 1/(max_bound-min_bound). If val is provided, it will check whether min_bound < val <= max_bound. if not: returns 0
    """

    if val:
        if not min_bound < val <= max_bound:
            print("out of bounds")
            prob = 0
            return prob
    prob = 1.0 / (max_bound - min_bound)
    return prob

def powerlaw_constant(
    min_val: Union[int, float], max_val: Union[int, float], k: Union[int, float]
    """
    Function that returns the constant to normalise a powerlaw

    TODO: what if k is -1?

    Args:
        min_val: lower bound of the range
        max_val: upper bound of the range
        k: powerlaw slope

    Returns:
        constant to normalize the given powerlaw between the min_val and max_val range
    """

    k1 = k + 1.0
    # print(
    #     "Powerlaw consts from {} to {}, k={} where k1={}".format(
    #         min_val, max_val, k, k1
    #     )
    # )

    powerlaw_const = k1 / (max_val ** k1 - min_val ** k1)
    return powerlaw_const

def powerlaw(
    min_val: Union[int, float],
    max_val: Union[int, float],
    k: Union[int, float],
    x: Union[int, float],
    """
    Single powerlaw with index k at x from min to max

    Args:
        min_val: lower bound of the powerlaw
        max_val: upper bound of the powerlaw
        k: slope of the power law
        x: position at which we want to evaluate

    Returns:
        `probability` at the given position(x)
    """

    # Handle faulty value
    if k == -1:
        print("wrong value for k")

    if (x < min_val) or (x > max_val):
        print("input value is out of bounds!")
        return 0

    powerlaw_const = powerlaw_constant(min_val, max_val, k)

    # powerlaw
    prob = powerlaw_const * (x ** k)
    # print(
    #     "Power law from {} to {}: const = {}, y = {}".format(
    #         min_val, max_val, const, y
    #     )
    # )

def calculate_constants_three_part_powerlaw(
    m0: Union[int, float],
    m1: Union[int, float],
    m2: Union[int, float],
    m_max: Union[int, float],
    p1: Union[int, float],
    p2: Union[int, float],
    p3: Union[int, float],
    """
    Function to calculate the constants for a three-part powerlaw

    TODO: use the powerlaw_constant function to calculate all these values

    Args:
        m0: lower bound mass
        m1: second boundary, between the first slope and the second slope
        m2: third boundary, between the second slope and the third slope
        m_max: upper bound mass
        p1: first slope
        p2: second slope
        p3: third slope

    Returns:
        array of normalisation constants
    """

    # print("Initialising constants for the three-part powerlaw: m0={} m1={} m2={}\
    # m_max={} p1={} p2={} p3={}\n".format(m0, m1, m2, m_max, p1, p2, p3))

    array_constants_three_part_powerlaw = [0, 0, 0]

    array_constants_three_part_powerlaw[1] = (
        ((m1 ** p2) * (m1 ** (-p1)))
        * (1.0 / (1.0 + p1))
        * (m1 ** (1.0 + p1) - m0 ** (1.0 + p1))
    )
    array_constants_three_part_powerlaw[1] += (
        (m2 ** (1.0 + p2) - m1 ** (1.0 + p2))
    ) * (1.0 / (1.0 + p2))
        ((m2 ** p2) * (m2 ** (-p3)))
        * (1.0 / (1.0 + p3))
        * (m_max ** (1.0 + p3) - m2 ** (1.0 + p3))
    )
    array_constants_three_part_powerlaw[1] = 1.0 / (
        array_constants_three_part_powerlaw[1] + 1e-50
    )

    array_constants_three_part_powerlaw[0] = array_constants_three_part_powerlaw[1] * (
        (m1 ** p2) * (m1 ** (-p1))
    )
    array_constants_three_part_powerlaw[2] = array_constants_three_part_powerlaw[1] * (
        (m2 ** p2) * (m2 ** (-p3))
    )

    return array_constants_three_part_powerlaw
    # $$array[1]=(($m1**$p2)*($m1**(-$p1)))*
    # (1.0/(1.0+$p1))*
    # ($m1**(1.0+$p1)-$m0**(1.0+$p1))+
    # (($m2**(1.0+$p2)-$m1**(1.0+$p2)))*
    # (1.0/(1.0+$p2))+
    # (($m2**$p2)*($m2**(-$p3)))*
    # (1.0/(1.0+$p3))*
    # ($mmax**(1.0+$p3)-$m2**(1.0+$p3));
    # $$array[1]=1.0/($$array[1]+1e-50);
    # $$array[0]=$$array[1]*$m1**$p2*$m1**(-$p1);
    # $$array[2]=$$array[1]*$m2**$p2*$m2**(-$p3);
    # #print "ARRAY SET @_ => @$array\n";
    # $threepart_powerlaw_consts{"@_"}=[@$array];

def three_part_powerlaw(
    m: Union[int, float],
    """
    Generalized three-part power law, usually used for mass distributions

    Args:
        m: mass at which we want to evaluate the distribution.

    Returns:
        'probability' at given mass m
    """

    # TODO: add check on whether the values exist

    three_part_powerlaw_constants = calculate_constants_three_part_powerlaw(
        m0, m1, m2, m_max, p1, p2, p3
    )

    #
    if m < m0:
        prob = 0  # Below lower bound
    elif m0 < m <= m1:
        prob = three_part_powerlaw_constants[0] * (m ** p1)  # Between M0 and M1
    elif m1 < m <= m2:
        prob = three_part_powerlaw_constants[1] * (m ** p2)  # Between M1 and M2
    elif m2 < m <= m_max:
        prob = three_part_powerlaw_constants[2] * (m ** p3)  # Between M2 and M_MAX
    else:
        prob = 0  # Above M_MAX

def gaussian_normalizing_const(
    mean: Union[int, float],
    sigma: Union[int, float],
    gmin: Union[int, float],
    gmax: Union[int, float],
    """
    Function to calculate the normalisation constant for the gaussian

    Args:
        mean: mean of the gaussian
        sigma: standard deviation of the gaussian
        gmin: lower bound of the range to calculate the probabilities in
        gmax: upper bound of the range to calculate the probabilities in

    Returns:
        normalisation constant for the gaussian distribution(mean, sigma) between gmin and gmax
    """

    # First time; calculate multipllier for given mean and sigma
    ptot = 0
    resolution = 1000
    d = (gmax - gmin) / resolution

    for i in range(resolution):
        y = gmin + i * d
        ptot += d * gaussian_func(y, mean, sigma)

    # TODO: Set value in global
    return ptot

def gaussian_func(
    x: Union[int, float], mean: Union[int, float], sigma: Union[int, float]
    """
    Function to evaluate a gaussian at a given point, but this time without any boundaries.

    Args:
        x: location at which to evaluate the distribution

    Returns:
        value of the gaussian at x
    """
    gaussian_prefactor = 1.0 / math.sqrt(2.0 * math.pi)

    r = 1.0 / (sigma)
    y = (x - mean) * r
    return gaussian_prefactor * r * math.exp(-0.5 * y ** 2)

def gaussian(
    """
    Gaussian distribution function. used for e..g Duquennoy + Mayor 1991

    Args:

    Returns:
        'probability' of the gaussian distribution between the boundaries, evaluated at x
    """

    # # location (X value), mean and sigma, min and max range
    # my ($x,$mean,$sigma,$gmin,$gmax) = @_;

    if (x < gmin) or (x > gmax):
        prob = 0
    else:
        # normalize over given range
        # TODO: add loading into global var
        normalisation = gaussian_normalizing_const(mean, sigma, gmin, gmax)
        prob = normalisation * gaussian_func(x, mean, sigma)

#####
# Mass distributions
#####

def Kroupa2001(m: Union[int, float], newopts: dict = None) -> Union[int, float]:
    """
    Probability distribution function for kroupa 2001 IMF, where the default values to the three_part_powerlaw are: default = {"m0": 0.1, "m1": 0.5, "m2": 1, "mmax": 100, "p1": -1.3, "p2": -2.3,"p3": -2.3}

    Args:
        m: mass to evaluate the distribution at
        newopts: optional dict to override the default values.

    Returns:
        'probability' of distribution function evaluated at m
    """

    # Default params and override them
    default = {
        "m0": 0.1,
        "m1": 0.5,
        "m2": 1,
        "mmax": 100,
        "p1": -1.3,
        "p2": -2.3,
        "p3": -2.3,
    }

    value_dict = default.copy()

        value_dict.update(newopts)

    return three_part_powerlaw(
        m,
        value_dict["m0"],
        value_dict["m1"],
        value_dict["m2"],
        value_dict["mmax"],
        value_dict["p1"],
        value_dict["p2"],
        value_dict["p3"],
    )

def ktg93(m: Union[int, float], newopts: dict = None) -> Union[int, float]:
    """
    Probability distribution function for KTG93 IMF, where the default values to the three_part_powerlaw are: default = {"m0": 0.1, "m1": 0.5, "m2": 1, "mmax": 80, "p1": -1.3, "p2": -2.2,"p3": -2.7}

    Args:

    Returns:
    """
    # TODO: ask rob what this means

    # if($m eq 'uncertainties')
    # {
    # # return (pointer to) the uncertainties hash
    # return {
    #     m0=>{default=>0.1,
    #      fixed=>1},
    #     m1=>{default=>0.5,
    #     m2=>{default=>1.0,
    #     mmax=>{default=>80.0,
    #        fixed=>1},
    #     p1=>{default=>-1.3,
    #      low=>-1.3,
    #      high=>-1.3},
    #     p2=>{default=>-2.2,
    #      low=>-2.2,
    #      high=>-2.2},
    #     p3=>{default=>-2.7,
    #      low=>-2.7,
    #      high=>-2.7}
    # };
    # }

    # set options
    # opts = set_opts({'m0':0.1, 'm1':0.5, 'm2':1.0, 'mmax':80, 'p1':-1.3, 'p2':-2.2, 'p3':-2.7},
    # newopts)

        "m0": 0.1,
        "m1": 0.5,
        "m2": 1.0,
        "mmax": 80,
        "p2": -2.2,
        "p3": -2.7,
    }
    value_dict = defaults.copy()

        m,
    )

# sub ktg93_lnspace
# {
#     # wrapper for KTG93 on a ln(m) grid
#     my $m=$_[0];
#     return ktg93(@_) * $m;
# }

def imf_tinsley1980(m: Union[int, float]) -> Union[int, float]:
    """
    Probability distribution function for tinsley 1980 IMF (defined up until 80Msol): three_part_powerlaw(m, 0.1, 2.0, 10.0, 80.0, -2.0, -2.3, -3.3)

    Args:

    Returns:
    """

    return three_part_powerlaw(m, 0.1, 2.0, 10.0, 80.0, -2.0, -2.3, -3.3)

def imf_scalo1986(m: Union[int, float]) -> Union[int, float]:
    """
    Probability distribution function for Scalo 1986 IMF (defined up until 80Msol): three_part_powerlaw(m, 0.1, 1.0, 2.0, 80.0, -2.35, -2.35, -2.70)

    Args:

    Returns:
    """
    return three_part_powerlaw(m, 0.1, 1.0, 2.0, 80.0, -2.35, -2.35, -2.70)

def imf_scalo1998(m: Union[int, float]) -> Union[int, float]:
    """
    From scalo 1998

    Probability distribution function for Scalo 1998 IMF (defined up until 80Msol): three_part_powerlaw(m, 0.1, 1.0, 10.0, 80.0, -1.2, -2.7, -2.3)

    Args:

    Returns:
    """

    return three_part_powerlaw(m, 0.1, 1.0, 10.0, 80.0, -1.2, -2.7, -2.3)

def imf_chabrier2003(m: Union[int, float]) -> Union[int, float]:
    """
    Probability distribution function for IMF of Chabrier 2003 PASP 115:763-795

    Args:

    Returns:
    """

    chabrier_logmc = math.log10(0.079)
    chabrier_sigma2 = 0.69 * 0.69
    chabrier_a1 = 0.158
    chabrier_a2 = 4.43e-2
    chabrier_x = -1.3
    if m <= 0:
        print("below bounds")
    if 0 < m < 1.0:
        A = 0.158
        dm = math.log10(m) - chabrier_logmc
        prob = chabrier_a1 * math.exp(-(dm ** 2) / (2.0 * chabrier_sigma2))
    else:
        prob = chabrier_a2 * (m ** chabrier_x)
    prob = prob / (0.1202462 * m * math.log(10))

########################################################################
# Binary fractions

def Arenou2010_binary_fraction(m: Union[int, float]) -> Union[int, float]:
    """
    Arenou 2010 function for the binary fraction as f(M1)

    GAIA-C2-SP-OPM-FA-054
    www.rssd.esa.int/doc_fetch.php?id=2969346

    Args:

    Returns:
        binary fraction at m
    """

    return 0.8388 * math.tanh(0.688 * m + 0.079)

# print(Arenou2010_binary_fraction(0.4))

def raghavan2010_binary_fraction(m: Union[int, float]) -> Union[int, float]:
    """
    Fit to the Raghavan 2010 binary fraction as a function of
    spectral type (Fig 12). Valid for local stars (Z=Zsolar).

    The spectral type is converted  mass by use of the ZAMS
    effective temperatures from binary_c/BSE (at Z=0.02)
    and the new "long_spectral_type" function of binary_c
    (based on Jaschek+Jaschek's Teff-spectral type table).

    Rob then fitted the result

    Args:

    Returns:
    """

    return min(
        1.0,
        max(
            (m ** 0.1) * (5.12310e-01) + (-1.02070e-01),
            (1.10450e00) * (m ** (4.93670e-01)) + (-6.95630e-01),
        ),
    )

# print(raghavan2010_binary_fraction(2))

# Period distributions

def duquennoy1991(logper: Union[int, float]) -> Union[int, float]:
    """
    Period distribution from Duquennoy + Mayor 1991. Evaluated the function gaussian(logper, 4.8, 2.3, -2, 12)

    Args:
        logper: logarithm of period to evaluate the distribution at

    Returns:
        'probability' at gaussian(logper, 4.8, 2.3, -2, 12)
    """
    return gaussian(logper, 4.8, 2.3, -2, 12)

def sana12(
    M1: Union[int, float],
    M2: Union[int, float],
    a: Union[int, float],
    P: Union[int, float],
    amin: Union[int, float],
    amax: Union[int, float],
    x0: Union[int, float],
    x1: Union[int, float],
    p: Union[int, float],
    """
    distribution of initial orbital periods as found by Sana et al. (2012)
    which is a flat distribution in ln(a) and ln(P) respectively for stars
    * less massive than 15Msun (no O-stars)
    * mass ratio q=M2/M1<0.1
    * log(P)<0.15=x0 and log(P)>3.5=x1
    and is be given by dp/dlogP ~ (logP)^p for all other binary configurations (default p=-0.55)

    arguments are M1, M2, a, Period P, amin, amax, x0=log P0, x1=log P1, p

    example args: 10, 5, sep(M1, M2, P), sep, ?, -2, 12, -0.55

    # TODO: Fix this function!

    Args:
        M1: Mass of primary
        M2: Mass of secondary
        a: separation of binary
        P: period of binary
        amin: minimum separation of the distribution (lower bound of the range)
        amax: maximum separation of the distribution (upper bound of the range)
        x0: log of minimum period of the distribution (lower bound of the range)
        x1: log of maximum period of the distribution (upper bound of the range)
        p: slope of the distributoon

    Returns:
        'probability' of orbital period P given the other parameters
    """

    res = 0
    if (M1 < 15) or (M2 / M1 < 0.1):
        res = 1.0 / (math.log(amax) - math.log(amin))
    else:
        p1 = 1.0 + p

        # For more details see the LyX document of binary_c for this distribution
        # where the variables and normalizations are given
        # we use the notation x=log(P), xmin=log(Pmin), x0=log(P0), ... to determine the
        x = LOG_LN_CONVERTER * math.log(P)
        xmin = LOG_LN_CONVERTER * math.log(calc_period_from_sep(M1, M2, amin))
        xmax = LOG_LN_CONVERTER * math.log(calc_period_from_sep(M1, M2, amax))

        # print("M1 M2 amin amax P x xmin xmax")
        # print(M1, M2, amin, amax, P, x, xmin, xmax)
        # my $x0 = 0.15;
        # my $x1 = 3.5;

        A1 = 1.0 / (
            x0 ** p * (x0 - xmin) + (x1 ** p1 - x0 ** p1) / p1 + x1 ** p * (xmax - x1)
        )
        A0 = A1 * x0 ** p
        A2 = A1 * x1 ** p

        if x < x0:
            res = 3.0 / 2.0 * LOG_LN_CONVERTER * A0
        elif x > x1:
            res = 3.0 / 2.0 * LOG_LN_CONVERTER * A2
        else:
            res = 3.0 / 2.0 * LOG_LN_CONVERTER * A1 * x ** p

    return res

# print(sana12(10, 2, 10, 100, 1, 1000, math.log(10), math.log(1000), 6))

def interpolate_in_mass_izzard2012(
    M: Union[int, float], high: Union[int, float], low: Union[int, float]
    """
    Function to interpolate in mass

    TODO: fix this function.
    TODO: describe the args
    high: at M=16.3
    low: at 1.15

    Args:
        M: mass
        high:
        low:

    Returns:

    """

    log_interpolation = False

    if log_interpolation:
        return (high - low) / (math.log10(16.3) - math.log10(1.15)) * (
            math.log10(M) - math.log10(1.15)
        ) + low
    else:
        return (high - low) / (16.3 - 1.15) * (M - 1.15) + low

def Izzard2012_period_distribution(
    P: Union[int, float], M1: Union[int, float], log10Pmin: Union[int, float] = 1
    """
    period distribution which interpolates between
    Duquennoy and Mayor 1991 at low mass (G/K spectral type <~1.15Msun)
    and Sana et al 2012 at high mass (O spectral type >~16.3Msun)

    This gives dN/dlogP, i.e. DM/Raghavan's Gaussian in log10P at low mass
    and Sana's power law (as a function of logP) at high mass

    TODO: fix this function

    Args:
        P: period
        M1: Primary star mass
        log10Pmin: minimum period in base log10 (optional)

    Returns:
        'probability' of interpolated distribution function at P and M1

    """

    # Check if there is input and force it to be at least 1
    log10Pmin //= -1.0
    log10Pmin = max(-1.0, log10Pmin)

    # save mass input and limit mass used (M1 from now on) to fitted range
    Mwas = M1
    M1 = max(1.15, min(16.3, M1))
    # print("Izzard2012 called for M={} (trunc'd to {}), P={}\n".format(Mwas, M1, P))

    # Calculate the normalisations
    # need to normalize the distribution for this mass
    # (and perhaps secondary mass)
    prepare_dict(distribution_constants, ["Izzard2012", M1])
    if not distribution_constants["Izzard2012"][M1].get(log10Pmin):
        distribution_constants["Izzard2012"][M1][
            log10Pmin
        ] = 1  # To prevent this loop from going recursive
        N = 200.0  # Resolution for normalisation. I hope 1000 is enough
        dlP = (10.0 - log10Pmin) / N
        C = 0  # normalisation const.
        for lP in np.arange(log10Pmin, 10, dlP):
            C += dlP * Izzard2012_period_distribution(10 ** lP, M1, log10Pmin)

        distribution_constants["Izzard2012"][M1][log10Pmin] = 1.0 / C
    # print(
    #     "Normalization constant for Izzard2012 M={} (log10Pmin={}) is\
    #     {}\n".format(
    #         M1, log10Pmin, distribution_constants["Izzard2012"][M1][log10Pmin]
    #     )
    # )

    lP = math.log10(P)
    # log period

    # # fits
    mu = interpolate_in_mass_izzard2012(M1, -17.8, 5.03)
    sigma = interpolate_in_mass_izzard2012(M1, 9.18, 2.28)
    K = interpolate_in_mass_izzard2012(M1, 6.93e-2, 0.0)
    nu = interpolate_in_mass_izzard2012(M1, 0.3, -1)
    g = 1.0 / (1.0 + 1e-30 ** (lP - nu))

    lPmu = lP - mu
    # print(
    #     "M={} ({}) P={} : mu={} sigma={} K={} nu={} norm=%g\n".format(
    #         Mwas, M1, P, mu, sigma, K, nu
    #     )
    # )

    # print "FUNC $distdata{Izzard2012}{$M}{$log10Pmin} * (exp(- (x-$mu)**2/(2.0*$sigma*$sigma) ) + $K/MAX(0.1,$lP)) * $g;\n";

    if (lP < log10Pmin) or (lP > 10.0):
        return 0

    else:
        return (
            distribution_constants["Izzard2012"][M1][log10Pmin]
            * (math.exp(-lPmu * lPmu / (2.0 * sigma * sigma)) + K / max(0.1, lP))
            * g
        )

# Mass ratio distributions

def flatsections(x: float, opts: dict) -> Union[float, int]:
    """
    Function to generate flat distributions, possibly in multiple sections

    Args:
        x: mass ratio value
        opts: list containing the flat sections. Which are themselves dictionaries, with keys "max": upper bound, "min": lower bound and "height": value

    Returns:
        probability of that mass ratio.
    """

    c = 0
    y = 0

    for opt in opts:
        dc = (opt["max"] - opt["min"]) * opt["height"]
        # print("added flatsection ({}-{})*{} = {}\n".format(
        #   opt['max'], opt['min'], opt['height'], dc))
        c += dc
        if opt["min"] <= x <= opt["max"]:
            y = opt["height"]
            # print("Use this\n")

    c = 1.0 / c
    y = y * c

    # print("flatsections gives C={}: y={}\n",c,y)
    return y

# print(flatsections(1, [{'min': 0, 'max': 2, 'height': 3}]))

# Eccentricity distributions

# Star formation histories

def cosmic_SFH_madau_dickinson2014(z):
    """
    Cosmic star formation history distribution from Madau & Dickonson 2014 (https://arxiv.org/pdf/1403.0007.pdf)
    
    Args:
        z: redshift
    
    Returns:
        Cosmic star formation rate in Solarmass year^-1 megaparsec^-3
    """

    CSFH = 0.015 * ((1+z)**2.7)/(1+(((1+z)/2.9)**5.6))

    return CSFH

# Metallicity distributions

"""
Module containing most of the utility functions for the binarycpython package

Functions here are mostly functions used in other classes/functions, or
useful functions for the user

Tasks:
    - TODO: change all prints to verbose_prints
"""

import json
import os
import tempfile
import copy
import inspect
import ast
from typing import Union, Any
from collections import defaultdict

import h5py

from binarycpython import _binary_c_bindings

########################################################
# utility functions

def verbose_print(message: str, verbosity: int, minimal_verbosity: int) -> None:
    """
    Function that decides whether to print a message based on the current verbosity
    and its minimum verbosity

    if verbosity is equal or higher than the minimum, then we print

    Args:
        message: message to print
        verbosity: current verbosity level
        minimal_verbosity: threshold verbosity above which to print
    """

    if verbosity >= minimal_verbosity:
        print(message)

def remove_file(file: str, verbosity: int = 0) -> None:
    """
    Function to remove files but with verbosity

    Args:
        file: full filepath to the file that will be removed.
        verbosity: current verbosity level (Optional)

    Returns:
        the path of a subdirectory called binary_c_python in the TMP of the filesystem

    """

    if os.path.exists(file):
        if not os.path.isfile(file):
            verbose_print(
                "This path ({}) is a directory, not a file".format(file), verbosity, 0
            )

        try:
            verbose_print("Removed {}".format(file), verbosity, 1)
            os.remove(file)

        except FileNotFoundError as inst:
            print("Error while deleting file {}: {}".format(file, inst))
    else:
        verbose_print(
            "File/directory {} doesn't exist. Can't remove it.".format(file),
            verbosity,
            1,
        )

def temp_dir(*args: str) -> str:
    """
    Function to create directory within the TMP directory of the filesystem

    Makes use of os.makedirs exist_ok which requires python 3.2+

    Args:
        function arguments: str input where each next input will be a child of the previous full_path. e.g. temp_dir('tests', 'grid') will become '/tmp/binary_c_python/tests/grid'

    Returns:
    """

    tmp_dir = tempfile.gettempdir()
    path = os.path.join(tmp_dir, "binary_c_python")

    # loop over the other paths if there are any:
    if args:
        for extra_dir in args:
            path = os.path.join(path, extra_dir)

    #
    os.makedirs(path, exist_ok=True)

    return path

def create_hdf5(data_dir: str, name: str) -> None:
    """
    Function to create an hdf5 file from the contents of a directory:
     - settings file is selected by checking on files ending on settings
     - data files are selected by checking on files ending with .dat

    TODO: fix missing settingsfiles

    Args:
        data_dir: directory containing the data files and settings file
        name: name of hdf5file.

    """

    # Make HDF5:
    # Create the file
    hdf5_filename = os.path.join(data_dir, "{}".format(name))
    print("Creating {}".format(hdf5_filename))
    hdf5_file = h5py.File(hdf5_filename, "w")

    # Get content of data_dir
    content_data_dir = os.listdir(data_dir)

    # Settings
    if any([file.endswith("_settings.json") for file in content_data_dir]):
        print("Adding settings to HDF5 file")
        settings_file = os.path.join(
            data_dir,
            [file for file in content_data_dir if file.endswith("_settings.json")][0],
        )

        with open(settings_file, "r") as settings_file:
            settings_json = json.load(settings_file)

        # Create settings group
        settings_grp = hdf5_file.create_group("settings")

        # Write version_string to settings_group
        settings_grp.create_dataset("used_settings", data=json.dumps(settings_json))

    # Get data files
    data_files = [el for el in content_data_dir if el.endswith(".dat")]
    if data_files:
        print("Adding data to HDF5 file")

        # Create the data group
        data_grp = hdf5_file.create_group("data")

        # Write the data to the file:
        # Make sure:
        for data_file in data_files:
            # filename stuff
            filename = data_file
            full_path = os.path.join(data_dir, filename)
            base_name = os.path.splitext(os.path.basename(filename))[0]

            # Get header info
            header_name = "{base_name}_header".format(base_name=base_name)
            data_headers = np.genfromtxt(full_path, dtype="str", max_rows=1)
            data_headers = np.char.encode(data_headers)
            data_grp.create_dataset(header_name, data=data_headers)

            # Add data
            data = np.loadtxt(full_path, skiprows=1)
            data_grp.create_dataset(base_name, data=data)

        hdf5_file.close()

# version_info functions

def return_binary_c_version_info(parsed: bool = False) -> Union[str, dict]:
    """
    Function that returns the version information of binary_c. This function calls the function _binary_c_bindings.return_version_info()

    Args:
        parsed: Boolean flag whether to parse the version_info output of binary_c. default = False

    Returns:
        Either the raw string of binary_c or a parsed version of this in the form of a nested dictionary
    """

    found_prev = False
    if "BINARY_C_MACRO_HEADER" in os.environ:
        # the envvar is already present. lets save that and put that back later
        found_prev = True
        prev_value = os.environ["BINARY_C_MACRO_HEADER"]

    #
    os.environ["BINARY_C_MACRO_HEADER"] = "macroxyz"

    # Get version_info
    version_info = _binary_c_bindings.return_version_info().strip()

    # parse if wanted
    if parsed:
        version_info = parse_binary_c_version_info(version_info)

    # delete value
    del os.environ["BINARY_C_MACRO_HEADER"]

    # put stuff back if we found a previous one
    if found_prev:
        os.environ["BINARY_C_MACRO_HEADER"] = prev_value

    return version_info

def parse_binary_c_version_info(version_info_string: str) -> dict:
    """
    Function that parses the binary_c version info. Long function with a lot of branches

    TODO: fix this function. stuff is missing: isotopes, macros, nucleosynthesis_sources

    Args:
        version_info_string: raw output of version_info call to binary_c

    Returns:
        Parsed version of the version info, which is a dictionary containing the keys: 'isotopes' for isotope info, 'argpairs' for argument pair info (TODO: explain), 'ensembles' for ensemble settings/info, 'macros' for macros, 'elements' for atomic element info, 'DTlimit' for (TODO: explain), 'nucleosynthesis_sources' for nucleosynthesis sources, and 'miscellaneous' for all those that were not caught by the previous groups. 'git_branch', 'git_build', 'revision' and 'email' are also keys, but its clear what those contain.
    """

    version_info_dict = {}

    # Clean data and put in correct shape
    splitted = version_info_string.strip().splitlines()
    cleaned = set([el.strip() for el in splitted if not el == ""])

    ##########################
    # Isotopes:
    # Split off
    isotopes = set([el for el in cleaned if el.startswith("Isotope ")])
    cleaned = cleaned - isotopes

    isotope_dict = {}
    for el in isotopes:
        split_info = el.split("Isotope ")[-1].strip().split(" is ")

        isotope_info = split_info[-1]
        name = isotope_info.split(" ")[0].strip()

        # Get details
        mass_g = float(
            isotope_info.split(",")[0].split("(")[1].split("=")[-1][:-2].strip()
        )
        mass_amu = float(
            isotope_info.split(",")[0].split("(")[-1].split("=")[-1].strip()
        )
        mass_mev = float(
            isotope_info.split(",")[-3].split("=")[-1].replace(")", "").strip()
        )
        A = int(isotope_info.split(",")[-1].strip().split("=")[-1].replace(")", ""))
        Z = int(isotope_info.split(",")[-2].strip().split("=")[-1])

        #
        isotope_dict[int(split_info[0])] = {
            "name": name,
            "Z": Z,
            "A": A,
            "mass_mev": mass_mev,
            "mass_g": mass_g,
            "mass_amu": mass_amu,
        }
    version_info_dict["isotopes"] = isotope_dict if isotope_dict else None

    # Argpairs:
    # Split off
    argpairs = set([el for el in cleaned if el.startswith("ArgPair")])
    cleaned = cleaned - argpairs

    argpair_dict = {}
    for el in sorted(argpairs):
        split_info = el.split("ArgPair ")[-1].split(" ")

        if not argpair_dict.get(split_info[0], None):
            argpair_dict[split_info[0]] = {split_info[1]: split_info[2]}
        else:
            argpair_dict[split_info[0]][split_info[1]] = split_info[2]

    version_info_dict["argpairs"] = argpair_dict if argpair_dict else None

    # ensembles:
    # Split off
    ensembles = set([el for el in cleaned if el.startswith("Ensemble")])
    cleaned = cleaned - ensembles

    ensemble_dict = {}
    for el in ensembles:
        split_info = el.split("Ensemble ")[-1].split(" is ")
        if len(split_info) > 1:
            ensemble_dict[int(split_info[0])] = split_info[-1]
    version_info_dict["ensembles"] = ensemble_dict if ensemble_dict else None

    # macros:
    # Split off
    macros = set([el for el in cleaned if el.startswith("macroxyz")])
    cleaned = cleaned - macros

    param_type_dict = {
        "STRING": str,
        "FLOAT": float,
        "MACRO": str,
        "INT": int,
        "LONG_INT": int,
    }

    macros_dict = {}
    for el in macros:
        split_info = el.split("macroxyz ")[-1].split(" : ")
        param_type = split_info[0]

        new_split = "".join(split_info[1:]).split(" is ")
        param_name = new_split[0]
        param_value = " is ".join(new_split[1:])
        # Sometimes the macros have extra information behind it. Needs an update in outputting by binary_c
        try:
            macros_dict[param_name] = param_type_dict[param_type](param_value)
        except ValueError:
            macros_dict[param_name] = str(param_value)
    version_info_dict["macros"] = macros_dict if macros_dict else None

    # Elements:
    # Split off:
    elements = set([el for el in cleaned if el.startswith("Element")])
    cleaned = cleaned - elements

    # Fill dict:
    elements_dict = {}
    for el in elements:
        split_info = el.split("Element ")[-1].split(" : ")
        name_info = split_info[0].split(" is ")

        # get isotope info
        isotopes = {}
        if not split_info[-1][0] == "0":
            isotope_string = split_info[-1].split(" = ")[-1]
            isotopes = {
                int(split_isotope.split("=")[0]): split_isotope.split("=")[1]
                for split_isotope in isotope_string.split(" ")
            }

        elements_dict[int(name_info[0])] = {
            "name": name_info[-1],
            "atomic_number": int(name_info[0]),
            "amt_isotopes": len(isotopes),
            "isotopes": isotopes,
        }
    version_info_dict["elements"] = elements_dict if elements_dict else None

    # dt_limits:
    # split off
    dt_limits = set([el for el in cleaned if el.startswith("DTlimit")])
    cleaned = cleaned - dt_limits

    # Fill dict
    dt_limits_dict = {}
    for el in dt_limits:
        split_info = el.split("DTlimit ")[-1].split(" : ")
        dt_limits_dict[split_info[1].strip()] = {
            "index": int(split_info[0]),
            "value": float(split_info[-1]),
        }

    version_info_dict["dt_limits"] = dt_limits_dict if dt_limits_dict else None

    # Nucleosynthesis sources:
    # Split off
    nucsyn_sources = set([el for el in cleaned if el.startswith("Nucleosynthesis")])
    cleaned = cleaned - nucsyn_sources

    # Fill dict
    nucsyn_sources_dict = {}
    for el in nucsyn_sources:
        split_info = el.split("Nucleosynthesis source")[-1].strip().split(" is ")
        nucsyn_sources_dict[int(split_info[0])] = split_info[-1]

    version_info_dict["nucleosynthesis_sources"] = (
        nucsyn_sources_dict if nucsyn_sources_dict else None
    )

    # miscellaneous:
    # All those that I didnt catch with the above filters. Could try to get some more out though.
    # TODO: filter a bit more.

    misc_dict = {}

    # Filter out git revision
    git_revision = [el for el in cleaned if el.startswith("git revision")]
    misc_dict["git_revision"] = (
        git_revision[0].split("git revision ")[-1].replace('"', "")
    )
    cleaned = cleaned - set(git_revision)

    # filter out git url
    git_url = [el for el in cleaned if el.startswith("git URL")]
    misc_dict["git_url"] = git_url[0].split("git URL ")[-1].replace('"', "")
    cleaned = cleaned - set(git_url)

    # filter out version
    version = [el for el in cleaned if el.startswith("Version")]
    misc_dict["version"] = str(version[0].split("Version ")[-1])
    cleaned = cleaned - set(version)

    git_branch = [el for el in cleaned if el.startswith("git branch")]
    misc_dict["git_branch"] = git_branch[0].split("git branch ")[-1].replace('"', "")
    cleaned = cleaned - set(git_branch)

    build = [el for el in cleaned if el.startswith("Build")]
    misc_dict["build"] = build[0].split("Build: ")[-1].replace('"', "")
    cleaned = cleaned - set(build)

    email = [el for el in cleaned if el.startswith("Email")]
    misc_dict["email"] = email[0].split("Email ")[-1].split(",")
    cleaned = cleaned - set(email)

    other_items = set([el for el in cleaned if " is " in el])
    cleaned = cleaned - other_items

    for el in other_items:
        split = el.split(" is ")
        key = split[0].strip()
        val = " is ".join(split[1:]).strip()
        misc_dict[key] = val

    misc_dict["uncaught"] = list(cleaned)

    version_info_dict["miscellaneous"] = misc_dict if misc_dict else None
    return version_info_dict

# binary_c output functions

def output_lines(output: str) -> list:
    """
    Function that outputs the lines that were recieved from the binary_c run, but now as an iterator.

    Args:
        output: raw binary_c output

    Returns:
        Iterator over the lines of the binary_c output
    """

    if output:
        return output.splitlines()
    else:
        return []

def example_parse_output(output: str, selected_header: str) -> dict:
    """
    Function that parses output of binary_c. This version serves as an example and is quite detailed. Custom functions can be easier:

    This function works in two cases:
    if the caught line contains output like 'example_header time=12.32 mass=0.94 ..'
    or if the line contains output like 'example_header 12.32 0.94'
    Please dont the two cases.

    You can give a 'selected_header' to catch any line that starts with that.
    Then the values will be put into a dictionary.

    Tasks:
        - TODO: Think about exporting to numpy array or pandas instead of a defaultdict
        - TODO: rethink whether this function is necessary at all
        - TODO: check this function again

    Args:
        output: binary_c output string
        selected_header: string header of the output (the start of the line that you want to process)

    Returns:
        dictionary containing parameters as keys and lists for the values
    """

    value_dicts = []

    # split output on newlines
    for line in output.split("\n"):
        # Skip any blank lines
        if not line == "":
            split_line = line.split()

            # Select parts
            header = split_line[0]
            values_list = split_line[1:]

            # print(values_list)
            # Catch line starting with selected header
            if header == selected_header:
                # Check if the line contains '=' symbols:
                value_dict = {}
                if all("=" in value for value in values_list):
                    for value in values_list:
                        key, val = value.split("=")
                        value_dict[key.strip()] = val.strip()
                    value_dicts.append(value_dict)
                else:
                    if any("=" in value for value in values_list):
                        raise ValueError(
                            "Caught line contains some = symbols but not \
                            all of them do. aborting run"
                        )

                    for j, val in enumerate(values_list):
                        value_dict[j] = val

    if len(value_dicts) == 0:
        print(
            "Sorry, didnt find any line matching your header {}".format(selected_header)
        )

    keys = value_dicts[0].keys()

    # Construct final dict.
    final_values_dict = defaultdict(list)
    for value_dict in value_dicts:
        for key in keys:
            final_values_dict[key].append(value_dict[key])

    return final_values_dict

# Argument and default value functions

def get_defaults(filter_values: bool = False) -> dict:
    """
    Function that calls the binaryc get args function and cast it into a dictionary.

    All the values are strings

    Args:
        filter_values: whether to filter out NULL and Function defaults.

    Returns:
        dictionary containing the parameter name as key and the parameter default as value
    """

    default_output = _binary_c_bindings.return_arglines()
    default_dict = {}

    for default in default_output.split("\n"):
        if not default in ["__ARG_BEGIN", "__ARG_END", ""]:
            key, value = default.split(" = ")
            default_dict[key] = value

    if filter_values:
        default_dict = filter_arg_dict(default_dict)

    return default_dict

def get_arg_keys() -> list:
    """
    Function that return the list of possible keys to give in the arg string. This function calls get_defaults()

    Returns:
        list of all the parameters that binary_c accepts (and has default values for, since we call get_defaults())
    """

    return list(get_defaults().keys())

def filter_arg_dict(arg_dict: dict) -> dict:
    """
    Function to filter out keys that contain values included in ['NULL', 'Function', '']

    This function is called by get_defaults()

    Args:
        arg_dict: dictionary containing the argument + default keypairs of binary_c

    Returns:
        filtered dictionary (pairs with NULL and Function values are removed)
    """

    old_dict = arg_dict.copy()
    new_dict = {}

    for key in old_dict.keys():
        if not old_dict[key] in ["NULL", "Function"]:
            if not old_dict[key] == "":
                new_dict[key] = old_dict[key]

    return new_dict

def create_arg_string(
    arg_dict: dict, sort: bool = False, filter_values: bool = False
) -> str:
    """
    Function that creates the arg string for binary_c. Takes a dictionary containing the arguments and writes them to a string
    This string is missing the 'binary_c ' at the start.

    Args:
        arg_dict: dictionary
        sort: (optional, default = False) Boolean whether to sort the order of the keys.
        filter_values: (optional, default = False) filters the input dict on keys that have NULL or `function` as value.

    Returns:
        The string built up by combining all the key + value's.
    """

    arg_string = ""

    # Whether to filter the arguments
        arg_dict = filter_arg_dict(arg_dict)

    #
    keys = sorted(arg_dict.keys()) if sort else arg_dict.keys()

    #
    for key in keys:
        arg_string += "{key} {value} ".format(key=key, value=arg_dict[key])

    arg_string = arg_string.strip()
    return arg_string

# Help functions

def get_help(
    param_name: str = "", print_help: bool = True, fail_silently: bool = False
) -> Union[dict, None]:
    """
    Function that returns the help info for a given parameter, by interfacing with binary_c

    Will check whether it is a valid parameter.

    Binary_c will output things in the following order;
    - Did you mean?
    - binary_c help for variable
    - default
    - available macros

    This function reads out that structure and catches the different components of this output

    Tasks:
        - TODO: consider not returning None, but return empty dict

    Args:
        param_name: name of the parameter that you want info from. Will get checked whether its a valid parameter name
        print_help: (optional, default = True) whether to print out the help information
        fail_silently: (optional, default = False) Whether to print the errors raised if the parameter isn't valid

    Returns:
        Dictionary containing the help info. This dictionary contains 'parameter_name', 'parameter_value_input_type', 'description', optionally 'macros'
    """

    available_arg_keys = get_arg_keys()

    if not param_name:
        print(
            "Please set the param_name to any of the following:\n {}".format(
                sorted(available_arg_keys)
            )
        )

    if param_name in available_arg_keys:
        help_info = _binary_c_bindings.return_help(param_name)
        cleaned = [el for el in help_info.split("\n") if not el == ""]

        # Get line numbers
        did_you_mean_nr = [
            i for i, el in enumerate(cleaned) if el.startswith("Did you mean")
        ]
        parameter_line_nr = [
            i for i, el in enumerate(cleaned) if el.startswith("binary_c help")
        ]
        default_line_nr = [
            i for i, el in enumerate(cleaned) if el.startswith("Default")
        ]
        macros_line_nr = [
            i for i, el in enumerate(cleaned) if el.startswith("Available")
        ]

        help_info_dict = {}

        # Get alternatives
        if did_you_mean_nr:
            alternatives = cleaned[did_you_mean_nr[0] + 1 : parameter_line_nr[0]]
            alternatives = [el.strip() for el in alternatives]
            help_info_dict["alternatives"] = alternatives

        # Information about the parameter
        parameter_line = cleaned[parameter_line_nr[0]]
        parameter_name = parameter_line.split(":")[1].strip().split(" ")[0]
        parameter_value_input_type = (
            " ".join(parameter_line.split(":")[1].strip().split(" ")[1:])
            .replace("<", "")
            .replace(">", "")
        )

        help_info_dict["parameter_name"] = parameter_name
        help_info_dict["parameter_value_input_type"] = parameter_value_input_type

        description_line = " ".join(
            cleaned[parameter_line_nr[0] + 1 : default_line_nr[0]]
        )
        help_info_dict["description"] = description_line

        # Default:
        default_line = cleaned[default_line_nr[0]]
        default_value = default_line.split(":")[-1].strip()

        help_info_dict["default"] = default_value

        # Get Macros:
        if macros_line_nr:
            macros = cleaned[macros_line_nr[0] + 1 :]
            help_info_dict["macros"] = macros

        if print_help:
            for key in help_info_dict:
                print("{}:\n\t{}".format(key, help_info_dict[key]))

        return help_info_dict

    else:
        if not fail_silently:
            print(
                "{} is not a valid parameter name. Please choose from the \
                following parameters:\n\t{}".format(
                    param_name, list(available_arg_keys)
                )
            )

def get_help_all(print_help: bool = True) -> dict:
    """
    Function that reads out the output of the return_help_all api call to binary_c. This return_help_all binary_c returns all the information for the parameters, their descriptions and other properties. The output is categorized in sections.

    Args:
        print_help: (optional, default = Tru) prints all the parameters and their descriptions.

    Returns:
        returns a dictionary containing dictionaries per section. These dictionaries contain the parameters and descriptions etc for all the parameters in that section
    """

    # Call function
    help_all = _binary_c_bindings.return_help_all()

    # String manipulation
    split = help_all.split(
        "############################################################\n"
    )
    cleaned = [el for el in split if not el == "\n"]

    section_nums = [i for i in range(len(cleaned)) if cleaned[i].startswith("#####")]

    # Create dicts
    help_all_dict = {}

    # Select the section name and the contents of that section. Note, not all sections have content!
    for i in range(len(section_nums)):
        if not i == len(section_nums) - 1:
            params = cleaned[section_nums[i] + 1 : section_nums[i + 1]]
        else:
            params = cleaned[section_nums[i] + 1 : len(cleaned)]
        section_name = (
            cleaned[section_nums[i]]
            .lstrip("#####")
            .strip()
            .replace("Section ", "")
            .lower()
        )

        #
        params_dict = {}

        if params:

            # Clean it, replace in-text newlines with a space and then split on newlines.
            split_params = params[0].strip().replace("\n ", " ").split("\n")

            # Process params and descriptions per section
            for split_param in split_params:
                split_param_info = split_param.split(" : ")
                if not len(split_param_info) == 3:
                    # there are ocassions where the semicolon
                    # is used in the description text itself.
                    if len(split_param_info) == 4:
                        split_param_info = [
                            split_param_info[0],
                            ": ".join([split_param_info[1], split_param_info[2]]),
                            split_param_info[3],
                        ]

                    # other occassions?

                # Put the information in a dict
                param_name = split_param_info[0]
                param_description = split_param_info[1]

                if len(split_param_info) > 2:
                    rest = split_param_info[2:]
                else:
                    rest = None

                params_dict[param_name] = {
                    "param_name": param_name,
                    "description": param_description,
                    "rest": "".join(rest) if rest else "",
                }

            # make section_dict
            section_dict = {
                "section_name": section_name,
                "parameters": params_dict.copy(),
            }

            # Put in the total dict
            help_all_dict[section_name] = section_dict.copy()

    # Print things
    if print_help:
        for section in sorted(help_all_dict.keys()):
            print(
                "##################\n###### Section {}\n##################".format(
                    section
                )
            )
            section_dict = help_all_dict[section]
            for param_name in sorted(section_dict["parameters"].keys()):
                param = section_dict["parameters"][param_name]
                print(
                    "\n{}:\n\t{}: {}".format(
                        param["param_name"], param["description"], param["rest"]
                    )
                )

    # # Loop over all the parameters an call the help() function on it.
    # # Takes a long time but this is for testing
    # for section in help_all_dict.keys():
    #     section_dict = help_all_dict[section]
    #     for param in section_dict['parameters'].keys():
    #         get_help(param)

    return help_all_dict

def get_help_super(print_help: bool = False, fail_silently: bool = True) -> dict:
    """
    Function that first runs get_help_all, and then per argument also run
    the help function to get as much information as possible.

    Args:
        print_help: (optional, default = False) Whether to print the information
        fail_silently: (optional, default = True) Whether to fail silently or to print the errors

    Returns:
        dictionary containing all dictionaries per section, which then contain as much info as possible per parameter.
    """

    # Get help_all information
    help_all_dict = get_help_all(print_help=False)
    for section_name in help_all_dict:
        section = help_all_dict[section_name]

        # print(section_name)
        # for parameter_name in section["parameters"].keys():
        #     print("\t", parameter_name)

    help_all_super_dict = help_all_dict.copy()

    # Loop over all sections and stuff
        # Skipping the section i/o because that one shouldn't be available to python anyway
        if not section_name == "i/o":
            section = help_all_dict[section_name]

            for parameter_name in section["parameters"].keys():
                parameter = section["parameters"][parameter_name]

                # Get detailed help info
                detailed_help = get_help(
                    parameter_name,
                    print_help=False,
                    fail_silently=fail_silently,
                )

                if detailed_help:
                    # check whether the descriptions of help_all and detailed help are the same
                    if not fail_silently:
                        if not parameter["description"] == detailed_help["description"]:
                            print(json.dumps(parameter, indent=4))

                    ## put values into help all super dict
                    # input type
                    parameter["parameter_value_input_type"] = detailed_help[
                        "parameter_value_input_type"
                    ]

                    # default
                    parameter["default"] = detailed_help["default"]

                    # macros
                    if "macros" in detailed_help.keys():
                        parameter["macros"] = detailed_help["macros"]

                section["parameters"][parameter_name] = parameter

        print(json.dumps(help_all_super_dict, indent=4))

    return help_all_super_dict

def make_build_text() -> str:
    """
    Function to make build text

    Returns:
        string containing information about the build and the git branch
    """

    version_info = return_binary_c_version_info(parsed=True)

    git_revision = version_info["miscellaneous"]["git_revision"]
    git_branch = version_info["miscellaneous"]["git_branch"]
    build_datetime = version_info["miscellaneous"]["build"]

    info_string = "This information was obtained by the following binary_c build: \n\t**binary_c git branch**: {}\t**binary_c git revision**: {}\t**Built on**: {}".format(
        git_branch, git_revision, build_datetime
    )

    return info_string

def write_binary_c_parameter_descriptions_to_rst_file(output_file: str) -> None:
    """
    Function that calls the get_help_super() to get the help text/descriptions for all the parameters available in that build.
    Writes the results to a .rst file that can be included in the docs.

    Tasks:
        - TODO: add the specific version git branch, git build, git commit, and binary_c version to this document

    Args:
        output_file: name of the output .rst faile containing the ReStructuredText formatted output of all the binary_c parameters.
    """

    # Get the whole arguments dictionary
    arguments_dict = get_help_super()

    build_info = make_build_text()

    if not output_file.endswith(".rst"):
        print("Filename doesn't end with .rst, please provide a proper filename")

    with open(output_file, "w") as f:

        print("Binary\\_c parameters", file=f)
        print("{}".format("=" * len("Binary\\_c parameters")), file=f)
        print(
            "The following chapter contains all the parameters that the current version of binary\\_c can handle, along with their descriptions and other properties.",
            file=f,
        )
        print("\n", file=f)
        print(build_info, file=f)

        for el in arguments_dict.keys():
            print("Section: {}".format(el), file=f)
            print("{}\n".format("-" * len("Section: {}".format(el))), file=f)
            # print(arguments_dict[el]['parameters'].keys())

            for arg in arguments_dict[el]["parameters"].keys():
                argdict = arguments_dict[el]["parameters"][arg]

                print("| **Parameter**: {}".format(argdict["param_name"]), file=f)
                print("| **Description**: {}".format(argdict["description"]), file=f)
                if "parameter_value_input_type" in argdict:
                    print(
                        "| **Parameter input type**: {}".format(
                            argdict["parameter_value_input_type"]
                        ),
                        file=f,
                    )
                if "default" in argdict:
                    print("| **Default value**: {}".format(argdict["default"]), file=f)
                if "macros" in argdict:
                    print("| **Macros**: {}".format(argdict["macros"]), file=f)
                if not argdict["rest"] == "(null)":
                    print("| **Extra**: {}".format(argdict["rest"]), file=f)
                print("", file=f)

# logfile functions

def load_logfile(logfile: str) -> None:
    """
    Experimental function that parses the generated logfile of binary_c.

    This function is not finished and shouldn't be used yet.

    Tasks:
        - TODO:

    Args:
        - logfile: filename of the logfile you want to parse

    Returns:

    """

    with open(logfile, "r") as file:
        logfile_data = file.readlines()

    time_list = []
    m1_list = []
    m2_list = []
    k1_list = []
    k2_list = []
    sep_list = []
    ecc_list = []
    rel_r1_list = []
    rel_r2_list = []
    event_list = []

    # random_seed = logfile_data[0].split()[-2]
    # random_count = logfile_data[0].split()[-1]
    # probability = logfile_data[-1].split()

    for line in logfile_data[1:-1]:
        split_line = line.split()

        time_list.append(split_line[0])
        m1_list.append(split_line[1])
        m2_list.append(split_line[2])
        k1_list.append(split_line[3])
        k2_list.append(split_line[4])
        sep_list.append(split_line[5])
        ecc_list.append(split_line[6])
        rel_r1_list.append(split_line[7])
        rel_r2_list.append(split_line[8])
        event_list.append(" ".join(split_line[9:]))

    print(event_list)

# Ensemble dict functions

def inspect_dict(
    input_dict: dict, indent: int = 0, print_structure: bool = True
) -> dict:
    """
    Function to (recursively) inspect a (nested) dictionary.
    The object that is returned is a dictionary containing the key of the input_dict, but as value it will return the type of what the value would be in the input_dict

    In this way we inspect the structure of these dictionaries, rather than the exact contents.

    Args:
        input_dict: dictionary you want to inspect
        print_structure: (optional, default = True)
        indent: (optional, default = 0) indent of the first output

    Returns:
        Dictionary that has the same structure as the input_dict, but as values it has the type(input_dict[key]) (except if the value is a dict)
    """

    structure_dict = {}

    #
    for key, value in input_dict.items():
        structure_dict[key] = type(value)

        if print_structure:
            print("\t" * indent, key, type(value))

        if isinstance(value, dict):
            structure_dict[key] = inspect_dict(
                value, indent=indent + 1, print_structure=print_structure
            )

    return structure_dict

def merge_dicts(dict_1: dict, dict_2: dict) -> dict:
    """
    Function to merge two dictionaries in a custom way.

    Behaviour:

    When dict keys are only present in one of either:
        - we just add the content to the new dict

    When dict keys are present in both, we decide based on the value types how to combine them:
        - dictionaries will be merged by calling recursively calling this function again
        - numbers will be added
        - (opt) lists will be appended
        - In the case that the instances do not match: for now I will raise an error

    Args:
        dict_1: first dictionary
        dict_2: second dictionary

    Returns:
        Merged dictionary

    """

    # Set up new dict

    #
    keys_1 = dict_1.keys()
    keys_2 = dict_2.keys()

    # Find overlapping keys of both dicts
    overlapping_keys = set(keys_1).intersection(set(keys_2))

    # Find the keys that are unique
    unique_to_dict_1 = set(keys_1).difference(set(keys_2))
    unique_to_dict_2 = set(keys_2).difference(set(keys_1))

    # Add the unique keys to the new dict
    for key in unique_to_dict_1:
        # If these items are ints or floats, then just put them in
        if isinstance(dict_1[key], (float, int)):
            new_dict[key] = dict_1[key]
        # Else, to be safe we should deepcopy them
        else:
            copy_dict = copy.deepcopy(dict_1[key])
            new_dict[key] = copy_dict

    for key in unique_to_dict_2:
        if isinstance(dict_2[key], (float, int)):
            new_dict[key] = dict_2[key]
        else:
            copy_dict = copy.deepcopy(dict_2[key])

    # Go over the common keys:
    for key in overlapping_keys:

        # See whether the types are actually the same
        if not type(dict_1[key]) is type(dict_2[key]):
            # Exceptions:
            if (type(dict_1[key]) in [int, float]) and (
                type(dict_2[key]) in [int, float]
            ):
                new_dict[key] = dict_1[key] + dict_2[key]

            else:
                print(
                    "Error key: {} value: {} type: {} and key: {} value: {} type: {} are not of the same type and cannot be merged".format(
                        key,
                        dict_1[key],
                        type(dict_1[key]),
                        key,
                        dict_2[key],
                        type(dict_2[key]),
                    )
                )
                raise ValueError

        # Here we check for the cases that we want to explicitly catch. Ints will be added,
        # floats will be added, lists will be appended (though that might change) and dicts will be
        # dealt with by calling this function again.
        else:
            # ints
            # Booleans (has to be the type Bool, not just a 0 or 1)
            if isinstance(dict_1[key], bool) and isinstance(dict_2[key], bool):
                new_dict[key] = dict_1[key] or dict_2[key]

            elif isinstance(dict_1[key], int) and isinstance(dict_2[key], int):

            # floats
            elif isinstance(dict_1[key], float) and isinstance(dict_2[key], float):

            # lists
            elif isinstance(dict_1[key], list) and isinstance(dict_2[key], list):

            # dicts
            elif isinstance(dict_1[key], dict) and isinstance(dict_2[key], dict):
                new_dict[key] = merge_dicts(dict_1[key], dict_2[key])

            else:
                print(
                    "Object types {},{} not supported".format(
                        type(dict_1[key]), type(dict_2[key])
                    )
                )

    #

def extract_ensemble_json_from_string(binary_c_output: str) -> dict:
    """
    Function to extract the ensemble_json information from a raw binary_c output string

    Args:
        binary_c_output: raw binary_c output string

    Returns:
        json dictionary with the parsed ENSEMBLE_JSON data
    """

    json = None

    try:
        ensemble_jsons_strings = [
            line
            for line in binary_c_output.splitlines()
            if line.startswith("ENSEMBLE_JSON")
        ]

        json = handle_ensemble_string_to_json(
            ensemble_jsons_strings[0][len("ENSEMBLE_JSON ") :]
        )

        if len(ensemble_jsons_strings) > 1:
                "Warning: There is more than one line starting with ENSEMBLE_JSON. Taking the first, but you should check this out.",
                1,
                0,
            )
    except IndexError:
            "Error: Couldn't extract the ensemble information from the output string",
            0,
            0,
        )

    return json

class binarycDecoder(json.JSONDecoder):
    """
    Custom decoder to transform the numbers that are strings to actual floats
    """

    def decode(self, s):
        """
        Entry point function for decoding
        """

        result = super().decode(
            s
        )  # result = super(Decoder, self).decode(s) for Python 2.x
        return self._decode(result)

    def _decode(self, o):
        """
        Depending on the type of object is will determine whether to loop over the elements,
        or try to change the type of the object from string to float

        The try except might be a somewhat rough solution but it catches all cases.
        """

        # Check if we can turn it into a float
        # if isinstance(o, str) or isinstance(o, unicode):
        if isinstance(o, str):
            try:
                return float(o)
            except ValueError:
                return o
        elif isinstance(o, dict):
            return {k: self._decode(v) for k, v in o.items()}
        elif isinstance(o, list):
            return [self._decode(v) for v in o]
        else:
            return o

class BinaryCEncoder(json.JSONEncoder):
    """
    Encoding class function to attempt to convert things to strings.
    """

    def default(self, o):
        """
        Converting function. Well, could be more precise. look at the json module
        """
        try:
            str_repr = str(o)
        except TypeError:
            pass
        else:
            return str_repr
        # Let the base class default method raise the TypeError
        return JSONEncoder.default(self, o)

def binaryc_json_serializer(obj: Any) -> Any:
    """
    Custom serializer for binary_c to use when functions are present in the dictionary
    that we want to export.

    Function objects will be turned into str representations of themselves

    Args:
        obj: obj being process

    Returns:
        Either string representation of object if the object is a function, or the object itself
    """

    if inspect.isfunction(obj):
        return str(obj)
    else:
        return obj

def handle_ensemble_string_to_json(raw_output):
    """
    Function that deals with the raw output of the ensemble and
    creates a working JSON dictionary out of it.

    Having this wrapper makes it easy to

    Args:
        raw_output: raw output of the ensemble dump by binary_c

    Returns:
        json.loads(raw_output, cls=binarycDecoder)

    """

    # return json.loads(json.dumps(ast.literal_eval(raw_output)), cls=binarycDecoder)
    return json.loads(raw_output, cls=binarycDecoder)

"""
Collection of useful functions.

Part of this is copied/inspired by Rob's binary_stars module

Functions:
    - calc_period_from_sep(m1, m2, sep) calculate the period given the separation.
    - calc_sep_from_period(m1, m2, per) does the inverse.
    - rzams(m, z) gives you the ZAMS radius of a star
    - ZAMS_collision(m1, m2, e, sep, z) returns 1 if stars collide on the ZAMS
    - roche_lobe(q): returns roche lobe radius in units of separation
    - ragb(m, z): radius at first thermal pulse

Tasks:
    - TODO: check whether these functions are correct
"""

import re
from typing import Union

from binarycpython.utils.functions import create_arg_string

AURSUN = 2.150445198804013386961742071435e02
YEARDY = 3.651995478818308811241877265275e02

def minimum_period_for_RLOF(M1, M2, metallicity, store_memaddr=-1):
    """
    Wrapper function for _binary_c_bindings.return_minimum_orbit_for_RLOF

    Handles the output and returns the minimum orbital period at which RLOF just does not occur at zams

    Args:
        M1: Primary mass in solar mass
        M2: Secondary mass in solar mass
        metallicity: metallicity
        store_memaddr (optional): store memory adress
    Returns:
        minimum orbital_period that just does not cause a RLOF at ZAMS

    """

    bse_dict = {"M_1": M1, "M_2": M2, "metallicity": metallicity, "minimum_orbital_period_for_instant_RLOF": 1, "minimum_separation_for_instant_RLOF": 1}

    argstring = "binary_c " + create_arg_string(bse_dict)

    output = _binary_c_bindings.return_minimum_orbit_for_RLOF(argstring, store_memaddr)

    minimum_period = float(re.search('MINIMUM PERIOD (.*)', output).group(1))

    return minimum_period

# print(minimum_period_for_RLOF(10, 5, 0.02))
# print(minimum_period_for_RLOF(10, 2, 0.02))

def minimum_separation_for_RLOF(M1, M2, metallicity, store_memaddr=-1):
    """

    Handles the output and returns the minimum separation at which RLOF just does not occur at ZAMS

    Args:
    Returns:
        minimum separation that just does not cause a RLOF at ZAMS

    """

    minimum_separation = float(re.search('MINIMUM SEPARATION (.*)', output).group(1))

    return minimum_separation

# print(minimum_separation_for_RLOF(0.08, 0.08, 0.00002))
# print(minimum_separation_for_RLOF(10, 2, 0.02))

def calc_period_from_sep(
    M1: Union[int, float], M2: Union[int, float], sep: Union[int, float]
    """
    calculate period from separation

    Args:
        sep: Separation in solar radii

    Returns:
        period in days
    """

    return YEARDY * (sep / AURSUN) * math.sqrt(sep / (AURSUN * (M1 + M2)))

def calc_sep_from_period(
    M1: Union[int, float], M2: Union[int, float], period: Union[int, float]
    """
    Calculate separation from period.

    Args:
        period: Period of binary in days

    Returns:
        Separation in solar radii
    """

    return AURSUN * (period * period * (M1 + M2) / (YEARDY * YEARDY)) ** (1.0 / 3.0)

def roche_lobe(q: Union[int, float]) -> Union[int, float]:
    """
    A function to evaluate R_L/a(q), Eggleton 1983.

    # TODO: check the definition of the mass ratio
    # TODO: check whether the logs are correct

    Args:
        q: mass ratio of the binary (secondary/primary). If you input: q = mass_accretor/mass_donor, you will get the rochelobe radius of the accretor. And vice versa for the donor.

    Returns:
        Roche lobe radius in units of the separation
    """

    p = q ** (1.0 / 3.0)
    return 0.49 * p * p / (0.6 * p * p + math.log(1.0 + p))

def ragb(m: Union[int, float], z: Union[int, float]) -> Union[int, float]:
    """
    Function to calculate radius of a star in units of solar radii at first thermal pulse as a function of mass (Z=0.02 only, but also good for Z=0.0001)

    TODO: ask rob about this function

    Args:
        m: mass of star in units of solar mass
        z: metallicity of star

    Returns:
        radius at first thermal pulse in units of solar radii
    """

    return m * 40.0 + 20.0

def zams_collision(
    sep: Union[int, float],
    e: Union[int, float],
    z: Union[int, float],
    """
    given m1,m2, separation and eccentricity (and metallicity)
    determine if two stars collide on the ZAMS

    Args:
        m1: Primary mass in solar mass
        m2: Secondary mass in solar mass
        sep: separation in solar radii
        e: eccentricity
        z: metallicity

    Returns:
        integer boolean whether the binary stars will collide at pericenter
    """

    # calculate periastron distance
    peri_distance = (1.0 - e) * sep

    # calculate star radii
    r1 = rzams(m1, z)
    r2 = rzams(m2, z)

    if r1 + r2 > peri_distance:
        return 1
    return 0

def rzams(m, z):
    """
    Function to determine the radius of a ZAMS star as a function of m and z:

    Based on the fits of Tout et al., 1996, MNRAS, 281, 257

    Args:
        m: mass of star in solar mass

    Returns:
        radius of star at ZAMS, in solar radii
    """

    lzs = math.log10(z / 0.02)

    #
    # A function to evaluate Rzams
    # ( from Tout et al., 1996, MNRAS, 281, 257 ).
    #

    p = {}
    xz = (
        666,
        0.39704170,
        -0.32913574,
        0.34776688,
        0.37470851,
        0.09011915,
        8.52762600,
        -24.41225973,
        56.43597107,
        37.06152575,
        5.45624060,
        0.00025546,
        -0.00123461,
        -0.00023246,
        0.00045519,
        0.00016176,
        5.43288900,
        -8.62157806,
        13.44202049,
        14.51584135,
        3.39793084,
        5.56357900,
        -10.32345224,
        19.44322980,
        18.97361347,
        4.16903097,
        0.78866060,
        -2.90870942,
        6.54713531,
        4.05606657,
        0.53287322,
        0.00586685,
        -0.01704237,
        0.03872348,
        0.02570041,
        0.00383376,
        1.71535900,
        0.62246212,
        -0.92557761,
        -1.16996966,
        -0.30631491,
        6.59778800,
        -0.42450044,
        -12.13339427,
        -10.73509484,
        -2.51487077,
        10.08855000,
        -7.11727086,
        -31.67119479,
        -24.24848322,
        -5.33608972,
        1.01249500,
        0.32699690,
        -0.00923418,
        -0.03876858,
        -0.00412750,
        0.07490166,
        0.02410413,
        0.07233664,
        0.03040467,
        0.00197741,
        0.01077422,
        3.08223400,
        0.94472050,
        -2.15200882,
        -2.49219496,
        -0.63848738,
        17.84778000,
        -7.45345690,
        -48.96066856,
        -40.05386135,
        -9.09331816,
        0.00022582,
        -0.00186899,
        0.00388783,
        0.00142402,
        -0.00007671,
    )

    p["8"] = xz[36] + lzs * (xz[37] + lzs * (xz[38] + lzs * (xz[39] + lzs * xz[40])))
    p["9"] = xz[41] + lzs * (xz[42] + lzs * (xz[43] + lzs * (xz[44] + lzs * xz[45])))
    p["10"] = xz[46] + lzs * (xz[47] + lzs * (xz[48] + lzs * (xz[49] + lzs * xz[50])))
    p["11"] = xz[51] + lzs * (xz[52] + lzs * (xz[53] + lzs * (xz[54] + lzs * xz[55])))
    p["12"] = xz[56] + lzs * (xz[57] + lzs * (xz[58] + lzs * (xz[59] + lzs * xz[60])))
    p["13"] = xz[61]
    p["14"] = xz[62] + lzs * (xz[63] + lzs * (xz[64] + lzs * (xz[65] + lzs * xz[66])))
    p["15"] = xz[67] + lzs * (xz[68] + lzs * (xz[69] + lzs * (xz[70] + lzs * xz[71])))
    p["16"] = xz[72] + lzs * (xz[73] + lzs * (xz[74] + lzs * (xz[75] + lzs * xz[76])))

    m195 = m ** 19.5
    rad = (
        p["8"] * (m ** 2.5)
        + p["9"] * (m ** 6.5)
        + p["10"] * (m ** 11)
        + p["11"] * (m ** 19)
        + p["12"] * m195
    ) / (
        p["13"]
        + p["14"] * (m * m)
        + p["15"] * (m ** 8.5)
        + (m ** 18.5)
        + p["16"] * m195
    )
    return rad

"""
Module that contains the default options for the population grid code along with the description for these options, in the form of dictionaries: 
    - grid_options_defaults_dict: dictionary containing the default values for all the options
    - grid_options_descriptions: dictionary containing the description for these options.

There are several other functions in this module, mostly to generate help texts or documents:
    - grid_options_help: interactive function for the user to get descriptions for options 
    - grid_options_description_checker: function that checks that checks which options have a description.
    - write_grid_options_to_rst_file: function to generate the .rst document for the docs

With this its also possible to automatically generate a document containing all the setting names + descriptions. 

All the options starting with _ should not be changed by the user except when you really know what you're doing (which is probably hacking the code :P)
"""

import os

from binarycpython.utils.custom_logging_functions import temp_dir

# Options dict
grid_options_defaults_dict = {
    # general (or unordered..)
    "amt_cores": 1,  # total amount of cores used to evolve the population
    "binary": 0,  # FLag on whether the systems are binary systems or single systems.
    "parse_function": None,  # FUnction to parse the output with.
    "tmp_dir": temp_dir(),  # Setting the temp dir of the program
    "_main_pid": -1,  # Placeholder for the main process id of the run.
    "combine_ensemble_with_thread_joining": True,  # Flag on whether to combine everything and return it to the user or if false: write it to data_dir/ensemble_output_{popuation_id}_{thread_id}.json
    # "output_dir":
    "_commandline_input": "",
    "log_runtime_systems": 0, # whether to log the runtime of the systems (1 file per thread. stored in the tmp_dir)
    # Execution log:
    "verbosity": 0,  # Level of verbosity of the simulation. 0=INFO,
    "log_file": os.path.join(
        temp_dir(), "binary_c_python.log"
    ),  # Set to None to not log to file. The directory will be created
    # binary_c files
    "_binary_c_executable": os.path.join(
        os.environ["BINARY_C"], "binary_c"
    ),  # TODO: make this more robust
    "_binary_c_shared_library": os.path.join(
        os.environ["BINARY_C"], "src", "libbinary_c.so"
    "_binary_c_config_executable": os.path.join(
        os.environ["BINARY_C"], "binary_c-config"
    "_binary_c_dir": os.environ["BINARY_C"],
    # Custom logging
    "C_auto_logging": None,  # Should contain a dictionary where the kes are they headers
    # and the values are lists of parameters that should be logged.
    # This will get parsed by autogen_C_logging_code in custom_loggion_functions.py
    "C_logging_code": None,  # Should contain a string which holds the logging code.
    "custom_logging_func_memaddr": -1,  # Contains the custom_logging functions memory address
    "_custom_logging_shared_library_file": None,  # file containing the .so file
    # Store pre-loading:
    "_store_memaddr": -1,  # Contains the store object memory adress, useful for preloading.
    # defaults to -1 and isnt used if thats the default then.
    # Log args: logging of arguments
    "log_args": 0,  # unused
    "log_args_dir": "/tmp/",  # unused
    # Population evolution
    ## General
    "evolution_type": "grid",  # Flag for type of population evolution
    "_evolution_type_options": [
        "grid",
    ],  # available choices for type of population evolution. # TODO: fill later with monte carlo, sourcefile
    "_system_generator": None,  # value that holds the function that generates the system
    # (result of building the grid script)
    "source_file_filename": None,  # filename for the source
    "_count": 0,  # count of systems
    "_total_starcount": 0,  # Total count of systems in this generator
    "_probtot": 0,  # total probability
    "weight": 1.0,  # weighting for the probability
    "repeat": 1,  # number of times to repeat each system (probability is adjusted to be 1/repeat)
    "results": {},  # dict to store the results. Every process fills this on its own and then it will be joined later
    "ensemble_results": {},  # Dict to store the ensemble results
    "_start_time_evolution": 0,  # Start time of the grid
    "_end_time_evolution": 0,  # end time of the grid
    "_errors_found": False,  # Flag whether there are any errors from binary_c
    "_errors_exceeded": False,  # Flag whether the amt of errors have exceeded the limit
    "_failed_count": 0,  # amt of failed systems
    "_failed_prob": 0,  # Summed probability of failed systems
    "failed_systems_threshold": 20,  # Maximum failed systems per process allowed to fail before the process stops logging the failing systems.
    "_failed_systems_error_codes": [],  # List to store the unique error codes
    "_population_id": 0,  # Random id of this grid/population run, Unique code for the population. Should be set only once by the controller process.
    "modulo": 1,  # run modulo n of the grid. #TODO: fix this
    ## Grid type evolution
    "_grid_variables": {},  # grid variables
    "gridcode_filename": None,  # filename of gridcode
    ## Monte carlo type evolution
    # TODO: make MC options
    ## Evolution from source file
    # TODO: make run from sourcefile options.
    ## Other no yet implemented parts for the population evolution part
    #     # start at this model number: handy during debugging
    #     # to skip large parts of the grid
    #     start_at => 0
    #     global_error_string => undef,
    #     monitor_files => [],
    #     nextlogtime   => 0,
    #     nthreads      => 1, # number of threads
    #     # start at model offset (0-based, so first model is zero)
    #     offset        => 0,
    #     resolution=>{
    #         shift   =>0,
    #         previous=>0,
    #         n       =>{} # per-variable resolution
    #     },
    #     thread_q      => undef,
    #     threads       => undef, # array of threads objects
    #     tstart        => [gettimeofday], # flexigrid start time
    #     __nvar        => 0, # number of grid variables
    #     _varstub      => undef,
    #     _lock         => undef,
    #     _evcode_pids  => [],
    # };
    ########################################
    # Slurm stuff
    "slurm": 0,  # dont use the slurm by default. 1 = use slurm
    # "slurm_ntasks": 1,  # CPUs required per array job: usually only need this
    # "slurm_command": "",  # Command that slurm runs (e.g. evolve or join_datafiles)
    # "slurm_dir": "",  # working directory containing scripts output logs etc.
    # "slurm_njobs": 0,  # number of scripts; set to 0 as default
    # "slurm_jobid": "",  # slurm job id (%A)
    # "slurm_memory": 512,  # in MB, the memory use of the job
    # "slurm_warn_max_memory": 1024,  # in MB : warn if mem req. > this
    # "slurm_use_all_node_CPUs": 0,  # 1 = use all of a node's CPUs. 0 = use a given amount of CPUs
    # "slurm_postpone_join": 0,  # if 1 do not join on slurm, join elsewhere. want to do it off the slurm grid (e.g. with more RAM)
    # "slurm_jobarrayindex": "",  # slurm job array index (%a)
    # "slurm_jobname": "binary_grid",  # default
    # "slurm_partition": None,
    # "slurm_time": 0,  # total time. 0 = infinite time
    # "slurm_postpone_sbatch": 0,  # if 1: don't submit, just make the script
    # "slurm_array": None,  # override for --array, useful for rerunning jobs
    # "slurm_use_all_node_CPUs": 0,  # if given nodes, set to 1
    # # if given CPUs, set to 0
    # # you will want to use this if your Slurm SelectType is e.g. linear
    # # which means it allocates all the CPUs in a node to the job
    # "slurm_control_CPUs": 0,  # if so, leave this many for Pythons control (0)
    # "slurm_partition": None,  # MUST be defined
    # "slurm_extra_settings": {},  # Place to put extra configuration for the SLURM batch file. The key and value of the dict will become the key and value of the line in te slurm batch file. Will be put in after all the other settings (and before the command). Take care not to overwrite something without really meaning to do so.
    # Condor stuff
    "condor": 0,  # 1 to use condor, 0 otherwise
    # "condor_command": "",  # condor command e.g. "evolve", "join"
    # "condor_dir": "",  # working directory containing e.g. scripts, output, logs (e.g. should be NFS available to all)
    # "condor_njobs": "",  # number of scripts/jobs that CONDOR will run in total
    # "condor_jobid": "",  # condor job id
    # "condor_postpone_join": 0,  # if 1, data is not joined, e.g. if you want to do it off the condor grid (e.g. with more RAM)
    # # "condor_join_machine": None, # if defined then this is the machine on which the join command should be launched (must be sshable and not postponed)
    # "condor_join_pwd": "",  # directory the join should be in (defaults to $ENV{PWD} if undef)
    # "condor_memory": 1024,  # in MB, the memory use (ImageSize) of the job
    # "condor_universe": "vanilla",  # usually vanilla universe
    # "condor_extra_settings": {},  # Place to put extra configuration for the CONDOR submit file. The key and value of the dict will become the key and value of the line in te slurm batch file. Will be put in after all the other settings (and before the command). Take care not to overwrite something without really meaning to do so.
    # snapshots and checkpoints
    # condor_snapshot_on_kill=>0, # if 1 snapshot on SIGKILL before exit
    # condor_load_from_snapshot=>0, # if 1 check for snapshot .sv file and load it if found
    # condor_checkpoint_interval=>0, # checkpoint interval (seconds)
    # condor_checkpoint_stamp_times=>0, # if 1 then files are given timestamped names
    # (warning: lots of files!), otherwise just store the lates
    # condor_streams=>0, # stream stderr/stdout by default (warning: might cause heavy network load)
    # condor_save_joined_file=>0, # if 1 then results/joined contains the results
    # (useful for debugging, otherwise a lot of work)
    # condor_requirements=>'', # used?
    #     # resubmit options : if the status of a condor script is
    #     # either 'finished','submitted','running' or 'crashed',
    #     # decide whether to resubmit it.
    #     # NB Normally the status is empty, e.g. on the first run.
    #     # These are for restarting runs.
    #     condor_resubmit_finished=>0,
    # condor_resubmit_submitted=>0,
    # condor_resubmit_running=>0,
    # condor_resubmit_crashed=>0,
    # Unordered. Need to go through this. Copied from the perl implementation.
    ##
    # return_array_refs=>1, # quicker data parsing mode
    # sort_args=>1,
    # save_args=>1,
    # nice=>'nice -n +20',  # nice command e.g. 'nice -n +10' or ''
    # timeout=>15, # seconds until timeout
    # log_filename=>"/scratch/davidh/results_simulations/tmp/log.txt",
    # # current_log_filename=>"/scratch/davidh/results_simulations/tmp/grid_errors.log",
    ############################################################
    # Set default grid properties (in %self->{_grid_options}}
    # and %{$self->{_bse_options}})
    # This is the first thing that should be called by the user!
    # # set signal handlers for timeout
    # $self->set_class_signal_handlers();
    # # set operating system
    # my $os = rob_misc::operating_system();
    # %{$self->{_grid_options}}=(
    #     # save operating system
    # operating_system=>$os,
    #     # process name
    #     process_name => 'binary_grid'.$VERSION,
    # grid_defaults_set=>1, # so we know the grid_defaults function has been called
    # # grid suspend files: assume binary_c by default
    # suspend_files=>[$tmp.'/force_binary_c_suspend',
    #         './force_binary_c_suspend'],
    # snapshot_file=>$tmp.'/binary_c-snapshot',
    # ########################################
    # # infomration about the running grid script
    # working_directory=>cwd(), # the starting directory
    # perlscript=>$0, # the name of the perlscript
    # perlscript_arguments=>join(' ',@ARGV), # arguments as a string
    # perl_executable=>$^X, # the perl executable
    # command_line=>join(' ',$0,@ARGV), # full command line
    # process_ID=>$$, # process ID of the main perl script
    # # GRID
    #     # if undef, generate gridcode, otherwise load the gridcode
    #     # from this file. useful for debugging
    #     gridcode_from_file => undef,
    #     # assume binary_grid perl backend by default
    #     backend =>
    #     $self->{_grid_options}->{backend} //
    #     $binary_grid2::backend //
    #     'binary_grid::Perl',
    #     # custom C function for output : this automatically
    #     # binds if a function is available.
    #     C_logging_code => undef,
    #     C_auto_logging => undef,
    #     custom_output_C_function_pointer => binary_c_function_bind(),
    # # control flow
    # rungrid=>1, # usually run the grid, but can be 0
    # # to skip it (e.g. for condor/slurm runs)
    # merge_datafiles=>'',
    # merge_datafiles_filelist=>'',
    # # parameter space options
    # binary=>0, # set to 0 for single stars, 1 for binaries
    #     # if use_full_resolution is 1, then run a dummy grid to
    #     # calculate the resolution. this could be slow...
    #     use_full_resolution => 1,
    # # the probability in any distribution must be within
    # # this tolerance of 1.0, ignored if undef (if you want
    # # to run *part* of the parameter space then this *must* be undef)
    # probability_tolerance=>undef,
    # # how to deal with a failure of the probability tolerance:
    # # 0 = nothing
    # # 1 = warning
    # # 2 = stop
    # probability_tolerance_failmode=>1,
    # # add up and log system error count and probability
    # add_up_system_errors=>1,
    # log_system_errors=>1,
    # # codes, paths, executables etc.
    # # assume binary_c by default, and set its defaults
    # code=>'binary_c',
    # arg_prefix=>'--',
    # prog=>'binary_c', # executable
    # nice=>'nice -n +0', # nice command
    # ionice=>'',
    # # compress output?
    # binary_c_compression=>0,
    #     # get output as array of pre-split array refs
    #     return_array_refs=>1,
    # # environment
    # shell_environment=>undef,
    # libpath=>undef, # for backwards compatibility
    # # where is binary_c? need this to get the values of some counters
    # rootpath=>$self->okdir($ENV{BINARY_C_ROOTPATH}) //
    # $self->okdir($ENV{HOME}.'/progs/stars/binary_c') //
    # '.' , # last option is a fallback ... will fail if it doesn't exist
    # srcpath=>$self->okdir($ENV{BINARY_C_SRCPATH}) //
    # $self->okdir($ENV{BINARY_C_ROOTPATH}.'/src') //
    # $self->okdir($ENV{HOME}.'/progs/stars/binary_c/src') //
    # './src' , # last option is fallback... will fail if it doesn't exist
    # # stack size per thread in megabytes
    # threads_stack_size=>50,
    # # thread sleep time between starting the evolution code and starting
    # # the grid
    # thread_presleep=>0,
    # # threads
    # # Max time a thread can sit looping (with calls to tbse_line)
    # # before a warning is issued : NB this does not catch real freezes,
    # # just infinite loops (which still output)
    # thread_max_freeze_time_before_warning=>10,
    # # run all models by default: modulo=1, offset=0
    # modulo=>1,
    # offset=>0,
    #     # max number of stars on the queue
    #     maxq_per_thread => 100,
    # # data dump file : undef by default (do nothing)
    # results_hash_dumpfile => '',
    # # compress files with bzip2 by default
    # compress_results_hash => 1,
    # # CPU
    # cpu_cap=>0, # if 1, limits to one CPU
    # cpu_affinity => 0, # do not bind to a CPU by default
    # # Code, Timeouts, Signals
    # binary_grid_code_filtering=>1, #  you want this, it's (MUCH!) faster
    # pre_filter_file=>undef, # dump pre filtered code to this file
    # post_filter_file=>undef,  # dump post filtered code to this file
    # timeout=>30, # timeout in seconds
    # timeout_vb=>0, # no timeout logging
    # tvb=>0, # no thread logging
    # nfs_sleep=>1, # time to wait for NFS to catch up with file accesses
    # # flexigrid checks the timeouts every
    # # flexigrid_timeout_check_interval seconds
    # flexigrid_timeout_check_interval=>0.01,
    # # this is set to 1 when the grid is finished
    # flexigrid_finished=>0,
    # # allow signals by default
    # 'no signals'=>0,
    # # but perhaps disable specific signals?
    # 'disable signal'=>{INT=>0,ALRM=>0,CONT=>0,USR1=>0,STOP=>0},
    # # dummy variables
    # single_star_period=>1e50,  # orbital period of a single star
    # #### timers : set timers to 0 (or empty list) to ignore,
    # #### NB these must be given context (e.g. main::xyz)
    # #### for functions not in binary_grid
    # timers=>0,
    # timer_subroutines=>[
    #     # this is a suggested default list
    #     'flexigrid',
    #         'set_next_alarm',
    #     'vbout',
    #         'vbout_fast',
    #     'run_flexigrid_thread',
    #         'thread_vb'
    # ],
    # # INPUT/OUTPUT
    # blocking=>undef, # not yet set
    # # prepend command with stdbuf to stop buffering (if available)
    # stdbuf_command=>`stdbuf --version`=~/stdbuf \(GNU/ ? ' stdbuf -i0 -o0 -e0 ' : undef,
    # vb=>("@ARGV"=~/\Wvb=(\d+)\W/)[0] // 0, # set to 1 (or more) for verbose output to the screen
    # log_dt_secs=>1, # log output to stdout~every log_dt_secs seconds
    # nmod=>10, # every nmod models there is output to the screen,
    # # if log_dt_secs has been exceeded also (ignored if 0)
    # colour=>1, # set to 1 to use the ANSIColor module for colour output
    # log_args=>0, # do not log args in files
    # log_fins=>0, # log end of runs too
    #     sort_args=>0, # do not sort args
    # save_args=>0, # do not save args in a string
    # log_args_dir=>$tmp, # where to output the args files
    # always_reopen_arg_files=>0, # if 1 then arg files are always closed and reopened
    #   (may cause a lot of disk I/O)
    # lazy_arg_sending=>1, # if 1, the previous args are remembered and
    # # only args that changed are sent (except M1, M2 etc. which always
    # # need sending)
    # # force output files to open on a local disk (not an NFS partion)
    # # not sure how to do this on another OS
    # force_local_hdd_use=>($os eq 'unix'),
    # # for verbose output, define the newline
    # # For terminals use "\x0d", for files use "\n", in the
    # # case of multiple threads this will be set to \n
    # newline=> "\x0d",
    #     # use reset_stars_defaults
    #     reset_stars_defaults=>1,
    # # set signal captures: argument determines behaviour when the code locks up
    # # 0: exit
    # # 1: reset and try the next star (does this work?!)
    # alarm_procedure=>1,
    # # exit on eval failure?
    # exit_on_eval_failure=>1,
    # ## functions: these should be set by perl lexical name
    # ## (they are automatically converted to function pointers
    # ## at runtime)
    # # function to be called just before a thread is created
    # thread_precreate_function=>undef,
    #     thread_precreate_function_pointer=>undef,
    # # function to be called just after a thread is created
    # # (from inside the thread just before *grid () call)
    # threads_entry_function=>undef,
    #     threads_entry_function_pointer=>undef,
    # # function to be called just after a thread is finished
    # # (from inside the thread just after *grid () call)
    # threads_flush_function=>undef,
    # threads_flush_function_pointer=>undef,
    # # (but external to the thread)
    # thread_postrun_function=>undef,
    # thread_postrun_function_pointer=>undef,
    # # function to be called just before a thread join
    # # (external to the thread)
    # thread_prejoin_function=>undef,
    # thread_prejoin_function_pointer=>undef,
    # # default to using the internal join_flexigrid_thread function
    # threads_join_function=>'binary_grid2::join_flexigrid_thread',
    # threads_join_function_pointer=>sub{return $self->join_flexigrid_thread(@_)},
    # # function to be called just after a thread join
    # thread_postjoin_function=>undef,
    # thread_postjoin_function_pointer=>undef,
    # # usually, parse_bse in the main script is called
    # parse_bse_function=>'main::parse_bse',
    #     parse_bse_function_pointer=>undef,
    # # if starting_snapshot_file is defined, load initial
    # # values for the grid from the snapshot file rather
    # # than a normal initiation: this enables you to
    # # stop and start a grid
    # starting_snapshot_file=>undef,
}

# Grid containing the descriptions of the options # TODO: add input types for all of them
grid_options_descriptions = {
    "tmp_dir": "Directory where certain types of output are stored. The grid code is stored in that directory, as well as the custom logging libraries. Log files and other diagnostics will usually be written to this location, unless specified otherwise",  # TODO: improve this
    "_binary_c_dir": "Director where binary_c is stored. This options are not really used",
    "_binary_c_config_executable": "Full path of the binary_c-config executable. This options is not used in the population object.",
    "_binary_c_executable": "Full path to the binary_c executable. This options is not used in the population object.",
    "_binary_c_shared_library": "Full path to the libbinary_c file. This options is not used in the population object",
    "verbosity": "Verbosity of the population code. Default is 0, by which only errors will be printed. Higher values will show more output, which is good for debugging.",
    "binary": "Set this to 1 if the population contains binaries. Input: int",  # TODO: write what effect this has.
    "amt_cores": "The amount of cores that the population grid will use. The multiprocessing is useful but make sure to figure out how many logical cores the machine has. The core is multiprocessed, not multithreaded, and will gain no extra speed when amt_cores exceeds the amount of logical cores. Input: int",
    "_start_time_evolution": "Variable storing the start timestamp of the population evolution. Set by the object itself.",  # TODO: make sure this is logged to a file
    "_end_time_evolution": "Variable storing the end timestamp of the population evolution. Set by the object itself",  # TODO: make sure this is logged to a file
    "_total_starcount": "Variable storing the total amount of systems in the generator. Used and set by the population object.",
    "_custom_logging_shared_library_file": "filename for the custom_logging shared library. Used and set by the population object",
    "_errors_found": "Variable storing a boolean flag whether errors by binary_c are encountered.",
    "_errors_exceeded": "Variable storing a boolean flag whether the amount of errors was higher than the set threshold (failed_systems_threshold). If True, then the commandline arguments of the failing systems will not be stored in the failed_system_log files.",
    "source_file_filename": "Variable containing the source file containing lines of binary_c commandline calls. These all have to start with binary_c.",  # TODO: Expand
    "results": "Dictionary in which the user can place their results. This dictionary gets merged at the end of a mulitprocessing simulation.",
    "C_auto_logging": "Dictionary containing parameters to be logged by binary_c. The structure of this dictionary is as follows: the key is used as the headline which the user can then catch. The value at that key is a list of binary_c system parameters (like star[0].mass)",
    "C_logging_code": "Variable to store the exact code that is used for the custom_logging. In this way the user can do more complex logging, as well as putting these logging strings in files.",
    "_failed_count": "Variable storing the amount of failed systems.",
    "_evolution_type_options": "List containing the evolution type options.",
    "_failed_prob": "Variable storing the total probability of all the failed systems",
    "_failed_systems_error_codes": "List storing the unique error codes raised by binary_c of the failed systems",
    "_grid_variables": "Dictionary storing the grid_variables. These contain properties which are accessed by the _generate_grid_code function",
    "_population_id": "Variable storing a unique 32-char hex string.",
    "_commandline_input": "String containing the arguments passed to the population object via the command line. Set and used by the population object.",
    "_system_generator": "Function object that contains the system generator function. This can be from a grid, or a source file, or a montecarlo grid.",
    "gridcode_filename": "Filename for the grid code. Set and used by the population object. TODO: allow the user to provide their own function, rather than only a generated function.",
    "log_args": "Boolean to log the arguments. Unused ",  # TODO: fix the functionality for this and describe it properly
    "log_args_dir": "Directory to log the arguments to. Unused",  # TODO: fix the functionality for this and describe it properly
    "log_file": "Log file for the population object. Unused",  # TODO: fix the functionality for this and describe it properly
    "custom_logging_func_memaddr": "Memory adress where the custom_logging_function is stored. Input: int",
    "_count": "Counter tracking which system the generator is on.",
    "_probtot": "Total probability of the population.",  # TODO: check whether this is used properly throughout
    "_main_pid": "Main process ID of the master process. Used and set by the population object.",
    "_store_memaddr": "Memory adress of the store object for binary_c.",
    "failed_systems_threshold": "Variable storing the maximum amount of systems that are allowed to fail before logging their commandline arguments to failed_systems log files",
    "parse_function": "Function that the user can provide to handle the output the binary_c. This function has to take the arguments (self, output). Its best not to return anything in this function, and just store stuff in the grid_options['results'] dictionary, or just output results to a file",
    "condor": "Int flag whether to use a condor type population evolution. Not implemented yet.",  # TODO: describe this in more detail
    "slurm": "Int flag whether to use a slurm type population evolution.",  # TODO: describe this in more detail
    "weight": "Weight factor for each system. The calculated probability is mulitplied by this. If the user wants each system to be repeated several times, then this variable should not be changed, rather change the _repeat variable instead, as that handles the reduction in probability per system. This is useful for systems that have a process with some random element in it.",  # TODO: add more info here, regarding the evolution splitting.
    "repeat": "Factor of how many times a system should be repeated. Consider the evolution splitting binary_c argument for supernovae kick repeating.",  # TODO: make sure this is used.
    "evolution_type": "Variable containing the type of evolution used of the grid. Multiprocessing or linear processing",
    "combine_ensemble_with_thread_joining": "BOolean flag on whether to combine everything and return it to the user or if false: write it to data_dir/ensemble_output_{popuation_id}_{thread_id}.json",
    "ensemble_results": "Dictinary that stores the ensemble results if combine_ensemble_with_thread_joining==True",
    "log_runtime_systems": "Whether to log the runtime of the systems . Each systems run by the thread is logged to a file and is stored in the tmp_dir. (1 file per thread). Don't use this if you are planning to run alot of systems. This is mostly for debugging and finding systems that take long to run. Integer, default = 0. if value is 1 then the systems are logged",
}

#################################
# Grid options functions

# Utility functions
def grid_options_help(option: str) -> dict:
    """
    Function that prints out the description of a grid option. Useful function for the user.

    Args:
        option: which option you want to have the description of

    returns:
        dict containg the option, the description if its there, otherwise empty string. And if the key doesnt exist, the dict is empty
    """

    option_keys = grid_options_defaults_dict.keys()
    description_keys = grid_options_descriptions.keys()

    if not option in option_keys:
        print(
            "Error: This is an invalid entry. Option does not exist, please choose from the following options:\n\t{}".format(
                ", ".join(option_keys)
            )
        )
        return {}

    else:
        if not option in description_keys:
            print(
                "This option has not been described properly yet. Please contact on of the authors"
            )
            return {option: ""}
        else:
            print(grid_options_descriptions[option])
            return {option: grid_options_descriptions[option]}

def grid_options_description_checker(print_info: bool = True) -> int:
    """
    Function that checks which descriptions are missing

    Args:
        print_info: whether to print out information about which options contain proper descriptions and which do not

    Returns:
        the amount of undescribed keys
    """

    # Get the keys

    #
    undescribed_keys = list(set(option_keys) - set(description_keys))

    if undescribed_keys:
        if print_info:
            print(
                "Warning: the following keys have no description yet:\n\t{}".format(
                    ", ".join(sorted(undescribed_keys))
                )
            )
            print(
                "Total description progress: {:.2f}%%".format(
                    100 * len(description_keys) / len(option_keys)
                )
            )
    return len(undescribed_keys)

def write_grid_options_to_rst_file(output_file: str) -> None:
    """
    Function that writes the descriptions of the grid options to a rst file

    Tasks:
        TODO: seperate things into private and public options

    Args:
        output_file: target file where the grid options descriptions are written to
    """

    # Get the options and the description
    options = grid_options_defaults_dict
    descriptions = grid_options_descriptions

    # Get those that do not have a description
    not_described_yet = list(set(options) - set(descriptions))

    # separate public and private options
    public_options = [key for key in options if not key.startswith("_")]
    private_options = [key for key in options if key.startswith("_")]

    # Check input

        print("Population grid code options", file=f)
        print("{}".format("=" * len("Population grid code options")), file=f)
        print(
            "The following chapter contains all grid code options, along with their descriptions",
            file=f,
        )
        print(
            "There are {} options that are not described yet.".format(
                len(not_described_yet)
            ),
            file=f,
        )

        # Start public options part
        print("Public options", file=f)
        print("{}".format("-" * len("Public options")), file=f)
        print("The following options are meant to be changed by the user.", file=f)

        for public_option in sorted(public_options):
            if public_option in descriptions:
                print(
                    "| **{}**: {}".format(public_option, descriptions[public_option]),
                    file=f,
                )
            else:
                print(
                    "| **{}**: No description available yet".format(public_option),
                    file=f,
                )
            print("", file=f)

        # Start private options part
        print("Private options", file=f)
        print("{}".format("-" * len("Private options")), file=f)
        print(
            "The following options are not meant to be changed by the user, as these options are used and set internally by the object itself. The description still is provided, but just for documentation purposes.",
            file=f,
        )

        for private_option in sorted(private_options):
            if private_option in descriptions:
                print(
                    "| **{}**: {}".format(private_option, descriptions[private_option]),
                    file=f,
                )
            else:
                print(
                    "| **{}**: No description available yet".format(private_option),
                    file=f,
                )

"""
Module containing the spacing functions for the binarycpython package. Very under-populated at the moment, but more are likely to come soon

Tasks:
    TODO: add more spacing functions to this module.
"""

def const(
    min_bound: Union[int, float], max_bound: Union[int, float], steps: int
) -> list:
    """
    Samples a range linearly. Uses numpy linspace.

    Args:
        min_bound: lower bound of range
        max_bound: upper bound of range
        steps: amount of segments between min_bound and max_bound

    Returns:
        np.linspace(min_bound, max_bound, steps)
    """

    return np.linspace(min_bound, max_bound, steps)

"""
Module that contains functionality to plot some properties of (binary) systems. 

Different routines are defined here to plot orbits, masses, angular momenta etc. 

Structure of file:
    custom logging strings
    parsing functions
    plotting functions
    master function: plot_system

All the loose components here can ofcourse be used in other routines if you want.

There is no preloaded matplotlib rc, you should do that yourself

These plotting routines are designed for binary systems, and admittingly they are here mostly for 
inspirational purposes, since one would problably want to customize the plots. 
Regardless, having some plotting routines in here seemed like a nice idea

Tasks
    TODO: This module is not finished yet.
    TODO: Make modules for single system
    TODO: Put all the plotting functions in here
"""

import pandas as pd
import astropy.constants as const

import matplotlib.pyplot as plt

from binarycpython.utils.functions import output_lines
from binarycpython.utils.run_system_wrapper import run_system
from binarycpython.utils.custom_logging_functions import binary_c_log_code

# Define the custom_logging_strings.
# These are kept to the minimum necessary for each plotting routine.

CUSTOM_LOGGING_STRING_MASSES = """
Printf("MASS_PLOTTING %30.12e %g %g %g %g\\n",
    //
    stardata->model.time, // 1
    
    // masses
    stardata->common.zero_age.mass[0], //
    stardata->common.zero_age.mass[1], //

    stardata->star[0].mass,
    stardata->star[1].mass
    );
"""

CUSTOM_LOGGING_STRING_ORBIT = """
Printf("ORBIT_PLOTTING %30.12e %g %g %g\\n",
    //
    
    // orbit elements
    stardata->common.orbit.period, //2
    stardata->common.orbit.separation, //3
    stardata->common.orbit.eccentricity //4
    );
"""

CUSTOM_LOGGING_STRING_HR_DIAGRAM = """
Printf("HR_PLOTTING %30.12e %d %d %g %g %g %g %g %g\\n",
    //
    
    // stellar types
    stardata->star[0].stellar_type, //2
    stardata->star[1].stellar_type, //3

    // luminosity and radii
    stardata->star[0].luminosity, // 4
    stardata->star[1].luminosity, // 5
    stardata->star[0].radius,     // 6
    stardata->star[1].radius,     // 7

    // masses
    stardata->common.zero_age.mass[0], // 8
    stardata->common.zero_age.mass[1]  // 9
    );
"""

def color_by_index(row, column, colors):
    """
    Function that returns a color based on row and column information. Used to color the stellar types
    """

    return colors[int(row[column])]

def plot_HR_diagram(
    df,
    show_stellar_types: bool = False,
    show_plot: bool = True,
    use_astropy_values: bool = True,
):
    """
    Function to plot the HR diagram evolution of the system. Assumes its a binary system.

    For a single star see plot_HR_diagram_single

    Plot shows Log luminosity on y axis,
    log temperature on x axis (reversed)

    requires:
        - time
        - stellar_type_1
        - stellar_type_2
        - radius_1
        - radius_2
        - luminosity_1
        - luminosity_2

    Plots:
        - luminosity_1 vs teff_1
        - luminosity_2 vs teff_2

    Tasks:
        - TODO: add HD limit
        - TODO: add lines of constant radius

    Args:
        df: pandas dataframe with the required columns
        show_stellar_types: whether to color code the tracks and show the stellar types
        show_plot: whether to actually show the plot. If false: returns the figure object
        use_astropy_values: Whether to use astropy values for for Rsun, Lsun and stefan boltzman constant.

    Returns
        returns a figure object if show_plot is false

    """

    # prefactor = (1/(4 * math.pi * omega_sb))**(1.0/4)
    # prefactor_approx = 1000 * (1130)**(1.0/4) * ((R_SUN**2)/L_SUN)**(1.0/4)

    if use_astropy_values:
        R_SUN = const.R_sun.cgs.value
        L_SUN = const.L_sun.cgs.value
        omega_sb = const.sigma_sb.cgs.value
        print(
            "Using astropy values: R_SUN= {} L_SUN = {} omega_sb = {}".format(
                R_SUN, L_SUN, omega_sb
            )
        )
    else:
        R_SUN = 6.956600000000000000000000000000e10
        L_SUN = 3.851500000000000274321803705319e33
        omega_sb = 5.670352798655924736991040813194e-05
        print(
            "Using binary_c values: R_SUN= {} L_SUN = {} omega_sb = {}".format(
            )
        )

    prefactor = (1 / (4 * math.pi * omega_sb)) ** (1.0 / 4)

    if show_stellar_types:
        fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(20, 20))
        fig, colors = add_stellar_types_bar(df, fig, ax_index=-1, only_colorbar=True)
    else:

    df = df.assign(
        teff_1=prefactor
        * ((df["luminosity_1"] * L_SUN) / ((df["radius_1"] * R_SUN) ** 2)) ** (1.0 / 4)
    )
        teff_2=prefactor
        * ((df["luminosity_2"] * L_SUN) / ((df["radius_2"] * R_SUN) ** 2)) ** (1.0 / 4)
    )

    # # Add colors to dataframe
    # df = df.assign(colors_1=df.apply(color_by_index, axis=1, args=('stellar_type_1', colors)))
    # df = df.assign(colors_2=df.apply(color_by_index, axis=1, args=('stellar_type_2', colors)))

    # Star 1:
    fig.axes[0].scatter(
        df["teff_1"],
        df["luminosity_1"],
        label="Star 1 (M={})".format(df["pms_mass_1"].values.tolist()[0]),
        # color=df['colors_1']
    )

    # Star 2:
        df["teff_2"],
        df["luminosity_2"],
        label="Star 2 (M={})".format(df["pms_mass_2"].values.tolist()[0]),
        # color=df['colors_2']
    )

    margin_fraction_x = 0.1
    margin_fraction_y = 0.9

    # Fix axes
    min_y, max_y = (
        df[["luminosity_1", "luminosity_2"]].min().min(),
        df[["luminosity_1", "luminosity_2"]].max().max(),
    )
    min_x, max_x = (
        df[["teff_1", "teff_2"]].min().min(),
        df[["teff_1", "teff_2"]].max().max(),
    )
    fig.axes[0].set_xlim(
        min_x * (1 - margin_fraction_x), max_x * (1 + margin_fraction_x)
    )
    fig.axes[0].set_ylim(
        min_y * (1 - margin_fraction_y), max_y * (1 + margin_fraction_y)
    )
    fig.axes[0].set_yscale("log")
    fig.axes[0].set_xscale("log")
    fig.axes[0].invert_xaxis()

    # Other stuff
    fig.axes[0].set_title("HR diagram")
    fig.axes[0].legend(loc="best")
    fig.axes[0].set_ylabel(r"Luminosity [$L_{star}$/$L_{\odot}$]")
    fig.axes[0].set_xlabel(r"$T_{eff}$ (K)")

    # Show or return
    if show_plot:
        plt.show()
    else:
        return fig

def plot_orbit(df, show_stellar_types: bool = False, show_plot: bool = True):
    """
    Function to plot the orbital elements of the system

    Plots the following quantities:
        - Orbital period
        - Separation
        - eccentricity

    TODO: fix stellar_types plot

    Args:

    Returns
    """

        fig, ax = plt.subplots(ncols=1, nrows=4, figsize=(20, 10))
        fig.subplots_adjust(hspace=0)
    else:
        fig, ax = plt.subplots(ncols=1, nrows=3, figsize=(20, 10), sharex=True)

    #
    fig.axes[0].plot(df["time"], df["orbital_period"], label="Orbital period")

    fig.axes[1].plot(df["time"], df["separation"], label="Separation orbit")

    fig.axes[2].plot(df["time"], df["eccentricity"], label="Eccentricity orbit")

    # Make up
    fig.axes[0].set_title("Orbital elements evolution")

    fig.axes[1].legend(loc="best")
    fig.axes[2].legend(loc="best")

    # fig.axes[0].set_ylim(0, 1.1*max_total_mass)
    fig.axes[0].set_ylabel(r"Orbital period")
    fig.axes[1].set_ylabel(r"Separation")
    fig.axes[2].set_ylabel(r"Eccentricity")
    fig.axes[1].set_yscale("log")

    fig.axes[2].set_xlabel(r"Time (Myr)")

    else:
        return fig

def plot_masses(df, show_stellar_types: bool = False, show_plot: bool = True):
    """
    Function to plot the masses of the system.

    Function requires the following keys:
        - time
        - pms_mass_1
        - pms_mass_2
        - mass_1
        - mass_2

        - Total mass
        - Mass star 1
        - Mass star 2
        - Pms mass 1
        - pms mass 2
        - pms total mass
        - (maybe?) core and env masses

    Args:

    Returns
    """

        fig, ax = plt.subplots(ncols=1, nrows=2, figsize=(20, 10))
    else:
        fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(20, 10))

    max_total_mass = (
        df["pms_mass_1"].values.tolist()[0] + df["pms_mass_2"].values.tolist()[0]
    )

    df["total_mass_temp"] = df["mass_1"] + df["mass_2"]
    df["pms_total_mass_temp"] = max_total_mass

    # total mass
    fig.axes[0].plot(df["time"], df["total_mass_temp"], label="Total mass")
    fig.axes[0].axhline(
        df["pms_total_mass_temp"].values.tolist()[0],
        label="Initial total mass",
        linestyle="--",
        alpha=0.5,
    )

    # Mass 1
    fig.axes[0].plot(df["time"], df["mass_1"], label="star 1")
        df["pms_mass_1"].values.tolist()[0],
        color="red",
        linewidth=2,
        label="Initial mass 1",
    )

    # mass 2
    fig.axes[0].plot(df["time"], df["mass_2"], color="orange", label="star 2")
        df["pms_mass_2"].values.tolist()[0],
        label="Initial mass 2",
    )

    # Make up
    fig.axes[0].set_title("Stellar mass evolution")
    fig.axes[0].set_ylim(0, 1.1 * max_total_mass)
    fig.axes[0].set_ylabel(r"Mass [$M_{\odot}$]")
    fig.axes[0].set_xlabel(r"Time (Myr)")

    else:
        return fig

# Define the parse functions for the plotting routines
def dummy():
    """Placeholder"""
    pass

def parse_function_hr_diagram(output: str):
    """
    Parsing function for the HR plotting routine

    Args:
        output: raw binary_c output string

    Returns:
        pandas dataframe containing the columns for the HR diagram plotting routine
    """

    # extract info from the single evolution

    values_list = []

    parameters = [
        "time",
        "stellar_type_1",
        "stellar_type_2",
        "luminosity_1",
        "luminosity_2",
        "radius_1",
        "radius_2",
        "pms_mass_1",
        "pms_mass_2",
    ]

    # Go over the output.
    for line in output_lines(output):
        headerline = line.split()[0]

        # Check the header and act accordingly
        if headerline == "HR_PLOTTING":
            values = line.split()[1:]
            values_list.append(values)

    df = pd.DataFrame(values_list)
    df.columns = parameters

    df = df.astype(np.float64)
    df["stellar_type_1"] = df["stellar_type_1"].astype(np.int64)
    df["stellar_type_2"] = df["stellar_type_2"].astype(np.int64)

    return df

def parse_function_orbit(output: str):
    """
    Parsing function for the orbit plotting routine

    Args:

    Returns:
        pandas dataframe containing the columns for the orbit plotting routine
    """

        "time",
        "orbital_period",
        "separation",
        "eccentricity",
    ]

        if headerline == "ORBIT_PLOTTING":

    return df

def parse_function_masses(output: str):
    """

    Args:

    Returns:
        pandas dataframe containing the columns for the masses plotting routine
    """

        "time",
        "mass_1",
        "mass_2",
    ]

        if headerline == "MASS_PLOTTING":

    return df

def plot_system(plot_type, **kwargs):
    """
    Function to plot the different quantities of a system.

    This goes (in general) via the following steps:
        - a preset custom logging for a specific plotting routine is loaded, depending on the choice of plot_type
        - This is used for the run_system call
        - The output of this run_system is loaded into a dataframe by
        parsing it with a corresponding parsing function
        - The dataframe is passed to the plotting routine
        - plot is shown or returned.

    There are several pre-set plots to choose from:
        - mass_evolution
        - orbit_evolution
        - hr_diagram

    Tasks:
        - TODO: Complex Function!
        - TODO: make sure this way of passing args works correctly.
        - TODO: make the plotting specific keywords available via the inspect stuff

    All keywords are considered kwargs, except for plot_type
    Args:
        plot_type: string input should be one of ['mass_evolution', 'orbit_evolution', 'hr_diagram'].
            Input will be matched against this, and then go through a dictionary to pick the correct plotting function.

        show_plot: boolean whether to show the plot. If False it returns the figure object
            (makes so that you can customize it)

        show_stellar_types: whether to plot the stellar type evolution on a second pane.
            This is not included in all the plotting routines.

        Other input: other kwargs that are passed to run_system
            (inspect the docstring of run_system for more info)

    Returns:
        returns a object figure if show_plot = false
    """

    # set defaults and options
    show_plot = False
    show_stellar_types = False

    plot_types_dict = {
        "mass_evolution": {
            "plot_function": plot_masses,
            "custom_logging_string": CUSTOM_LOGGING_STRING_MASSES,
            "parse_function": parse_function_masses,
        },
        "orbit_evolution": {
            "plot_function": plot_orbit,
            "custom_logging_string": CUSTOM_LOGGING_STRING_ORBIT,
            "parse_function": parse_function_orbit,
        },
        "hr_diagram": {
            "plot_function": plot_HR_diagram,
            "custom_logging_string": CUSTOM_LOGGING_STRING_HR_DIAGRAM,
            "parse_function": parse_function_hr_diagram,
        },
    }
    plot_system_specific_keywords = ["plot_type", "show_plot", "show_stellar_types"]

    # First check on the plot_type input
    if not plot_type in plot_types_dict:
        print(
            "Warning, the provided plot type is not known. \
            Please choose one from the following:\n\t{}".format(
                plot_types_dict.keys()
            )
        )

    # First: check all the arguments. Chosen to not check all the keywords for run_system
    #   and binary_c specifically, but just to pick out the ones needed for this routine.
    #   run_system will handle the rest
    run_system_arg_dict = {}

    for key in kwargs.keys():
        if key == "show_plot":
            show_plot = kwargs[key]
        elif key == "show_stellar_types":
            show_stellar_types = kwargs[key]

        # The rest will be passed to run_system
        else:
            run_system_arg_dict[key] = kwargs[key]

    # TODO: When a list of plot_types is passed, make it so that the strings are chained,
    #   and that the output of the binary_c call is handled by multiple parsers
    custom_logging_code = binary_c_log_code(
        plot_types_dict[plot_type]["custom_logging_string"]
    )
    run_system_arg_dict["custom_logging_code"] = custom_logging_code

    run_system_arg_dict["parse_function"] = plot_types_dict[plot_type]["parse_function"]

    # Run and get the output of the parse stellar_type_dict_shortfunction
    binary_c_output_df = run_system(**run_system_arg_dict)

    fig = plot_types_dict[plot_type]["plot_function"](
        binary_c_output_df, show_plot=show_plot, show_stellar_types=show_stellar_types
    )

    if not show_plot:
        return fig

# from david_phd_functions.plotting.custom_mpl_settings import load_mpl_rc
# load_mpl_rc()

# fig = plot_system(
#     plot_type="mass_evolution",
#     M_1=10,
#     M_2=5,
#     separation=1000000,
#     orbital_period=100000000,
#     max_evolution_time=15000,
#     show_plot=True,
# )

# fig.axes[0].set_xlim(0, 150)
# plt.show()

"""
File containing functions for HPC computing, distributed tasks on clusters etc.

Functions that the slurm and condor subroutines of the population object use.

Mainly divided in 2 sections: Slurm and Condor
"""

import os
import sys
import time
import __main__ as main

def get_slurm_version() -> Union[str, None]:
    """
    Function that checks whether slurm is installed and returns the version if its installed.

    Only tested this with slurm v17+

    Returns:
        slurm version, or None
    """

    slurm_version = None

    try:
        slurm_version = (
            subprocess.run(["sinfo", "-V"], stdout=subprocess.PIPE, check=True)
            .stdout.decode("utf-8")
            .split()
        )[1]
    except FileNotFoundError as err:
        print(err)
        print(err.args)
        print("Slurm is not installed or not loaded")
    except Exception as err:
        print("Unknown error, contact me about this")

    return slurm_version

def get_condor_version() -> Union[str, None]:
    """

    otherwise returns None

    Result has to be condor v8 or higher

    Returns:
        condor version, or None
    """

    condor_version = None

    try:
        condor_version = (
            subprocess.run(
                ["condor_q", "--version"], stdout=subprocess.PIPE, check=True
            )
            .split()
        )[1]
        print("Slurm is not installed or not loaded: ")
        print("Unknown error, contact me about this: ")

    return condor_version

def create_directories_hpc(working_dir: str) -> None:
    """
    Function to create a set of directories, given a root directory

    These directories will contain stuff for the HPC runs

    Args:
        working_dir: main working directory of the run. Under this directory all the dirs will be created
    """

    # Check if working_dir exists
    if not os.path.isdir(working_dir):
        print("Error. Working directory {} does not exist! Aborting")

    directories_list = [
        "scripts",
        "stdout",
        "stderr",
        "results",
        "logs",
        "status",
        "joining",
    ]

    # Make directories.
    for subdir in directories_list:
        full_path = os.path.join(working_dir, subdir)
        os.makedirs(full_path, exist_ok=True)

    # Since the directories are probably made on some mount which has to go over NFS
    # we should explicitly check if they are created
    print("Checking if creating the directories has finished...")
    directories_exist = False
    while directories_exist:
        directories_exist = True

        for subdir in directories_list:
            full_path = os.path.join(working_dir, subdir)

            if not os.path.isdir(full_path):
                time.sleep(1)
                directories_exist = False
    print("..Finished! Directories exist.")

def path_of_calling_script() -> str:
    """
    Function to get the name of the script the user executes.
    TODO: fix this function. seems not to work properly.
    """

    return main.__file__

def get_python_details() -> dict:
    """
    Function to get some info about the used python version and virtualenv etc

    Returns:
        dictionary with python executable, virtual environment and version information.
    """

    python_info_dict = {}

    #
    python_info_dict["virtualenv"] = os.getenv("VIRTUAL_ENV")
    python_info_dict["executable"] = sys.executable
    python_info_dict["version"] = sys.version

    return python_info_dict

"""
Module containing the utility function run_system,
which handles a lot of things by analysing the passed kwargs
"""

from binarycpython.utils.functions import (
    create_arg_string,
    get_arg_keys,
    remove_file,
)

from binarycpython.utils.custom_logging_functions import (
    create_and_load_logging_function,
)

def run_system(**kwargs):
    """
    Function that runs a system.
    Mostly as a useful utility function that handles all the setup of argument lists etc.

    Examples:
        - run_system(M_1=10): will run a system with ZAMS mass 1 = 10
        - run_system(M_1=10, log_filename="~/example_log.txt"): Will run a system
            and write the logfile too
        - run_system(M_1=10, parse_function=fancy_parsing_function)

    Tasks:
        - TODO: Expand functionality.
        - TODO: Notify user when an unknown keyword is passed.

    All~ the arguments known to binary_c can be passed to this function as kwargs.
    Several extra arguments can be passed through the kwargs:
    Kwargs:
        custom_logging_code (string):
            Should contain a string containing the c-code for the shared library.
            If this is provided binary_c will use that custom logging code to output its data
        log_filename (string):
            Should contain name of the binary_c system logfile.
            Passing this will make sure that the filename gets written for a run
            (its default behaviour is NOT to write a logfile for a system)
        parse_function (function):
            should contain a function that parses the output.
            The parse function should take 1 required parameter: the output of the binaryc run
            Passing this will call the parse_function by passing it the output of the binary_c call
            and returns what the parse_function returns

    Returns:
        Either returns the raw output of binary_c, or the output of a parse_function if parse_function is given
    """

    # Load available arg keywords
    available_binary_c_arg_keywords = get_arg_keys()

    other_keywords = ["custom_logging_code", "log_filename", "parse_function"]

    # Set default values
    func_memaddr = -1
    write_logfile = 0

    # Create dict to pass as argstring
    binary_c_args = {}

    # Check which binary_c arguments have been passed and put them into a dict
    for key in kwargs:
        if key in available_binary_c_arg_keywords:
            binary_c_args[key] = kwargs[key]

        # Notify user when this key wont be used
        else:
            if not key in other_keywords:
                print(
                    "The following keyword was not recognized and wont be used:\n\t{}".format(
                        key
                    )
                )

    # Check if custom logging is required
    if "custom_logging_code" in kwargs:
        func_memaddr, shared_lib_filename = create_and_load_logging_function(
            kwargs["custom_logging_code"]
        )

    # Check if writing logfile is required:
    if "log_filename" in kwargs:
        write_logfile = 1

    # Construct arguments string and final execution string
    arg_string = create_arg_string(binary_c_args)
    binary_c_command = "binary_c {}".format(arg_string)

    # Call binary_c
    output = _binary_c_bindings.run_system(
        binary_c_command,
        custom_logging_func_memaddr=func_memaddr,
        write_logfile=write_logfile,
    )

    # Remove and unload files. .
        remove_file(shared_lib_filename)

    if "parse_function" in kwargs:
        return kwargs["parse_function"](output)
    return output

"""
Module containing the Population grid class object.

Here all the functionality of a Population object is defined.

Useful for the user to understand the functionality,
but copying functionality isn't recommended except if you know what you are doing

Tasks:
    - TODO: add functionality to 'on-init' set arguments
    - TODO: add functionality to return the initial_abundance_hash
    - TODO: add functionality to return the isotope_hash
    - TODO: add functionality to return the isotope_list
    - TODO: add functionality to return the nuclear_mass_hash
    - TODO: add functionality to return the nuclear_mass_list
    - TODO: add functionality to return the source_list
    - TODO: add functionality to return the ensemble_list
    - TODO: consider spreading the functions over more files.
    - TODO: type the private functions
    - TODO: fix the correct object types for the default values of the bse_options
    - TODO: uncomment and implement the HPC functionality
    - TODO: think of a clean and nice way to unload and remove the custom_logging_info library from memory (and from disk)
    - TODO: think of a nice way to remove the loaded grid_code/ generator from memory. 
    - TODO: Create a designated dict for results
"""

import os
import sys
import datetime
import logging
import argparse
import importlib.util
from pathos.helpers import mp as pathos_multiprocess

from pathos.pools import _ProcessPool as Pool

from binarycpython.utils.grid_options_defaults import grid_options_defaults_dict
    autogen_C_logging_code,
    binary_c_log_code,
)
    get_defaults,
    parse_binary_c_version_info,
    filter_arg_dict,
    get_help_all,
    return_binary_c_version_info,
    binaryc_json_serializer,
    verbose_print,
    binarycDecoder,
    merge_dicts,
    BinaryCEncoder,
    extract_ensemble_json_from_string,
)

# from binarycpython.utils.hpc_functions import (
#     get_condor_version,
#     get_slurm_version,
#     create_directories_hpc,
#     path_of_calling_script,
#     get_python_details,
# )

class Population:
    """
    Population Object. Contains all the necessary functions to set up, run and process a
    population of systems
    """

    def __init__(self):
        """
        Initialisation function of the population class
        """

        # Different sections of options

        # get binary_c defaults and create a cleaned up dict
        # Setting stuff will check against the defaults to see if the input is correct.
        self.defaults = get_defaults()
        self.cleaned_up_defaults = self._cleanup_defaults()

        # make the input dictionary
        self.bse_options = {}  # bse_options is just empty.

        # Grid options
        self.grid_options = copy.deepcopy(grid_options_defaults_dict)

        # Custom options
        self.custom_options = {}

        # Argline dict
        self.argline_dict = {}

        # Set main process id
        self.grid_options["_main_pid"] = os.getpid()

        # Set some memory dicts
        self.persistent_data_memory_dict = {}

        #
        self.process_ID = 0

        # Create location to store results. Users should write to this dictionary. 
        self.grid_results = {}

        # Create location where ensemble results are written to
        self.grid_ensemble_results = {}

    ###################################################
    # Argument functions

    # General flow of generating the arguments for the binary_c call:
    # - user provides parameter and value via set (or manually but that is risky)
    # - The parameter names of these input get compared to the parameter names in the self.defaults;
    #    with this, we know that its a valid parameter to give to binary_c.
    # - For a single system, the bse_options will be written as a arg line
    # - For a population the bse_options will get copied to a temp_bse_options dict and updated with
    #   all the parameters generated by the grid

    # I will NOT create the argument line by fully writing ALL the defaults and overriding user
    # input, that seems not necessary because by using the get_defaults() function we already
    # know for sure which parameter names are valid for the binary_c version
    # And because binary_c uses internal defaults, its not necessary to explicitly pass them.
    # I do however suggest everyone to export the binary_c defaults to a file, so that you know
    # exactly which values were the defaults.

    def set(self, **kwargs) -> None:
        """
        Function to set the values of the population. This is the preferred method to set values
        of functions, as it provides checks on the input.

        the bse_options will get populated with all the those that have a key that is present
        in the self.defaults

        the grid_options will get updated with all the those that have a key that is present
        in the self.grid_options

        If neither of above is met; the key and the value get stored in a custom_options dict.

        Args:
            via kwargs all the arguments are either set to binary_c parameters, grid_options or custom_options (see above)
        """

        # Select the params that end with %d
        special_params = [el for el in list(self.defaults.keys()) if el.endswith("%d")]

        # Go over all the input
        for key in kwargs:
            # Filter out keys for the bse_options
            if key in self.defaults.keys():
                verbose_print(
                    "adding: {}={} to BSE_options".format(key, kwargs[key]),
                    self.grid_options["verbosity"],
                    1,
                )
                self.bse_options[key] = kwargs[key]

            # Extra check to check if the key fits one of parameter names that end with %d
            elif any(
                [
                    True
                    if (key.startswith(param[:-2]) and len(param[:-2]) < len(key))
                    else False
                    for param in special_params
                ]
            ):
                    "adding: {}={} to BSE_options by catching the %d".format(
                        key, kwargs[key]
                    ),
                    1,
                )

            # Filter out keys for the grid_options
            elif key in self.grid_options.keys():
                    "adding: {}={} to grid_options".format(key, kwargs[key]),
                    1,
                )
                self.grid_options[key] = kwargs[key]

            # The of the keys go into a custom_options dict
            else:
                print(
                    "!! Key doesnt match previously known parameter: \
                    adding: {}={} to custom_options".format(
                    )
                )
                self.custom_options[key] = kwargs[key]

    def parse_cmdline(self) -> None:
        """
        Function to handle settings values via the command line.
        Best to be called after all the .set(..) lines, and just before the .evolve() is called

        If you input any known parameter (i.e. contained in grid_options, defaults/bse_options or custom_options),
        this function will attempt to convert the input from string (because everything is string) to the type of
        the value that option had before.

        The values of the bse_options are initially all strings, but after user input they can change to ints.

        The value of any new parameter (which will go to custom_options) will be a string.

        Tasks:
            - TODO: remove the need for --cmdline
        """

        parser = argparse.ArgumentParser()
        parser.add_argument(
            "--cmdline",
            help='Setting values via the commandline. Input like --cmdline "metallicity=0.02"',
        )
        args = parser.parse_args()

        # How its set up now is that as input you need to give --cmdline "metallicity=0.002"
        # Its checked if this exists and handled accordingly.
        if args.cmdline:
                "Found cmdline args. Parsing them now",
                self.grid_options["verbosity"],
                1,
            )

            # Grab the input and split them up, while accepting only non-empty entries
            cmdline_args = args.cmdline
            self.grid_options["_commandline_input"] = cmdline_args
            split_args = [
                cmdline_arg
                for cmdline_arg in cmdline_args.split(" ")
                if not cmdline_arg == ""
            ]

            # Make dict and fill it
            cmdline_dict = {}
            for cmdline_arg in split_args:
                split = cmdline_arg.split("=")
                parameter = split[0]
                value = split[1]
                old_value_found = False

                # Find an old value
                if parameter in self.grid_options:
                    old_value = self.grid_options[parameter]
                    old_value_found = True

                elif parameter in self.defaults:
                    old_value = self.defaults[parameter]

                elif parameter in self.custom_options:
                    old_value = self.custom_options[parameter]

                # (attempt to) convert
                if old_value_found:
                    try:
                        verbose_print(
                            "Converting type of {} from {} to {}".format(
                                parameter, type(value), type(old_value)
                            ),
                            self.grid_options["verbosity"],
                            1,
                        )
                        value = type(old_value)(value)
                        verbose_print("Success!", self.grid_options["verbosity"], 1)

                    except ValueError as e:
                            "Tried to convert the given parameter {}/value {} to its correct type {} (from old value {}). But that wasn't possible.".format(
                                parameter, value, type(old_value), old_value
                            ),
                            0,
                        )

                # Add to dict
                cmdline_dict[parameter] = value

            # unpack the dictionary into the setting function that handles where the values are set
            self.set(**cmdline_dict)

    def _return_argline(self, parameter_dict=None):
        """
        Function to create the string for the arg line from a parameter dict
        """

        if not parameter_dict:
            parameter_dict = self.bse_options

        argline = "binary_c "

        for param_name in sorted(parameter_dict):
            argline += "{} {} ".format(param_name, parameter_dict[param_name])
        argline = argline.strip()
        return argline

    def add_grid_variable(
        self,
        name: str,
        longname: str,
        valuerange: Union[list, str],
        resolution: str,
        spacingfunc: str,
        probdist: str,
        dphasevol: str,
        parameter_name: str,
        precode: Union[str, None] = None,
        condition: Union[str, None] = None,
    ) -> None:
        """
        Function to add grid variables to the grid_options.

        TODO: Fix this complex function.

        The execution of the grid generation will be through a nested forloop.
        Each of the grid variables will get create a deeper for loop.

        The real function that generates the numbers will get written to a new file in the TMP_DIR, and then loaded imported and evaluated.
        beware that if you insert some destructive piece of code, it will be executed anyway. Use at own risk.

        Args:
            name:
                name of parameter. This is evaluated as a parameter and you can use it throughout the rest of the function
                example: name = 'lnm1'
            longname:
                Long name of parameter
                example: longname = 'Primary mass'
            range:
                Range of values to take. Does not get used really, the spacingfunction is used to get the values from
                example: range = [math.log(m_min), math.log(m_max)]
            resolution:
                Resolution of the sampled range (amount of samples). TODO: check if this is used anywhere
                example: resolution = resolution["M_1"]
            spacingfunction:
                Function determining how the range is sampled. You can either use a real function, or a string representation of a function call. Will get written to a file and then evaluated.
                example: spacingfunction = "const(math.log(m_min), math.log(m_max), {})".format(resolution['M_1'])
            precode:
                Extra room for some code. This code will be evaluated within the loop of the sampling function (i.e. a value for lnm1 is chosen already)
                example: precode = 'M_1=math.exp(lnm1);'
            probdist:
                FUnction determining the probability that gets asigned to the sampled parameter
                example: probdist = 'Kroupa2001(M_1)*M_1'
            dphasevol:
                part of the parameter space that the total probability is calculated with
                example: dphasevol = 'dlnm1'
            condition:
                condition that has to be met in order for the grid generation to continue
                example: condition = 'self.grid_options['binary']==1'
        """

        # Add grid_variable
        grid_variable = {
            "longname": longname,
            "valuerange": valuerange,
            "resolution": resolution,
            "spacingfunc": spacingfunc,
            "precode": precode,
            "probdist": probdist,
            "dphasevol": dphasevol,
            "parameter_name": parameter_name,
            "condition": condition,
            "grid_variable_number": len(self.grid_options["_grid_variables"]),
        }

        # Load it into the grid_options
        self.grid_options["_grid_variables"][grid_variable["name"]] = grid_variable
            "Added grid variable: {}".format(json.dumps(grid_variable, indent=4)),
            self.grid_options["verbosity"],
            1,
        )

    # Return functions

    def return_population_settings(self) -> dict:
        """
        Function that returns all the options that have been set.

        Can be combined with json to make a nice file.

        Returns:
            dictionary containing "bse_options", "grid_options", "custom_options"
        """

        options = {
            "bse_options": self.bse_options,
            "grid_options": self.grid_options,
            "custom_options": self.custom_options,
        }

        return options

    def _return_binary_c_version_info(self, parsed=False):
        """
        Function that returns the version information of binary_c
        """

        version_info = return_binary_c_version_info(parsed=parsed)

        return version_info

    def _return_binary_c_defaults(self):
        """
        Function that returns the defaults of the binary_c version that is used.
        """

        return self.defaults

    def return_all_info(
        self,
        include_population_settings: bool = True,
        include_binary_c_defaults: bool = True,
        include_binary_c_version_info: bool = True,
        include_binary_c_help_all: bool = True,
    ) -> dict:
        """
        Function that returns all the information about the population and binary_c

        Args:
            include_population_settings: whether to include the population_settings (see function return_population_settings)
            include_binary_c_defaults: whether to include a dict containing the binary_c parameters and their default values
            include_binary_c_version_info: whether to include a dict containing all the binary_c version info (see return_binary_c_version_info)
            include_binary_c_help_all: whether to include a dict containing all the information about the binary_c parameters (see get_help_all)

        Return:
            dictionary containing all, or part of, the above dictionaries
        """

        #
        all_info = {}

        #
        if include_population_settings:
            population_settings = self.return_population_settings()
            all_info["population_settings"] = population_settings

        #
        if include_binary_c_defaults:
            binary_c_defaults = self._return_binary_c_defaults()
            all_info["binary_c_defaults"] = binary_c_defaults

        if include_binary_c_version_info:
            binary_c_version_info = return_binary_c_version_info(parsed=True)
            all_info["binary_c_version_info"] = binary_c_version_info

        if include_binary_c_help_all:
            binary_c_help_all_info = get_help_all(print_help=False)
            all_info["binary_c_help_all"] = binary_c_help_all_info

        return all_info

    def export_all_info(
        self,
        use_datadir: bool = True,
        outfile: Union[str, None] = None,
    ) -> Union[str, None]:
        """
        Function that exports the all_info to a json file

        Tasks:
            - TODO: if any of the values in the dicts here is of a not-serializable form, then we need
                to change that to a string or something so, use a recursive function that goes over the
                all_info dict and finds those that fit
            - TODO: Fix to write things to the directory. which options do which etc
            - TODO: theres flawed logic here. rewrite this part pls
            - TODO: consider actually just removing the whole 'output to file' part and let the user do this.

        Args:
            use_datadir: boolean whether to use the custom_options['data_dir'] to write the file to. If the  custom_options["base_filename"] is set, the output file will be called <custom_options["base_filename"]>_settings.json. Otherwise a file called simulation_<date+time>_settings.json will be created
            outfile: if use_datadir is false, a custom filename will be used
        """

        all_info = self.return_all_info(
            include_population_settings=include_population_settings,
            include_binary_c_defaults=include_binary_c_defaults,
            include_binary_c_version_info=include_binary_c_version_info,
            include_binary_c_help_all=include_binary_c_help_all,
        )

        # Copy dict
        all_info_cleaned = copy.deepcopy(all_info)

        if use_datadir:
            if not self.custom_options.get("base_filename", None):
                base_name = "simulation_{}".format(
                    datetime.datetime.strftime(datetime.datetime.now(), "%Y%m%d_%H%M%S")
                )
            else:
                base_name = os.path.splitext(self.custom_options["base_filename"])[0]

            settings_name = base_name + "_settings.json"

            # Check directory, make if necessary
            os.makedirs(self.custom_options["data_dir"], exist_ok=True)

            settings_fullname = os.path.join(
                self.custom_options["data_dir"], settings_name
            )

                "Writing settings to {}".format(settings_fullname),
                1,
            )
            # if not outfile.endswith('json'):
            with open(settings_fullname, "w") as file:
                file.write(
                    json.dumps(
                        all_info_cleaned,
                        indent=4,
                        default=binaryc_json_serializer,
                    )
                )
            return settings_fullname
        else:
                "Writing settings to {}".format(outfile),
                1,
            )
            if not outfile.endswith("json"):
                    "Error: outfile ({}) must end with .json".format(outfile),
                    0,
                )

            with open(outfile, "w") as file:
                        all_info_cleaned, indent=4, default=binaryc_json_serializer
                    )
                )
            return outfile

    def _set_custom_logging(self):
        """
        Function/routine to set all the custom logging so that the function memory pointer
        is known to the grid.
        """

        # C_logging_code gets priority of C_autogen_code
            "Creating and loading custom logging functionality",
            1,
        )
        if self.grid_options["C_logging_code"]:
            # Generate entire shared lib code around logging lines
            custom_logging_code = binary_c_log_code(
                self.grid_options["C_logging_code"],
                verbose=self.grid_options["verbosity"],
            )

            # Load memory adress
            (
                self.grid_options["custom_logging_func_memaddr"],
                self.grid_options["_custom_logging_shared_library_file"],
            ) = create_and_load_logging_function(
                custom_logging_code, verbose=self.grid_options["verbosity"]
            )

        elif self.grid_options["C_auto_logging"]:
            # Generate real logging code
            logging_line = autogen_C_logging_code(
                self.grid_options["C_auto_logging"],
            )

                logging_line, verbose=self.grid_options["verbosity"]
            )

            (
            )

    # Ensemble functions

    # Now they are stored in the _process_run_population thing.
    # Needed less code since they all

    # Evolution functions

    def _pre_run_cleanup(self):
        """
        Function to clean up some stuff in the grid before a run (like results, ensemble results etc)
        """

        # empty results
        self.grid_options["results"] = {}

    def evolve(self) -> None:
        """
        Entrypoint function of the whole object. From here, based on the settings,
        we set up a SLURM or CONDOR grid, or if no setting is given we go straight
        to evolving the population

        There are no direct arguments to this function, rather it is based on the grid_options settings:
            grid_options['slurm']: integer boolean whether to use a slurm_grid evolution
            grid_options['condor']: integer boolean whether to use a condor_grid evolution

        If neither of the above is set, we continue without using HPC routines
        (that doesn't mean this cannot be run on a server with many cores)
        """

        # Just to make sure we don't have stuff from a previous run hanging around
        self._pre_run_cleanup()

        # Check which type:
        if self.grid_options["slurm"] == 1:
            # Execute slurm subroutines
            self._slurm_grid()

        elif self.grid_options["condor"] == 1:
            # Execute condor subroutines
            self._condor_grid()
        else:
            # Execute population evolution subroutines
            self.evolve_population()

        # Put all interesting stuff in a variable and output that afterwards, as analytics of the run.
        analytics_dict = {
            "population_name": self.grid_options["_population_id"],
            "evolution_type": self.grid_options["evolution_type"],
            "failed_count": self.grid_options["_failed_count"],
            "failed_prob": self.grid_options["_failed_prob"],
            "failed_systems_error_codes": self.grid_options[
                "_failed_systems_error_codes"
            ].copy(),
            "errors_exceeded": self.grid_options["_errors_exceeded"],
            "errors_found": self.grid_options["_errors_found"],
            "total_probability": self.grid_options["_probtot"],
            "total_count": self.grid_options["_count"],
            "start_timestamp": self.grid_options["_start_time_evolution"],
            "end_timestamp": self.grid_options["_end_time_evolution"],
        }

        ##
        # Clean up code: remove files, unset values. This is placed in the general evolve function, because that makes for easier control
        self._cleanup()

        return analytics_dict

    def evolve_population(self):
        """
        Function to evolve populations. This handles the setting up, evolving
        and cleaning up of a population of stars.

        Choices here are:
            to evolve a population via multiprocessing or linearly on 1 core.
            NOT IMPLEMENTED YET to evolve a population via a variable grid, a source file or MC

        Tasks:
            - TODO: include options for different ways of generating a population here. (i.e. MC or source file)
        """

        # Reset some settings: population_id, results, ensemble_results etc
        self.grid_options["_population_id"] = uuid.uuid4().hex

        ##
        # Prepare code/initialise grid.
        # set custom logging, set up store_memaddr, build grid code. dry run grid code.
        self._setup()

        ##
        # Evolve systems: via grid_options one can choose to do this linearly, or
        # multiprocessing method.
        if (
            self.grid_options["evolution_type"]
            in self.grid_options["_evolution_type_options"]
        ):
            if self.grid_options["evolution_type"] == "grid":
                self._evolve_population_grid()
            # elif self.grid_options["evolution_type"] == "mc":
            #     # TODO: add MC option
        else:
            print(
                "Warning. you chose a wrong option for the grid evolution types.\
                Please choose from the following: {}.".format(
                    self.grid_options["_evolution_type_options"]
                )
            )

        #
        self.grid_options["_end_time_evolution"] = time.time()

        # Log and print some information
            "Population-{} finished! The total probability was: {}. It took a total of {}s to run {} systems on {} cores".format(
                self.grid_options["_population_id"],
                self.grid_options["_probtot"],
                self.grid_options["_end_time_evolution"]
                - self.grid_options["_start_time_evolution"],
                self.grid_options["_total_starcount"],
                self.grid_options["amt_cores"],
            ),
            0,
        )

        if self.grid_options["_errors_found"]:
            # Some information afterwards
                "During the run {} failed systems were found, with a total probability of {} and with the following unique error codes: {} ".format(
                    self.grid_options["_failed_count"],
                    self.grid_options["_failed_prob"],
                    self.grid_options["_failed_systems_error_codes"],
                ),
                0,
            )
                "The full argline commands for {} these systems have been written to {}".format(
                    "ALL"
                    if not self.grid_options["_errors_exceeded"]
                    else "SOME (only the first ones, as there were too many to log all of them)",
                    os.path.join(
                        self.grid_options["tmp_dir"],
                        "failed_systems_{}_X.txt".format(
                            self.grid_options["_population_id"]
                        ),
                    ),
                ),
                0,
            )
        else:
                "There were no errors found in this run.",
                0,
            )

    def _evolve_population_grid(self):
        """
        Function to evolve the population with multiprocessing approach.
        Using pathos to be able to include class-owned functions.

        This function will create a pool with <self.grid_options["amt_cores"]> processes, and
        perform an imap_unordered to run the different `threads`.
        Before this was done by giving a generator as the iterable, and have the processes get a
        certain chunksize each round.
        Later on this seemed to be a bad decision, because it is difficult to pass information
        back to the main controller, and because with each new batch of systems a new object instance was created.

        What I do now is I spawn these X amount of processes, and pass a range(self.grid_options["amt_cores"]) as iterable.
        In that way, only once do they fetch a `job`, but that job is just a ID number.
        With this ID number each thread/process loops over the whole generator,
        but only runs the one <ID>'th system (if (localcounter+ID) % self.grid_options["amt_cores"]==0)'

        When they are finished, these jobs are instructed to return a set of information
        (the result dict, TODO: describe what more)

        These resultation dictionaries are then merged and stored as object properties again.
        """

        # TODO: make further use of a queue to handle jobs or at least
        #   get information on the process ids etc
        # https://stackoverflow.com/questions/10190981/get-a-unique-id-for-worker-in-python-multiprocessing-pool
        # https://stackoverflow.com/questions/8640367/python-manager-dict-in-multiprocessing/9536888
        #   for muting values through dicts
        # https://python-forum.io/Thread-Dynamic-updating-of-a-nested-dictionary-in-multiprocessing-pool
        # https://stackoverflow.com/questions/28740955/working-with-pathos-multiprocessing-tool-in-python-and

        # TODO: make good example of how to deal with a result_dict
        # https://www.programcreek.com/python/example/58176/multiprocessing.Value
        # https://stackoverflow.com/questions/17377426/shared-variable-in-pythons-multiprocessing

        # Create the pool
        pool = Pool(processes=self.grid_options["amt_cores"])

        # start the processes by giving them an ID value
        result = list(
            pool.imap_unordered(
                self._process_run_population_grid, range(self.grid_options["amt_cores"])
            )
        )

        # Handle clean termination of the whole multiprocessing (making sure there are no zombie
        # processes (https://en.wikipedia.org/wiki/Zombie_process))
        pool.close()
        pool.join()

        # Handle the results by merging all the dictionaries. How that merging happens exactly is
        # described in the merge_dicts description.
        combined_output_dict = {}
        for output_dict in result:
            combined_output_dict = merge_dicts(combined_output_dict, output_dict)

        # Put the values back as object properties
        self.grid_results = combined_output_dict["results"]
        self.grid_ensemble_results = combined_output_dict['ensemble_results']   # Ensemble results are also passed as output from that dictionary
        self.grid_options["_failed_count"] = combined_output_dict["_failed_count"]
        self.grid_options["_failed_prob"] = combined_output_dict["_failed_prob"]
        self.grid_options["_failed_systems_error_codes"] = list(
            set(combined_output_dict["_failed_systems_error_codes"])
        )
        self.grid_options["_errors_exceeded"] = combined_output_dict["_errors_exceeded"]
        self.grid_options["_errors_found"] = combined_output_dict["_errors_found"]
        self.grid_options["_probtot"] = combined_output_dict["_probtot"]
        self.grid_options["_count"] = combined_output_dict["_count"]

    def _evolve_system_mp(self, full_system_dict):
        """
        Function that the multiprocessing evolution method calls to evolve a system

        this function is called by _process_run_population_grid
        """

        binary_cmdline_string = self._return_argline(full_system_dict)
        print(binary_cmdline_string)

        persistent_data_memaddr = -1
        if self.bse_options.get("ensemble", 0) == 1:
            persistent_data_memaddr = self.persistent_data_memory_dict[self.process_ID]
            # print("thread {}: persistent_data_memaddr: ".format(self.process_ID), persistent_data_memaddr)

        # Get results binary_c
        out = _binary_c_bindings.run_system(
            argstring=binary_cmdline_string,
            custom_logging_func_memaddr=self.grid_options[
                "custom_logging_func_memaddr"
            ],
            store_memaddr=self.grid_options["_store_memaddr"],
            population=1,  # since this system is part of a population, we set this flag to prevent the store from being freed
            persistent_data_memaddr=persistent_data_memaddr,
        )

        # Check for errors
        _ = self._check_binary_c_error(out, full_system_dict)

        # Have some user-defined function do stuff with the data.
        if self.grid_options["parse_function"]:
            self.grid_options["parse_function"](self, out)

    def _process_run_population_grid(self, ID):
        """
        Function that loops over the whole generator, but only runs
        systems that fit to: if (localcounter+ID) % self.grid_options["amt_cores"] == 0

        That way with e.g. 4 processes, process 1 runs sytem 0, 4, 8... process 2 runs system 1, 5, 9..., etc

        This function is called by _evolve_population_grid
        """

        # set start timer
        start_process_time = datetime.datetime.now()

        self.process_ID = (
            ID  # Store the ID as a object property again, lets see if that works.
        )

        # lets try out making stores for all the grids:
        self.grid_options["_store_memaddr"] = _binary_c_bindings.return_store_memaddr()

            "Process {} started at {}.\tUsing store memaddr {}".format(ID, datetime.datetime.now().isoformat(), self.grid_options["_store_memaddr"]), self.grid_options["verbosity"], 0
        )

        # Set the ensemble memaddr
            # set persistent data memaddr if necessary.
            persistent_data_memaddr = (
                _binary_c_bindings.return_persistent_data_memaddr()
            )

            self.persistent_data_memory_dict = {
                self.process_ID: persistent_data_memaddr
            }

                "\tUsing persistent_data memaddr: {}".format(persistent_data_memaddr), self.grid_options["verbosity"], 0
            )

        # apparently we have to re-load this for every process, otherwise NameErrors arise (seems like a bug but I'm not sure)
        self._load_grid_function()

        # Set up generator
        generator = self.grid_options["_system_generator"](self, print_results=False)

        # Set up local variables
        running = True
        localcounter = (
            0  # global counter for the whole loop. (need to be ticked every loop)
        )
        probability_of_systems_run = (
            0  # counter for the probability of the actual systems this tread ran
        )
        number_of_systems_run = (
            0  # counter for the actual amt of systems this thread ran
        )

        round_number_mod = 0 # rotating modulo

        total_time_calling_binary_c = 0

        # Go over the generator
        while running:
            # round_number_mod = (localcounter+1)%self.grid_options["amt_cores"]

            try:
                # Get the system
                system = next(generator)

                # Check if the ID is the correct one for this process
                if (localcounter + (ID + round_number_mod)) % self.grid_options["amt_cores"] == 0:

                    # Combine that with the other settings
                    full_system_dict = self.bse_options.copy()
                    full_system_dict.update(system)

                    # self._print_info(
                    #     i + 1, self.grid_options["_total_starcount"], full_system_dict
                    # )

                    #
                    verbose_print(
                        "Process {} is handling system {}".format(ID, localcounter), self.grid_options["verbosity"], 2
                    )

                    # In some cases, the whole run crashes. To be able to figure out which system that was on, we log each current system to a file (each thread has one). Each new system overrides the previous
                    with open(os.path.join(self.grid_options["tmp_dir"], "thread_{}_current_system.txt".format(self.process_ID)), 'w') as f:
                        binary_cmdline_string = self._return_argline(full_system_dict)
                        f.write(binary_cmdline_string)

                    start_runtime_binary_c = time.time()

                    # Evolve the system
                    self._evolve_system_mp(full_system_dict)

                    end_runtime_binary_c = time.time()

                    total_time_calling_binary_c += end_runtime_binary_c - start_runtime_binary_c # keep track of total binary_c call time

                    # Debug line: logging all the lines
                    if self.grid_options['log_runtime_systems']==1:
                        with open(os.path.join(self.grid_options["tmp_dir"], "thread_{}_runtime_systems.txt".format(self.process_ID)), 'a+') as f:
                            binary_cmdline_string = self._return_argline(full_system_dict)
                            f.write("{} {}\n".format(end_runtime_binary_c-start_runtime_binary_c, binary_cmdline_string))

                    # Keep track of systems:
                    probability_of_systems_run += full_system_dict["probability"]
                    number_of_systems_run += 1

            except StopIteration:
                running = False

            if (localcounter+1)%self.grid_options["amt_cores"]==0:
                round_number_mod += 1

            # print("thread {} round_nr_mod {}. localcounter {}".format(ID, round_number_mod, localcounter))

            # Has to be here because this one is used for the (localcounter+ID) % (self..)
            localcounter += 1

        # Handle ensemble output: is ensemble==1, then either directly write that data to a file, or combine everything into 1 file.
        ensemble_json = {}  # Make sure it exists already
                "Process {}: is freeing ensemble output (using persisten_data memaddr {})".format(ID, self.persistent_data_memory_dict[self.process_ID]), self.grid_options["verbosity"], 2
            )

            ensemble_raw_output = (
                _binary_c_bindings.free_persistent_data_memaddr_and_return_json_output(
                    self.persistent_data_memory_dict[self.process_ID]
                )
            )
            if ensemble_raw_output == None:
                    "Process {}: Warning! Ensemble output is empty. ".format(ID), self.grid_options["verbosity"], 2
                )

            #
            if self.grid_options["combine_ensemble_with_thread_joining"] == True:
                    "Process {}: Extracting ensemble info from raw string".format(ID), self.grid_options["verbosity"], 2
                )

                ensemble_json['ensemble'] = extract_ensemble_json_from_string(
                    ensemble_raw_output
                )  # Load this into a dict so that we can combine it later

            else:
                # If we do not allow this, automatically we will export this to the data_dir, in some formatted way
                output_file = os.path.join(
                    self.custom_options["data_dir"],
                    "ensemble_output_{}_{}.json".format(
                        self.grid_options["_population_id"], self.process_ID
                    ),
                )

                # Write to file
                with open(output_file, "w") as f:
                    f.write(ensemble_raw_output)

                print(
                    "Thread {}: Wrote ensemble results directly to file: {}".format(
                        self.process_ID, output_file
                    )
                )

        # free store mem:
        _binary_c_bindings.free_store_memaddr(self.grid_options["_store_memaddr"])

        # Return a set of results and errors
        output_dict = {
            "results": self.grid_results,
            "ensemble_results": ensemble_json,
            "_failed_count": self.grid_options["_failed_count"],
            "_failed_prob": self.grid_options["_failed_prob"],
            "_failed_systems_error_codes": self.grid_options[
            ],
            "_errors_exceeded": self.grid_options["_errors_exceeded"],
            "_errors_found": self.grid_options["_errors_found"],
            "_probtot": probability_of_systems_run,
            "_count": number_of_systems_run,
        }

        end_process_time = datetime.datetime.now()

            "Process {} finished:\n\tgenerator started at {}, done at {} (total: {}s of which {}s interfacing with binary_c).\n\tRan {} systems with a total probability of {}.\n\tThis thread had {} failing systems with a total probability of {}".format(
                ID,
                start_process_time.isoformat(),
                end_process_time.isoformat(),
                (end_process_time-start_process_time).total_seconds(),
                total_time_calling_binary_c,
                number_of_systems_run,
                probability_of_systems_run,
                self.grid_options["_failed_count"],
                self.grid_options["_failed_prob"],
            ),
            0,
        )

        return output_dict

    # Single system
    def evolve_single(self, clean_up_custom_logging_files: bool = True) -> Any:
        """
        Function to run a single system, based on the settings in the grid_options

        The output of the run gets returned, unless a parse function is given to this function.

        Args:
            clean_up_custom_logging_files: whether the clean up all the custom_logging files.

        returns:
            either returns the raw binary_c output, or whatever the parse_function does
        """

        ### Custom logging code:
        self._set_custom_logging()

        # Get argument line
        argline = self._return_argline(self.bse_options)
        verbose_print("Running {}".format(argline), self.grid_options["verbosity"], 1)

        # Run system
            argstring=argline,
            ],
            population=0,
        )

        # TODO: add call to function that cleans up the temp customlogging dir,
        #   and unloads the loaded libraries.
        # TODO: make a switch to turn this off

        if clean_up_custom_logging_files:
            self._clean_up_custom_logging(evol_type="single")

        # Parse
            return self.grid_options["parse_function"](self, out)
        return out

    def _setup(self):
        """
        Function to set up the necessary stuff for the population evolution.

        The idea is to do all the stuff that is necessary for a population to run.
        Since we have different methods of running a population, this setup function
        will do different things depending on different settings

        Tasks:
            TODO: Make other kinds of populations possible. i.e, read out type of grid,
                and set up accordingly
            TODO: make this function more general. Have it explicitly set the system_generator
                function
        """

        if not self.grid_options["parse_function"]:
            print("Warning: No parse function set. Make sure you intended to do this.")

        # #######################
        # ### Custom logging code:

        ### Load store: Make sure this actually works.

        ### ensemble: make some checks for this
        ## check the settings and set all the warnings.
        if self.bse_options.get("ensemble", None):
            if not self.bse_options.get("ensemble_defer", 0) == 1:
                    "Error, if you want to run an ensemble in a population, the output needs to be deferred. Please set 'ensemble_defer' to 1",
                    0,
                )

            if not any(
                [key.startswith("ensemble_filter_") for key in self.bse_options]
            ):
                    "Warning: Running the ensemble without any filter requires alot of available RAM",
                    0,
                )

            if self.bse_options.get("ensemble_filters_off", 0) != 1:
                    0,
                )

            if self.grid_options["combine_ensemble_with_thread_joining"] == False:
                if not self.custom_options.get("data_dir", None):
                        "Error: chosen to write the ensemble output directly to files but data_dir isnt set",
                        self.grid_options["verbosity"],
                        0,
                    )
                    raise ValueError

        # Check which type of population generation
        if self.grid_options["evolution_type"] == "grid":
            #######################
            # Dry run and getting starcount
            self.grid_options["_probtot"] = 0

            # Put in check
            if len(self.grid_options["_grid_variables"]) == 0:
                print("Error: you havent defined any grid variables! Aborting")

            # Set up the grid code with a dry run option to see total probability
            self._generate_grid_code(dry_run=True)

            # Load the grid code
            self._load_grid_function()

            # Do a dry run
            self._dry_run()

            print(
                "Total starcount for this run will be: {}".format(
                    self.grid_options["_total_starcount"]
                )
            )

            # Reset values and prepare the grid function
            self.grid_options[
                "_probtot"
            ] = 0  # To make sure that the values are reset. TODO: fix this in a cleaner way
                "_start_time_evolution"
            ] = time.time()  # Setting start time of grid

            #
            self._generate_grid_code(dry_run=False)

            #

        # Source file
        elif self.grid_options["evolution_type"] == "source_file":

            # TODO: fix this function
            # self._load_source_file_function()

            self._dry_run_source_file()

            print(
                )
            )

            #
            # self._load_source_file_function(dry_run=False)

            #

        # elif self.grid_options["evolution_type"] == "mc":

        #######

    def _cleanup(self):
        """
        Function that handles all the cleaning up after the grid has been generated and/or run

        - reset values to 0
        - remove grid file
        - unload grid function/module
        - remove dry grid file
        - unload dry grid function/module
        """

        # Reset values
        self.grid_options["_count"] = 0
        self.grid_options["_probtot"] = 0
        self.grid_options["_system_generator"] = None
        self.grid_options["_failed_count"] = 0
        self.grid_options["_failed_prob"] = 0
        self.grid_options["_errors_found"] = False
        self.grid_options["_errors_exceeded"] = False
        self.grid_options["_failed_systems_error_codes"] = []

        # Remove files
        # TODO: remove files

        # Unload functions
        # TODO: unload functions

        # Unload store

        # Unload/free custom_logging_code
        # TODO: cleanup custom logging code.

    # Gridcode functions
    #
    # Function below are used to run populations with
    # a variable grid

    def _generate_grid_code(self, dry_run=False):
        """
        Function that generates the code from which the population will be made.

        dry_run: when True, it will return the starcount at the end so that we know
        what the total amount of systems is.

        The phasevol values are handled by generating a second array

        # TODO: Add correct logging everywhere
        # TODO: add part to handle separation if orbital_period is added. Idea. use default values
        #   for orbital parameters and possibly overwrite those or something.
        # TODO: add centering center left right for the spacing.
        # TODO: add sensible description to this function.
        # TODO: Check whether all the probability and phasevol values are correct.
        # TODO: import only the necessary packages/functions

        Results in a generated file that contains a system_generator function.
        """

        verbose_print("Generating grid code", self.grid_options["verbosity"], 1)

        # Some local values
        code_string = ""
        depth = 0
        indent = "    "
        total_grid_variables = len(self.grid_options["_grid_variables"])

        # Import packages
        code_string += "import math\n"
        code_string += "import numpy as np\n"
        code_string += "from binarycpython.utils.distribution_functions import *\n"
        code_string += "from binarycpython.utils.spacing_functions import *\n"
        code_string += "from binarycpython.utils.useful_funcs import *\n"
        code_string += "\n\n"

        # Make the function
        code_string += "def grid_code(self, print_results=True):\n"

        # Increase depth
        depth += 1

        # Write some info in the function
        code_string += (
            indent * depth
            + "# Grid code generated on {}\n".format(
                datetime.datetime.now().isoformat()
            )
            + indent * depth
            + "# This function generates the systems that will be evolved with binary_c\n\n"
        )

        # Set some values in the generated code:
        code_string += indent * depth + "# Setting initial values\n"
        code_string += indent * depth + "_total_starcount = 0\n"
        code_string += indent * depth + "starcounts = [0 for i in range({})]\n".format(
            total_grid_variables
        )
        code_string += indent * depth + "probabilities = {}\n"
            + "probabilities_list = [0 for i in range({})]\n".format(
                total_grid_variables
            )
        )
            + "probabilities_sum = [0 for i in range({})]\n".format(
            )
        )
        code_string += indent * depth + "parameter_dict = {}\n"
        code_string += indent * depth + "phasevol = 1\n"
        code_string += indent * depth + "\n"

        # Prepare the probability
        code_string += indent * depth + "# setting probability lists\n"
        for grid_variable_el in sorted(
            self.grid_options["_grid_variables"].items(),
            key=lambda x: x[1]["grid_variable_number"],
        ):
            # Make probabilities dict
            grid_variable = grid_variable_el[1]
            code_string += indent * depth + 'probabilities["{}"] = 0\n'.format(
                grid_variable["parameter_name"]
            )

        #################################################################################
        # Start of code generation
        # Generate code
        print("Generating grid code")
        for loopnr, grid_variable_el in enumerate(
            sorted(
                self.grid_options["_grid_variables"].items(),
                key=lambda x: x[1]["grid_variable_number"],
            )
        ):
            print("Constructing/adding: {}".format(grid_variable_el[0]))

            #################################################################################
            # Check condition and generate forloop

            # If the grid variable has a condition, write the check and the action
            if grid_variable["condition"]:
                # Add comment
                code_string += (
                    indent * depth
                    + "# Condition for {}".format(grid_variable["parameter_name"])
                    + "\n"
                )

                # Add condition check
                    + "if not {}:".format(grid_variable["condition"])
                    + "\n"
                )

                # Add condition failed action:
                    indent * (depth + 1)
                    + 'print("Condition for {} not met!")'.format(
                        grid_variable["parameter_name"]
                    )
                    + "\n"
                )
                code_string += indent * (depth + 1) + "raise ValueError" + "\n"

                # Add some whiteline
                code_string += indent * (depth + 1) + "\n"

            #########################
            # Setting up the forloop
            # Add comment for forloop
            code_string += (
                indent * depth
                + "# for loop for {}".format(grid_variable["parameter_name"])
                + "\n"
            )

                + "sampled_values_{} = {}".format(
                    grid_variable["name"], grid_variable["spacingfunc"]
                )
                + "\n"
            )

            # TODO: Make clear that the phasevol only works good
            # TODO: add option to ignore this phasevol calculation and set it to 1
            #   if you sample linearly in that thing.
                + "phasevol_{} = sampled_values_{}[1]-sampled_values_{}[0]".format(
                    grid_variable["name"], grid_variable["name"], grid_variable["name"]
                )
                + "\n"
            )

            # # Some print statement
            # code_string += (
            #     indent * depth
            #     + "print('phasevol_{}:', phasevol_{})".format(grid_variable["name"],
            #           grid_variable["name"])
            #     + "\n"
            # )

            # Adding for loop structure
                + "for {} in sampled_values_{}:".format(
                    grid_variable["name"], grid_variable["name"]
                )
                + "\n"
            )

            # Setting up pre-code and value in some cases
            # Add pre-code
            if grid_variable["precode"]:
                    + "{}".format(
                        grid_variable["precode"].replace(
                            "\n", "\n" + indent * (depth + 1)
                        )
                    )
                    + "\n"
                )

            # Set phasevol
            code_string += indent * (depth + 1) + "phasevol *= phasevol_{}\n".format(
                grid_variable["name"],
            )

            # Probabilities
            # Calculate probability
            code_string += indent * (depth + 1) + "\n"
            code_string += indent * (depth + 1) + "# Setting probabilities\n"
                indent * (depth + 1)
                + "d{} = phasevol_{} * {}".format(
                    grid_variable["name"],
                    grid_variable["probdist"],
                )
                + "\n"
            )

            # Saving probability sum
                + "probabilities_sum[{}] += d{}".format(
                    grid_variable["grid_variable_number"], grid_variable["name"]
                )
                + "\n"
            )

            if grid_variable["grid_variable_number"] == 0:
                    + "probabilities_list[0] = d{}".format(grid_variable["name"])
                    + "\n"
                )
            else:
                    + "probabilities_list[{}] = probabilities_list[{}] * d{}".format(
                        grid_variable["grid_variable_number"],
                        grid_variable["grid_variable_number"] - 1,
                        grid_variable["name"],
                    )
                    + "\n"
                )

            # Increment starcount for this parameter
            code_string += "\n"
            code_string += indent * (
                depth + 1
            ) + "# Increment starcount for {}\n".format(grid_variable["parameter_name"])

                + "starcounts[{}] += 1".format(
                    grid_variable["grid_variable_number"],
                )
                + "\n"
            )

            # Add value to dict
                + 'parameter_dict["{}"] = {}'.format(
                    grid_variable["parameter_name"], grid_variable["parameter_name"]
                )
                + "\n"
            )

            # Add some space

            # The final parts of the code, where things are returned, are within the deepest loop,
            # but in some cases code from a higher loop needs to go under it again
            # SO I think its better to put an ifstatement here that checks
            # whether this is the last loop.
            if loopnr == len(self.grid_options["_grid_variables"]) - 1:

                #################################################################################
                # Here are the calls to the queuing or other solution. this part is for every system
                code_string += indent * (depth + 1) + "#" * 40 + "\n"
                    + "# Code below will get evaluated for every generated system\n"
                )

                # Calculate value
                    + 'probability = self.grid_options["weight"] * probabilities_list[{}]'.format(
                        grid_variable["grid_variable_number"]
                    )
                    + "\n"
                )
                # TODO: ask rob if just replacing this with probability is enough
                    # + 'repeat_probability = probability / self.grid_options["repeat"]'
                    + 'probability = probability / self.grid_options["repeat"]'
                    + "\n"
                )

                # For each repeat of the system this has to be done yes.
                    + 'for _ in range(self.grid_options["repeat"]):'
                    + "\n"
                )

                code_string += indent * (depth + 2) + "_total_starcount += 1\n"

                # set probability and phasevol values
                    indent * (depth + 2)
                    + 'parameter_dict["{}"] = {}'.format("probability", "probability")
                    + "\n"
                )
                    + 'parameter_dict["{}"] = {}'.format("phasevol", "phasevol")
                    + "\n"
                )

                # Some prints. will be removed
                # code_string += indent * (depth + 1) + "print(probabilities)\n"
                # code_string += (
                #     indent * (depth + 1) + 'print("_total_starcount: ", _total_starcount)\n'
                # )

                # code_string += indent * (depth + 1) + "print(probability)\n"

                # Increment total probability
                    indent * (depth + 2) + "self._increment_probtot(probability)\n"
                )

                if not dry_run:
                    # Handling of what is returned, or what is not.
                    # TODO: think of whether this is a good method
                    code_string += indent * (depth + 2) + "yield(parameter_dict)\n"

                    # The below solution might be a good one to add things to specific queues
                    # $self->queue_evolution_code_run($self->{_flexigrid}->{thread_q},
                    # $system);

                # If its a dry run, dont do anything with it
                else:
                    code_string += indent * (depth + 2) + "pass\n"

            # increment depth
            depth += 1

        depth -= 1
        code_string += "\n"
        # Write parts to write below the part that yield the results.
        # this has to go in a reverse order:
        # Here comes the stuff that is put after the deepest nested part that calls returns stuff.
            sorted(
                reverse=True,
            )
        ):
            code_string += indent * (depth + 1) + "#" * 40 + "\n"
                + "# Code below is for finalising the handling of this iteration of the parameter\n"
            )

            # TODO: fix. this isnt supposed to be the value that we give it here. discuss
            code_string += indent * (depth + 1) + "phasevol /= phasevol_{}\n".format(
                grid_variable["name"]
            )

            depth -= 1

        ################
        # Finalising print statements
        #
        # code_string += indent * (depth + 1) + "\n"
        code_string += indent * (depth + 1) + "#" * 40 + "\n"
        code_string += indent * (depth + 1) + "if print_results:\n"
            indent * (depth + 2)
            + "print('Grid has handled {} stars'.format(_total_starcount))\n"
        )
            + "print('with a total probability of {}'.format(self.grid_options['_probtot']))\n"
        )

        # Finalising return statement for dry run.
        #
        if dry_run:
            code_string += indent * (depth + 1) + "return _total_starcount\n"

        # Stop of code generation. Here the code is saved and written

        # Save the gridcode to the grid_options
            "Saving grid code to grid_options", self.grid_options["verbosity"], 1
        )

        self.grid_options["code_string"] = code_string

        # Write to file
        gridcode_filename = os.path.join(
            self.grid_options["tmp_dir"],
            "binary_c_grid_{}.py".format(self.grid_options["_population_id"]),
        )
        self.grid_options["gridcode_filename"] = gridcode_filename

            "Writing grid code to {}".format(gridcode_filename),
            1,
        )

        with open(gridcode_filename, "w") as file:
            file.write(code_string)

    def _load_grid_function(self):
        """
        Functon that loads the script containing the grid code.

        TODO: Update this description
        Test function to run grid stuff. mostly to test the import
        """

        # Code to load the
            message="Loading grid code function from {}".format(
                self.grid_options["gridcode_filename"]
            ),
            verbosity=self.grid_options["verbosity"],
            minimal_verbosity=1,
        )

        spec = importlib.util.spec_from_file_location(
            "binary_c_python_grid",
            os.path.join(self.grid_options["gridcode_filename"]),
        )
        grid_file = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(grid_file)
        generator = grid_file.grid_code

        self.grid_options["_system_generator"] = generator

        verbose_print("Grid code loaded", self.grid_options["verbosity"], 1)

    def _dry_run(self):
        """
        Function to dry run the grid and know how many stars it will run

        Requires the grid to be built as a dry run grid
        """

        system_generator = self.grid_options["_system_generator"]
        total_starcount = system_generator(self)
        self.grid_options["_total_starcount"] = total_starcount

    def _print_info(self, run_number, total_systems, full_system_dict):
        """
        Function to print info about the current system and the progress of the grid.

        # color info tricks from https://ozzmaker.com/add-colour-to-text-in-python/
        https://stackoverflow.com/questions/287871/how-to-print-colored-text-in-terminal-in-python
        """

        # Define frequency
        if self.grid_options["verbosity"] == 1:
            print_freq = 1
        else:
            print_freq = 10

        # Calculate amount of time left

        # calculate amount of time passed
        # time_passed = time.time() - self.grid_options["_start_time_evolution"]

        if run_number % print_freq == 0:
            binary_cmdline_string = self._return_argline(full_system_dict)
            info_string = "{color_part_1} \
            {text_part_1}{end_part_1}{color_part_2} \
            {text_part_2}{end_part_2}".format(
                color_part_1="\033[1;32;41m",
                text_part_1="{}/{}".format(run_number, total_systems),
                end_part_1="\033[0m",
                color_part_2="\033[1;32;42m",
                text_part_2="{}".format(binary_cmdline_string),
                end_part_2="\033[0m",
            )
            print(info_string)

    # Montecarlo functions
    #
    # Functions below are used to run populations with
    # Monte carlo

    # Population from file functions
    #
    # Functions below are used to run populations from
    # a file containing binary_c calls

    def _dry_run_source_file(self):
        """
        Function to go through the source_file and count the amount of lines and the total probability
        """

        total_starcount = 0
        total_probability = 0

        contains_probability = False

        for line in system_generator:
            total_starcount += 1

    def _load_source_file(self, check=False):
        """
        Function that loads the source_file that contains a binary_c calls
        """

        if not os.path.isfile(self.grid_options["source_file_filename"]):
            verbose_print("Source file doesnt exist", self.grid_options["verbosity"], 0)

            message="Loading source file from {}".format(
            ),
        )

        # We can choose to perform a check on the sourcefile, which checks if the lines start with 'binary_c'
        if check:
            source_file_check_filehandle = open(
                self.grid_options["source_file_filename"], "r"
            )
            for line in source_file_check_filehandle:
                if not line.startswith("binary_c"):
                    failed = True
                    break
            if failed:
                    "Error, sourcefile contains lines that do not start with binary_c",
                    0,
                )

        source_file_filehandle = open(self.grid_options["source_file_filename"], "r")

        self.grid_options["_system_generator"] = source_file_filehandle

        verbose_print("Source file loaded", self.grid_options["verbosity"], 1)

    def _dict_from_line_source_file(self, line):
        """
        Function that creates a dict from a binary_c argline
        """

        if line.startswith("binary_c "):
            line = line.replace("binary_c ", "")

        arg_dict = {}

        for i in range(0, len(split_line), 2):
            if "." in split_line[i + 1]:
                arg_dict[split_line[i]] = float(split_line[i + 1])
            else:
                arg_dict[split_line[i]] = int(split_line[i + 1])

        return arg_dict

    # SLURM functions
    #
    # subroutines to run SLURM grids

    # def _slurm_grid(self):
    #     """
    #     Main function that manages the SLURM setup.

    #     Has three stages:

    #     - setup
    #     - evolve
    #     - join

    #     Which stage is used is determined by the value of grid_options['slurm_command']:

    #     <empty>: the function will know its the user that executed the script and
    #     it will set up the necessary condor stuff

    #     'evolve': evolve_population is called to evolve the population of stars

    #     'join': We will attempt to join the output
    #     """

    #     # Check version
    #     # TODO: Put in function
    #     slurm_version = get_slurm_version()
    #     if not slurm_version:
    #         verbose_print(
    #             "SLURM: Error: No installation of slurm found",
    #             self.grid_options["verbosity"],
    #             0,
    #         )
    #     else:
    #         major_version = int(slurm_version.split(".")[0])
    #         minor_version = int(slurm_version.split(".")[1])

    #         if major_version > 17:
    #             verbose_print(
    #                 "SLURM: Found version {} which is new enough".format(slurm_version),
    #                 self.grid_options["verbosity"],
    #                 1,
    #             )
    #         else:
    #                 "SLURM: Found version {} which is too old (we require 17+)".format(
    #                     slurm_version
    #                 ),
    #                 0,
    #             )

    #     verbose_print(
    #         "SLURM: Running slurm grid. command={}".format(
    #             self.grid_options["slurm_command"]
    #         ),
    #         self.grid_options["verbosity"],
    #         1,
    #     )

    #     if not self.grid_options["slurm_command"]:
    #         # Setting up
    #             "SLURM: Main controller script. Setting up",
    #             1,
    #         )

    #         # Set up working directories:
    #             "SLURM: creating working directories", self.grid_options["verbosity"], 1
    #         )
    #         create_directories_hpc(self.grid_options["slurm_dir"])

    #         # Create command
    #         python_details = get_python_details()
    #         scriptname = path_of_calling_script()
    #         command = "{} {}".format(python_details["executable"], scriptname)
    #         command += ' --cmdline "{}"'.format(
    #             " ".join(
    #                 [
    #                     "{}".format(self.grid_options["_commandline_input"]),
    #                     "offset=$jobarrayindex",
    #                     "modulo={}".format(self.grid_options["slurm_njobs"]),
    #                     "vb={}".format(self.grid_options["verbosity"]),
    #                     "slurm_jobid=$jobid",
    #                     "slurm_jobarrayindex=$jobarrayindex",
    #                     "slurm_jobname='binary_grid_'$jobid'.'$jobarrayindex",
    #                     "slurm_njobs={}".format(self.grid_options["slurm_njobs"]),
    #                     "slurm_dir={}".format(self.grid_options["slurm_dir"]),
    #                     "rungrid=1",
    #                     "slurm_command=evolve",
    #                 ]
    #             ).strip()
    #         )

    #         # Construct dict with settings for the script while checking the settings at the same time
    #         # Check settings:
    #         # TODO: check settings
    #         # Create SLURM_DIR script:
    #         slurm_script_options = {}
    #         slurm_script_options["n"] = self.grid_options["slurm_njobs"]
    #         slurm_script_options["njobs"] = self.grid_options["slurm_njobs"]
    #         slurm_script_options["dir"] = self.grid_options["slurm_dir"]
    #         slurm_script_options["memory"] = self.grid_options["slurm_memory"]
    #         slurm_script_options["working_dir"] = self.grid_options[
    #             "slurm_dir"
    #         ]  # TODO: check this
    #         slurm_script_options["command"] = command
    #         # slurm_script_options['streams'] = self.grid_options['streams']

    #         # Construct the script
    #         slurm_script_contents = ""
    #         slurm_script_contents += "#!/bin/bash\n"
    #         slurm_script_contents += "# Slurm file for binary_grid and slurm\n"
    #         slurm_script_contents += "#SBATCH --error={}/stderr/%A.%a\n".format(
    #             self.grid_options["slurm_dir"]
    #         )
    #         slurm_script_contents += "#SBATCH --output={}/stdout/%A.%a\n".format(
    #         )
    #         slurm_script_contents += "#SBATCH --job-name={}\n".format(
    #             self.grid_options["slurm_jobname"]
    #         )
    #         slurm_script_contents += "#SBATCH --partition={}\n".format(
    #             self.grid_options["slurm_partition"]
    #         )
    #         slurm_script_contents += "#SBATCH --time={}\n".format(
    #             self.grid_options["slurm_time"]
    #         )
    #         slurm_script_contents += "#SBATCH --mem={}\n".format(
    #             self.grid_options["slurm_memory"]
    #         )
    #         slurm_script_contents += "#SBATCH --ntasks={}\n".format(
    #             self.grid_options["slurm_ntasks"]
    #         )
    #         slurm_script_contents += "#SBATCH --array={}\n".format(
    #             self.grid_options["slurm_array"]
    #         )
    #         slurm_script_contents += "\n"

    #         if self.grid_options["slurm_extra_settings"]:
    #             slurm_script_contents += "# Extra settings by user:"
    #             slurm_script_contents += "\n".join(
    #                 [
    #                     "--{}={}".format(
    #                         key, self.grid_options["slurm_extra_settings"][key]
    #                     )
    #                     for key in self.grid_options["slurm_extra_settings"]
    #                 ]
    #             )

    #         slurm_script_contents += '# set status to "running"\n'
    #         slurm_script_contents += (
    #             'echo "running" > {}/status/$jobid.$jobarrayindex\n\n'.format(
    #                 self.grid_options["slurm_dir"]
    #             )
    #         )
    #         slurm_script_contents += "# run grid of stars\n"
    #         slurm_script_contents += "{}\n\n".format(command)
    #         slurm_script_contents += '# set status to "finished"\n'
    #             'echo "finished" > {}/status/$jobid.$jobarrayindex\n'.format(
    #             )
    #         )

    #         if self.grid_options["slurm_postpone_join"]:
    #             slurm_script_contents += "{} rungrid=0 results_hash_dumpfile={}/results/$jobid.all slurm_command=join\n".format(
    #                 command, self.grid_options["slurm_dir"]
    #             )

    #         # Write script to file
    #         slurm_script_filename = os.path.join(
    #             self.grid_options["slurm_dir"], "slurm_script"
    #         )
    #         with open(slurm_script_filename, "w") as slurm_script_file:
    #             slurm_script_file.write(slurm_script_contents)

    #         # Execute or postpone
    #         if self.grid_options["slurm_postpone_sbatch"]:
    #             # Execute or postpone the real call to sbatch
    #             sbatch_command = "sbatch {}".format(slurm_script_filename)
    #                 "running slurm script {}".format(slurm_script_filename),
    #                 0,
    #             )
    #             # subprocess.Popen(sbatch_command, close_fds=True)
    #             # subprocess.Popen(sbatch_command, creationflags=subprocess.DETACHED_PROCESS)
    #             verbose_print("Submitted scripts.", self.grid_options["verbosity"], 0)
    #         else:
    #                 "Slurm script is in {} but hasnt been executed".format(
    #                     slurm_script_filename
    #                 ),
    #                 0,
    #             )

    #         verbose_print("all done!", self.grid_options["verbosity"], 0)
    #         exit()

    #     elif self.grid_options["slurm_command"] == "evolve":
    #         # Part to evolve the population.
    #         # TODO: decide how many CPUs
    #             "SLURM: Evolving population", self.grid_options["verbosity"], 1
    #         )

    #         #
    #         self.evolve_population()

    #     elif self.grid_options["slurm_command"] == "join":
    #         # Joining the output.
    #         verbose_print("SLURM: Joining results", self.grid_options["verbosity"], 1)

    # CONDOR functions
    #
    # subroutines to run CONDOR grids

    #     def _condor_grid(self):
    #         """
    #         Main function that manages the CONDOR setup.

    #         Has three stages:

    #         - setup
    #         - evolve
    #         - join

    #         Which stage is used is determined by the value of grid_options['condor_command']:

    #         <empty>: the function will know its the user that executed the script and
    #         it will set up the necessary condor stuff

    #         'evolve': evolve_population is called to evolve the population of stars

    #         'join': We will attempt to join the output
    #         """

    #         # TODO: Put in function
    #         condor_version = get_condor_version()
    #         if not condor_version:
    #                 "CONDOR: Error: No installation of condor found",
    #                 0,
    #             )
    #         else:
    #             major_version = int(condor_version.split(".")[0])
    #             minor_version = int(condor_version.split(".")[1])

    #             if (major_version == 8) and (minor_version > 4):
    #                 verbose_print(
    #                     "CONDOR: Found version {} which is new enough".format(
    #                         condor_version
    #                     ),
    #                     self.grid_options["verbosity"],
    #                     0,
    #                 )
    #             elif major_version > 9:
    #                     ),
    #                     0,
    #                 )
    #             else:
    #                     "CONDOR: Found version {} which is too old (we require 8.3/8.4+)".format(
    #                     ),
    #                     0,
    #                 )

    #             "Running Condor grid. command={}".format(
    #                 self.grid_options["condor_command"]
    #             ),
    #             1,
    #         )
    #         if not self.grid_options["condor_command"]:
    #             # Setting up
    #                 "CONDOR: Main controller script. Setting up",
    #                 1,
    #             )

    #             # Set up working directories:
    #                 "CONDOR: creating working directories",
    #                 1,
    #             )
    #             create_directories_hpc(self.grid_options["condor_dir"])

    #             # Create command
    #             current_workingdir = os.getcwd()
    #             python_details = get_python_details()
    #             scriptname = path_of_calling_script()
    #             # command = "".join([
    #             #     "{}".python_details['executable'],
    #             #     "{}".scriptname,
    #             #     "offset=$jobarrayindex",
    #             #     "modulo={}".format(self.grid_options['condor_njobs']),
    #             #     "vb={}".format(self.grid_options['verbosity'])

    #             #      "results_hash_dumpfile=$self->{_grid_options}{slurm_dir}/results/$jobid.$jobarrayindex",
    #             #      'slurm_jobid='.$jobid,
    #             #      'slurm_jobarrayindex='.$jobarrayindex,
    #             #      'slurm_jobname=binary_grid_'.$jobid.'.'.$jobarrayindex,
    #             #      "slurm_njobs=$njobs",
    #             #      "slurm_dir=$self->{_grid_options}{slurm_dir}",
    #             # );

    #             # Create directory with info for the condor script. By creating this directory we also check whether all the values are set correctly
    #             # TODO: create the condor script.
    #             condor_script_options = {}
    #             # condor_script_options['n'] =
    #             condor_script_options["njobs"] = self.grid_options["condor_njobs"]
    #             condor_script_options["dir"] = self.grid_options["condor_dir"]
    #             condor_script_options["memory"] = self.grid_options["condor_memory"]
    #             condor_script_options["working_dir"] = self.grid_options[
    #                 "condor_working_dir"
    #             ]
    #             condor_script_options["command"] = self.grid_options["command"]
    #             condor_script_options["streams"] = self.grid_options["streams"]

    #             # TODO: condor works with running an executable.

    #             # Create script contents
    #             condor_script_contents = ""
    #             condor_script_contents += """
    # #################################################
    # #
    # # Condor script to run a binary_grid via python
    # #
    # """
    #             condor_script_contents += "Executable\t= {}".format(executable)
    #             condor_script_contents += "arguments\t= {}".format(arguments)
    #             condor_script_contents += "environment\t= {}".format(environment)
    #             condor_script_contents += "universe\t= {}".format(
    #                 self.grid_options["condor_universe"]
    #             )
    #             condor_script_contents += "\n"
    #             condor_script_contents += "output\t= {}/stdout/$id\n".format(
    #                 self.grid_options["condor_dir"]
    #             )
    #             condor_script_contents += "error\t={}/sterr/$id".format(
    #             )
    #             condor_script_contents += "log\t={}\n".format(
    #             )
    #             condor_script_contents += "initialdir\t={}\n".format(current_workingdir)
    #             condor_script_contents += "remote_initialdir\t={}\n".format(
    #                 current_workingdir
    #             )
    #             condor_script_contents += "steam_output\t={}".format(stream)
    #             condor_script_contents += "steam_error\t={}".format(stream)
    #             condor_script_contents += "+WantCheckpoint = False"
    #             condor_script_contents += "request_memory\t={}".format(
    #                 self.grid_options["condor_memory"]
    #             )
    #             condor_script_contents += "ImageSize\t={}".format(
    #             )

    #             if self.grid_options["condor_extra_settings"]:
    #                 slurm_script_contents += "# Extra settings by user:"
    #                 slurm_script_contents += "\n".join(
    #                     [
    #                         "{}\t={}".format(
    #                             key, self.grid_options["condor_extra_settings"][key]
    #                         )
    #                         for key in self.grid_options["condor_extra_settings"]
    #                     ]
    #                 )

    #             #   request_memory = $_[0]{memory}
    #             #   ImageSize = $_[0]{memory}

    #             #   Requirements = (1) \&\& (".
    #             #   $self->{_grid_options}{condor_requirements}.")\n";

    #             #
    #             # file name:  my_program.condor
    #             # Condor submit description file for my_program
    #             # Executable      = my_program
    #             # Universe        = vanilla
    #             # Error           = logs/err.$(cluster)
    #             # Output          = logs/out.$(cluster)
    #             # Log             = logs/log.$(cluster)

    #             # should_transfer_files = YES
    #             # when_to_transfer_output = ON_EXIT
    #             # transfer_input_files = files/in1,files/in2

    #             # Arguments       = files/in1 files/in2 files/out1
    #             # Queue

    #             # Write script contents to file
    #             if self.grid_options["condor_postpone_join"]:
    #                 condor_script_contents += "{} rungrid=0 results_hash_dumpfile={}/results/$jobid.all condor_command=join\n".format(
    #                     command, self.grid_options["condor_dir"]
    #                 )

    #             condor_script_filename = os.path.join(
    #                 self.grid_options["condor_dir"], "condor_script"
    #             )
    #             with open(condor_script_filename, "w") as condor_script_file:
    #                 condor_script_file.write(condor_script_contents)

    #             if self.grid_options["condor_postpone_sbatch"]:
    #                 # Execute or postpone the real call to sbatch
    #                 submit_command = "condor_submit {}".format(condor_script_filename)
    #                     "running condor script {}".format(condor_script_filename),
    #                     0,
    #                 )
    #                 # subprocess.Popen(sbatch_command, close_fds=True)
    #                 # subprocess.Popen(sbatch_command, creationflags=subprocess.DETACHED_PROCESS)
    #                 verbose_print("Submitted scripts.", self.grid_options["verbosity"], 0)
    #             else:
    #                     "Condor script is in {} but hasnt been executed".format(
    #                         condor_script_filename
    #                     ),
    #                     0,
    #                 )

    #             verbose_print("all done!", self.grid_options["verbosity"], 0)
    #             exit()

    #         elif self.grid_options["condor_command"] == "evolve":
    #             # TODO: write this function
    #             # Part to evolve the population.
    #             # TODO: decide how many CPUs
    #                 "CONDOR: Evolving population", self.grid_options["verbosity"], 1
    #             )

    #             #
    #             self.evolve_population()

    #         elif self.grid_options["condor_command"] == "join":
    #             # Joining the output.
    #             verbose_print("CONDOR: Joining results", self.grid_options["verbosity"], 1)

    #             pass

    # Unordered functions
    #
    # Functions that arent ordered yet

    def write_binary_c_calls_to_file(
        self,
        output_dir: Union[str, None] = None,
        output_filename: Union[str, None] = None,
        include_defaults: bool = False,
    ) -> None:
        """
        Function that loops over the gridcode and writes the generated parameters to a file.
        In the form of a commandline call

        Only useful when you have a variable grid as system_generator. MC wouldnt be that useful

        Also, make sure that in this export there are the basic parameters
        like m1,m2,sep, orb-per, ecc, probability etc.

        On default this will write to the datadir, if it exists

        WARNING; dont use yet. not fully tested.

        Tasks:
            - TODO: test this function

        Args:
            output_dir: (optional, default = None) directory where to write the file to. If custom_options['data_dir'] is present, then that one will be used first, and then the output_dir
            output_filename: (optional, default = None) filename of the output. If not set it will be called "binary_c_calls.txt"
            include_defaults: (optional, default = None) whether to include the defaults of binary_c in the lines that are written. Beware that this will result in very long lines, and it might be better to just export the binary_c defaults and keep them in a seperate file.
        """

        # Check if there is no compiled grid yet. If not, lets try to build it first.
        if not self.grid_options["_system_generator"]:

            ## check the settings:
            if self.bse_options.get("ensemble", None):
                if self.bse_options["ensemble"] == 1:
                    if not self.bse_options.get("ensemble_defer", 0) == 1:
                            "Error, if you want to run an ensemble in a population, the output needs to be deferred",
                            0,
                        )
                        raise ValueError

            #

            #

        # then if the _system_generator is present, we go through it
        if self.grid_options["_system_generator"]:
            # Check if there is an output dir configured
            if self.custom_options.get("data_dir", None):
                binary_c_calls_output_dir = self.custom_options["data_dir"]
            # otherwise check if theres one passed to the function
            else:
                if not output_dir:
                    print(
                        "Error. No data_dir configured and you gave no output_dir. Aborting"
                    )
                binary_c_calls_output_dir = output_dir

            # check if theres a filename passed to the function
            if output_filename:
                binary_c_calls_filename = output_filename
            # otherwise use default value
            else:
                binary_c_calls_filename = "binary_c_calls.txt"

            binary_c_calls_full_filename = os.path.join(
                binary_c_calls_output_dir, binary_c_calls_filename
            )
            print("Writing binary_c calls to {}".format(binary_c_calls_full_filename))

            # Write to file
            with open(binary_c_calls_full_filename, "w") as file:
                # Get defaults and clean them, then overwrite them with the set values.
                if include_defaults:
                    # TODO: make sure that the defaults here are cleaned up properly
                    cleaned_up_defaults = self.cleaned_up_defaults
                    full_system_dict = cleaned_up_defaults.copy()
                    full_system_dict.update(self.bse_options.copy())
                else:

                for system in self.grid_options["_system_generator"](self):
                    # update values with current system values

                    binary_cmdline_string = self._return_argline(full_system_dict)
                    file.write(binary_cmdline_string + "\n")
        else:
            print("Error. No grid function found!")
            raise ValueError

    def _cleanup_defaults(self):
        """
        Function to clean up the default values:

        from a dictionary, removes the entries that have the following values:
        - "NULL"
        - ""
        - "Function"

        Uses the function from utils.functions

        TODO: Rethink this functionality. seems a bit double, could also be just outside of the class
        """

        binary_c_defaults = self._return_binary_c_defaults().copy()
        cleaned_dict = filter_arg_dict(binary_c_defaults)

        return cleaned_dict

    def _clean_up_custom_logging(self, evol_type):
        """
        Function to clean up the custom logging.
        Has two types:
            'single':
                - removes the compiled shared library
                    (which name is stored in grid_options['_custom_logging_shared_library_file'])
                - TODO: unloads/frees the memory allocated to that shared library
                    (which is stored in grid_options['custom_logging_func_memaddr'])
                - sets both to None
            'multiple':
                - TODO: make this and design this
        """

        if evol_type == "single":
                "Cleaning up the custom logging stuff. type: single",
                1,
            )

            # TODO: Unset custom logging code

            # TODO: Unset function memory adress
            # print(self.grid_options["custom_logging_func_memaddr"])

            # remove shared library files
            if self.grid_options["_custom_logging_shared_library_file"]:
                remove_file(
                    self.grid_options["_custom_logging_shared_library_file"],
                )

        if evol_type == "population":
                "Cleaning up the custom logging stuffs. type: population",
                1,
            )

            # TODO: make sure that these also work. not fully sure if necessary tho.
            #   whether its a single file, or a dict of files/memaddresses

        if evol_type == "MC":
            pass

    def _increment_probtot(self, prob):
        """
        Function to add to the total probability. For now not used
        """

        self.grid_options["_probtot"] += prob

    def _increment_count(self):
        """
        Function to add to the total amount of stars. For now not used
        """
        self.grid_options["_count"] += 1

    def _set_loggers(self):
        """
        Function to set the loggers for the execution of the grid
        """

        # Set logfile
        binary_c_logfile = self.grid_options["log_file"]

        # Create directory
        os.makedirs(os.path.dirname(binary_c_logfile), exist_ok=True)

        # Set up logger
        self.logger = logging.getLogger("binary_c_python_logger")
        self.logger.setLevel(self.grid_options["verbosity"])

        # Reset handlers
        self.logger.handlers = []

        # Set formatting of output
        log_formatter = logging.Formatter(
            "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
        )

        # Make and add filehandlers
        # make handler for output to file
        handler_file = logging.FileHandler(filename=os.path.join(binary_c_logfile))
        handler_file.setFormatter(log_formatter)
        handler_file.setLevel(logging.INFO)

        # Make handler for output to stdout
        handler_stdout = logging.StreamHandler(sys.stdout)
        handler_stdout.setFormatter(log_formatter)
        handler_stdout.setLevel(logging.INFO)

        # Add the loggers
        self.logger.addHandler(handler_file)
        self.logger.addHandler(handler_stdout)

    # def join_result_dicts(self):
    #     """
    #     Function to join the result dictionaries
    #     """

    def _check_binary_c_error(self, binary_c_output, system_dict):
        """
        Function to check whether binary_c throws an error and handle accordingly.
        """

        if binary_c_output:
            if (binary_c_output.splitlines()[0].startswith("SYSTEM_ERROR")) or (binary_c_output.splitlines()[-1].startswith("SSYSTEM_ERROR")):
                    "FAILING SYSTEM FOUND",
                    0,
                )

                # Keep track of the amount of failed systems and their error codes
                self.grid_options["_failed_prob"] += system_dict["probability"]
                self.grid_options["_failed_count"] += 1
                self.grid_options["_errors_found"] = True

                # Try catching the error code and keep track of the unique ones.
                try:
                    error_code = int(
                        binary_c_output.splitlines()[0]
                        .split("with error code")[-1]
                        .split(":")[0]
                        .strip()
                    )

                    if not error_code in self.grid_options["_failed_systems_error_codes"]:
                        self.grid_options["_failed_systems_error_codes"].append(error_code)
                except ValueError:
                        "Failed to extract the error-code",
                        1,
                    )

                # Check if we have exceeded the amount of errors
                if (
                    self.grid_options["_failed_count"]
                    > self.grid_options["failed_systems_threshold"]
                ):
                    if not self.grid_options["_errors_exceeded"]:
                            "Process {} exceeded the maximum ({}) amount of failing systems. Stopped logging them to files now".format(
                                self.process_ID,
                                self.grid_options["failed_systems_threshold"],
                            ),
                            1,
                        )
                        self.grid_options["_errors_exceeded"] = True

                # If not, write the failing systems to files unique to each process
                else:
                    # Write arglines to file
                    argstring = self._return_argline(system_dict)
                    with open(
                        os.path.join(
                            self.grid_options["tmp_dir"],
                            "failed_systems_{}_process_{}.txt".format(
                                self.grid_options["_population_id"], self.process_ID
                            ),
                        ),
                        "a+",
                    ) as f:
                        f.write(argstring + "\n")
        else:
                "binary_c had 0 output. Which is strange", self.grid_options['verbosity'], 2)

################################################################################################