Busy with loading the shared lib. got some errors about buffered_printf

TODO.org

@@ -131,14 +131,21 @@ I get this error when I am using the master version of binary_c with either bran
 
 That went very deep haha. alot of memory allocation stuff
 
-*** TODO Make sure this works with the last major release of binaryc
+*** DONE Make sure this works with the last major release of binaryc
+    CLOSED: [2019-11-08 Fri 15:00]
 *** DONE Finish testing a simpler case (see other repo)
     CLOSED: [2019-11-08 Fri 09:37]
-*** TODO Make master master work
-*** TODO Sync master with david_branch
-*** TODO make tag of old master branch for future reference
-*** TODO finish today
-
+*** DONE Make master master work
+    CLOSED: [2019-11-08 Fri 15:00]
+*** DONE Sync master with david_branch
+    CLOSED: [2019-11-08 Fri 15:00]
+*** DONE make tag of old master branch for future reference
+    CLOSED: [2019-11-08 Fri 15:00]
+*** TODO Implement the autogeneration of the library
+*** TODO Load all the things with the c-types
+*** TODO Implement new function for run_binary_with_custom_logging
+*** TODO Make new c function run_binary_with_custom_logging
+*** TODO Put in some new tests in the python test api
 ** General:
 *** DONE Get a more reliable way of loading the default values (running a ./tbse echo or something?)
     CLOSED: [2019-10-29 Tue 17:44]

binary_c_python.c

@@ -286,5 +286,3 @@ static PyObject* binary_c_return_arglines(PyObject *self, PyObject *args)
     return return_string;
 }
 
-
-

binaryc_python_utils/custom_logging_functions.py

+import os
+import textwrap
+import subprocess
+import socket
+
+# Functions for the automatic logging of stuff
+def autogen_C_logging_code(logging_dict):
+    # See example_perl.pm autologging
+    """
+    Function that autogenerates PRINTF statements for binaryc
+
+    Input:
+        dictionary where the key is the header of that logging line and items which are lists of parameters that will be put in that logging line
+
+        example: {'MY_STELLAR_DATA': 
+        [
+            'model.time',
+            'star[0].mass',
+            'model.probability',
+            'model.dt'
+        ']}
+    """
+
+    # Check if the input is of the correct form 
+    if not type(logging_dict)==dict:
+        print("Error: please use a dictionary as input")
+        return None
+
+    code = ''
+    # Loop over dict keys
+    for key in logging_dict:
+        logging_dict_entry = logging_dict[key]
+
+        # Check if item is of correct type:
+        if type(logging_dict_entry)==list:
+
+            # Construct print statement
+            code += 'Printf("{}'.format(key)
+            code += ' {}'.format('%g '*len(logging_dict_entry))
+            code = code.strip()
+            code += '\\n"'
+
+            # Add format keys
+            for param in logging_dict_entry:
+                code += ',((double)stardata->{})'.format(param)
+            code += ');\n'
+
+        else:
+            print('Error: please use a list for the list of parameters that you want to have logged')
+    code = code.strip()
+    # print("MADE AUTO CODE\n\n{}\n\n{}\n\n{}\n".format('*'*60, repr(code), '*'*60))
+
+    return code
+
+####################################################################################
+def binary_c_log_code(code):
+    """
+    Function to construct the code to construct the custom logging function
+    # see example_perl.pm binary_c_log_code in perl
+    """
+    custom_logging_function_string = """\
+#pragma push_macro(\"MAX\")
+#pragma push_macro(\"MIN\")
+#undef MAX
+#undef MIN
+#include \"binary_c.h\"
+
+void custom_output_function(struct stardata_t * stardata);
+void custom_output_function(struct stardata_t * stardata)
+{{
+    // struct stardata_t * stardata = (struct stardata_t *)x;
+    {};
+}}
+
+#undef MAX 
+#undef MIN
+#pragma pop_macro(\"MIN\")
+#pragma pop_macro(\"MAX\")\
+    """.format(code)
+     
+    # print(repr(textwrap.dedent(custom_logging_function_string)))
+    return textwrap.dedent(custom_logging_function_string)
+
+def binary_c_write_log_code(code, filename):
+    """
+    Function to write the generated logging code to a file
+    """
+
+    cwd = os.getcwd()
+
+    filePath = os.path.join(cwd, filename)
+    if os.path.exists(filePath):
+        try:
+            os.remove(filePath)
+        except:
+            print("Error while deleting file {}".format(filePath))
+
+    with open(filePath, 'w') as f:
+        f.write(code)
+
+def from_binary_c_config(config_file, flag):
+    """
+    Function to run the binaryc_config command with flags
+    """
+
+    res = subprocess.check_output('{config_file} {flag}'.format(config_file=config_file, flag=flag),
+        shell=True, stderr=subprocess.STDOUT)
+
+    # convert and chop off newline
+    res = res.decode('utf').rstrip()
+    return res
+
+def return_compilation_dict():
+    """
+    Function to build the compile command for the shared library
+
+    inspired by binary_c_inline_config command in perl
+
+    TODO: this function still has some cleaning up to do wrt default values for the compile command
+    # https://developers.redhat.com/blog/2018/03/21/compiler-and-linker-flags-gcc/
+
+
+    returns:
+     - string containing the command to build the shared library
+    """    
+
+    # use binary_c-config to get necessary flags
+    BINARY_C_DIR = os.getenv('BINARY_C')
+    if BINARY_C_DIR:
+        BINARY_C_CONFIG = os.path.join(BINARY_C_DIR, 'binary_c-config')
+        BINARY_C_SRC_DIR = os.path.join(BINARY_C_DIR, 'src')
+        # TODO: build in check to see whether the file exists
+    else:
+        raise NameError('Envvar BINARY_C doesnt exist')
+        return None
+
+    # TODO: make more options for the compiling
+    cc = from_binary_c_config(BINARY_C_CONFIG, 'cc')
+
+    # Check for binary_c
+    BINARY_C_EXE = os.path.join(BINARY_C_DIR, 'binary_c')
+    if not os.path.isfile(BINARY_C_EXE):
+        print("We require  binary_c executable; have you built binary_c?")
+        raise NameError('BINARY_C executable doesnt exist')
+
+    # TODO: debug
+    libbinary_c = '-lbinary_c'
+    binclibs = from_binary_c_config(BINARY_C_CONFIG, 'libs')
+    libdirs = "{} -L{}".format(from_binary_c_config(BINARY_C_CONFIG, 'libdirs'), BINARY_C_SRC_DIR)
+    bincflags = from_binary_c_config(BINARY_C_CONFIG, 'cflags')
+    bincincdirs = from_binary_c_config(BINARY_C_CONFIG, 'incdirs')
+
+    # combine 
+    binclibs = ' {} {} {}'.format(libdirs, libbinary_c, binclibs)
+
+    # setup defaults:
+    defaults = {
+        'cc': 'gcc', # default compiler
+        'ccflags': bincflags,
+        'ld': 'ld',         # 'ld': $Config{ld}, # default linker
+        'debug': 0,
+        'inc': '{} -I{}'.format(bincincdirs, BINARY_C_SRC_DIR),
+        # inc => ' '.($Config{inc}//' ').' '.$bincincdirs." -I$srcdir ", # include the defaults plus # GSL and binary_c
+        # 'libname': libname, # libname is usually just binary_c corresponding to libbinary_c.so
+        'libs': binclibs,
+    }
+
+    # set values with defaults. TODO: make other input possile.
+    ld = defaults['ld']
+    debug = defaults['debug']
+    inc = defaults['inc'] # = ($ENV{BINARY_GRID2_INC} // $defaults{inc}).' '.($ENV{BINARY_GRID2_EXTRAINC} // '');
+    libs = defaults['libs'] # = ($ENV{BINARY_GRID2_LIBS} // $defaults{libs}).' '.($ENV{BINARY_GRID2_EXTRALIBS}//'');
+    ccflags = defaults['ccflags'] #  = $ENV{BINARY_GRID2_CCFLAGS} // ($defaults{ccflags}) . ($ENV{BINARY_GRID2_EXTRACCFLAGS} // '');
+  
+    # you must define _SEARCH_H to prevent it being loaded twice
+    ccflags += ' -shared -D_SEARCH_H'
+
+    # ensure library paths to the front of the libs:
+    libs_content = libs.split(' ')
+    library_paths = [el for el in libs_content if el.startswith('-L')]
+    non_library_paths = [el for el in libs_content if (not el.startswith('-L') and not el=='')]
+    libs = "{} {}".format(' '.join(library_paths), ' '.join(non_library_paths))
+
+    print("Building shared library for custom logging with (binary_c.h) at {} on {}\n".format(BINARY_C_SRC_DIR, socket.gethostname()))
+    print("With options:\n\tcc = {cc}\n\tccflags = {ccflags}\n\tld = {ld}\n\tlibs = {libs}\n\tinc = {inc}\n\n".format(
+        cc=cc, ccflags=ccflags, ld=ld, libs=libs, inc=inc)
+    )
+
+    return {
+        'cc': cc,
+        'ld': ld,
+        'ccflags': ccflags,
+        'libs': libs,
+        'inc': inc
+        }
+
+def compile_shared_lib(code, sourcefile_name, outfile_name):
+    """
+    Function to write the custom logging code to a file and then compile it.
+    """
+
+    # Write code to file
+    binary_c_write_log_code(code, sourcefile_name)
+
+    # create compilation command
+    compilation_dict = return_compilation_dict()
+
+    # Construct full command
+    command = "{cc} {ccflags} {libs} -o {outfile_name} {sourcefile_name} {inc}".format(
+        cc=compilation_dict['cc'],
+        ccflags=compilation_dict['ccflags'],
+        libs=compilation_dict['libs'],
+        outfile_name=outfile_name,
+        sourcefile_name=sourcefile_name,
+        inc=compilation_dict['inc'])
+
+    # remove extra whitespaces:
+    command = ' '.join(command.split())
+
+    # Execute compilation
+    print('Executing following command:\n{command}'.format(command=command))
+    res = subprocess.check_output('{command}'.format(command=command),
+        shell=True)
+
+    if res:
+        print('Output of compilation command:\n{}'.format(res))
\ No newline at end of file

binaryc_python_utils/example_perl.pm

-# Autologging
-# Perl code for autogeneration
-sub autogen_C_logging_code
-{
-    # given a hash of arrays of variable names, where the hash
-    # key is the header, autogenerate PRINTF statements
-    my ($self) = @_;
-    my $code = undef;
-    if(defined $self->{_grid_options}->{C_auto_logging} &&
-       ref $self->{_grid_options}->{C_auto_logging} eq 'HASH'
-        )
-    {
-        $code = '';
-
-        foreach my $header (keys %{$self->{_grid_options}->{C_auto_logging}})
-        {
-            if(ref $self->{_grid_options}->{C_auto_logging}->{$header} eq 'ARRAY')
-            {
-                $code .= 'PRINTF("'.$header.' ';
-                foreach my $x (@{$self->{_grid_options}->{C_auto_logging}->{$header}})
-                {
-                    $code .= '%g ';
-                }
-                $code .= '\n"';
-
-                foreach my $x (@{$self->{_grid_options}->{C_auto_logging}->{$header}})
-                {
-                    $code .= ',((double)stardata->'.$x.')';
-                }
-                $code .= ');'
-            }
-        }
-    }
-    print "MADE AUTO CODE \n\n************************************************************\n\n$code\n\n************************************************************\n";
-
-    return $code;
-}
-
-Which is used in flexi-grid via this:
-$population->set(    C_auto_logging => {
-        'MY_STELLAR_DATA' =>
-            [
-             'model.time',
-             'star[0].mass',
-             'model.probability',
-             'model.dt'
-            ]
-    });
-
-# binary_c_log_code
-sub binary_c_log_code
-{
-    my ($code) = @_;
-    return "
-#pragma push_macro(\"MAX\")
-#pragma push_macro(\"MIN\")
-#undef MAX
-#undef MIN
-#include \"binary_c.h\"
-
-void custom_output_function(SV * x);
-SV * custom_output_function_pointer(void);
-
-SV * custom_output_function_pointer()
-{
-    /*
-     * use PTR2UV to convert the function pointer 
-     * &custom_output_function to an unsigned int,
-     * which is then converted to a Perl SV
-     */
-    return (SV*)newSVuv(PTR2UV(&custom_output_function));
-}
-
-void custom_output_function(SV * x)
-{
-    struct stardata_t * stardata = (struct stardata_t *)x;
-    $code;
-}
-#undef MAX 
-#undef MIN
-#pragma pop_macro(\"MIN\")
-#pragma pop_macro(\"MAX\")
-";
-}
-
-And then to use it via 
-    $population->set(
-        C_logging_code => '
-             PRINTF("MY_STELLAR_DATA %g %g %g %g\n",
-                 stardata->model.time,
-                 stardata->star[0].mass,
-                 stardata->model.probability,
-                 stardata->model.dt);
-                       '
-    );

binaryc_python_utils/functions.py

@@ -134,8 +134,6 @@ def parse_output(output, selected_header):
         print('Sorry, didnt find any line matching your header {}'.format(selected_header))
         return None
 
-
-
     keys = value_dicts[0].keys()
 
     # Construct final dict.

custom_logging.c

+#pragma push_macro("MAX")
+#pragma push_macro("MIN")
+#undef MAX
+#undef MIN
+#include "binary_c.h"
+
+void custom_output_function(struct stardata_t * stardata);
+void custom_output_function(struct stardata_t * stardata)
+{
+    // struct stardata_t * stardata = (struct stardata_t *)x;
+    Printf("MY_STELLAR_DATA %g %g\n",((double)stardata->model.time),((double)stardata->star[0].mass));
+Printf("my_sss2 %g %g\n",((double)stardata->model.time),((double)stardata->star[1].mass));;
+}
+
+#undef MAX 
+#undef MIN
+#pragma pop_macro("MIN")
+#pragma pop_macro("MAX")    
\ No newline at end of file

testing.py

+import ctypes
+
+from binaryc_python_utils.custom_logging_functions import autogen_C_logging_code, binary_c_log_code, compile_shared_lib
+
+
+# generate logging lines
+logging_line = autogen_C_logging_code(
+    {
+        'MY_STELLAR_DATA': ['model.time', 'star[0].mass'], 
+        'my_sss2': ['model.time', 'star[1].mass']
+    }
+)
+
+# Generate code around logging lines
+created_code = binary_c_log_code(logging_line)
+
+# 
+compile_shared_lib(created_code, sourcefile_name='custom_logging.c', outfile_name='libcustom_logging.so')
+
+# Loading library
+libmean = ctypes.CDLL("libcustom_logging.so") # loads the shared library
+
+# Get memory adress of function. mimicking a pointer
+mem = ctypes.cast(libmean.custom_output_function, ctypes.c_void_p).value
\ No newline at end of file