Make legacy scopmath solvers header only, drop kinderiv.py and dimplic.cpp (#809)

* Solvers used for derivimplicit, sparse and euler methods were
   compiled as a separate library. These solvers were called from
   generated .mod to .cpp files.
* But as GPU programming models don't support function pointers,
   we use a very convoluted way via macros (+ _kinderiv.py) to achieve
   similar functionality as function pointers. This also needed to use
   `acc routine seq` for all the solver routines. But this is not compatible
   when we want to support shared libraries for python wheels (see #141)
* With this PR we make all solvers inline-able by making them part of headers
* Finally drop kinderiv.py !
* Drop acc routine seq
* Reallocate net receive buffers using ecalloc_align/free_memory for consistency.
* fix-up test, update submodules
* Make the NetReceiveBuffer initial size at least 8.

Author: olupton

coreneuron/CMakeLists.txt

@@ -31,9 +31,8 @@ file(
   "utils/*.cpp"
   "utils/*/*.c"
   "utils/*/*.cpp")
-set(SCOPMATH_CODE_FILES
-    "sim/scopmath/abort.cpp" "sim/scopmath/crout_thread.cpp" "sim/scopmath/newton_thread.cpp"
-    "sim/scopmath/sparse_thread.cpp" "sim/scopmath/ssimplic_thread.cpp")
+set(SCOPMATH_CODE_FILES "sim/scopmath/abort.cpp" "sim/scopmath/crout_thread.cpp"
+                        "sim/scopmath/newton_thread.cpp")
 set(MPI_LIB_FILES "mpi/lib/mpispike.cpp" "mpi/lib/nrnmpi.cpp")
 set(MPI_CORE_FILES "mpi/core/nrnmpi_def_cinc.cpp")
 if(CORENRN_ENABLE_MPI)
@@ -44,19 +43,11 @@ file(COPY ${CORENEURON_PROJECT_SOURCE_DIR}/external/Random123/include/Random123
      DESTINATION ${CMAKE_BINARY_DIR}/include)
 list(APPEND CORENEURON_CODE_FILES ${PROJECT_BINARY_DIR}/coreneuron/config/config.cpp)
 
-set(DIMPLIC_CODE_FILE "mechanism/mech/dimplic.cpp")
 set(ENGINEMECH_CODE_FILE "mechanism/mech/enginemech.cpp")
 
 # for external mod files we need to generate modl_ref function in mod_func.c
 set(MODFUNC_PERL_SCRIPT "mechanism/mech/mod_func.c.pl")
 
-# ~~~
-# as openacc limits passing function pointers as arguments, mod2c translated files
-# depend on a _kinderiv.h file that is constructed by kinderiv.py
-# ~~~
-set(KINDERIV_PYTHON_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/kinderiv.py")
-set(KINDERIV_HEADER_FILE "${CMAKE_CURRENT_BINARY_DIR}/_kinderiv.h")
-
 set(NMODL_UNITS_FILE "${CMAKE_BINARY_DIR}/share/mod2c/nrnunits.lib")
 
 # =============================================================================
@@ -66,23 +57,12 @@ cpp_cc_build_time_copy(
   INPUT "${CMAKE_CURRENT_SOURCE_DIR}/${MODFUNC_PERL_SCRIPT}"
   OUTPUT "${CMAKE_BINARY_DIR}/share/coreneuron/mod_func.c.pl"
   NO_TARGET)
-cpp_cc_build_time_copy(
-  INPUT "${KINDERIV_PYTHON_SCRIPT}"
-  OUTPUT "${CMAKE_BINARY_DIR}/share/coreneuron/kinderiv.py"
-  NO_TARGET)
-cpp_cc_build_time_copy(
-  INPUT "${CMAKE_CURRENT_SOURCE_DIR}/${DIMPLIC_CODE_FILE}"
-  OUTPUT "${CMAKE_BINARY_DIR}/share/coreneuron/dimplic.cpp"
-  NO_TARGET)
 cpp_cc_build_time_copy(
   INPUT "${CMAKE_CURRENT_SOURCE_DIR}/${ENGINEMECH_CODE_FILE}"
   OUTPUT "${CMAKE_BINARY_DIR}/share/coreneuron/enginemech.cpp"
   NO_TARGET)
-set(nrnivmodl_core_dependencies
-    "${CMAKE_BINARY_DIR}/share/coreneuron/mod_func.c.pl"
-    "${CMAKE_BINARY_DIR}/share/coreneuron/kinderiv.py"
-    "${CMAKE_BINARY_DIR}/share/coreneuron/dimplic.cpp"
-    "${CMAKE_BINARY_DIR}/share/coreneuron/enginemech.cpp")
+set(nrnivmodl_core_dependencies "${CMAKE_BINARY_DIR}/share/coreneuron/mod_func.c.pl"
+                                "${CMAKE_BINARY_DIR}/share/coreneuron/enginemech.cpp")
 # Set up build rules that copy builtin mod files from
 # {source}/coreneuron/mechanism/mech/modfile/*.mod to {build_dir}/share/modfile/
 file(GLOB builtin_modfiles
@@ -150,17 +130,6 @@ set_property(
   APPEND
   PROPERTY COMPILE_DEFINITIONS "CORENRN_USE_LEGACY_UNITS=${CORENRN_USE_LEGACY_UNITS}")
 
-# =============================================================================
-# run KINDERIV_PYTHON_SCRIPT to generate _kinderiv.h
-# =============================================================================
-add_custom_command(
-  OUTPUT "${KINDERIV_HEADER_FILE}"
-  DEPENDS ${NMODL_INBUILT_MOD_OUTPUTS} "${KINDERIV_PYTHON_SCRIPT}"
-  COMMAND ${PYTHON_EXECUTABLE} ${KINDERIV_PYTHON_SCRIPT} ${CMAKE_CURRENT_BINARY_DIR}
-  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-  COMMENT "Generating kinderiv.h by inspecting MOD files")
-add_custom_target(kin_deriv_header DEPENDS "${KINDERIV_HEADER_FILE}")
-
 # =============================================================================
 # create libraries
 # =============================================================================
@@ -275,9 +244,7 @@ set_target_properties(coreneuron PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
 # Suppress some compiler warnings. Note in GPU builds this library includes CUDA files.
 target_compile_options(coreneuron
                        PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${CORENEURON_CXX_WARNING_SUPPRESSIONS}>)
-
-# need to have _kinderiv.h for mod2c generated files and nrnivmodl-core and nmodl building
-add_dependencies(coreneuron kin_deriv_header nrnivmodl-core)
+add_dependencies(coreneuron nrnivmodl-core)
 
 # scopmath is created separately for nrnivmodl-core workflow
 add_library(scopmath STATIC ${CORENEURON_HEADER_FILES} ${SCOPMATH_CODE_FILES})
@@ -400,8 +367,7 @@ install(
   PATTERN "*.h*"
   PATTERN "*.ipp"
   PATTERN "*.ispc")
-install(FILES ${KINDERIV_PYTHON_SCRIPT} ${MODFUNC_PERL_SCRIPT} ${DIMPLIC_CODE_FILE}
-              ${ENGINEMECH_CODE_FILE} DESTINATION share/coreneuron)
+install(FILES ${MODFUNC_PERL_SCRIPT} ${ENGINEMECH_CODE_FILE} DESTINATION share/coreneuron)
 
 # copy mod2c/nmodl for nrnivmodl-core
 install(PROGRAMS ${CORENRN_MOD2CPP_BINARY} DESTINATION bin)

coreneuron/apps/corenrn_parameters.cpp

@@ -1,6 +1,6 @@
 /*
 # =============================================================================
-# Copyright (c) 2016 - 2021 Blue Brain Project/EPFL
+# Copyright (c) 2016 - 2022 Blue Brain Project/EPFL
 #
 # See top-level LICENSE file for details.
 # =============================================================================.
@@ -184,13 +184,21 @@ void corenrn_parameters::parse(int argc, char** argv) {
                   << std::endl;
         app.exit(e);
         throw e;
-
     } catch (const CLI::ParseError& e) {
         // use --help is also ParseError; in this case exit by showing all options
         app.exit(e);
         exit(0);
     }
 
+#ifndef CORENEURON_ENABLE_GPU
+    if (gpu) {
+        std::cerr
+            << "Error: GPU support was not enabled at build time but GPU execution was requested."
+            << std::endl;
+        exit(42);
+    }
+#endif
+
     // is user has asked for version info, print it and exit
     if (show_version) {
         std::cout << "CoreNEURON Version : " << cnrn_version() << std::endl;

coreneuron/gpu/nrn_acc_manager.cpp

@@ -1,6 +1,6 @@
 /*
 # =============================================================================
-# Copyright (c) 2016 - 2021 Blue Brain Project/EPFL
+# Copyright (c) 2016 - 2022 Blue Brain Project/EPFL
 #
 # See top-level LICENSE file for details.
 # =============================================================================
@@ -626,44 +626,50 @@ void realloc_net_receive_buffer(NrnThread* nt, Memb_list* ml) {
         cnrn_target_delete(nrb->_nrb_index, nrb->_size);
     }
 #endif
-
-    // Reallocate host
+    // Reallocate host buffers using ecalloc_align (as in phase2.cpp) and
+    // free_memory (as in nrn_setup.cpp)
+    auto const realloc = [old_size = nrb->_size, nrb](auto*& ptr, std::size_t extra_size = 0) {
+        using T = std::remove_pointer_t<std::remove_reference_t<decltype(ptr)>>;
+        static_assert(std::is_trivial<T>::value,
+                      "Only trivially constructible and copiable types are supported.");
+        static_assert(std::is_same<decltype(ptr), T*&>::value,
+                      "ptr should be reference-to-pointer");
+        auto* const new_data = static_cast<T*>(ecalloc_align((nrb->_size + extra_size), sizeof(T)));
+        std::memcpy(new_data, ptr, (old_size + extra_size) * sizeof(T));
+        free_memory(ptr);
+        ptr = new_data;
+    };
     nrb->_size *= 2;
-    nrb->_pnt_index = (int*) erealloc(nrb->_pnt_index, nrb->_size * sizeof(int));
-    nrb->_weight_index = (int*) erealloc(nrb->_weight_index, nrb->_size * sizeof(int));
-    nrb->_nrb_t = (double*) erealloc(nrb->_nrb_t, nrb->_size * sizeof(double));
-    nrb->_nrb_flag = (double*) erealloc(nrb->_nrb_flag, nrb->_size * sizeof(double));
-    nrb->_displ = (int*) erealloc(nrb->_displ, (nrb->_size + 1) * sizeof(int));
-    nrb->_nrb_index = (int*) erealloc(nrb->_nrb_index, nrb->_size * sizeof(int));
-
+    realloc(nrb->_pnt_index);
+    realloc(nrb->_weight_index);
+    realloc(nrb->_nrb_t);
+    realloc(nrb->_nrb_flag);
+    realloc(nrb->_displ, 1);
+    realloc(nrb->_nrb_index);
 #ifdef CORENEURON_ENABLE_GPU
     if (nt->compute_gpu) {
-        int *d_weight_index, *d_pnt_index, *d_displ, *d_nrb_index;
-        double *d_nrb_t, *d_nrb_flag;
-
         // update device copy
         nrn_pragma_acc(update device(nrb));
         nrn_pragma_omp(target update to(nrb));
 
-        NetReceiveBuffer_t* d_nrb = cnrn_target_deviceptr(nrb);
-
+        NetReceiveBuffer_t* const d_nrb{cnrn_target_deviceptr(nrb)};
         // recopy the vectors in the buffer
-        d_pnt_index = cnrn_target_copyin(nrb->_pnt_index, nrb->_size);
+        int* const d_pnt_index{cnrn_target_copyin(nrb->_pnt_index, nrb->_size)};
         cnrn_target_memcpy_to_device(&(d_nrb->_pnt_index), &d_pnt_index);
 
-        d_weight_index = cnrn_target_copyin(nrb->_weight_index, nrb->_size);
+        int* const d_weight_index{cnrn_target_copyin(nrb->_weight_index, nrb->_size)};
         cnrn_target_memcpy_to_device(&(d_nrb->_weight_index), &d_weight_index);
 
-        d_nrb_t = cnrn_target_copyin(nrb->_nrb_t, nrb->_size);
+        double* const d_nrb_t{cnrn_target_copyin(nrb->_nrb_t, nrb->_size)};
         cnrn_target_memcpy_to_device(&(d_nrb->_nrb_t), &d_nrb_t);
 
-        d_nrb_flag = cnrn_target_copyin(nrb->_nrb_flag, nrb->_size);
+        double* const d_nrb_flag{cnrn_target_copyin(nrb->_nrb_flag, nrb->_size)};
         cnrn_target_memcpy_to_device(&(d_nrb->_nrb_flag), &d_nrb_flag);
 
-        d_displ = cnrn_target_copyin(nrb->_displ, nrb->_size + 1);
+        int* const d_displ{cnrn_target_copyin(nrb->_displ, nrb->_size + 1)};
         cnrn_target_memcpy_to_device(&(d_nrb->_displ), &d_displ);
 
-        d_nrb_index = cnrn_target_copyin(nrb->_nrb_index, nrb->_size);
+        int* const d_nrb_index{cnrn_target_copyin(nrb->_nrb_index, nrb->_size)};
         cnrn_target_memcpy_to_device(&(d_nrb->_nrb_index), &d_nrb_index);
     }
 #endif
@@ -947,8 +953,11 @@ void delete_nrnthreads_on_device(NrnThread* threads, int nthreads) {
 #ifdef CORENEURON_ENABLE_GPU
     for (int i = 0; i < nthreads; i++) {
         NrnThread* nt = threads + i;
-        cnrn_target_delete(nt->_fornetcon_weight_perm);
-        cnrn_target_delete(nt->_fornetcon_perm_indices);
+        if (!nt->compute_gpu) {
+            continue;
+        }
+        cnrn_target_delete(nt->_fornetcon_weight_perm, nt->_fornetcon_weight_perm_size);
+        cnrn_target_delete(nt->_fornetcon_perm_indices, nt->_fornetcon_perm_indices_size);
         {
             TrajectoryRequests* tr = nt->trajec_requests;
             if (tr) {

coreneuron/io/phase2.cpp

@@ -482,12 +482,8 @@ void Phase2::set_net_send_buffer(Memb_list** ml_list, const std::vector<int>& pn
             ml->_net_receive_buffer = nrb;
             nrb->_pnt_offset = pnt_offset[type];
 
-            // begin with a size of 5% of the number of instances
-            nrb->_size = ml->nodecount;
-            // or at least 8
-            nrb->_size = std::max(8, nrb->_size);
-            // but not more than nodecount
-            nrb->_size = std::min(ml->nodecount, nrb->_size);
+            // begin with a size equal to the number of instances, or at least 8
+            nrb->_size = std::max(8, ml->nodecount);
             nrb->_pnt_index = (int*) ecalloc_align(nrb->_size, sizeof(int));
             nrb->_displ = (int*) ecalloc_align(nrb->_size + 1, sizeof(int));
             nrb->_nrb_index = (int*) ecalloc_align(nrb->_size, sizeof(int));