Add C++ DynamicArrays in Brian2 for runtime mode

.github/workflows/testsuite.yml

@@ -25,8 +25,8 @@ jobs:
           persist-credentials: false
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          python-version: '3.12'
-          cache: 'pip'
+          python-version: "3.12"
+          cache: "pip"
           cache-dependency-path: .devcontainer/dev-requirements.txt
       - name: Install deps
         run: pip3 install -r .devcontainer/dev-requirements.txt
@@ -40,19 +40,22 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [{image: ubuntu-24.04, triplet: x64-linux},
-             {image: windows-2022, triplet: x64-windows},
-             {image: macOS-13, triplet: x64-osx},
-             {image: macOS-14, triplet: arm64-osx}]
+        os:
+          [
+            { image: ubuntu-24.04, triplet: x64-linux },
+            { image: windows-2022, triplet: x64-windows },
+            { image: macOS-13, triplet: x64-osx },
+            { image: macOS-14, triplet: arm64-osx },
+          ]
         standalone: [false, true]
         float_dtype_32: [false, true]
         python-version: ["${{ needs.get_python_versions.outputs.max-python }}"]
         include:
-          - os: {image: ubuntu-24.04, triplet: x64-linux}
+          - os: { image: ubuntu-24.04, triplet: x64-linux }
             standalone: false
             python-version: "${{ needs.get_python_versions.outputs.min-python }} < 3.11.9 || ${{ needs.get_python_versions.outputs.min-python }} >= 3.12"
             float_dtype_32: false
-          - os: {image: ubuntu-24.04, triplet: x64-linux}
+          - os: { image: ubuntu-24.04, triplet: x64-linux }
             standalone: true
             python-version: "${{ needs.get_python_versions.outputs.min-python }} < 3.11.9 || ${{ needs.get_python_versions.outputs.min-python }} >= 3.12"
             float_dtype_32: false
@@ -86,20 +89,23 @@ jobs:
         id: python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          cache: 'pip'
+          cache: "pip"
           python-version: ${{ matrix.python-version }}
       - name: Install Brian2 and dependencies
         env:
           PYTHON_BINARY: ${{ steps.python.outputs.python-path }}
         run: |
           "$PYTHON_BINARY" -m pip install .[test]
-      
+
       - name: Determine Cython cache dir
         id: cython-cache
         run: |
-          CACHE_DIR=$(python -c 'from brian2.codegen.runtime.cython_rt.extension_manager import get_cython_cache_dir; print(get_cython_cache_dir())')
+          cd  $GITHUB_WORKSPACE/.. # move out of the workspace to avoid direct import
+          CACHE_DIR=$("$PYTHON_BINARY" -c 'from brian2.codegen.runtime.cython_rt.extension_manager import get_cython_cache_dir; print(get_cython_cache_dir())')
           echo "Cython cache dir: $CACHE_DIR"
           echo "cachedir=$CACHE_DIR" >> "$GITHUB_OUTPUT"
+        env:
+          PYTHON_BINARY: ${{ steps.python.outputs.python-path }}
 
       - name: restore Cython cache
         uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
@@ -108,7 +114,7 @@ jobs:
           key: cython-extensions-${{ matrix.os.image }}-${{ matrix.python-version }}-32bit-${{ matrix.float_dtype_32 }}
           path: ${{ steps.cython-cache.outputs.cachedir }}
 
-      - name: Run Tests      
+      - name: Run Tests
         run: |
           cd  $GITHUB_WORKSPACE/.. && \
           "$PYTHON_BINARY" $GITHUB_WORKSPACE/$SCRIPT_NAME && \
@@ -120,7 +126,7 @@ jobs:
           STANDALONE: ${{ matrix.standalone }}
           FLOAT_DTYPE_32: ${{ matrix.float_dtype_32 }}
           PYTHON_BINARY: ${{ steps.python.outputs.python-path }}
-          DO_NOT_RESET_PREFERENCES: true  # Make sure that GSL setting is used
+          DO_NOT_RESET_PREFERENCES: true # Make sure that GSL setting is used
       - name: Send coverage to Coveralls (parallel)
         if: ${{ startsWith(matrix.os.image, 'ubuntu-') && matrix.python-version == needs.get_python_versions.outputs.max-python }}
         uses: coverallsapp/github-action@648a8eb78e6d50909eff900e4ec85cab4524a45b # v2.3.6
@@ -146,32 +152,32 @@ jobs:
       run:
         shell: bash -l {0}
     steps:
-    - name: Checkout Repository
-      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-      with:
-        fetch-depth: 0
-        persist-credentials: false
-        submodules: true
+      - name: Checkout Repository
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+          submodules: true
 
-    - name: Setup Conda and Python
-      uses: conda-incubator/setup-miniconda@835234971496cad1653abb28a638a281cf32541f # v3.2.0
-      with:
-        conda-remove-defaults: true
-        auto-update-conda: true
-        auto-activate-base: false
-        miniforge-version: latest
-        activate-environment: 'test_env'
-        python-version: "${{ needs.get_python_versions.outputs.max-python }}"
+      - name: Setup Conda and Python
+        uses: conda-incubator/setup-miniconda@835234971496cad1653abb28a638a281cf32541f # v3.2.0
+        with:
+          conda-remove-defaults: true
+          auto-update-conda: true
+          auto-activate-base: false
+          miniforge-version: latest
+          activate-environment: "test_env"
+          python-version: "${{ needs.get_python_versions.outputs.max-python }}"
 
-    - name: Install dependencies
-      run: pip install -r rtd-requirements.txt
+      - name: Install dependencies
+        run: pip install -r rtd-requirements.txt
 
-    - name: Install brian2
-      run: pip install .
+      - name: Install brian2
+        run: pip install .
 
-    - name: Build HTML documentation
-      run: |
-        cd docs_sphinx
-        sphinx-build -b html . ../docs
-      env:
-        READTHEDOCS: True
+      - name: Build HTML documentation
+        run: |
+          cd docs_sphinx
+          sphinx-build -b html . ../docs
+        env:
+          READTHEDOCS: True

brian2/codegen/generators/cython_generator.py

@@ -5,6 +5,7 @@
 from brian2.core.variables import (
     AuxiliaryVariable,
     Constant,
+    DynamicArrayVariable,
     Subexpression,
     Variable,
     get_dtype_str,
@@ -44,6 +45,36 @@ def get_numpy_dtype(obj):
     return numpy_dtype[get_dtype_str(obj)]
 
 
+def get_dynamic_array_cpp_type(var):
+    """Get the full templated C++ type for a DynamicArrayVariable"""
+    cpp_dtype = get_cpp_dtype(var.dtype)  # e.g., 'double', 'int32_t', 'float'
+
+    # Special handling for bool - use char instead
+    if get_dtype_str(var.dtype) == "bool":
+        cpp_dtype = "char"
+
+    if var.ndim == 1:
+        return f"DynamicArray1DCpp[{cpp_dtype}]"  # Returns "DynamicArray1DCpp[double]"
+    elif var.ndim == 2:
+        return f"DynamicArray2DCpp[{cpp_dtype}]"  # Returns "DynamicArray2DCpp[double]"
+    else:
+        raise ValueError(
+            f"Unsupported dynamic array dimension: {var.ndim}. Only 1D and 2D arrays are supported."
+        )
+
+
+def get_capsule_type(var):
+    """Get the capsule type name for PyCapsule_GetPointer"""
+    if var.ndim == 1:
+        return "DynamicArray1D"
+    elif var.ndim == 2:
+        return "DynamicArray2D"
+    else:
+        raise ValueError(
+            f"Unsupported dynamic array dimension: {var.ndim}. Only 1D and 2D arrays are supported."
+        )
+
+
 class CythonNodeRenderer(NodeRenderer):
     def render_NameConstant(self, node):
         return {True: "1", False: "0"}.get(node.value, node.value)
@@ -178,6 +209,7 @@ def translate_to_write_arrays(self, write):
             line = (
                 f"{self.get_array_name(var, self.variables)}[{index_var}] = {varname}"
             )
+
             lines.append(line)
         return lines
 
@@ -348,40 +380,81 @@ def determine_keywords(self):
                 load_namespace.append(line)
             elif isinstance(var, Variable):
                 if var.dynamic:
-                    pointer_name = self.get_array_name(var, False)
-                    load_namespace.append(
-                        f'{pointer_name} = _namespace["{pointer_name}"]'
-                    )
+                    if isinstance(var, DynamicArrayVariable):
+                        # Uses direct C++ pointers for fast access, avoiding Python overhead.
+                        # We define unique names for the array object, its pointer, and the capsule.
+                        dyn_array_name = self.get_array_name(var, access_data=False)
+
+                        if dyn_array_name in handled_pointers:
+                            continue
+
+                        capsule_name = f"{dyn_array_name}_capsule"
+
+                        # Get the C++ type for accurate casting (e.g., "DynamicArray1DCpp[double]").
+                        cpp_type = get_dynamic_array_cpp_type(var)
+
+                        # Generate Cython code to retrieve the C++ pointer from the capsule.
+                        # This is a two-step process in Cython:
+                        # 1. Look up the capsule object in the provided namespace.
+                        # 2. Unwrap the raw C++ pointer from the capsule for direct C++ object access.
+                        load_namespace.append(
+                            f'cdef object {capsule_name} = _namespace["{capsule_name}"]'
+                        )
+                        load_namespace.append(
+                            f"cdef {cpp_type}* {dyn_array_name}_ptr = "
+                            f'<{cpp_type}*>PyCapsule_GetPointer({capsule_name}, "{get_capsule_type(var)}")'
+                        )
+                        handled_pointers.add(dyn_array_name)
+                    else:
+                        pointer_name = self.get_array_name(var, False)
+                        load_namespace.append(
+                            f'{pointer_name} = _namespace["{pointer_name}"]'
+                        )
 
                 # This is the "true" array name, not the restricted pointer.
                 array_name = device.get_array_name(var)
                 pointer_name = self.get_array_name(var)
+                dyn_array_name = self.get_array_name(var, access_data=False)
                 if pointer_name in handled_pointers:
                     continue
-                if getattr(var, "ndim", 1) > 1:
-                    continue  # multidimensional (dynamic) arrays have to be treated differently
-                if get_dtype_str(var.dtype) == "bool":
+                if isinstance(var, DynamicArrayVariable):
+                    # For Dynamic Arrays, we get the data pointer directly from the C++ object
+                    # The C++ DynamicArray class handles bool types correctly when we provide
+                    # them as char (unlike non-dynamic arrays which require special Cython buffer handling).
+                    cpp_dtype = get_cpp_dtype(var.dtype)
+                    if get_dtype_str(var.dtype) == "bool":
+                        # Use char for boolean dynamic arrays
+                        cpp_dtype = "char"
+
                     newlines = [
                         (
-                            "cdef _numpy.ndarray[char, ndim=1, mode='c', cast=True]"
-                            " _buf_{array_name} = _namespace['{array_name}']"
-                        ),
-                        (
-                            "cdef {cpp_dtype} * {array_name} = <{cpp_dtype} *>"
-                            " _buf_{array_name}.data"
-                        ),
+                            f"cdef {cpp_dtype}* {array_name} = <{cpp_dtype}*>"
+                            f" {dyn_array_name}_ptr.get_data_ptr()"
+                        )
                     ]
                 else:
-                    newlines = [
-                        (
-                            "cdef _numpy.ndarray[{cpp_dtype}, ndim=1, mode='c']"
-                            " _buf_{array_name} = _namespace['{array_name}']"
-                        ),
-                        (
-                            "cdef {cpp_dtype} * {array_name} = <{cpp_dtype} *>"
-                            " _buf_{array_name}.data"
-                        ),
-                    ]
+                    if get_dtype_str(var.dtype) == "bool":
+                        newlines = [
+                            (
+                                "cdef _numpy.ndarray[char, ndim=1, mode='c', cast=True]"
+                                " _buf_{array_name} = _namespace['{array_name}']"
+                            ),
+                            (
+                                "cdef {cpp_dtype} * {array_name} = <{cpp_dtype} *>"
+                                " _buf_{array_name}.data"
+                            ),
+                        ]
+                    else:
+                        newlines = [
+                            (
+                                "cdef _numpy.ndarray[{cpp_dtype}, ndim=1, mode='c']"
+                                " _buf_{array_name} = _namespace['{array_name}']"
+                            ),
+                            (
+                                "cdef {cpp_dtype} * {array_name} = <{cpp_dtype} *>"
+                                " _buf_{array_name}.data"
+                            ),
+                        ]
 
                 if not var.scalar:
                     newlines += [
@@ -399,6 +472,7 @@ def determine_keywords(self):
                         numpy_dtype=get_numpy_dtype(var.dtype),
                         pointer_name=pointer_name,
                         array_name=array_name,
+                        dyn_array_name=dyn_array_name,
                         varname=varname,
                     )
                     load_namespace.append(line)

brian2/codegen/runtime/cython_rt/cython_rt.py

@@ -246,6 +246,11 @@ def variables_to_namespace(self):
                 dyn_array_name = self.generator_class.get_array_name(
                     var, access_data=False
                 )
+                # Adding logic to pass in the capsule object to namespace , we already have code set in generator to
+                # take the passed in object and work on it
+                capsule_name = f"{dyn_array_name}_capsule"
+                capsule = self.device.get_capsule(var)
+                self.namespace[capsule_name] = capsule
                 self.namespace[dyn_array_name] = self.device.get_value(
                     var, access_data=False
                 )
@@ -264,7 +269,6 @@ def variables_to_namespace(self):
         self.namespace = {
             k: v for k, v in self.namespace.items() if k in all_identifiers
         }
-
         # There is one type of objects that we have to inject into the
         # namespace with their current value at each time step: dynamic
         # arrays that change in size during runs, where the size change is not

brian2/codegen/runtime/cython_rt/extension_manager.py

@@ -264,6 +264,13 @@ def _load_module(
             synapses_dir = os.path.dirname(synapses.__file__)
             c_include_dirs.append(synapses_dir)
 
+            import brian2
+
+            brian2_base_dir = os.path.dirname(brian2.__file__)
+            brianlib_dir = os.path.join(
+                brian2_base_dir, "devices", "cpp_standalone", "brianlib"
+            )
+            c_include_dirs.append(brianlib_dir)
             pyx_file = os.path.join(lib_dir, f"{module_name}.pyx")
             # ignore Python 3 unicode stuff for the moment
             # pyx_file = py3compat.cast_bytes_py2(pyx_file, encoding=sys.getfilesystemencoding())

brian2/codegen/runtime/cython_rt/templates/common_group.pyx

@@ -46,6 +46,29 @@ cdef extern from "stdint_compat.h":
     cdef int int_(float)
     cdef int int_(double)
     cdef int int_(long double)
+
+# PyCapsule support for direct C++ pointer access
+from cpython.pycapsule cimport PyCapsule_GetPointer
+
+# Dynamic array C++ interface declarations
+cdef extern from "dynamic_array.h":
+    cdef cppclass DynamicArray1DCpp "DynamicArray1D"[T]:
+        void resize(size_t) except +
+        void shrink_to_fit()
+        T& operator[](size_t)
+        T* get_data_ptr()
+        size_t size()
+        size_t capacity()
+
+    cdef cppclass DynamicArray2DCpp "DynamicArray2D"[T]:
+        void resize(size_t, size_t) except +
+        void resize_along_first(size_t) except +
+        void shrink_to_fit()
+        T& operator()(size_t, size_t)
+        T* get_data_ptr()
+        size_t rows()
+        size_t cols()
+        size_t stride()
 {% endmacro %}
 
 {% macro before_run() %}