Merge pull request #3257 from nest/add-boostless-ci-build

Add more minimal CI test

Author: terhorstd

.github/workflows/nestbuildmatrix.yml

@@ -579,8 +579,9 @@ jobs:
         cpp_compiler: ["gcc"]
 
         # available use flags (all default to "OFF"):
-        # openmp, mpi, python, gsl, ltdl, boost, sionlib, libneurosim, optimize, warning, userdoc, music
+        # openmp, mpi, python, gsl, ltdl, boost, sionlib, libneurosim, optimize, warning, userdoc, music, readline
         use:
+          - "optimize, warning"
           - "boost, optimize, warning"
           - "openmp, python, gsl, ltdl, boost, optimize, warning"
           - "mpi, python, gsl, ltdl, boost, optimize, warning"
@@ -619,26 +620,41 @@ jobs:
 
       - name: "Install Linux system dependencies"
         run: |
-          sudo apt-get update
           # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md
-          sudo apt-get install ccache
-          sudo apt-get install libltdl-dev libreadline6-dev libncurses5-dev libgsl0-dev python3-all-dev jq libpcre3 libpcre3-dev
-          sudo apt-get install tcl8.6 tcl8.6-dev tk8.6-dev
-          # Install MPI dependencies regardless of whether we compile NEST with or without MPI, so the installation of MPI4Py works
-          sudo apt-get install openmpi-bin libopenmpi-dev
-          sudo apt-get install libboost-filesystem-dev libboost-regex-dev libboost-wave-dev libboost-python-dev libboost-program-options-dev libboost-test-dev
-          sudo apt-get install pkg-config
+          sudo apt-get update
+          sudo apt-get install \
+                           ccache \
+                           python3-all-dev \
+                           jq \
+                           libpcre3 libpcre3-dev \
+                           pkg-config
+          # not sure why these would be required:
+          #   sudo apt-get install tcl8.6 tcl8.6-dev tk8.6-dev
           sudo ldconfig
 
+      - name: "Install Readline dependencies"
+        if: "${{ contains(matrix.use, 'readline') }}"
+        run: "sudo apt-get install libreadline6-dev libncurses5-dev"
+
+      - name: "Install LTDL dependencies"
+        if: "${{ contains(matrix.use, 'ltdl') }}"
+        run: "sudo apt-get install libltdl-dev"
+
+      - name: "Install Boost dependencies"
+        if: "${{ contains(matrix.use, 'boost') }}"
+        run: "sudo apt-get install libboost-filesystem-dev libboost-regex-dev libboost-wave-dev libboost-python-dev libboost-program-options-dev libboost-test-dev"
+
+      - name: "Install OpenMPI development dependencies"
+        if: "${{ contains(matrix.use, 'mpi') }}"
+        run: "sudo apt-get install openmpi-bin libopenmpi-dev"
+
       - name: "Install GSL system dependencies"
         if: "${{ contains(matrix.use, 'gsl') }}"
-        run: |
-          sudo apt-get install libgsl0-dev
+        run: "sudo apt-get install libgsl0-dev"
 
       - name: "Install HDF5 system dependencies"
         if: "${{ contains(matrix.use, 'hdf5') }}"
-        run: |
-          sudo apt-get install libhdf5-dev
+        run: "sudo apt-get install libhdf5-dev"
 
       - name: "Restore ccache"
         # Using CCache to re-use compiled objects from prior runs that have the same
@@ -674,13 +690,24 @@ jobs:
       - name: "Install Python dependencies"
         run: |
           python -m pip install --upgrade pip setuptools
+          python -m pip install --force-reinstall --upgrade cython  # this finds a broken preinstall if not explicitly forced
           python -c "import setuptools; print('package location:', setuptools.__file__)"
-          python -m pip install --force-reinstall --upgrade scipy 'junitparser>=2' numpy pytest pytest-timeout pytest-xdist cython matplotlib terminaltables pandoc pandas
-          # Install mpi4py regardless of whether we compile NEST with or without MPI, so regressiontests/issue-1703.py will run in both cases
-          python -m pip install --force-reinstall --upgrade mpi4py
+          pip list
+
+      - name: "Install Python test dependencies"
+        # These could go to a separate venv…
+        run: |
+          python -m pip install 'junitparser>=2' pytest pytest-timeout pytest-xdist terminaltables
+          python -m pip install numpy scipy pandas matplotlib
           test \! -e "=2"   # assert junitparser is correctly quoted and '>' is not interpreted as shell redirect
           python -c "import pytest; print('package location:', pytest.__file__)"
-          pip list
+
+      - name: "Install Python doc dependencies"
+        run: "python -m pip install pandoc"
+
+      - name: "Install Python mpi dependencies"
+        if: "${{ contains(matrix.use, 'mpi') }}"
+        run: "python -m pip install --force-reinstall --upgrade mpi4py"
 
       - name: "Install h5py"
         if: "${{ contains(matrix.use, 'hdf5') }}"
@@ -706,6 +733,26 @@ jobs:
           PYLIB_DIR="$(python3 -c "import sysconfig; print(sysconfig.get_path('include'))" | sed 's/include/lib/')"
           ./build_support/install_csa-libneurosim.sh $PYLIB_DIR
 
+      - name: "Uninstall Boost development libraries"
+        if: "${{ ! contains(matrix.use, 'boost') }}"
+        run: |
+          apt list --installed 'libboost*'
+          sudo apt-get remove --autoremove 'libboost*-dev'
+
+      - name: "List available packages"
+        if: "${{ ! contains(matrix.use, 'boost') }}"
+        run: |
+          gcc --version
+          python --version
+          echo "INSTALLED PYTHON PACKAGES:"
+          pip list
+          echo "INSTALLED BOOST PACKAGES:"
+          dpkg -l 'libboost*' | grep "^ii" || true
+          echo "INSTALLED BOOST HEADERS:"
+          dpkg -l 'libboost*-dev' | grep "^ii"  || true
+          echo "LIST HAS TO BE ZERO"
+          test -z "$(dpkg -l 'libboost*-dev' | grep "^ii")"
+
       - name: "Configure NEST build"
         env:
           CMAKE_C_COMPILER_LAUNCHER: ccache

testsuite/pytests/conftest.py

@@ -159,10 +159,15 @@ def have_plotting():
 
 @pytest.fixture(scope="session")
 def subprocess_compatible_mpi():
-    """Until at least OpenMPI 4.1.6, the following fails due to a bug in OpenMPI, from 5.0.7 is definitely safe."""
-
-    res = subprocess.run(["mpirun", "-np", "1", "echo"])
-    return res.returncode == 0
+    """
+    Until at least OpenMPI 4.1.6, the following fails due to a bug in OpenMPI,
+    from 5.0.7 is definitely safe.
+    """
+    try:
+        res = subprocess.run(["mpirun", "-np", "1", "echo"])
+        return 0 == res.returncode
+    except FileNotFoundError:
+        return False
 
 
 @pytest.fixture(autouse=True)

testsuite/pytests/connect_test_base.py

@@ -25,12 +25,16 @@
 import numpy as np
 import scipy.stats
 
+# Check that NEST is installed with MPI support and mpi4py is available.
 try:
-    from mpi4py import MPI
+    import mpi4py
 
-    haveMPI4Py = True
+    HAVE_MPI4PY = nest.ll_api.sli_func("statusdict/have_mpi ::")
 except ImportError:
-    haveMPI4Py = False
+    HAVE_MPI4PY = False
+
+if HAVE_MPI4PY:
+    from mpi4py import MPI
 
 
 class ConnectTestBase(unittest.TestCase):
@@ -309,7 +313,7 @@ def gather_data(data_array):
     None otherwise.
 
     """
-    if haveMPI4Py:
+    if HAVE_MPI4PY:
         data_array_list = MPI.COMM_WORLD.gather(data_array, root=0)
         if MPI.COMM_WORLD.Get_rank() == 0:
             if isinstance(data_array, list):
@@ -327,7 +331,7 @@ def bcast_data(data):
     """
     Broadcasts data from the root MPI node to all other nodes.
     """
-    if haveMPI4Py:
+    if HAVE_MPI4PY:
         data = MPI.COMM_WORLD.bcast(data, root=0)
     return data
 
@@ -340,7 +344,7 @@ def is_array(data):
 
 
 def mpi_barrier():
-    if haveMPI4Py:
+    if HAVE_MPI4PY:
         MPI.COMM_WORLD.Barrier()
 
 

testsuite/pytests/test_connect_all_to_all.py

@@ -27,9 +27,11 @@
 import numpy as np
 import scipy.stats
 
+HAVE_MPI = nest.ll_api.sli_func("statusdict/have_mpi ::")
 HAVE_OPENMP = nest.ll_api.sli_func("is_threaded")
 
 
+@unittest.skipIf(not HAVE_MPI, "NEST was compiled without MPI")
 @unittest.skipIf(not HAVE_OPENMP, "NEST was compiled without multi-threading")
 @nest.ll_api.check_stack
 class TestAllToAll(connect_test_base.ConnectTestBase):

testsuite/pytests/test_connect_tripartite_bernoulli.py

@@ -25,12 +25,16 @@
 import pytest
 import scipy.stats
 
+# Check that NEST is installed with MPI support and mpi4py is available.
 try:
-    from mpi4py import MPI
+    import mpi4py
 
-    haveMPI4Py = True
+    HAVE_MPI4PY = nest.ll_api.sli_func("statusdict/have_mpi ::")
 except ImportError:
-    haveMPI4Py = False
+    HAVE_MPI4PY = False
+
+if HAVE_MPI4PY:
+    from mpi4py import MPI
 
 
 def setup_network(
@@ -209,7 +213,7 @@ def gather_data(data_array):
         available, the original array is returned.
 
     """
-    if haveMPI4Py:
+    if HAVE_MPI4PY:
         data_array_list = MPI.COMM_WORLD.gather(data_array, root=0)
         if MPI.COMM_WORLD.Get_rank() == 0:
             if isinstance(data_array, list):
@@ -263,7 +267,7 @@ def chi_squared_check(degrees, expected):
 
 def mpi_barrier():
     """Forms a barrier for MPI processes until all of them call the function."""
-    if haveMPI4Py:
+    if HAVE_MPI4PY:
         MPI.COMM_WORLD.Barrier()
 
 

testsuite/pytests/test_sp/test_disconnect.py

@@ -23,15 +23,18 @@
 
 import nest
 
+# Check that NEST is installed with MPI support and mpi4py is available.
 try:
-    from mpi4py import MPI
+    import mpi4py
 
-    have_mpi4py = True
+    HAVE_MPI4PY = nest.ll_api.sli_func("statusdict/have_mpi ::")
 except ImportError:
-    have_mpi4py = False
+    HAVE_MPI4PY = False
+
+if HAVE_MPI4PY:
+    from mpi4py import MPI
 
-have_mpi = nest.ll_api.sli_func("statusdict/have_mpi ::")
-test_with_mpi = have_mpi and have_mpi4py and nest.num_processes > 1
+test_with_mpi = HAVE_MPI4PY and nest.num_processes > 1
 
 
 class TestDisconnectSingle(unittest.TestCase):

testsuite/regressiontests/issue-1703.py

@@ -31,6 +31,8 @@
 import subprocess
 import sys
 
+import nest
+
 EXIT_CODE_SUCCESS = 0
 EXIT_CODE_ERROR = 1
 EXIT_CODE_SEGFAULT = -11
@@ -40,12 +42,16 @@
 EXIT_FAILURE = 127
 EXIT_SEGFAULT = 139
 
-# Check that NEST is installed with Python support, and that mpi4py is available.
+# Check that NEST is installed with MPI support and mpi4py is available.
 # If not, the test is skipped.
 try:
-    import nest
-    from mpi4py import MPI  # noqa: F401
+    import mpi4py
+
+    HAVE_MPI4PY = nest.ll_api.sli_func("statusdict/have_mpi ::")
 except ImportError:
+    HAVE_MPI4PY = False
+
+if not HAVE_MPI4PY:
     sys.exit(EXIT_SKIPPED)
 
 # Attempt to import MPI from mpi4py before NEST. Running the script in a separate process,