Add OpenPMD support

.github/workflows/ci-macos.yml

@@ -28,10 +28,14 @@ jobs:
         cache-dependency-path: '**/requirements.txt'
     - run: pip install -r requirements.txt
 
-    - name: Install dependencies
+    - name: Install dependencies (Homebrew)
       run: |
         brew install openmpi hdf5-mpi adios2 || true
 
+    - name: Install OpenPMD
+      run: |
+        openPMD_USE_MPI=ON python3 -m pip install openpmd-api --no-binary openpmd-api
+
     - name: Install parthenon_tools
       run: |
         pip install scripts/python/packages/parthenon_tools

.github/workflows/ci-short.yml

@@ -32,8 +32,9 @@ jobs:
       - name: cpplint
         run: python3 ./tst/style/cpplint.py --counting=detailed --recursive src example tst
       - name: copyright
+        # using variant mpi because openpmd lib is only installed with mpi support (no serial)
         run: |
-          cmake -DCMAKE_CXX_FLAGS=-Werror -Bbuild-copyright-check
+          cmake -DCMAKE_CXX_FLAGS=-Werror -Bbuild-copyright-check -DMACHINE_VARIANT=mpi
           cmake --build build-copyright-check -t check-copyright
       - uses: actions/upload-artifact@v4
         with:
@@ -104,6 +105,7 @@ jobs:
           # make sure parthenon tools is in our python path
           export PYTHONPATH="$PYTHONPATH:$(python -m site --user-site)"
           ctest -R regression_mpi_test:output_hdf5
+          ctest -R regression_mpi_test:restart_opmd
       # Test example with swarms
       - name: particle-leapfrog
         run: |
@@ -174,6 +176,7 @@ jobs:
           # make sure parthenon tools is in our python path
           export PYTHONPATH="$PYTHONPATH:$(python -m site --user-site)"
           ctest -R regression_mpi_test:output_hdf5
+          ctest -R regression_mpi_test:restart_opmd
       # Test example with swarms
       - name: particle-leapfrog
         run: |

.github/workflows/docs.yml

@@ -36,6 +36,7 @@ jobs:
             mkdir -p build && cd build
             cmake -DPARTHENON_DISABLE_HDF5=ON \
                   -DPARTHENON_DISABLE_MPI=ON \
+                  -DPARTHENON_DISABLE_OPENPMD=ON \
                   -DREGRESSION_GOLD_STANDARD_SYNC=OFF \
                   -DBUILD_TESTING=OFF \
                   ..

CMakeLists.txt

@@ -36,6 +36,8 @@ option(PARTHENON_DISABLE_MPI "MPI is enabled by default if found, set this to Tr
 option(PARTHENON_ENABLE_HOST_COMM_BUFFERS "CUDA/HIP Only: Allocate communication buffers on host (may be slower)" OFF)
 option(PARTHENON_DISABLE_HDF5 "HDF5 is enabled by default if found, set this to True to disable HDF5" OFF)
 option(PARTHENON_DISABLE_HDF5_COMPRESSION "HDF5 compression is enabled by default, set this to True to disable compression in HDF5 output/restart files" OFF)
+option(PARTHENON_DISABLE_OPENPMD "OpenPMD is enabled by default if found, set this to ON to disable OpenPMD" OFF)
+option(PARTHENON_USE_SYSTEM_OPENPMD "OpenPMD API is downloaded and installed automatically by default. Set this to ON to find and use a version installed on the system." OFF)
 option(PARTHENON_DISABLE_SPARSE "Sparse capability is enabled by default, set this to True to compile-time disable all sparse capability" OFF)
 option(PARTHENON_ENABLE_ASCENT "Enable Ascent for in situ visualization and analysis" OFF)
 option(PARTHENON_LINT_DEFAULT "Linting is turned off by default, use the \"lint\" target or set \
@@ -207,6 +209,51 @@ if (NOT PARTHENON_DISABLE_HDF5)
   install(TARGETS HDF5_C EXPORT parthenonTargets)
 endif()
 
+set(PARTHENON_ENABLE_OPENPMD OFF)
+if (NOT PARTHENON_DISABLE_OPENPMD)
+  if (PARTHENON_USE_SYSTEM_OPENPMD)
+    # we want to enforce the use of ADIOS2 with this backend
+    set(PARTHENON_OPENPMD_COMPONENTS "ADIOS2")
+    if (ENABLE_MPI)
+      list(APPEND PARTHENON_OPENPMD_COMPONENTS "MPI")
+    else()
+      list(APPEND PARTHENON_OPENPMD_COMPONENTS "NOMPI")
+    endif()
+    find_package(openPMD 0.16.1 CONFIG COMPONENTS ${PARTHENON_OPENPMD_COMPONENTS})
+    if (NOT openPMD_FOUND)
+      message(FATAL_ERROR "OpenPMD API requested to be used from the environment but it was "
+      "not found. Either append the path to the installed OpenPMD lib via the "
+      "openPMD_DIR or CMAKE_PREFIX_PATH, or use the version shipped with Parthenon by keeping "
+      "PARTHENON_USE_SYSTEM_OPENPMD=OFF")
+    endif()
+  else()
+    include(FetchContent)
+    set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
+    set(openPMD_BUILD_CLI_TOOLS OFF)
+    set(openPMD_BUILD_EXAMPLES OFF)
+    set(openPMD_BUILD_TESTING OFF)
+    set(openPMD_BUILD_SHARED_LIBS OFF)
+    set(openPMD_INSTALL OFF)
+    set(openPMD_USE_ADIOS2 ON)  # we definitely want ADIOS2
+    if( NOT Python3_Interpreter_FOUND)
+      find_package(Python3 REQUIRED COMPONENTS Interpreter)
+    endif()
+    # Why not build the python interface for easier testing when Python is available
+    if(Python3_Interpreter_FOUND)
+      set(openPMD_USE_PYTHON ON)
+    else()
+      set(openPMD_USE_PYTHON OFF)
+    endif()
+    FetchContent_Declare(openPMD
+      GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git"
+      GIT_TAG        "3a60e77") # Release 0.16.1
+    FetchContent_MakeAvailable(openPMD)
+    install(TARGETS openPMD EXPORT parthenonTargets)
+  endif()
+
+  set(PARTHENON_ENABLE_OPENPMD ON)
+endif()
+
 # Kokkos recommendatation resulting in not using default GNU extensions
 set(CMAKE_CXX_EXTENSIONS OFF)
 

cmake/machinecfg/CI.cmake

@@ -1,6 +1,6 @@
 #========================================================================================
 # Parthenon performance portable AMR framework
-# Copyright(C) 2020 The Parthenon collaboration
+# Copyright(C) 2020-2025 The Parthenon collaboration
 # Licensed under the 3-clause BSD License, see LICENSE file for details
 #========================================================================================
 # (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved.
@@ -38,3 +38,5 @@ else()
   set(HDF5_ROOT /usr/local/hdf5/serial CACHE STRING "HDF5 path")
   set(PARTHENON_DISABLE_MPI ON CACHE BOOL "Disable MPI")
 endif()
+
+set(PARTHENON_USE_SYSTEM_OPENPMD ON CACHE BOOL "Use API in container")

cmake/machinecfg/GitHubActions.cmake

@@ -1,6 +1,6 @@
 #========================================================================================
 # Parthenon performance portable AMR framework
-# Copyright(C) 2021 The Parthenon collaboration
+# Copyright(C) 2021-2025 The Parthenon collaboration
 # Licensed under the 3-clause BSD License, see LICENSE file for details
 #========================================================================================
 # (C) (or copyright) 2021. Triad National Security, LLC. All rights reserved.
@@ -50,4 +50,6 @@ else()
   set(PARTHENON_DISABLE_MPI ON CACHE BOOL "Disable MPI")
 endif()
 
+set(PARTHENON_USE_SYSTEM_OPENPMD ON CACHE BOOL "Use API in container")
+
 set(CMAKE_CXX_FLAGS "${MACHINE_CXX_FLAGS}" CACHE STRING "Default flags for this config")

src/CMakeLists.txt

@@ -1,4 +1,8 @@
 #=========================================================================================
+# Parthenon performance portable AMR framework
+# Copyright(C) 2020-2025 The Parthenon collaboration
+# Licensed under the 3-clause BSD License, see LICENSE file for detail
+#=========================================================================================
 # (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved.
 #
 # This program was produced under U.S. Government contract 89233218CNA000001 for Los
@@ -196,6 +200,7 @@ add_library(parthenon
   outputs/history.cpp
   outputs/io_wrapper.cpp
   outputs/io_wrapper.hpp
+  outputs/output_attr.hpp
   outputs/output_utils.cpp
   outputs/output_utils.hpp
   outputs/outputs.cpp
@@ -360,6 +365,16 @@ if (ENABLE_HDF5)
   target_link_libraries(parthenon PUBLIC HDF5_C)
 endif()
 
+if (PARTHENON_ENABLE_OPENPMD)
+  target_sources(parthenon PRIVATE
+    outputs/parthenon_opmd.cpp
+    outputs/parthenon_opmd.hpp
+    outputs/restart_opmd.cpp
+    outputs/restart_opmd.hpp
+  )
+  target_link_libraries(parthenon PUBLIC openPMD::openPMD)
+endif()
+
 # For Cuda with NVCC (<11.2) and C++17 Kokkos currently does not work/compile with
 # relaxed-constexpr, see https://github.com/kokkos/kokkos/issues/3496
 # However, Parthenon heavily relies on it and there is no harm in compiling Kokkos

src/config.hpp.in

@@ -45,6 +45,8 @@
 // defne ENABLE_HDF5 or not at all
 #cmakedefine ENABLE_HDF5
 
+#cmakedefine PARTHENON_ENABLE_OPENPMD
+
 // define PARTHENON_DISABLE_HDF5_COMPRESSION or not at all
 #cmakedefine PARTHENON_DISABLE_HDF5_COMPRESSION
 

src/interface/params.hpp

@@ -122,6 +122,12 @@ class Params {
     return myParams_.at(key)->type();
   }
 
+  const Mutability &GetMutability(const std::string &key) const {
+    auto const it = myMutable_.find(key);
+    PARTHENON_REQUIRE_THROWS(it != myMutable_.end(), "Key " + key + " doesn't exist");
+    return it->second;
+  }
+
   std::vector<std::string> GetKeys() const {
     std::vector<std::string> keys;
     for (auto &x : myParams_) {
@@ -130,7 +136,6 @@ class Params {
     return keys;
   }
 
-  auto GetMutability(const std::string &key) const { return myMutable_.at(key); }
   bool IsMutable(const std::string &key) const {
     return static_cast<bool>(myMutable_.at(key));
   }

src/outputs/output_attr.hpp

@@ -0,0 +1,47 @@
+//========================================================================================
+// Parthenon performance portable AMR framework
+// Copyright(C) 2023-2024 The Parthenon collaboration
+// Licensed under the 3-clause BSD License, see LICENSE file for details
+//========================================================================================
+// (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved.
+//
+// This program was produced under U.S. Government contract 89233218CNA000001 for Los
+// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
+// for the U.S. Department of Energy/National Nuclear Security Administration. All rights
+// in the program are reserved by Triad National Security, LLC, and the U.S. Department
+// of Energy/National Nuclear Security Administration. The Government is granted for
+// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide
+// license in this material to reproduce, prepare derivative works, distribute copies to
+// the public, perform publicly and display publicly, and to permit others to do so.
+//========================================================================================
+
+#ifndef OUTPUTS_OUTPUT_ATTR_HPP_
+#define OUTPUTS_OUTPUT_ATTR_HPP_
+
+#include <vector>
+
+// JMM: This could probably be done with template magic but I think
+// using a macro is honestly the simplest and cleanest solution here.
+// Template solution would be to define a variatic class to conain the
+// list of types and then a hierarchy of structs/functions to turn
+// that into function calls. Preprocessor seems easier, given we're
+// not manipulating this list in any way.
+// The following types are the ones we allow to be stored as attributes in outputs
+// (specifically within Params).
+#define PARTHENON_ATTR_VALID_VEC_TYPES(T)                                                \
+  T, std::vector<T>, ParArray1D<T>, ParArray2D<T>, ParArray3D<T>, HostArray1D<T>,        \
+      HostArray2D<T>, HostArray3D<T>, Kokkos::View<T *>, Kokkos::View<T **>,             \
+      ParArrayND<T>, ParArrayHost<T>
+// JMM: This is the list of template specializations we
+// "pre-instantiate" We only pre-instantiate device memory, not host
+// memory. The reason is that when building with the Kokkos serial
+// backend, DevMemSpace and HostMemSpace are the same and so this
+// resolves to the same type in the macro, which causes problems.
+#define PARTHENON_ATTR_FOREACH_VECTOR_TYPE(T)                                            \
+  PARTHENON_ATTR_APPLY(T);                                                               \
+  PARTHENON_ATTR_APPLY(Kokkos::View<T *, LayoutWrapper, DevMemSpace>);                   \
+  PARTHENON_ATTR_APPLY(Kokkos::View<T **, LayoutWrapper, DevMemSpace>);                  \
+  PARTHENON_ATTR_APPLY(Kokkos::View<T ***, LayoutWrapper, DevMemSpace>);                 \
+  PARTHENON_ATTR_APPLY(device_view_t<T>)
+
+#endif // OUTPUTS_OUTPUT_ATTR_HPP_