From 49027c7aadf79ff0efa1093fb7de78dc3e8fda62 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 8 Jan 2024 20:58:04 +0100 Subject: [PATCH 001/125] Add OpenPMD as external lib --- CMakeLists.txt | 19 +++++++++++++++++++ src/CMakeLists.txt | 4 ++++ src/config.hpp.in | 3 +++ 3 files changed, 26 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 994399d4bdc0..dfe1ccf3c2ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,7 @@ option(PARTHENON_DISABLE_MPI "MPI is enabled by default if found, set this to Tr option(PARTHENON_ENABLE_HOST_COMM_BUFFERS "CUDA/HIP Only: Allocate communication buffers on host (may be slower)" OFF) option(PARTHENON_DISABLE_HDF5 "HDF5 is enabled by default if found, set this to True to disable HDF5" OFF) option(PARTHENON_DISABLE_HDF5_COMPRESSION "HDF5 compression is enabled by default, set this to True to disable compression in HDF5 output/restart files" OFF) +option(PARTHENON_DISABLE_OPENPMD "OpenPMD is enabled by default if found, set this to True to disable OpenPMD" OFF) option(PARTHENON_DISABLE_SPARSE "Sparse capability is enabled by default, set this to True to compile-time disable all sparse capability" OFF) option(PARTHENON_ENABLE_ASCENT "Enable Ascent for in situ visualization and analysis" OFF) option(PARTHENON_LINT_DEFAULT "Linting is turned off by default, use the \"lint\" target or set \ @@ -184,6 +185,24 @@ if (NOT PARTHENON_DISABLE_HDF5) install(TARGETS HDF5_C EXPORT parthenonTargets) endif() +if (NOT PARTHENON_DISABLE_OPENPMD) +#TODO(pgrete) add logic for serial/parallel +#TODO(pgrete) add logic for internal/external build + include(FetchContent) + set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) + set(openPMD_BUILD_CLI_TOOLS OFF) + set(openPMD_BUILD_EXAMPLES OFF) + set(openPMD_BUILD_TESTING OFF) + set(openPMD_BUILD_SHARED_LIBS OFF) # precedence over BUILD_SHARED_LIBS if needed + set(openPMD_INSTALL OFF) # or instead use: + # set(openPMD_INSTALL ${BUILD_SHARED_LIBS}) # only install if used as a shared library + set(openPMD_USE_PYTHON OFF) + FetchContent_Declare(openPMD + GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git" + GIT_TAG "0.15.2") + FetchContent_MakeAvailable(openPMD) +endif() + # Kokkos recommendatation resulting in not using default GNU extensions set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0843dc725dad..35a35279776c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -300,6 +300,10 @@ if (ENABLE_HDF5) target_link_libraries(parthenon PUBLIC HDF5_C) endif() +if (PARTHENON_ENABLE_OPENPMD) + target_link_libraries(parthenon PUBLIC openPMD::openPMD) +endif() + # For Cuda with NVCC (<11.2) and C++17 Kokkos currently does not work/compile with # relaxed-constexpr, see https://github.com/kokkos/kokkos/issues/3496 # However, Parthenon heavily relies on it and there is no harm in compiling Kokkos diff --git a/src/config.hpp.in b/src/config.hpp.in index 299fe0936978..a19033bddbcf 100644 --- a/src/config.hpp.in +++ b/src/config.hpp.in @@ -45,6 +45,9 @@ // defne ENABLE_HDF5 or not at all #cmakedefine ENABLE_HDF5 +// defne ENABLE_HDF5 or not at all +#cmakedefine PARTHENON_ENABLE_OPENPMD + // define PARTHENON_DISABLE_HDF5_COMPRESSION or not at all #cmakedefine PARTHENON_DISABLE_HDF5_COMPRESSION From 4525e86262a0fdcb145ae6eac636ee12d07efe23 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 11 Jan 2024 14:16:56 +0100 Subject: [PATCH 002/125] Add OpenPMD skeleto --- CMakeLists.txt | 7 +- src/CMakeLists.txt | 1 + src/config.hpp.in | 1 - src/outputs/openpmd.cpp | 213 ++++++++++++++++++++++++++++++++++++++++ src/outputs/outputs.hpp | 12 +++ 5 files changed, 230 insertions(+), 4 deletions(-) create mode 100644 src/outputs/openpmd.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index dfe1ccf3c2ff..b031bef6a060 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,12 +31,12 @@ include(CTest) # Compile time constants # Compile Options -option(PARTHENON_SINGLE_PRECISION "Run in single precision" OFF) +option(PARTHENON_SINGLE_PRENSION "Run in single precision" OFF) option(PARTHENON_DISABLE_MPI "MPI is enabled by default if found, set this to True to disable MPI" OFF) option(PARTHENON_ENABLE_HOST_COMM_BUFFERS "CUDA/HIP Only: Allocate communication buffers on host (may be slower)" OFF) option(PARTHENON_DISABLE_HDF5 "HDF5 is enabled by default if found, set this to True to disable HDF5" OFF) option(PARTHENON_DISABLE_HDF5_COMPRESSION "HDF5 compression is enabled by default, set this to True to disable compression in HDF5 output/restart files" OFF) -option(PARTHENON_DISABLE_OPENPMD "OpenPMD is enabled by default if found, set this to True to disable OpenPMD" OFF) +option(PARTHENON_ENABLE_OPENPMD "OpenPMD is enabled by default if found, set this to True to disable OpenPMD" ON) option(PARTHENON_DISABLE_SPARSE "Sparse capability is enabled by default, set this to True to compile-time disable all sparse capability" OFF) option(PARTHENON_ENABLE_ASCENT "Enable Ascent for in situ visualization and analysis" OFF) option(PARTHENON_LINT_DEFAULT "Linting is turned off by default, use the \"lint\" target or set \ @@ -185,7 +185,7 @@ if (NOT PARTHENON_DISABLE_HDF5) install(TARGETS HDF5_C EXPORT parthenonTargets) endif() -if (NOT PARTHENON_DISABLE_OPENPMD) +if (PARTHENON_ENABLE_OPENPMD) #TODO(pgrete) add logic for serial/parallel #TODO(pgrete) add logic for internal/external build include(FetchContent) @@ -201,6 +201,7 @@ if (NOT PARTHENON_DISABLE_OPENPMD) GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git" GIT_TAG "0.15.2") FetchContent_MakeAvailable(openPMD) + install(TARGETS openPMD EXPORT parthenonTargets) endif() # Kokkos recommendatation resulting in not using default GNU extensions diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 35a35279776c..b9e48fd9db92 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -178,6 +178,7 @@ add_library(parthenon outputs/history.cpp outputs/io_wrapper.cpp outputs/io_wrapper.hpp + outputs/openpmd.cpp outputs/output_utils.cpp outputs/output_utils.hpp outputs/outputs.cpp diff --git a/src/config.hpp.in b/src/config.hpp.in index a19033bddbcf..8f67ed63bc5e 100644 --- a/src/config.hpp.in +++ b/src/config.hpp.in @@ -45,7 +45,6 @@ // defne ENABLE_HDF5 or not at all #cmakedefine ENABLE_HDF5 -// defne ENABLE_HDF5 or not at all #cmakedefine PARTHENON_ENABLE_OPENPMD // define PARTHENON_DISABLE_HDF5_COMPRESSION or not at all diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp new file mode 100644 index 000000000000..94d9b7e1365a --- /dev/null +++ b/src/outputs/openpmd.cpp @@ -0,0 +1,213 @@ +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2024 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +// (C) (or copyright) 2024. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== +//! \file openpmd.cpp +// \brief Output for OpenPMD https://www.openpmd.org/ (supporting various backends) + +#include +#include +#include +#include +#include +#include +#include + +// Parthenon headers +#include "coordinates/coordinates.hpp" +#include "defs.hpp" +#include "globals.hpp" +#include "interface/variable_state.hpp" +#include "mesh/mesh.hpp" +#include "outputs/output_utils.hpp" +#include "outputs/outputs.hpp" +#include "utils/error_checking.hpp" + +// OpenPMD headers +#ifdef PARTHENON_ENABLE_OPENPMD +#include +#endif // ifdef PARTHENON_ENABLE_OPENPMD + +namespace parthenon { + +using namespace OutputUtils; + +//---------------------------------------------------------------------------------------- +//! \fn void OpenPMDOutput:::WriteOutputFile(Mesh *pm) +// \brief Expose mesh and all Cell variables for processing with Ascent +void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, + const SignalHandler::OutputSignal signal) { +#ifndef PARTHENON_ENABLE_OPENPMD + if (Globals::my_rank == 0) { + PARTHENON_WARN("OpenPMD output requested by input file, but OpenPMD support not " + "compiled in. Skipping this output type."); + } +#else + + using conduit::Node; + + // Ascent needs the MPI communicator we are using + ascent::Ascent ascent; + Node ascent_opts; + ascent_opts["mpi_comm"] = MPI_Comm_c2f(MPI_COMM_WORLD); + ascent_opts["actions_file"] = pin->GetString(output_params.block_name, "actions_file"); + // Only publish fields that are used within actions to reduce memory footprint. + // A user might need to override this, e.g., in a runtime ascent_options.yaml, if + // the required fields cannot be resolved by Ascent. + // See https://ascent.readthedocs.io/en/latest/AscentAPI.html#field-filtering + // TODO(some in mid 2023) Reenable this as this currently only works in develop of + // Ascent and not in published release (expected in 0.9.1), see + // https://github.com/Alpine-DAV/ascent/pull/1109 + // ascent_opts["field_filtering"] = "true"; + ascent.open(ascent_opts); + + // create root node for the whole mesh + Node root; + + for (auto &pmb : pm->block_list) { + // create a unique id for this MeshBlock + const std::string &meshblock_name = "domain_" + std::to_string(pmb->gid); + Node &mesh = root[meshblock_name]; + + // add basic state info + mesh["state/domain_id"] = pmb->gid; + mesh["state/cycle"] = tm->ncycle; + mesh["state/time"] = tm->time; + + auto &bounds = pmb->cellbounds; + auto ib = bounds.GetBoundsI(IndexDomain::entire); + auto jb = bounds.GetBoundsJ(IndexDomain::entire); + auto kb = bounds.GetBoundsK(IndexDomain::entire); + auto ni = ib.e - ib.s + 1; + auto nj = jb.e - jb.s + 1; + auto nk = kb.e - kb.s + 1; + uint64_t ncells = ni * nj * nk; + + auto ib_int = bounds.GetBoundsI(IndexDomain::interior); + auto jb_int = bounds.GetBoundsJ(IndexDomain::interior); + auto kb_int = bounds.GetBoundsK(IndexDomain::interior); + + auto &coords = pmb->coords; + Real dx1 = coords.CellWidth(ib.s, jb.s, kb.s); + Real dx2 = coords.CellWidth(ib.s, jb.s, kb.s); + Real dx3 = coords.CellWidth(ib.s, jb.s, kb.s); + std::array corner = coords.GetXmin(); + + // create the coordinate set + mesh["coordsets/coords/type"] = "uniform"; + PARTHENON_REQUIRE_THROWS(typeid(Coordinates_t) == typeid(UniformCartesian), + "Ascent currently only supports Cartesian coordinates."); + + mesh["coordsets/coords/dims/i"] = ni + 1; + mesh["coordsets/coords/dims/j"] = nj + 1; + if (nk > 1) { + mesh["coordsets/coords/dims/k"] = nk + 1; + } + + // add origin and spacing to the coordset (optional) + mesh["coordsets/coords/origin/x"] = corner[0]; + mesh["coordsets/coords/origin/y"] = corner[1]; + if (nk > 1) { + mesh["coordsets/coords/origin/z"] = corner[2]; + } + + mesh["coordsets/coords/spacing/dx"] = dx1; + mesh["coordsets/coords/spacing/dy"] = dx2; + if (nk > 1) { + mesh["coordsets/coords/spacing/dz"] = dx3; + } + + // add the topology + mesh["topologies/topo/type"] = "uniform"; + mesh["topologies/topo/coordset"] = "coords"; + + // indicate ghost zones with ascent_ghosts set to 1 + Node &n_field = mesh["fields/ascent_ghosts"]; + n_field["association"] = "element"; + n_field["topology"] = "topo"; + + // allocate ghost mask if not already done + if (ghost_mask_.data() == nullptr) { + ghost_mask_ = ParArray1D("Ascent ghost mask", ncells); + + const int njni = nj * ni; + auto &ghost_mask = ghost_mask_; // redef to lambda capture class member + pmb->par_for( + "Set ascent ghost mask", 0, ncells - 1, KOKKOS_LAMBDA(const int &idx) { + const int k = idx / (njni); + const int j = (idx - k * njni) / ni; + const int i = idx - k * njni - j * nj; + + if ((i < ib_int.s) || (ib_int.e < i) || (j < jb_int.s) || (jb_int.e < j) || + ((nk > 1) && ((k < kb_int.s) || (kb_int.e < k)))) { + ghost_mask(idx) = 1; + } else { + ghost_mask(idx) = 0; + } + }); + } + // Set ghost mask + n_field["values"].set_external(ghost_mask_.data(), ncells); + + // create a field for each component of each variable pack + auto &mbd = pmb->meshblock_data.Get(); + + for (const auto &var : mbd->GetVariableVector()) { + // ensure that only cell vars are added (for now) as the topology above is only + // valid for cell centered vars + if (!var->IsSet(Metadata::Cell)) { + continue; + } + const auto var_info = VarInfo(var); + + for (int icomp = 0; icomp < var_info.num_components; ++icomp) { + auto const data = Kokkos::subview(var->data, 0, 0, icomp, Kokkos::ALL(), + Kokkos::ALL(), Kokkos::ALL()); + const std::string varname = var_info.component_labels.at(icomp); + mesh["fields/" + varname + "/association"] = "element"; + mesh["fields/" + varname + "/topology"] = "topo"; + mesh["fields/" + varname + "/values"].set_external(data.data(), ncells); + } + } + } + + // make sure we conform: + Node verify_info; + if (!conduit::blueprint::mesh::verify(root, verify_info)) { + if (parthenon::Globals::my_rank == 0) { + PARTHENON_WARN("Ascent output: blueprint::mesh::verify failed!"); + } + verify_info.print(); + } + ascent.publish(root); + + // Create dummy action as we need to "execute" to override the actions defined in the + // yaml file. + Node actions; + // execute the actions + ascent.execute(actions); + + // close ascent + ascent.close(); +#endif // ifndef PARTHENON_ENABLE_OPENPMD + + // advance output parameters + output_params.file_number++; + output_params.next_time += output_params.dt; + pin->SetInteger(output_params.block_name, "file_number", output_params.file_number); + pin->SetReal(output_params.block_name, "next_time", output_params.next_time); +} + +} // namespace parthenon diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index 4fd64236a19c..bd24a2d95e4c 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -189,6 +189,18 @@ class AscentOutput : public OutputType { ParArray1D ghost_mask_; }; +//---------------------------------------------------------------------------------------- +//! \class OpenPMDOutput +// \brief derived OutputType class for OpenPMD based output + +class OpenPMDOutput : public OutputType { + public: + explicit OpenPMDOutput(const OutputParameters &oparams) : OutputType(oparams) {} + void WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, + const SignalHandler::OutputSignal signal) override; + +}; + #ifdef ENABLE_HDF5 //---------------------------------------------------------------------------------------- //! \class PHDF5Output From 4de250caa11c3115a210216305f59f92e0a18fb3 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 11 Jan 2024 23:41:01 +0100 Subject: [PATCH 003/125] WIP more Open PMD --- src/outputs/openpmd.cpp | 58 +++++++++++++++++++++++++---------------- src/outputs/outputs.cpp | 11 ++++++++ 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 94d9b7e1365a..7943a3e0efe6 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -31,6 +31,8 @@ #include "globals.hpp" #include "interface/variable_state.hpp" #include "mesh/mesh.hpp" +#include "openPMD/IO/Access.hpp" +#include "openPMD/Series.hpp" #include "outputs/output_utils.hpp" #include "outputs/outputs.hpp" #include "utils/error_checking.hpp" @@ -48,38 +50,50 @@ using namespace OutputUtils; //! \fn void OpenPMDOutput:::WriteOutputFile(Mesh *pm) // \brief Expose mesh and all Cell variables for processing with Ascent void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, - const SignalHandler::OutputSignal signal) { + const SignalHandler::OutputSignal signal) { #ifndef PARTHENON_ENABLE_OPENPMD if (Globals::my_rank == 0) { PARTHENON_WARN("OpenPMD output requested by input file, but OpenPMD support not " "compiled in. Skipping this output type."); } #else - - using conduit::Node; - - // Ascent needs the MPI communicator we are using - ascent::Ascent ascent; - Node ascent_opts; - ascent_opts["mpi_comm"] = MPI_Comm_c2f(MPI_COMM_WORLD); - ascent_opts["actions_file"] = pin->GetString(output_params.block_name, "actions_file"); - // Only publish fields that are used within actions to reduce memory footprint. - // A user might need to override this, e.g., in a runtime ascent_options.yaml, if - // the required fields cannot be resolved by Ascent. - // See https://ascent.readthedocs.io/en/latest/AscentAPI.html#field-filtering - // TODO(some in mid 2023) Reenable this as this currently only works in develop of - // Ascent and not in published release (expected in 0.9.1), see - // https://github.com/Alpine-DAV/ascent/pull/1109 - // ascent_opts["field_filtering"] = "true"; - ascent.open(ascent_opts); - - // create root node for the whole mesh - Node root; + using openPMD::Access; + using openPMD::Series; + + // TODO(pgrete) .h5 for hd5 and .bp for ADIOS2 or .json for JSON + // TODO(pgrete) check if CREATE is the correct pattern (for not overwriting the series + // but an interation) This just describes the pattern of the filename. The correct file + // will be accessed through the iteration idx below. The file suffix maps to the chosen + // backend. + Series series = Series("test_%05T.h5", Access::CREATE); + + // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? + // TODO(pgrete) Should we update for restart or only set this once? Or make it per + // iteration? + // ... = pin->GetString(output_params.block_name, "actions_file"); + series.setAuthor("My Name file naming + auto it = series.iterations[42]; + it.open(); // explicit open() is important when run in parallel + + it.setTime(tm->time); + it.setDt(tm->dt); for (auto &pmb : pm->block_list) { // create a unique id for this MeshBlock const std::string &meshblock_name = "domain_" + std::to_string(pmb->gid); - Node &mesh = root[meshblock_name]; + auto mesh = it.meshes[meshblock_name]; // add basic state info mesh["state/domain_id"] = pmb->gid; diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index 33b35a24f284..028dad6513f3 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -246,6 +246,17 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { pnew_type = new VTKOutput(op); } else if (op.file_type == "ascent") { pnew_type = new AscentOutput(op); + } else if (op.file_type == "openpmd") { +#ifdef PARTHENON_ENABLE_OPENPMD + pnew_type = new OpenPMDOutput(op, pin); +#else + msg << "### FATAL ERROR in Outputs constructor" << std::endl + << "Executable not configured for OpenPMD outputs, but OpenPMD file format " + << "is requested in output/restart block '" << op.block_name << "'. " + << "You can disable this block without deleting it by setting a dt < 0." + << std::endl; + PARTHENON_FAIL(msg); +#endif // ifdef PARTHENON_ENABLE_OPENPMD } else if (op.file_type == "histogram") { #ifdef ENABLE_HDF5 pnew_type = new HistogramOutput(op, pin); From b906af7a87c27bb21a4931834644db02633bfe09 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 12 Jan 2024 17:55:04 +0100 Subject: [PATCH 004/125] WIP OpenPMD use file id --- src/outputs/openpmd.cpp | 60 +++++------------------------------------ 1 file changed, 7 insertions(+), 53 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 7943a3e0efe6..01330cb48ec3 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -84,12 +84,11 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // open iteration (corresponding to a timestep in OpenPMD naming) // TODO(pgrete) fix iteration name <-> file naming - auto it = series.iterations[42]; + auto it = series.iterations[output_params.file_number]; it.open(); // explicit open() is important when run in parallel it.setTime(tm->time); it.setDt(tm->dt); - for (auto &pmb : pm->block_list) { // create a unique id for this MeshBlock const std::string &meshblock_name = "domain_" + std::to_string(pmb->gid); @@ -143,38 +142,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, mesh["coordsets/coords/spacing/dz"] = dx3; } - // add the topology - mesh["topologies/topo/type"] = "uniform"; - mesh["topologies/topo/coordset"] = "coords"; - - // indicate ghost zones with ascent_ghosts set to 1 - Node &n_field = mesh["fields/ascent_ghosts"]; - n_field["association"] = "element"; - n_field["topology"] = "topo"; - - // allocate ghost mask if not already done - if (ghost_mask_.data() == nullptr) { - ghost_mask_ = ParArray1D("Ascent ghost mask", ncells); - - const int njni = nj * ni; - auto &ghost_mask = ghost_mask_; // redef to lambda capture class member - pmb->par_for( - "Set ascent ghost mask", 0, ncells - 1, KOKKOS_LAMBDA(const int &idx) { - const int k = idx / (njni); - const int j = (idx - k * njni) / ni; - const int i = idx - k * njni - j * nj; - - if ((i < ib_int.s) || (ib_int.e < i) || (j < jb_int.s) || (jb_int.e < j) || - ((nk > 1) && ((k < kb_int.s) || (kb_int.e < k)))) { - ghost_mask(idx) = 1; - } else { - ghost_mask(idx) = 0; - } - }); - } - // Set ghost mask - n_field["values"].set_external(ghost_mask_.data(), ncells); - // create a field for each component of each variable pack auto &mbd = pmb->meshblock_data.Get(); @@ -196,25 +163,12 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } } } - - // make sure we conform: - Node verify_info; - if (!conduit::blueprint::mesh::verify(root, verify_info)) { - if (parthenon::Globals::my_rank == 0) { - PARTHENON_WARN("Ascent output: blueprint::mesh::verify failed!"); - } - verify_info.print(); - } - ascent.publish(root); - - // Create dummy action as we need to "execute" to override the actions defined in the - // yaml file. - Node actions; - // execute the actions - ascent.execute(actions); - - // close ascent - ascent.close(); + // The iteration can be closed in order to help free up resources. + // The iteration's content will be flushed automatically. + // An iteration once closed cannot (yet) be reopened. + it.close(); + // No need to close series as it's done in the desctructor of the object when it runs + // out of scope. #endif // ifndef PARTHENON_ENABLE_OPENPMD // advance output parameters From 79660a2605cdd8b70afd2bfaa3992006eb9c0898 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 29 Feb 2024 16:48:13 +0100 Subject: [PATCH 005/125] Write blocks --- src/outputs/openpmd.cpp | 120 ++++++++++++++++++++++++---------------- src/outputs/outputs.cpp | 2 +- 2 files changed, 73 insertions(+), 49 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 01330cb48ec3..2868460ef09c 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -17,6 +17,7 @@ //! \file openpmd.cpp // \brief Output for OpenPMD https://www.openpmd.org/ (supporting various backends) +#include #include #include #include @@ -24,6 +25,8 @@ #include #include #include +#include +#include // Parthenon headers #include "coordinates/coordinates.hpp" @@ -31,10 +34,14 @@ #include "globals.hpp" #include "interface/variable_state.hpp" #include "mesh/mesh.hpp" +#include "openPMD/Datatype.hpp" #include "openPMD/IO/Access.hpp" +#include "openPMD/Mesh.hpp" #include "openPMD/Series.hpp" +#include "openPMD/backend/MeshRecordComponent.hpp" #include "outputs/output_utils.hpp" #include "outputs/outputs.hpp" +#include "parthenon_array_generic.hpp" #include "utils/error_checking.hpp" // OpenPMD headers @@ -65,7 +72,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // but an interation) This just describes the pattern of the filename. The correct file // will be accessed through the iteration idx below. The file suffix maps to the chosen // backend. - Series series = Series("test_%05T.h5", Access::CREATE); + Series series = Series("adios_test_%05T.bp", Access::CREATE); // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? // TODO(pgrete) Should we update for restart or only set this once? Or make it per @@ -75,13 +82,16 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, series.setComment("Hello world!"); series.setMachine("bla"); series.setSoftware("Parthenon + Downstream info"); - series.setDate("today"); + series.setDate("2024-02-29"); // TODO(pgrete) Units? // TODO(pgrete) We probably want this for params! series.setAttribute("bla", true); + // In line with existing outputs, we write one file per iteration/snapshot + series.setIterationEncoding(openPMD::IterationEncoding::fileBased); + // open iteration (corresponding to a timestep in OpenPMD naming) // TODO(pgrete) fix iteration name <-> file naming auto it = series.iterations[output_params.file_number]; @@ -89,64 +99,64 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, it.setTime(tm->time); it.setDt(tm->dt); - for (auto &pmb : pm->block_list) { + + // TODO(pgrete) check var name standard compatiblity + // e.g., description: names of records and their components are only allowed to contain + // the characters a-Z, the numbers 0-9 and the underscore _ + + const int num_blocks_local = static_cast(pm->block_list.size()); + // TODO(pgrete) adjust for single prec output + // openPMD::Datatype dtype = openPMD::determineDatatype(); + // using OutT = typename std::conditional::type; + // Need to create the buffer outside so that it's persistent until the data is flushed. + // Dynamically allocating the inner vector should not be a performance bottleneck given + // that the hdf5 backend also uses an explicit per block and per variable + // GetHostMirrorAndCopy, but still this could be optimized. + std::vector> buffer_list; + + for (size_t b_idx = 0; b_idx < num_blocks_local; ++b_idx) { + const auto &pmb = pm->block_list[b_idx]; // create a unique id for this MeshBlock - const std::string &meshblock_name = "domain_" + std::to_string(pmb->gid); - auto mesh = it.meshes[meshblock_name]; + const std::string &meshblock_name = "block_" + std::to_string(pmb->gid); + auto block_record = it.meshes[meshblock_name]; - // add basic state info - mesh["state/domain_id"] = pmb->gid; - mesh["state/cycle"] = tm->ncycle; - mesh["state/time"] = tm->time; + // TODO(pgrete) check if we should update the logic (e.g., defining axes) for 1D and + // 2D outputs auto &bounds = pmb->cellbounds; - auto ib = bounds.GetBoundsI(IndexDomain::entire); - auto jb = bounds.GetBoundsJ(IndexDomain::entire); - auto kb = bounds.GetBoundsK(IndexDomain::entire); - auto ni = ib.e - ib.s + 1; - auto nj = jb.e - jb.s + 1; - auto nk = kb.e - kb.s + 1; - uint64_t ncells = ni * nj * nk; - - auto ib_int = bounds.GetBoundsI(IndexDomain::interior); - auto jb_int = bounds.GetBoundsJ(IndexDomain::interior); - auto kb_int = bounds.GetBoundsK(IndexDomain::interior); + auto ib = bounds.GetBoundsI(IndexDomain::interior); + auto jb = bounds.GetBoundsJ(IndexDomain::interior); + auto kb = bounds.GetBoundsK(IndexDomain::interior); + uint64_t ni = ib.e - ib.s + 1; + uint64_t nj = jb.e - jb.s + 1; + uint64_t nk = kb.e - kb.s + 1; + uint64_t ncells = ni * nj * nk; auto &coords = pmb->coords; Real dx1 = coords.CellWidth(ib.s, jb.s, kb.s); Real dx2 = coords.CellWidth(ib.s, jb.s, kb.s); Real dx3 = coords.CellWidth(ib.s, jb.s, kb.s); - std::array corner = coords.GetXmin(); - // create the coordinate set - mesh["coordsets/coords/type"] = "uniform"; + // These attributes are shared across all components. + // In general, we should check if there's a performance bottleneck with writing so + // many meshes rather than writing one mesh per level and dump all data there (which + // relies on support for sparse storage by the backend.). + + // TODO(pgrete) check if this should be tied to the MemoryLayout + block_record.setDataOrder(openPMD::Mesh::DataOrder::C); + block_record.setGridSpacing(std::vector{dx3, dx2, dx1}); + block_record.setAxisLabels({"z", "y", "x"}); + std::array corner = coords.GetXmin(); + block_record.setGridGlobalOffset({corner[2], corner[1], corner[0]}); PARTHENON_REQUIRE_THROWS(typeid(Coordinates_t) == typeid(UniformCartesian), "Ascent currently only supports Cartesian coordinates."); - - mesh["coordsets/coords/dims/i"] = ni + 1; - mesh["coordsets/coords/dims/j"] = nj + 1; - if (nk > 1) { - mesh["coordsets/coords/dims/k"] = nk + 1; - } - - // add origin and spacing to the coordset (optional) - mesh["coordsets/coords/origin/x"] = corner[0]; - mesh["coordsets/coords/origin/y"] = corner[1]; - if (nk > 1) { - mesh["coordsets/coords/origin/z"] = corner[2]; - } - - mesh["coordsets/coords/spacing/dx"] = dx1; - mesh["coordsets/coords/spacing/dy"] = dx2; - if (nk > 1) { - mesh["coordsets/coords/spacing/dz"] = dx3; - } + block_record.setGeometry(openPMD::Mesh::Geometry::cartesian); // create a field for each component of each variable pack auto &mbd = pmb->meshblock_data.Get(); for (const auto &var : mbd->GetVariableVector()) { - // ensure that only cell vars are added (for now) as the topology above is only + // ensure that only cell vars are added (for now) as the topology below is only // valid for cell centered vars if (!var->IsSet(Metadata::Cell)) { continue; @@ -154,12 +164,26 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, const auto var_info = VarInfo(var); for (int icomp = 0; icomp < var_info.num_components; ++icomp) { - auto const data = Kokkos::subview(var->data, 0, 0, icomp, Kokkos::ALL(), - Kokkos::ALL(), Kokkos::ALL()); + // Pick a subview of the active cells of this component + auto const data = Kokkos::subview( + var->data, 0, 0, icomp, std::make_pair(kb.s, kb.e + 1), + std::make_pair(jb.s, jb.e + 1), std::make_pair(ib.s, ib.e + 1)); + + // Map a view onto a host allocation (so that we can call deep_copy) + auto component_buffer = buffer_list.emplace_back(ncells); + Kokkos::View> + component_buffer_view(component_buffer.data(), nk, nj, ni); + Kokkos::deep_copy(component_buffer_view, data); + const std::string varname = var_info.component_labels.at(icomp); - mesh["fields/" + varname + "/association"] = "element"; - mesh["fields/" + varname + "/topology"] = "topo"; - mesh["fields/" + varname + "/values"].set_external(data.data(), ncells); + auto component_record = block_record[varname]; + // TODO(pgrete) needs to be updated for face and edges etc + component_record.setPosition(std::vector{0.5, 0.5, 0.5}); + auto const dataset = + openPMD::Dataset(openPMD::determineDatatype(), {nk, nj, ni}); + component_record.resetDataset(dataset); + + component_record.storeChunkRaw(component_buffer.data(), {0, 0, 0}, {nk, nj, ni}); } } } diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index ef197177f957..bce7ac4fbffa 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -248,7 +248,7 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { pnew_type = new AscentOutput(op); } else if (op.file_type == "openpmd") { #ifdef PARTHENON_ENABLE_OPENPMD - pnew_type = new OpenPMDOutput(op, pin); + pnew_type = new OpenPMDOutput(op); #else msg << "### FATAL ERROR in Outputs constructor" << std::endl << "Executable not configured for OpenPMD outputs, but OpenPMD file format " From 8d40c91d806504632c05e0aeadeaa483178d642d Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 8 Mar 2024 10:29:44 +0100 Subject: [PATCH 006/125] Centralize getting var info for output --- src/outputs/output_utils.cpp | 28 ++++++++++++++++++++++++++++ src/outputs/output_utils.hpp | 10 ++++++++++ src/outputs/parthenon_hdf5.cpp | 24 +++--------------------- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/src/outputs/output_utils.cpp b/src/outputs/output_utils.cpp index 64c099d85617..baeacf8cbbe2 100644 --- a/src/outputs/output_utils.cpp +++ b/src/outputs/output_utils.cpp @@ -208,5 +208,33 @@ std::size_t MPISum(std::size_t val) { return val; } +VariableVector GetVarsToWrite(const std::shared_ptr pmb, + const bool restart, + const std::vector &variables) { + const auto &var_vec = pmb->meshblock_data.Get()->GetVariableVector(); + auto vars_to_write = GetAnyVariables(var_vec, variables); + if (restart) { + // get all vars with flag Independent OR restart + auto restart_vars = GetAnyVariables( + var_vec, {parthenon::Metadata::Independent, parthenon::Metadata::Restart}); + for (auto restart_var : restart_vars) { + vars_to_write.emplace_back(restart_var); + } + } + return vars_to_write; +} + +std::vector GetAllVarsInfo(const VariableVector &vars) { + std::vector all_vars_info; + for (auto &v : vars) { + all_vars_info.emplace_back(v); + } + + // sort alphabetically + std::sort(all_vars_info.begin(), all_vars_info.end(), + [](const VarInfo &a, const VarInfo &b) { return a.label < b.label; }); + return all_vars_info; +} + } // namespace OutputUtils } // namespace parthenon diff --git a/src/outputs/output_utils.hpp b/src/outputs/output_utils.hpp index db6353090cbf..20c9a8117ff6 100644 --- a/src/outputs/output_utils.hpp +++ b/src/outputs/output_utils.hpp @@ -263,6 +263,16 @@ std::vector ComputeIDsAndFlags(Mesh *pm); std::size_t MPIPrefixSum(std::size_t local, std::size_t &tot_count); std::size_t MPISum(std::size_t local); +// Return all variables to write, i.e., for restarts all indpendent variables and ones +// with explicit Restart flag, but also variables explicitly defined to output in the +// input file. +VariableVector GetVarsToWrite(const std::shared_ptr pmb, + const bool restart, + const std::vector &variables); + +// Returns a sorted vector of VarInfo associated with vars +std::vector GetAllVarsInfo(const VariableVector &vars); + } // namespace OutputUtils } // namespace parthenon diff --git a/src/outputs/parthenon_hdf5.cpp b/src/outputs/parthenon_hdf5.cpp index d7677dfee4fd..21d82a8cfac9 100644 --- a/src/outputs/parthenon_hdf5.cpp +++ b/src/outputs/parthenon_hdf5.cpp @@ -239,28 +239,10 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm // All blocks have the same list of variable metadata that exist in the entire // simulation, but not all variables may be allocated on all blocks - auto get_vars = [=](const std::shared_ptr pmb) { - auto &var_vec = pmb->meshblock_data.Get()->GetVariableVector(); - if (restart_) { - // get all vars with flag Independent OR restart - return GetAnyVariables( - var_vec, {parthenon::Metadata::Independent, parthenon::Metadata::Restart}); - } else { - return GetAnyVariables(var_vec, output_params.variables); - } - }; - // get list of all vars, just use the first block since the list is the same for all // blocks - std::vector all_vars_info; - const auto vars = get_vars(pm->block_list.front()); - for (auto &v : vars) { - all_vars_info.emplace_back(v); - } - - // sort alphabetically - std::sort(all_vars_info.begin(), all_vars_info.end(), - [](const VarInfo &a, const VarInfo &b) { return a.label < b.label; }); + auto all_vars_info = GetAllVarsInfo( + GetVarsToWrite(pm->block_list.front(), restart_, output_params.variables)); // We need to add information about the sparse variables to the HDF5 file, namely: // 1) Which variables are sparse @@ -391,7 +373,7 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm bool is_allocated = false; // for each variable that this local meshblock actually has - const auto vars = get_vars(pmb); + const auto vars = GetVarsToWrite(pmb, restart_, output_params.variables); for (auto &v : vars) { // For reference, if we update the logic here, there's also // a similar block in parthenon_manager.cpp From 4f20c26ebbb922e59d10f105dd653e1b6a1fd769 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 8 Mar 2024 16:36:34 +0100 Subject: [PATCH 007/125] WIP openpmd, chunks don't work yet plus check dimensionality --- src/outputs/openpmd.cpp | 236 +++++++++++++++++++++++---------- src/outputs/parthenon_hdf5.cpp | 30 ++--- 2 files changed, 182 insertions(+), 84 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 2868460ef09c..9d9245a68947 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -17,6 +17,8 @@ //! \file openpmd.cpp // \brief Output for OpenPMD https://www.openpmd.org/ (supporting various backends) +#include +#include #include #include #include @@ -29,11 +31,13 @@ #include // Parthenon headers +#include "basic_types.hpp" #include "coordinates/coordinates.hpp" #include "defs.hpp" #include "globals.hpp" #include "interface/variable_state.hpp" #include "mesh/mesh.hpp" +#include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/IO/Access.hpp" #include "openPMD/Mesh.hpp" @@ -105,65 +109,105 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // the characters a-Z, the numbers 0-9 and the underscore _ const int num_blocks_local = static_cast(pm->block_list.size()); + + // -------------------------------------------------------------------------------- // + // WRITING VARIABLES DATA // + // -------------------------------------------------------------------------------- // + Kokkos::Profiling::pushRegion("write all variable data"); + + // get list of all vars, just use the first block since the list is the same for all + // blocks + // TODO(pgrete) add restart_ var to output + auto all_vars_info = GetAllVarsInfo( + GetVarsToWrite(pm->block_list.front(), true, output_params.variables)); + + // We're currently writing (flushing) one var at a time. This saves host memory but + // results more smaller write. Might be updated in the future. + // Allocate space for largest size variable + int var_size_max = 0; + for (auto &vinfo : all_vars_info) { + const auto var_size = vinfo.Size(); + var_size_max = std::max(var_size_max, var_size); + } + // TODO(pgrete) adjust for single prec output // openPMD::Datatype dtype = openPMD::determineDatatype(); - // using OutT = typename std::conditional::type; - // Need to create the buffer outside so that it's persistent until the data is flushed. - // Dynamically allocating the inner vector should not be a performance bottleneck given - // that the hdf5 backend also uses an explicit per block and per variable - // GetHostMirrorAndCopy, but still this could be optimized. - std::vector> buffer_list; - - for (size_t b_idx = 0; b_idx < num_blocks_local; ++b_idx) { - const auto &pmb = pm->block_list[b_idx]; - // create a unique id for this MeshBlock - const std::string &meshblock_name = "block_" + std::to_string(pmb->gid); - auto block_record = it.meshes[meshblock_name]; - - // TODO(pgrete) check if we should update the logic (e.g., defining axes) for 1D and - // 2D outputs - - auto &bounds = pmb->cellbounds; - auto ib = bounds.GetBoundsI(IndexDomain::interior); - auto jb = bounds.GetBoundsJ(IndexDomain::interior); - auto kb = bounds.GetBoundsK(IndexDomain::interior); - uint64_t ni = ib.e - ib.s + 1; - uint64_t nj = jb.e - jb.s + 1; - uint64_t nk = kb.e - kb.s + 1; - - uint64_t ncells = ni * nj * nk; - auto &coords = pmb->coords; - Real dx1 = coords.CellWidth(ib.s, jb.s, kb.s); - Real dx2 = coords.CellWidth(ib.s, jb.s, kb.s); - Real dx3 = coords.CellWidth(ib.s, jb.s, kb.s); - - // These attributes are shared across all components. - // In general, we should check if there's a performance bottleneck with writing so - // many meshes rather than writing one mesh per level and dump all data there (which - // relies on support for sparse storage by the backend.). - - // TODO(pgrete) check if this should be tied to the MemoryLayout - block_record.setDataOrder(openPMD::Mesh::DataOrder::C); - block_record.setGridSpacing(std::vector{dx3, dx2, dx1}); - block_record.setAxisLabels({"z", "y", "x"}); - std::array corner = coords.GetXmin(); - block_record.setGridGlobalOffset({corner[2], corner[1], corner[0]}); - PARTHENON_REQUIRE_THROWS(typeid(Coordinates_t) == typeid(UniformCartesian), - "Ascent currently only supports Cartesian coordinates."); - block_record.setGeometry(openPMD::Mesh::Geometry::cartesian); - - // create a field for each component of each variable pack - auto &mbd = pmb->meshblock_data.Get(); - - for (const auto &var : mbd->GetVariableVector()) { - // ensure that only cell vars are added (for now) as the topology below is only - // valid for cell centered vars - if (!var->IsSet(Metadata::Cell)) { - continue; + using OutT = + Real; // typename std::conditional::type; + std::vector tmp_data(var_size_max * num_blocks_local); + + // TODO(pgrete) This needs to be in the loop for non-cell-centered vars + auto &bounds = pm->block_list.front()->cellbounds; + auto ib = bounds.GetBoundsI(IndexDomain::interior); + auto jb = bounds.GetBoundsJ(IndexDomain::interior); + auto kb = bounds.GetBoundsK(IndexDomain::interior); + // for each variable we write + for (auto &vinfo : all_vars_info) { + PARTHENON_INSTRUMENT_REGION("Write variable loop") + + // Reset host write bufer. Not really necessary, but doesn't hurt. + memset(tmp_data.data(), 0, tmp_data.size() * sizeof(OutT)); + uint64_t tmp_offset = 0; + + const bool is_scalar = vinfo.nx4 == 1 && vinfo.nx5 == 1 && vinfo.nx6 == 1; + if (vinfo.is_vector) { + // sanity check + PARTHENON_REQUIRE_THROWS( + vinfo.nx4 == pm->ndim && vinfo.nx5 == 1 && vinfo.nx6 == 1, + "A 'standard' vector is expected to only have components matching the " + "dimensionality of the simulation.") + } + + // TODO(pgrete) need to make sure that var names are allowed within standard + const std::string var_name = vinfo.label; + for (auto &pmb : pm->block_list) { + // TODO(pgrete) check if we should skip the suffix for level 0 + const auto level = pmb->loc.level() - pm->GetRootLevel(); + const std::string &mesh_record_name = var_name + "_lvl" + std::to_string(level); + + // Create the mesh_record for this variable at the given level (if it doesn't exist + // yet) + if (!it.meshes.contains(mesh_record_name)) { + auto mesh_record = it.meshes[mesh_record_name]; + + // These following attributes are shared across all components of the record. + + PARTHENON_REQUIRE_THROWS( + typeid(Coordinates_t) == typeid(UniformCartesian), + "OpenPMD in Parthenon currently only supports Cartesian coordinates."); + mesh_record.setGeometry(openPMD::Mesh::Geometry::cartesian); + auto &coords = pmb->coords; + // For uniform Cartesian, all dxN are const across the block so we just pick the + // first index. + Real dx1 = coords.CellWidth(0, 0, 0); + Real dx2 = coords.CellWidth(0, 0, 0); + Real dx3 = coords.CellWidth(0, 0, 0); + + // TODO(pgrete) check if this should be tied to the MemoryLayout + mesh_record.setDataOrder(openPMD::Mesh::DataOrder::C); + mesh_record.setGridSpacing(std::vector{dx3, dx2, dx1}); + mesh_record.setAxisLabels({"z", "y", "x"}); + mesh_record.setGridGlobalOffset({ + pm->mesh_size.xmin(X3DIR), + pm->mesh_size.xmin(X2DIR), + pm->mesh_size.xmin(X1DIR), + }); + + // TODO(pgrete) need unitDimension and timeOffset for this record? } - const auto var_info = VarInfo(var); - for (int icomp = 0; icomp < var_info.num_components; ++icomp) { + auto mesh_record = it.meshes[mesh_record_name]; + + // Now that the mesh record exists, actually write the data + auto out_var = pmb->meshblock_data.Get()->GetVarPtr(var_name); + PARTHENON_REQUIRE_THROWS(out_var->metadata().Where() == + MetadataFlag(Metadata::Cell), + "Currently only cell centered vars are supported."); + + if (out_var->IsAllocated()) { + // TODO(pgrete) check if we can work with a direct copy from a subview to not + // duplicate the memory footprint here +#if 0 // Pick a subview of the active cells of this component auto const data = Kokkos::subview( var->data, 0, 0, icomp, std::make_pair(kb.s, kb.e + 1), @@ -174,25 +218,81 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, Kokkos::View> component_buffer_view(component_buffer.data(), nk, nj, ni); Kokkos::deep_copy(component_buffer_view, data); +#endif + auto out_var_h = out_var->data.GetHostMirrorAndCopy(); + int idx_component = 0; + const auto &Nt = out_var->GetDim(6); + const auto &Nu = out_var->GetDim(5); + const auto &Nv = out_var->GetDim(4); + // loop over all components + for (int t = 0; t < Nt; ++t) { + for (int u = 0; u < Nu; ++u) { + for (int v = 0; v < Nv; ++v) { + // Get the correct record + std::string comp_name; + if (is_scalar) { + comp_name = openPMD::MeshRecordComponent::SCALAR; + } else if (vinfo.is_vector) { + if (v == 0) { + comp_name = "x"; + } else if (v == 1) { + comp_name = "y"; + } else if (v == 2) { + comp_name = "z"; + } else { + PARTHENON_THROW("Expected v index doesn't match vector expectation."); + } + } else { + comp_name = vinfo.component_labels[idx_component]; + } + auto mesh_comp = mesh_record[comp_name]; - const std::string varname = var_info.component_labels.at(icomp); - auto component_record = block_record[varname]; - // TODO(pgrete) needs to be updated for face and edges etc - component_record.setPosition(std::vector{0.5, 0.5, 0.5}); - auto const dataset = - openPMD::Dataset(openPMD::determineDatatype(), {nk, nj, ni}); - component_record.resetDataset(dataset); + // TODO(pgrete) needs to be updated for face and edges etc + mesh_comp.setPosition(std::vector{0.5, 0.5, 0.5}); + // TODO(pgrete) needs to be updated for face and edges etc + // Also this feels wrong for deep hierachies... + auto effective_nx = static_cast(std::pow(2, level)); + openPMD::Extent global_extent = { + static_cast(pm->mesh_size.nx(X3DIR)) * effective_nx, + static_cast(pm->mesh_size.nx(X2DIR)) * effective_nx, + static_cast(pm->mesh_size.nx(X1DIR)) * effective_nx, + }; + auto const dataset = + openPMD::Dataset(openPMD::determineDatatype(), global_extent); + mesh_comp.resetDataset(dataset); + + const auto comp_offset = tmp_offset; + for (int k = kb.s; k <= kb.e; ++k) { + for (int j = jb.s; j <= jb.e; ++j) { + for (int i = ib.s; i <= ib.e; ++i) { + tmp_data[tmp_offset] = static_cast(out_var_h(t, u, v, k, j, i)); + tmp_offset++; + } + } + } + openPMD::Offset chunk_offset = { + pmb->loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), + pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), + pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; + openPMD::Extent chunk_extent = {static_cast(vinfo.nx3), + static_cast(vinfo.nx2), + static_cast(vinfo.nx1)}; + mesh_comp.storeChunkRaw(&tmp_data[comp_offset], chunk_offset, chunk_extent); + idx_component += 1; + } + } + } // loop over components + } // out_var->IsAllocated() + } // loop over blocks + it.seriesFlush(); + } // loop over vars + Kokkos::Profiling::popRegion(); // write all variable data - component_record.storeChunkRaw(component_buffer.data(), {0, 0, 0}, {nk, nj, ni}); - } - } - } // The iteration can be closed in order to help free up resources. // The iteration's content will be flushed automatically. // An iteration once closed cannot (yet) be reopened. it.close(); - // No need to close series as it's done in the desctructor of the object when it runs - // out of scope. + series.close(); #endif // ifndef PARTHENON_ENABLE_OPENPMD // advance output parameters diff --git a/src/outputs/parthenon_hdf5.cpp b/src/outputs/parthenon_hdf5.cpp index 21d82a8cfac9..75c61cc76196 100644 --- a/src/outputs/parthenon_hdf5.cpp +++ b/src/outputs/parthenon_hdf5.cpp @@ -372,22 +372,20 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm const auto &pmb = pm->block_list[b_idx]; bool is_allocated = false; - // for each variable that this local meshblock actually has - const auto vars = GetVarsToWrite(pmb, restart_, output_params.variables); - for (auto &v : vars) { - // For reference, if we update the logic here, there's also - // a similar block in parthenon_manager.cpp - if (v->IsAllocated() && (var_name == v->label())) { - auto v_h = v->data.GetHostMirrorAndCopy(); - OutputUtils::PackOrUnpackVar( - pmb.get(), v.get(), output_params.include_ghost_zones, index, tmpData, - [&](auto index, int t, int u, int v, int k, int j, int i) { - tmpData[index] = static_cast(v_h(t, u, v, k, j, i)); - }); - - is_allocated = true; - break; - } + // TODO(reviewers) Why was the loop originally there? Does the direct Get causes + // issue? + auto v = pmb->meshblock_data.Get()->GetVarPtr(var_name); + // For reference, if we update the logic here, there's also + // a similar block in parthenon_manager.cpp + if (v->IsAllocated() && (var_name == v->label())) { + auto v_h = v->data.GetHostMirrorAndCopy(); + OutputUtils::PackOrUnpackVar( + pmb.get(), v.get(), output_params.include_ghost_zones, index, tmpData, + [&](auto index, int t, int u, int v, int k, int j, int i) { + tmpData[index] = static_cast(v_h(t, u, v, k, j, i)); + }); + + is_allocated = true; } if (vinfo.is_sparse) { From f29e8d1d85bc7f4dca742f80c6f6bf4376748f2f Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 14 Mar 2024 15:23:46 +0100 Subject: [PATCH 008/125] Fix chunk extents --- src/outputs/openpmd.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 9d9245a68947..1702a6503a7e 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -274,9 +274,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, pmb->loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; - openPMD::Extent chunk_extent = {static_cast(vinfo.nx3), - static_cast(vinfo.nx2), - static_cast(vinfo.nx1)}; + openPMD::Extent chunk_extent = { + static_cast(pmb->block_size.nx(X3DIR)), + static_cast(pmb->block_size.nx(X2DIR)), + static_cast(pmb->block_size.nx(X1DIR))}; mesh_comp.storeChunkRaw(&tmp_data[comp_offset], chunk_offset, chunk_extent); idx_component += 1; } From a501a5dc8c2b90d2ebd36923830c0556f3ddd833 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 15 Mar 2024 12:31:12 +0100 Subject: [PATCH 009/125] Write Attributes --- src/outputs/openpmd.cpp | 86 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 3 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 1702a6503a7e..da04b1123c38 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -34,6 +34,7 @@ #include "basic_types.hpp" #include "coordinates/coordinates.hpp" #include "defs.hpp" +#include "driver/driver.hpp" #include "globals.hpp" #include "interface/variable_state.hpp" #include "mesh/mesh.hpp" @@ -47,6 +48,7 @@ #include "outputs/outputs.hpp" #include "parthenon_array_generic.hpp" #include "utils/error_checking.hpp" +#include "utils/instrument.hpp" // OpenPMD headers #ifdef PARTHENON_ENABLE_OPENPMD @@ -76,7 +78,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // but an interation) This just describes the pattern of the filename. The correct file // will be accessed through the iteration idx below. The file suffix maps to the chosen // backend. - Series series = Series("adios_test_%05T.bp", Access::CREATE); + Series series = Series("opmd.%05T.bp", Access::CREATE); // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? // TODO(pgrete) Should we update for restart or only set this once? Or make it per @@ -101,8 +103,86 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, auto it = series.iterations[output_params.file_number]; it.open(); // explicit open() is important when run in parallel - it.setTime(tm->time); - it.setDt(tm->dt); + auto const &first_block = *(pm->block_list.front()); + + // TODO(?) in principle, we could abstract this to a more general WriteAttributes place + // and reuse for hdf5 and OpenPMD output with corresponing calls + // -------------------------------------------------------------------------------- // + // WRITING ATTRIBUTES // + // -------------------------------------------------------------------------------- // + + // Note, that profiling is likely skewed as data is actually written to disk/flushed + // only later. + Kokkos::Profiling::pushRegion("write Attributes"); + // First the ones required by the OpenPMD standard + if (tm != nullptr) { + it.setTime(tm->time); + it.setDt(tm->dt); + it.setAttribute("NCycle", tm->ncycle); + } else { + it.setTime(-1.0); + it.setDt(-1.0); + } + // Then our own + { + PARTHENON_INSTRUMENT_REGION("write input"); + // write input key-value pairs + std::ostringstream oss; + pin->ParameterDump(oss); + it.setAttribute("InputFile", oss.str()); + } + + { + it.setAttribute("WallTime", Driver::elapsed_main()); + it.setAttribute("NumDims", pm->ndim); + it.setAttribute("NumMeshBlocks", pm->nbtotal); + it.setAttribute("MaxLevel", pm->GetCurrentLevel() - pm->GetRootLevel()); + // write whether we include ghost cells or not + it.setAttribute("IncludesGhost", output_params.include_ghost_zones ? 1 : 0); + // write number of ghost cells in simulation + it.setAttribute("NGhost", Globals::nghost); + it.setAttribute("Coordinates", std::string(first_block.coords.Name()).c_str()); + + // restart info, write always + it.setAttribute("NBNew", pm->nbnew); + it.setAttribute("NBDel", pm->nbdel); + it.setAttribute("RootLevel", pm->GetRootLevel()); + it.setAttribute("Refine", pm->adaptive ? 1 : 0); + it.setAttribute("Multilevel", pm->multilevel ? 1 : 0); + + it.setAttribute("BlocksPerPE", pm->GetNbList()); + + // Mesh block size + const auto base_block_size = pm->GetBlockSize(); + it.setAttribute("MeshBlockSize", + std::vector{base_block_size.nx(X1DIR), base_block_size.nx(X2DIR), + base_block_size.nx(X3DIR)}); + + // RootGridDomain - float[9] array with xyz mins, maxs, rats (dx(i)/dx(i-1)) + it.setAttribute( + "RootGridDomain", + std::vector{pm->mesh_size.xmin(X1DIR), pm->mesh_size.xmax(X1DIR), + pm->mesh_size.xrat(X1DIR), pm->mesh_size.xmin(X2DIR), + pm->mesh_size.xmax(X2DIR), pm->mesh_size.xrat(X2DIR), + pm->mesh_size.xmin(X3DIR), pm->mesh_size.xmax(X3DIR), + pm->mesh_size.xrat(X3DIR)}); + + // Root grid size (number of cells at root level) + it.setAttribute("RootGridSize", + std::vector{pm->mesh_size.nx(X1DIR), pm->mesh_size.nx(X2DIR), + pm->mesh_size.nx(X3DIR)}); + + // Boundary conditions + std::vector boundary_condition_str(BOUNDARY_NFACES); + for (size_t i = 0; i < boundary_condition_str.size(); i++) { + boundary_condition_str[i] = GetBoundaryString(pm->mesh_bcs[i]); + } + + it.setAttribute("BoundaryConditions", boundary_condition_str); + Kokkos::Profiling::popRegion(); // write Info + } // Info section + + Kokkos::Profiling::popRegion(); // write Attributes // TODO(pgrete) check var name standard compatiblity // e.g., description: names of records and their components are only allowed to contain From c0be75c483d42a9ca386e3b754fe311edc2f6587 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 15 Mar 2024 14:50:52 +0100 Subject: [PATCH 010/125] Rename restart to restart_hdf5 --- src/outputs/{restart.cpp => restart_hdf5.cpp} | 0 src/outputs/{restart.hpp => restart_hdf5.hpp} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/outputs/{restart.cpp => restart_hdf5.cpp} (100%) rename src/outputs/{restart.hpp => restart_hdf5.hpp} (100%) diff --git a/src/outputs/restart.cpp b/src/outputs/restart_hdf5.cpp similarity index 100% rename from src/outputs/restart.cpp rename to src/outputs/restart_hdf5.cpp diff --git a/src/outputs/restart.hpp b/src/outputs/restart_hdf5.hpp similarity index 100% rename from src/outputs/restart.hpp rename to src/outputs/restart_hdf5.hpp From 7f03528da05b04e879fa0309293141b611128079 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 18 Mar 2024 15:47:29 +0100 Subject: [PATCH 011/125] WIP abstract RestartReader --- src/outputs/openpmd.cpp | 10 ++++++++++ src/outputs/restart_hdf5.hpp | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index da04b1123c38..818b1e2fe78f 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -31,6 +31,7 @@ #include // Parthenon headers +#include "Kokkos_Core_fwd.hpp" #include "basic_types.hpp" #include "coordinates/coordinates.hpp" #include "defs.hpp" @@ -123,6 +124,15 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, it.setTime(-1.0); it.setDt(-1.0); } + { // TESTING REMOVE + const auto view_d = Kokkos::View("blub", 5,3); + // Map a view onto a host allocation (so that we can call deep_copy) + auto host_vec = std::vector(view_d.size()); + Kokkos::View> + view_h(host_vec.data(), view_d.extent_int(0), view_d.extent_int(1)); + Kokkos::deep_copy(view_h, view_d); + it.setAttribute("blub", host_vec); + } // Then our own { PARTHENON_INSTRUMENT_REGION("write input"); diff --git a/src/outputs/restart_hdf5.hpp b/src/outputs/restart_hdf5.hpp index 69c301d30702..013800fc5d59 100644 --- a/src/outputs/restart_hdf5.hpp +++ b/src/outputs/restart_hdf5.hpp @@ -14,8 +14,8 @@ // license in this material to reproduce, prepare derivative works, distribute copies to // the public, perform publicly and display publicly, and to permit others to do so. //======================================================================================== -#ifndef OUTPUTS_RESTART_HPP_ -#define OUTPUTS_RESTART_HPP_ +#ifndef OUTPUTS_RESTART_HDF5_HPP_ +#define OUTPUTS_RESTART_HDF5_HPP_ //! \file io_wrapper.hpp // \brief defines a set of small wrapper functions for MPI versus Serial Output. @@ -342,4 +342,4 @@ class RestartReader { }; } // namespace parthenon -#endif // OUTPUTS_RESTART_HPP_ +#endif // OUTPUTS_RESTART_HDF5_HPP_ From 56795f2658f6f6b49088c3add2887bd56dbf1208 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 18 Mar 2024 23:39:15 +0100 Subject: [PATCH 012/125] WIP separating RestartReader --- src/CMakeLists.txt | 4 +- src/mesh/mesh.cpp | 1 + src/outputs/restart.hpp | 115 +++++++++++++++++++++++++++++++++++ src/outputs/restart_hdf5.cpp | 13 ++-- src/outputs/restart_hdf5.hpp | 5 +- src/parthenon_manager.cpp | 11 +++- 6 files changed, 139 insertions(+), 10 deletions(-) create mode 100644 src/outputs/restart.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e54b86d9be58..458fd6fea56e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -192,7 +192,9 @@ add_library(parthenon outputs/parthenon_xdmf.cpp outputs/parthenon_hdf5.hpp outputs/parthenon_xdmf.hpp - outputs/restart.cpp + outputs/restart.hpp + outputs/restart_hdf5.cpp + outputs/restart_hdf5.hpp outputs/vtk.cpp parthenon/driver.hpp diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index cee6af316fc8..2f327628c0d7 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -48,6 +48,7 @@ #include "mesh/meshblock.hpp" #include "mesh/meshblock_tree.hpp" #include "outputs/restart.hpp" +#include "outputs/restart_hdf5.hpp" #include "parameter_input.hpp" #include "parthenon_arrays.hpp" #include "prolong_restrict/prolong_restrict.hpp" diff --git a/src/outputs/restart.hpp b/src/outputs/restart.hpp new file mode 100644 index 000000000000..57cb8f10de48 --- /dev/null +++ b/src/outputs/restart.hpp @@ -0,0 +1,115 @@ +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2020-2024 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +// (C) (or copyright) 2020-2021. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== +#ifndef OUTPUTS_RESTART_HPP_ +#define OUTPUTS_RESTART_HPP_ +//! \file io_wrapper.hpp +// \brief defines a set of small wrapper functions for MPI versus Serial Output. + +#include +#include +#include +#include +#include + +#include "mesh/domain.hpp" +#include "utils/error_checking.hpp" + +namespace parthenon { + +class Mesh; +class Param; + +class RestartReader { + public: + RestartReader(); + virtual ~RestartReader() = default; + + struct SparseInfo { + // labels of sparse fields (full label, i.e. base name and sparse id) + std::vector labels; + + // allocation status of sparse fields (2D array outer dimension: block, inner + // dimension: sparse field) + // can't use std::vector here because std::vector is the same as + // std::vector and it doesn't have .data() member + std::unique_ptr allocated; + + int num_blocks = 0; + int num_sparse = 0; + + bool IsAllocated(int block, int sparse_field_idx) const { + PARTHENON_REQUIRE_THROWS(allocated != nullptr, + "Tried to get allocation status but no data present"); + PARTHENON_REQUIRE_THROWS((block >= 0) && (block < num_blocks), + "Invalid block index in SparseInfo::IsAllocated"); + PARTHENON_REQUIRE_THROWS((sparse_field_idx >= 0) && (sparse_field_idx < num_sparse), + "Invalid sparse field index in SparseInfo::IsAllocated"); + + return allocated[block * num_sparse + sparse_field_idx]; + } + }; + + SparseInfo GetSparseInfo() const; + + // Return output format version number. Return -1 if not existent. + int GetOutputFormatVersion() const; + + public: + // Gets data for all blocks on current rank. + // Assumes blocks are contiguous + // fills internal data for given pointer + template + void ReadBlocks(const std::string &name, IndexRange range, std::vector &dataVec, + const std::vector &bsize, int file_output_format_version, + MetadataFlag where, const std::vector &shape = {}) const; + + // Gets the data from a swarm var on current rank. Assumes all + // blocks are contiguous. Fills dataVec based on shape from swarmvar + // metadata. + template + void ReadSwarmVar(const std::string &swarmname, const std::string &varname, + const std::size_t count, const std::size_t offset, const Metadata &m, + std::vector &dataVec); + + // Reads an array dataset from file as a 1D vector. + template + std::vector ReadDataset(const std::string &name) const; + + template + std::vector GetAttrVec(const std::string &location, const std::string &name) const; + + template + T GetAttr(const std::string &location, const std::string &name) const; + + // Gets the counts and offsets for MPI ranks for the meshblocks set + // by the indexrange. Returns the total count on this rank. + std::size_t GetSwarmCounts(const std::string &swarm, const IndexRange &range, + std::vector &counts, + std::vector &offsets); + + void ReadParams(const std::string &name, Params &p); + + // closes out the restart file + // perhaps belongs in a destructor? + void Close(); + + // Does file have ghost cells? + int hasGhost; +}; + +} // namespace parthenon +#endif // OUTPUTS_RESTART_HDF5_HPP_ diff --git a/src/outputs/restart_hdf5.cpp b/src/outputs/restart_hdf5.cpp index d986d233bd1f..67eb99b1df85 100644 --- a/src/outputs/restart_hdf5.cpp +++ b/src/outputs/restart_hdf5.cpp @@ -1,6 +1,6 @@ //======================================================================================== // Parthenon performance portable AMR framework -// Copyright(C) 2020-2022 The Parthenon collaboration +// Copyright(C) 2020-2024 The Parthenon collaboration // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== // (C) (or copyright) 2020-2021. Triad National Security, LLC. All rights reserved. @@ -31,6 +31,7 @@ #include "outputs/parthenon_hdf5.hpp" #endif #include "outputs/restart.hpp" +#include "outputs/restart_hdf5.hpp" #include "utils/error_checking.hpp" namespace parthenon { @@ -38,7 +39,7 @@ namespace parthenon { //---------------------------------------------------------------------------------------- //! \fn void RestartReader::RestartReader(const std::string filename) // \brief Opens the restart file and stores appropriate file handle in fh_ -RestartReader::RestartReader(const char *filename) : filename_(filename) { +RestartReaderHDF5::RestartReaderHDF5(const char *filename) : filename_(filename) { #ifndef ENABLE_HDF5 std::stringstream msg; msg << "### FATAL ERROR in Restart (Reader) constructor" << std::endl @@ -54,7 +55,7 @@ RestartReader::RestartReader(const char *filename) : filename_(filename) { #endif // ENABLE_HDF5 } -int RestartReader::GetOutputFormatVersion() const { +int RestartReaderHDF5::GetOutputFormatVersion() const { #ifndef ENABLE_HDF5 PARTHENON_FAIL("Restart functionality is not available because HDF5 is disabled"); #else // HDF5 enabled @@ -69,7 +70,7 @@ int RestartReader::GetOutputFormatVersion() const { #endif // ENABLE_HDF5 } -RestartReader::SparseInfo RestartReader::GetSparseInfo() const { +RestartReaderHDF5::SparseInfo RestartReaderHDF5::GetSparseInfo() const { #ifndef ENABLE_HDF5 PARTHENON_FAIL("Restart functionality is not available because HDF5 is disabled"); #else // HDF5 enabled @@ -107,7 +108,7 @@ RestartReader::SparseInfo RestartReader::GetSparseInfo() const { // Gets the counts and offsets for MPI ranks for the meshblocks set // by the indexrange. Returns the total count on this rank. -std::size_t RestartReader::GetSwarmCounts(const std::string &swarm, +std::size_t RestartReaderHDF5::GetSwarmCounts(const std::string &swarm, const IndexRange &range, std::vector &counts, std::vector &offsets) { @@ -141,7 +142,7 @@ std::size_t RestartReader::GetSwarmCounts(const std::string &swarm, #endif // ENABLE_HDF5 } -void RestartReader::ReadParams(const std::string &name, Params &p) { +void RestartReaderHDF5::ReadParams(const std::string &name, Params &p) { #ifdef ENABLE_HDF5 p.ReadFromRestart(name, params_group_); #endif // ENABLE_HDF5 diff --git a/src/outputs/restart_hdf5.hpp b/src/outputs/restart_hdf5.hpp index 013800fc5d59..5b0f43fa4091 100644 --- a/src/outputs/restart_hdf5.hpp +++ b/src/outputs/restart_hdf5.hpp @@ -26,6 +26,7 @@ #include #include "config.hpp" +#include "outputs/restart.hpp" #ifdef ENABLE_HDF5 #include @@ -54,9 +55,9 @@ namespace parthenon { class Mesh; class Param; -class RestartReader { +class RestartReaderHDF5 : public RestartReader{ public: - explicit RestartReader(const char *theFile); + RestartReaderHDF5(const char *theFile); struct SparseInfo { // labels of sparse fields (full label, i.e. base name and sparse id) diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index 814a8c6f094f..853eb989d69e 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -29,6 +29,9 @@ #include "amr_criteria/refinement_package.hpp" #include "config.hpp" #include "driver/driver.hpp" +#include "outputs/restart.hpp" +#include "outputs/restart_hdf5.hpp" +#include FS_HEADER #include "globals.hpp" #include "interface/update.hpp" #include "mesh/domain.hpp" @@ -38,6 +41,8 @@ #include "utils/error_checking.hpp" #include "utils/utils.hpp" +namespace fs = FS_NAMESPACE; + namespace parthenon { ParthenonStatus ParthenonManager::ParthenonInitEnv(int argc, char *argv[]) { @@ -100,7 +105,11 @@ ParthenonStatus ParthenonManager::ParthenonInitEnv(int argc, char *argv[]) { pinput = std::make_unique(arg.input_filename); } else if (arg.res_flag != 0) { // Read input from restart file - restartReader = std::make_unique(arg.restart_filename); + if (fs::path(arg.restart_filename).extension() == ".rhdf") { + restartReader = std::make_unique(arg.restart_filename); + } else { + PARTHENON_FAIL("HELP!"); + } // Load input stream pinput = std::make_unique(); From 8587303c06426c2448730c09dfa22c45cf0ac689 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 19 Mar 2024 16:29:12 +0100 Subject: [PATCH 013/125] Make RestartReader abstract --- src/mesh/mesh.cpp | 35 +++++++++---------- src/outputs/restart.hpp | 62 +++++++++++++++++++-------------- src/outputs/restart_hdf5.cpp | 37 ++++++++++++++++++-- src/outputs/restart_hdf5.hpp | 67 ++++++++++++++++-------------------- src/parthenon_manager.cpp | 9 ++--- 5 files changed, 121 insertions(+), 89 deletions(-) diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index 2f327628c0d7..edd010ef72df 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -104,8 +104,8 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages, // private members: num_mesh_threads_(pin->GetOrAddInteger("parthenon/mesh", "num_threads", 1)), tree(this), use_uniform_meshgen_fn_{true, true, true, true}, lb_flag_(true), - lb_automatic_(), - lb_manual_(), MeshBndryFnctn{nullptr, nullptr, nullptr, nullptr, nullptr, nullptr} { + lb_automatic_(), lb_manual_(), + MeshBndryFnctn{nullptr, nullptr, nullptr, nullptr, nullptr, nullptr} { std::stringstream msg; RegionSize block_size; BoundaryFlag block_bcs[6]; @@ -486,8 +486,8 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr, // private members: num_mesh_threads_(pin->GetOrAddInteger("parthenon/mesh", "num_threads", 1)), tree(this), use_uniform_meshgen_fn_{true, true, true, true}, lb_flag_(true), - lb_automatic_(), - lb_manual_(), MeshBndryFnctn{nullptr, nullptr, nullptr, nullptr, nullptr, nullptr} { + lb_automatic_(), lb_manual_(), + MeshBndryFnctn{nullptr, nullptr, nullptr, nullptr, nullptr, nullptr} { std::stringstream msg; RegionSize block_size; BoundaryFlag block_bcs[6]; @@ -506,13 +506,14 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr, // read the restart file // the file is already open and the pointer is set to after + auto mesh_info = rr.GetMeshInfo(); // All ranks read HDF file - nbnew = rr.GetAttr("Info", "NBNew"); - nbdel = rr.GetAttr("Info", "NBDel"); - nbtotal = rr.GetAttr("Info", "NumMeshBlocks"); - root_level = rr.GetAttr("Info", "RootLevel"); + nbnew = mesh_info.nbnew; + nbdel = mesh_info.nbdel; + nbtotal = mesh_info.nbtotal; + root_level = mesh_info.root_level; - const auto bc = rr.GetAttrVec("Info", "BoundaryConditions"); + const auto bc = mesh_info.bound_cond; for (int i = 0; i < 6; i++) { block_bcs[i] = GetBoundaryFlag(bc[i]); } @@ -538,7 +539,7 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr, } EnrollBndryFncts_(app_in); - const auto grid_dim = rr.GetAttrVec("Info", "RootGridDomain"); + const auto grid_dim = mesh_info.grid_dim; mesh_size.xmin(X1DIR) = grid_dim[0]; mesh_size.xmax(X1DIR) = grid_dim[1]; mesh_size.xrat(X1DIR) = grid_dim[2]; @@ -554,14 +555,11 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr, // initialize loclist = std::vector(nbtotal); - const auto blockSize = rr.GetAttrVec("Info", "MeshBlockSize"); - const auto includesGhost = rr.GetAttr("Info", "IncludesGhost"); - const auto nGhost = rr.GetAttr("Info", "NGhost"); - for (auto &dir : {X1DIR, X2DIR, X3DIR}) { block_size.xrat(dir) = mesh_size.xrat(dir); - block_size.nx(dir) = - blockSize[dir - 1] - (blockSize[dir - 1] > 1) * includesGhost * 2 * nGhost; + block_size.nx(dir) = mesh_info.block_size[dir - 1] - + (mesh_info.block_size[dir - 1] > 1) * mesh_info.includes_ghost * + 2 * mesh_info.n_ghost; if (block_size.nx(dir) == 1) { block_size.symmetry(dir) = true; mesh_size.symmetry(dir) = true; @@ -595,9 +593,8 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr, InitUserMeshData(this, pin); // Populate logical locations - auto lx123 = rr.ReadDataset("/Blocks/loc.lx123"); - auto locLevelGidLidCnghostGflag = - rr.ReadDataset("/Blocks/loc.level-gid-lid-cnghost-gflag"); + auto lx123 = mesh_info.lx123; + auto locLevelGidLidCnghostGflag = mesh_info.level_gid_lid_cnghost_gflag; current_level = -1; for (int i = 0; i < nbtotal; i++) { loclist[i] = LogicalLocation(locLevelGidLidCnghostGflag[5 * i], lx123[3 * i], diff --git a/src/outputs/restart.hpp b/src/outputs/restart.hpp index 57cb8f10de48..2ee20afc01dc 100644 --- a/src/outputs/restart.hpp +++ b/src/outputs/restart.hpp @@ -35,7 +35,7 @@ class Param; class RestartReader { public: - RestartReader(); + RestartReader() = default; virtual ~RestartReader() = default; struct SparseInfo { @@ -63,45 +63,55 @@ class RestartReader { } }; - SparseInfo GetSparseInfo() const; + [[nodiscard]] virtual SparseInfo GetSparseInfo() const = 0; + + struct MeshInfo { + int nbnew, nbdel, nbtotal, root_level, includes_ghost, n_ghost; + std::vector bound_cond; + std::vector block_size; + std::vector grid_dim; + std::vector lx123; + std::vector level_gid_lid_cnghost_gflag; // what's this?! + }; + [[nodiscard]] virtual MeshInfo GetMeshInfo() const = 0; + + struct TimeInfo { + Real time, dt; + int ncycle; + }; + [[nodiscard]] virtual TimeInfo GetTimeInfo() const = 0; + + [[nodiscard]] virtual std::string GetInputString() const = 0; // Return output format version number. Return -1 if not existent. - int GetOutputFormatVersion() const; + [[nodiscard]] virtual int GetOutputFormatVersion() const = 0; - public: // Gets data for all blocks on current rank. // Assumes blocks are contiguous // fills internal data for given pointer - template - void ReadBlocks(const std::string &name, IndexRange range, std::vector &dataVec, - const std::vector &bsize, int file_output_format_version, - MetadataFlag where, const std::vector &shape = {}) const; + virtual void ReadBlocks(const std::string &name, IndexRange range, + std::vector &dataVec, const std::vector &bsize, + int file_output_format_version, MetadataFlag where, + const std::vector &shape = {}) const = 0; // Gets the data from a swarm var on current rank. Assumes all // blocks are contiguous. Fills dataVec based on shape from swarmvar // metadata. - template - void ReadSwarmVar(const std::string &swarmname, const std::string &varname, - const std::size_t count, const std::size_t offset, const Metadata &m, - std::vector &dataVec); - - // Reads an array dataset from file as a 1D vector. - template - std::vector ReadDataset(const std::string &name) const; - - template - std::vector GetAttrVec(const std::string &location, const std::string &name) const; - - template - T GetAttr(const std::string &location, const std::string &name) const; + virtual void ReadSwarmVar(const std::string &swarmname, const std::string &varname, + const std::size_t count, const std::size_t offset, + const Metadata &m, std::vector &dataVec) = 0; + virtual void ReadSwarmVar(const std::string &swarmname, const std::string &varname, + const std::size_t count, const std::size_t offset, + const Metadata &m, std::vector &dataVec) = 0; // Gets the counts and offsets for MPI ranks for the meshblocks set // by the indexrange. Returns the total count on this rank. - std::size_t GetSwarmCounts(const std::string &swarm, const IndexRange &range, - std::vector &counts, - std::vector &offsets); + [[nodiscard]] virtual std::size_t GetSwarmCounts(const std::string &swarm, + const IndexRange &range, + std::vector &counts, + std::vector &offsets) = 0; - void ReadParams(const std::string &name, Params &p); + virtual void ReadParams(const std::string &name, Params &p) = 0; // closes out the restart file // perhaps belongs in a destructor? diff --git a/src/outputs/restart_hdf5.cpp b/src/outputs/restart_hdf5.cpp index 67eb99b1df85..ceba14d72309 100644 --- a/src/outputs/restart_hdf5.cpp +++ b/src/outputs/restart_hdf5.cpp @@ -106,12 +106,43 @@ RestartReaderHDF5::SparseInfo RestartReaderHDF5::GetSparseInfo() const { #endif // ENABLE_HDF5 } +RestartReaderHDF5::MeshInfo RestartReaderHDF5::GetMeshInfo() const { + RestartReaderHDF5::MeshInfo mesh_info; + mesh_info.nbnew = GetAttr("Info", "NBNew"); + mesh_info.nbdel = GetAttr("Info", "NBDel"); + mesh_info.nbtotal = GetAttr("Info", "NumMeshBlocks"); + mesh_info.root_level = GetAttr("Info", "RootLevel"); + + mesh_info.bound_cond = GetAttrVec("Info", "BoundaryConditions"); + + mesh_info.block_size = GetAttrVec("Info", "MeshBlockSize"); + mesh_info.includes_ghost = GetAttr("Info", "IncludesGhost"); + mesh_info.n_ghost = GetAttr("Info", "NGhost"); + + mesh_info.grid_dim = GetAttrVec("Info", "RootGridDomain"); + + mesh_info.lx123 = ReadDataset("/Blocks/loc.lx123"); + mesh_info.level_gid_lid_cnghost_gflag = + ReadDataset("/Blocks/loc.level-gid-lid-cnghost-gflag"); + + return mesh_info; +} + +RestartReaderHDF5::TimeInfo RestartReaderHDF5::GetTimeInfo() const { + RestartReaderHDF5::TimeInfo time_info; + + time_info.time = GetAttr("Info", "Time"); + time_info.dt = GetAttr("Info", "dt"); + time_info.ncycle = GetAttr("Info", "NCycle"); + + return time_info; +} // Gets the counts and offsets for MPI ranks for the meshblocks set // by the indexrange. Returns the total count on this rank. std::size_t RestartReaderHDF5::GetSwarmCounts(const std::string &swarm, - const IndexRange &range, - std::vector &counts, - std::vector &offsets) { + const IndexRange &range, + std::vector &counts, + std::vector &offsets) { #ifndef ENABLE_HDF5 PARTHENON_FAIL("Restart functionality is not available because HDF5 is disabled"); return 0; diff --git a/src/outputs/restart_hdf5.hpp b/src/outputs/restart_hdf5.hpp index 5b0f43fa4091..65eafdec1c57 100644 --- a/src/outputs/restart_hdf5.hpp +++ b/src/outputs/restart_hdf5.hpp @@ -55,39 +55,22 @@ namespace parthenon { class Mesh; class Param; -class RestartReaderHDF5 : public RestartReader{ +class RestartReaderHDF5 : public RestartReader { public: - RestartReaderHDF5(const char *theFile); - - struct SparseInfo { - // labels of sparse fields (full label, i.e. base name and sparse id) - std::vector labels; - - // allocation status of sparse fields (2D array outer dimension: block, inner - // dimension: sparse field) - // can't use std::vector here because std::vector is the same as - // std::vector and it doesn't have .data() member - std::unique_ptr allocated; - - int num_blocks = 0; - int num_sparse = 0; - - bool IsAllocated(int block, int sparse_field_idx) const { - PARTHENON_REQUIRE_THROWS(allocated != nullptr, - "Tried to get allocation status but no data present"); - PARTHENON_REQUIRE_THROWS((block >= 0) && (block < num_blocks), - "Invalid block index in SparseInfo::IsAllocated"); - PARTHENON_REQUIRE_THROWS((sparse_field_idx >= 0) && (sparse_field_idx < num_sparse), - "Invalid sparse field index in SparseInfo::IsAllocated"); - - return allocated[block * num_sparse + sparse_field_idx]; - } - }; + RestartReaderHDF5(const char *filename); + + [[nodiscard]] SparseInfo GetSparseInfo() const override; + + [[nodiscard]] MeshInfo GetMeshInfo() const override; - SparseInfo GetSparseInfo() const; + [[nodiscard]] TimeInfo GetTimeInfo() const override; + + [[nodiscard]] std::string GetInputString() const override { + return GetAttr("Input", "File"); + }; // Return output format version number. Return -1 if not existent. - int GetOutputFormatVersion() const; + [[nodiscard]] int GetOutputFormatVersion() const override; private: struct DatasetHandle { @@ -144,14 +127,13 @@ class RestartReaderHDF5 : public RestartReader{ // Gets data for all blocks on current rank. // Assumes blocks are contiguous // fills internal data for given pointer - template - void ReadBlocks(const std::string &name, IndexRange range, std::vector &dataVec, + void ReadBlocks(const std::string &name, IndexRange range, std::vector &dataVec, const std::vector &bsize, int file_output_format_version, - MetadataFlag where, const std::vector &shape = {}) const { + MetadataFlag where, const std::vector &shape = {}) const override { #ifndef ENABLE_HDF5 PARTHENON_FAIL("Restart functionality is not available because HDF5 is disabled"); #else // HDF5 enabled - auto hdl = OpenDataset(name); + auto hdl = OpenDataset(name); constexpr int CHUNK_MAX_DIM = 7; @@ -315,14 +297,25 @@ class RestartReaderHDF5 : public RestartReader{ return res[0]; } + void ReadSwarmVar(const std::string &swarmname, const std::string &varname, + const std::size_t count, const std::size_t offset, const Metadata &m, + std::vector &dataVec) override { + ReadSwarmVar<>(swarmname, varname, count, offset, m, dataVec); + }; + void ReadSwarmVar(const std::string &swarmname, const std::string &varname, + const std::size_t count, const std::size_t offset, const Metadata &m, + std::vector &dataVec) override { + ReadSwarmVar<>(swarmname, varname, count, offset, m, dataVec); + }; // Gets the counts and offsets for MPI ranks for the meshblocks set // by the indexrange. Returns the total count on this rank. - std::size_t GetSwarmCounts(const std::string &swarm, const IndexRange &range, - std::vector &counts, - std::vector &offsets); + [[nodiscard]] std::size_t GetSwarmCounts(const std::string &swarm, + const IndexRange &range, + std::vector &counts, + std::vector &offsets) override; - void ReadParams(const std::string &name, Params &p); + void ReadParams(const std::string &name, Params &p) override; // closes out the restart file // perhaps belongs in a destructor? diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index 853eb989d69e..dadfde12dcab 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -113,7 +113,7 @@ ParthenonStatus ParthenonManager::ParthenonInitEnv(int argc, char *argv[]) { // Load input stream pinput = std::make_unique(); - auto inputString = restartReader->GetAttr("Input", "File"); + auto inputString = restartReader->GetInputString(); std::istringstream is(inputString); pinput->LoadFromStream(is); } @@ -176,13 +176,14 @@ void ParthenonManager::ParthenonInitPackagesAndMesh() { std::make_unique(pinput.get(), app_input.get(), *restartReader, packages); // Read simulation time and cycle from restart file and set in input - Real tNow = restartReader->GetAttr("Info", "Time"); + const auto time_info = restartReader->GetTimeInfo(); + Real tNow = time_info.time; pinput->SetReal("parthenon/time", "start_time", tNow); - Real dt = restartReader->GetAttr("Info", "dt"); + Real dt = time_info.dt; pinput->SetReal("parthenon/time", "dt", dt); - int ncycle = restartReader->GetAttr("Info", "NCycle"); + int ncycle = time_info.ncycle; pinput->SetInteger("parthenon/time", "ncycle", ncycle); // Read package data from restart file From e3ea8d796307e0b6b9e113f120fc14db77d03978 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 20 Mar 2024 14:46:19 +0100 Subject: [PATCH 014/125] Add OpenPMD restart skeleton --- src/CMakeLists.txt | 2 + src/outputs/restart_opmd.cpp | 94 ++++++++++++++++++++++++++++++++++++ src/outputs/restart_opmd.hpp | 79 ++++++++++++++++++++++++++++++ src/parthenon_manager.cpp | 3 ++ 4 files changed, 178 insertions(+) create mode 100644 src/outputs/restart_opmd.cpp create mode 100644 src/outputs/restart_opmd.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 458fd6fea56e..9174a5dd0be6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -195,6 +195,8 @@ add_library(parthenon outputs/restart.hpp outputs/restart_hdf5.cpp outputs/restart_hdf5.hpp + outputs/restart_opmd.cpp + outputs/restart_opmd.hpp outputs/vtk.cpp parthenon/driver.hpp diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp new file mode 100644 index 000000000000..48b27cc4e46c --- /dev/null +++ b/src/outputs/restart_opmd.cpp @@ -0,0 +1,94 @@ +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2024 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file restart_opmd.cpp +// \brief Restarts a simulation from an OpenPMD output with ADIOS2 backend + +#include +#include +#include + +#include "interface/params.hpp" +#include "openPMD/Iteration.hpp" +#include "openPMD/Series.hpp" +#include "outputs/restart.hpp" +#include "outputs/restart_opmd.hpp" +#include "utils/error_checking.hpp" + +namespace parthenon { + +//---------------------------------------------------------------------------------------- +//! \fn void RestartReader::RestartReader(const std::string filename) +// \brief Opens the restart file and stores appropriate file handle in fh_ +RestartReaderOPMD::RestartReaderOPMD(const char *filename) : filename_(filename) { + auto series = openPMD::Series(filename, openPMD::Access::READ_ONLY, MPI_COMM_WORLD); + PARTHENON_REQUIRE_THROWS( + series.iterations.size() == 1, + "Parthenon restarts should only contain one iteration/timestep."); + unsigned long idx; + for (const auto &i : series.iterations) { + idx = i.first; + } + it = std::make_unique(series.iterations[idx]); +} + +int RestartReaderOPMD::GetOutputFormatVersion() const { + // TODO(pgrete) move info to shared header and introduce constexpr var + if (it->containsAttribute("OutputFormatVersion")) { + return it->getAttribute("OutputFormatVersion").get(); + } else { + return -1; + } +} + +RestartReaderOPMD::SparseInfo RestartReaderOPMD::GetSparseInfo() const {} + +RestartReaderOPMD::MeshInfo RestartReaderOPMD::GetMeshInfo() const { + RestartReaderOPMD::MeshInfo mesh_info; + mesh_info.nbnew = it->getAttribute("NBNew").get(); + mesh_info.nbdel = it->getAttribute("NBDel").get(); + mesh_info.nbtotal = it->getAttribute("NumMeshBlocks").get(); + mesh_info.root_level = it->getAttribute("RootLevel").get(); + + mesh_info.bound_cond = + it->getAttribute("BoundaryConditions").get>(); + + mesh_info.block_size = it->getAttribute("MeshBlockSize").get>(); + mesh_info.includes_ghost = it->getAttribute("IncludesGhost").get(); + mesh_info.n_ghost = it->getAttribute("NGhost").get(); + + mesh_info.grid_dim = it->getAttribute("RootGridDomain").get>(); + // TODO(pgrete) need impl + // mesh_info.lx123 = ReadDataset("/Blocks/loc.lx123"); + // mesh_info.level_gid_lid_cnghost_gflag = + // ReadDataset("/Blocks/loc.level-gid-lid-cnghost-gflag"); + + return mesh_info; +} + +RestartReaderOPMD::TimeInfo RestartReaderOPMD::GetTimeInfo() const { + RestartReaderOPMD::TimeInfo time_info; + + time_info.time = it->time(); + time_info.dt = it->dt(); + time_info.ncycle = it->getAttribute("NCycle").get(); + + return time_info; +} +// Gets the counts and offsets for MPI ranks for the meshblocks set +// by the indexrange. Returns the total count on this rank. +std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, + const IndexRange &range, + std::vector &counts, + std::vector &offsets) {} + +void RestartReaderOPMD::ReadParams(const std::string &name, Params &p) {} +void RestartReaderOPMD::ReadBlocks(const std::string &name, IndexRange range, + std::vector &dataVec, + const std::vector &bsize, + int file_output_format_version, MetadataFlag where, + const std::vector &shape) const {} + +} // namespace parthenon diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp new file mode 100644 index 000000000000..ef4d5e9251c6 --- /dev/null +++ b/src/outputs/restart_opmd.hpp @@ -0,0 +1,79 @@ +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2024 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +#ifndef OUTPUTS_RESTART_OPMD_HPP_ +#define OUTPUTS_RESTART_OPMD_HPP_ +//! \file restart_opmd.hpp +// \brief Provides support for restarting from OpenPMD output + +#include +#include +#include + +#include "openPMD/Iteration.hpp" +#include "outputs/restart.hpp" + +#include "mesh/domain.hpp" + +namespace parthenon { + +class Mesh; +class Param; + +class RestartReaderOPMD : public RestartReader { + public: + explicit RestartReaderOPMD(const char *filename); + + [[nodiscard]] SparseInfo GetSparseInfo() const override; + + [[nodiscard]] MeshInfo GetMeshInfo() const override; + + [[nodiscard]] TimeInfo GetTimeInfo() const override; + + [[nodiscard]] std::string GetInputString() const override { + return it->getAttribute("InputFile").get(); + }; + + // Return output format version number. Return -1 if not existent. + [[nodiscard]] int GetOutputFormatVersion() const override; + + public: + // Gets data for all blocks on current rank. + // Assumes blocks are contiguous + // fills internal data for given pointer + void ReadBlocks(const std::string &name, IndexRange range, std::vector &dataVec, + const std::vector &bsize, int file_output_format_version, + MetadataFlag where, const std::vector &shape = {}) const override; + + void ReadSwarmVar(const std::string &swarmname, const std::string &varname, + const std::size_t count, const std::size_t offset, const Metadata &m, + std::vector &dataVec) override{}; + void ReadSwarmVar(const std::string &swarmname, const std::string &varname, + const std::size_t count, const std::size_t offset, const Metadata &m, + std::vector &dataVec) override{}; + + // Gets the counts and offsets for MPI ranks for the meshblocks set + // by the indexrange. Returns the total count on this rank. + [[nodiscard]] std::size_t GetSwarmCounts(const std::string &swarm, + const IndexRange &range, + std::vector &counts, + std::vector &offsets) override; + + void ReadParams(const std::string &name, Params &p) override; + + // closes out the restart file + // perhaps belongs in a destructor? + void Close(); + + // Does file have ghost cells? + int hasGhost; + + private: + const std::string filename_; + std::unique_ptr it; +}; + +} // namespace parthenon +#endif // OUTPUTS_RESTART_OPMD_HPP_ diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index 28a084bcd5e0..8507acd3d3fe 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -35,6 +35,7 @@ #include "outputs/output_utils.hpp" #include "outputs/restart.hpp" #include "outputs/restart_hdf5.hpp" +#include "outputs/restart_opmd.hpp" #include "utils/error_checking.hpp" #include "utils/utils.hpp" @@ -104,6 +105,8 @@ ParthenonStatus ParthenonManager::ParthenonInitEnv(int argc, char *argv[]) { // Read input from restart file if (fs::path(arg.restart_filename).extension() == ".rhdf") { restartReader = std::make_unique(arg.restart_filename); + } else if (fs::path(arg.restart_filename).extension() == ".bp") { + restartReader = std::make_unique(arg.restart_filename); } else { PARTHENON_FAIL("Unsupported restart file format."); } From 33b6261ed2f148d9df69cf2eb8d5c57094386d29 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 20 Mar 2024 16:32:25 +0100 Subject: [PATCH 015/125] WIP updating loc logic --- src/outputs/openpmd.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 818b1e2fe78f..253c9579e39f 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -124,8 +124,9 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, it.setTime(-1.0); it.setDt(-1.0); } - { // TESTING REMOVE - const auto view_d = Kokkos::View("blub", 5,3); + { // FIXME move this to dump params + const auto view_d = + Kokkos::View("blub", 5, 3); // Map a view onto a host allocation (so that we can call deep_copy) auto host_vec = std::vector(view_d.size()); Kokkos::View> @@ -194,6 +195,15 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, Kokkos::Profiling::popRegion(); // write Attributes + // Write block metadata + { + // TODO(pgrete) FIXME this is collective, so should be a dataset! + const int n_blocks_global = pm->nbtotal; + const int n_blocks_local = static_cast(pm->block_list.size()); + std::vector loc = OutputUtils::ComputeLocs(pm); + it.setAttribute("loc.lx123", loc); + } + // TODO(pgrete) check var name standard compatiblity // e.g., description: names of records and their components are only allowed to contain // the characters a-Z, the numbers 0-9 and the underscore _ From 62e54da614cbb39f3b849b29e7f8b47b25e1b1e4 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 11 Apr 2024 10:50:29 +0200 Subject: [PATCH 016/125] Fix interface from recent changes in develop --- src/outputs/openpmd.cpp | 13 ++++++++----- src/outputs/output_utils.cpp | 5 +++-- src/outputs/output_utils.hpp | 3 ++- src/outputs/restart_opmd.cpp | 4 ++-- src/outputs/restart_opmd.hpp | 3 ++- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 555df89cee66..b31d63148348 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -144,6 +144,8 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } { + // It's not clear we need all these attributes, but they mirror what's done in the + // hdf5 output. it.setAttribute("WallTime", Driver::elapsed_main()); it.setAttribute("NumDims", pm->ndim); it.setAttribute("NumMeshBlocks", pm->nbtotal); @@ -157,7 +159,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // restart info, write always it.setAttribute("NBNew", pm->nbnew); it.setAttribute("NBDel", pm->nbdel); - it.setAttribute("RootLevel", pm->GetRootLevel()); + it.setAttribute("RootLevel", pm->GetLegacyTreeRootLevel()); it.setAttribute("Refine", pm->adaptive ? 1 : 0); it.setAttribute("Multilevel", pm->multilevel ? 1 : 0); @@ -215,12 +217,13 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // -------------------------------------------------------------------------------- // Kokkos::Profiling::pushRegion("write all variable data"); + auto &bounds = pm->block_list.front()->cellbounds; // get list of all vars, just use the first block since the list is the same for all // blocks // TODO(pgrete) add restart_ var to output // TODO(pgrete) check if this needs to be updated/unifed with get_var logic in hdf5 auto all_vars_info = GetAllVarsInfo( - GetVarsToWrite(pm->block_list.front(), true, output_params.variables)); + GetVarsToWrite(pm->block_list.front(), true, output_params.variables), bounds); // We're currently writing (flushing) one var at a time. This saves host memory but // results more smaller write. Might be updated in the future. @@ -238,7 +241,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, std::vector tmp_data(var_size_max * num_blocks_local); // TODO(pgrete) This needs to be in the loop for non-cell-centered vars - auto &bounds = pm->block_list.front()->cellbounds; auto ib = bounds.GetBoundsI(IndexDomain::interior); auto jb = bounds.GetBoundsJ(IndexDomain::interior); auto kb = bounds.GetBoundsK(IndexDomain::interior); @@ -250,11 +252,12 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, memset(tmp_data.data(), 0, tmp_data.size() * sizeof(OutT)); uint64_t tmp_offset = 0; - const bool is_scalar = vinfo.nx4 == 1 && vinfo.nx5 == 1 && vinfo.nx6 == 1; + const bool is_scalar = + vinfo.GetDim(4) == 1 && vinfo.GetDim(5) == 1 && vinfo.GetDim(6) == 1; if (vinfo.is_vector) { // sanity check PARTHENON_REQUIRE_THROWS( - vinfo.nx4 == pm->ndim && vinfo.nx5 == 1 && vinfo.nx6 == 1, + vinfo.GetDim(4) == pm->ndim && vinfo.GetDim(5) == 1 && vinfo.GetDim(6) == 1, "A 'standard' vector is expected to only have components matching the " "dimensionality of the simulation.") } diff --git a/src/outputs/output_utils.cpp b/src/outputs/output_utils.cpp index a33dc03395a1..8b5d2e1c3cfe 100644 --- a/src/outputs/output_utils.cpp +++ b/src/outputs/output_utils.cpp @@ -329,10 +329,11 @@ VariableVector GetVarsToWrite(const std::shared_ptr pmb, return vars_to_write; } -std::vector GetAllVarsInfo(const VariableVector &vars) { +std::vector GetAllVarsInfo(const VariableVector &vars, + const IndexShape &cellbounds) { std::vector all_vars_info; for (auto &v : vars) { - all_vars_info.emplace_back(v); + all_vars_info.emplace_back(v, cellbounds); } // sort alphabetically diff --git a/src/outputs/output_utils.hpp b/src/outputs/output_utils.hpp index 2199edbc2e45..a6b4929e453d 100644 --- a/src/outputs/output_utils.hpp +++ b/src/outputs/output_utils.hpp @@ -354,7 +354,8 @@ VariableVector GetVarsToWrite(const std::shared_ptr pmb, const std::vector &variables); // Returns a sorted vector of VarInfo associated with vars -std::vector GetAllVarsInfo(const VariableVector &vars); +std::vector GetAllVarsInfo(const VariableVector &vars, + const IndexShape &cellbounds); } // namespace OutputUtils } // namespace parthenon diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 48b27cc4e46c..7e1732ea6743 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -68,8 +68,8 @@ RestartReaderOPMD::MeshInfo RestartReaderOPMD::GetMeshInfo() const { return mesh_info; } -RestartReaderOPMD::TimeInfo RestartReaderOPMD::GetTimeInfo() const { - RestartReaderOPMD::TimeInfo time_info; +SimTime RestartReaderOPMD::GetTimeInfo() const { + SimTime time_info{}; time_info.time = it->time(); time_info.dt = it->dt(); diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index ef4d5e9251c6..e795fe5123bd 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -12,6 +12,7 @@ #include #include +#include "basic_types.hpp" #include "openPMD/Iteration.hpp" #include "outputs/restart.hpp" @@ -30,7 +31,7 @@ class RestartReaderOPMD : public RestartReader { [[nodiscard]] MeshInfo GetMeshInfo() const override; - [[nodiscard]] TimeInfo GetTimeInfo() const override; + [[nodiscard]] SimTime GetTimeInfo() const override; [[nodiscard]] std::string GetInputString() const override { return it->getAttribute("InputFile").get(); From 63096633e52ac9cc561805150dae3a1bb5c1db9a Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 11 Apr 2024 13:52:43 +0200 Subject: [PATCH 017/125] Read and Write loc --- src/outputs/openpmd.cpp | 15 ++++++++++----- src/outputs/output_utils.cpp | 34 ++++++++++++++++++++++++++++++++++ src/outputs/output_utils.hpp | 5 +++++ src/outputs/restart_opmd.cpp | 8 ++++---- src/utils/mpi_types.hpp | 9 +++++++++ 5 files changed, 62 insertions(+), 9 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index b31d63148348..36eb4e84fbda 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -199,11 +199,16 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // Write block metadata { - // TODO(pgrete) FIXME this is collective, so should be a dataset! - const int n_blocks_global = pm->nbtotal; - const int n_blocks_local = static_cast(pm->block_list.size()); - std::vector loc = OutputUtils::ComputeLocs(pm); - it.setAttribute("loc.lx123", loc); + // Manually gather all block data first as it allows to use the (simpler) + // Attribute interface rather than writing a distributed dataset -- especially as all + // data is being read on restart by every rank anyway. + std::vector loc_local = OutputUtils::ComputeLocs(pm); + auto loc_global = FlattendedLocalToGlobal(pm, loc_local); + it.setAttribute("loc.lx123", loc_global); + + std::vector id_local = OutputUtils::ComputeIDsAndFlags(pm); + auto id_global = FlattendedLocalToGlobal(pm, id_local); + it.setAttribute("loc.level-gid-lid-cnghost-gflag", id_global); } // TODO(pgrete) check var name standard compatiblity diff --git a/src/outputs/output_utils.cpp b/src/outputs/output_utils.cpp index 8b5d2e1c3cfe..0e081744cd43 100644 --- a/src/outputs/output_utils.cpp +++ b/src/outputs/output_utils.cpp @@ -29,6 +29,7 @@ #include "mesh/mesh.hpp" #include "mesh/meshblock.hpp" #include "outputs/output_utils.hpp" +#include "utils/mpi_types.hpp" namespace parthenon { namespace OutputUtils { @@ -241,6 +242,39 @@ std::vector ComputeIDsAndFlags(Mesh *pm) { }); } +template +std::vector FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_local) { + const int n_blocks_global = pm->nbtotal; + const int n_blocks_local = static_cast(pm->block_list.size()); + + const int n_elem = data_local.size() / n_blocks_local; + PARTHENON_REQUIRE_THROWS(data_local.size() % n_blocks_local == 0, + "Results from flattened input vector does not evenly divide " + "into number of local blocks."); + std::vector data_global(n_elem * n_blocks_global); + + std::vector counts(Globals::nranks); + std::vector offsets(Globals::nranks); + + const auto &nblist = pm->GetNbList(); + counts[0] = n_elem * nblist[0]; + offsets[0] = 0; + for (int r = 1; r < Globals::nranks; r++) { + counts[r] = n_elem * nblist[r]; + offsets[r] = offsets[r - 1] + counts[r - 1]; + } + +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Allgatherv(data_local.data(), counts[Globals::my_rank], + MPITypeMap::type(), data_global.data(), + counts.data(), offsets.data(), MPITypeMap::type(), + MPI_COMM_WORLD)); +#else + return data_local; +#endif + return data_global; +} + // TODO(JMM): I could make this use the other loop // functionality/high-order functions. but it was more code than this // for, I think, little benefit. diff --git a/src/outputs/output_utils.hpp b/src/outputs/output_utils.hpp index a6b4929e453d..4f5c920da920 100644 --- a/src/outputs/output_utils.hpp +++ b/src/outputs/output_utils.hpp @@ -339,6 +339,11 @@ std::vector ComputeXminBlocks(Mesh *pm); std::vector ComputeLocs(Mesh *pm); std::vector ComputeIDsAndFlags(Mesh *pm); +// Takes a vector containing flattened data of all rank local blocks and returns the +// flattened data over all blocks. +template +std::vector FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_local); + // TODO(JMM): Potentially unsafe if MPI_UNSIGNED_LONG_LONG isn't a size_t // however I think it's probably safe to assume we'll be on systems // where this is the case? diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 7e1732ea6743..f49bdf47b56d 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include "interface/params.hpp" #include "openPMD/Iteration.hpp" @@ -60,10 +61,9 @@ RestartReaderOPMD::MeshInfo RestartReaderOPMD::GetMeshInfo() const { mesh_info.n_ghost = it->getAttribute("NGhost").get(); mesh_info.grid_dim = it->getAttribute("RootGridDomain").get>(); - // TODO(pgrete) need impl - // mesh_info.lx123 = ReadDataset("/Blocks/loc.lx123"); - // mesh_info.level_gid_lid_cnghost_gflag = - // ReadDataset("/Blocks/loc.level-gid-lid-cnghost-gflag"); + mesh_info.lx123 = it->getAttribute("loc.lx123").get>(); + mesh_info.level_gid_lid_cnghost_gflag = + it->getAttribute("loc.level-gid-lid-cnghost-gflag").get>(); return mesh_info; } diff --git a/src/utils/mpi_types.hpp b/src/utils/mpi_types.hpp index a990cecc7e3c..164ab66f1115 100644 --- a/src/utils/mpi_types.hpp +++ b/src/utils/mpi_types.hpp @@ -1,4 +1,8 @@ //======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2021-2024 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== // (C) (or copyright) 2021. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los @@ -34,6 +38,11 @@ inline MPI_Datatype MPITypeMap::type() { return MPI_PARTHENON_REAL; } +template <> +inline MPI_Datatype MPITypeMap::type() { + return MPI_INT64_T; +} + template <> inline MPI_Datatype MPITypeMap::type() { return MPI_INT; From 7224f42b703d8c2c2a2cb1f55c6bab9d16382178 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 11 Apr 2024 14:11:47 +0200 Subject: [PATCH 018/125] Houston, we have a build --- src/outputs/openpmd.cpp | 4 ++-- src/outputs/output_utils.cpp | 7 +++++++ src/outputs/restart_opmd.cpp | 15 ++++++++++----- src/outputs/restart_opmd.hpp | 6 +++--- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 36eb4e84fbda..7e5b88f022a8 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -203,11 +203,11 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // Attribute interface rather than writing a distributed dataset -- especially as all // data is being read on restart by every rank anyway. std::vector loc_local = OutputUtils::ComputeLocs(pm); - auto loc_global = FlattendedLocalToGlobal(pm, loc_local); + auto loc_global = FlattendedLocalToGlobal(pm, loc_local); it.setAttribute("loc.lx123", loc_global); std::vector id_local = OutputUtils::ComputeIDsAndFlags(pm); - auto id_global = FlattendedLocalToGlobal(pm, id_local); + auto id_global = FlattendedLocalToGlobal(pm, id_local); it.setAttribute("loc.level-gid-lid-cnghost-gflag", id_global); } diff --git a/src/outputs/output_utils.cpp b/src/outputs/output_utils.cpp index 0e081744cd43..a94a4160d789 100644 --- a/src/outputs/output_utils.cpp +++ b/src/outputs/output_utils.cpp @@ -15,6 +15,7 @@ // the public, perform publicly and display publicly, and to permit others to do so. //======================================================================================== +#include #include #include #include @@ -275,6 +276,12 @@ std::vector FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_loca return data_global; } +// explicit template instantiation +template std::vector +FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_local); +template std::vector FlattendedLocalToGlobal(Mesh *pm, + const std::vector &data_local); + // TODO(JMM): I could make this use the other loop // functionality/high-order functions. but it was more code than this // for, I think, little benefit. diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index f49bdf47b56d..d94da08e3332 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -44,7 +44,10 @@ int RestartReaderOPMD::GetOutputFormatVersion() const { } } -RestartReaderOPMD::SparseInfo RestartReaderOPMD::GetSparseInfo() const {} +RestartReaderOPMD::SparseInfo RestartReaderOPMD::GetSparseInfo() const { + // TODO(pgrete) needs impl + return {}; +} RestartReaderOPMD::MeshInfo RestartReaderOPMD::GetMeshInfo() const { RestartReaderOPMD::MeshInfo mesh_info; @@ -82,13 +85,15 @@ SimTime RestartReaderOPMD::GetTimeInfo() const { std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, const IndexRange &range, std::vector &counts, - std::vector &offsets) {} + std::vector &offsets) { + // TODO(pgrete) needs impl + return 0; +} void RestartReaderOPMD::ReadParams(const std::string &name, Params &p) {} void RestartReaderOPMD::ReadBlocks(const std::string &name, IndexRange range, + const OutputUtils::VarInfo &info, std::vector &dataVec, - const std::vector &bsize, - int file_output_format_version, MetadataFlag where, - const std::vector &shape) const {} + int file_output_format_version) const {}; } // namespace parthenon diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index e795fe5123bd..46a5658eb031 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -44,9 +44,9 @@ class RestartReaderOPMD : public RestartReader { // Gets data for all blocks on current rank. // Assumes blocks are contiguous // fills internal data for given pointer - void ReadBlocks(const std::string &name, IndexRange range, std::vector &dataVec, - const std::vector &bsize, int file_output_format_version, - MetadataFlag where, const std::vector &shape = {}) const override; + void ReadBlocks(const std::string &name, IndexRange range, + const OutputUtils::VarInfo &info, std::vector &dataVec, + int file_output_format_version) const override; void ReadSwarmVar(const std::string &swarmname, const std::string &varname, const std::size_t count, const std::size_t offset, const Metadata &m, From ae1f2417f92188d050f2d92a65278ea5833512f8 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 12 Apr 2024 09:34:06 +0200 Subject: [PATCH 019/125] Added OpenPMD restart ReadBlocks --- src/outputs/restart.hpp | 2 +- src/outputs/restart_hdf5.cpp | 4 +- src/outputs/restart_hdf5.hpp | 2 +- src/outputs/restart_opmd.cpp | 73 ++++++++++++++++++++++++++++++++++-- src/outputs/restart_opmd.hpp | 2 +- src/parthenon_manager.cpp | 2 +- 6 files changed, 76 insertions(+), 9 deletions(-) diff --git a/src/outputs/restart.hpp b/src/outputs/restart.hpp index e986acbaf028..9d452a786e07 100644 --- a/src/outputs/restart.hpp +++ b/src/outputs/restart.hpp @@ -89,7 +89,7 @@ class RestartReader { // fills internal data for given pointer virtual void ReadBlocks(const std::string &name, IndexRange range, const OutputUtils::VarInfo &info, std::vector &dataVec, - int file_output_format_version) const = 0; + int file_output_format_version, Mesh *pmesh) const = 0; // Gets the data from a swarm var on current rank. Assumes all // blocks are contiguous. Fills dataVec based on shape from swarmvar diff --git a/src/outputs/restart_hdf5.cpp b/src/outputs/restart_hdf5.cpp index a5016bc68e8d..8d83cbdf78fe 100644 --- a/src/outputs/restart_hdf5.cpp +++ b/src/outputs/restart_hdf5.cpp @@ -183,7 +183,8 @@ void RestartReaderHDF5::ReadParams(const std::string &name, Params &p) { void RestartReaderHDF5::ReadBlocks(const std::string &name, IndexRange range, const OutputUtils::VarInfo &info, std::vector &dataVec, - int file_output_format_version) const { + int file_output_format_version, + Mesh * /*pmesh*/) const { #ifndef ENABLE_HDF5 PARTHENON_FAIL("Restart functionality is not available because HDF5 is disabled"); #else // HDF5 enabled @@ -224,6 +225,7 @@ void RestartReaderHDF5::ReadBlocks(const std::string &name, IndexRange range, H5Sselect_hyperslab(hdl.dataspace, H5S_SELECT_SET, offset, NULL, count, NULL)); const H5S memspace = H5S::FromHIDCheck(H5Screate_simple(total_dim, count, NULL)); + // TODO(reviewer) What's going on here? The follow line is identical to the one above. PARTHENON_HDF5_CHECK( H5Sselect_hyperslab(hdl.dataspace, H5S_SELECT_SET, offset, NULL, count, NULL)); diff --git a/src/outputs/restart_hdf5.hpp b/src/outputs/restart_hdf5.hpp index 562b67daf78b..f43fbd58df84 100644 --- a/src/outputs/restart_hdf5.hpp +++ b/src/outputs/restart_hdf5.hpp @@ -113,7 +113,7 @@ class RestartReaderHDF5 : public RestartReader { // fills internal data for given pointer void ReadBlocks(const std::string &name, IndexRange range, const OutputUtils::VarInfo &info, std::vector &dataVec, - int file_output_format_version) const override; + int file_output_format_version, Mesh *pmesh) const override; // Gets the data from a swarm var on current rank. Assumes all // blocks are contiguous. Fills dataVec based on shape from swarmvar diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index d94da08e3332..e7bbbaec533f 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -11,6 +11,7 @@ #include #include +#include "basic_types.hpp" #include "interface/params.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" @@ -91,9 +92,73 @@ std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, } void RestartReaderOPMD::ReadParams(const std::string &name, Params &p) {} -void RestartReaderOPMD::ReadBlocks(const std::string &name, IndexRange range, - const OutputUtils::VarInfo &info, - std::vector &dataVec, - int file_output_format_version) const {}; +void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block_range, + const OutputUtils::VarInfo &vinfo, + std::vector &data_vec, + int file_output_format_version, Mesh *pm) const { + for (auto &pmb : pm->block_list) { + // TODO(pgrete) check if we should skip the suffix for level 0 + const auto level = pmb->loc.level() - pm->GetRootLevel(); + const std::string &mesh_record_name = var_name + "_lvl" + std::to_string(level); + + PARTHENON_REQUIRE_THROWS(it->meshes.contains(mesh_record_name), + "Missing mesh record in restart file."); + auto mesh_record = it->meshes[mesh_record_name]; + + int64_t comp_offset = 0; // offset data_vector to store component data + int idx_component = 0; // used in label for non-vector variables + const bool is_scalar = + vinfo.GetDim(4) == 1 && vinfo.GetDim(5) == 1 && vinfo.GetDim(6) == 1; + const auto &Nt = vinfo.GetDim(6); + const auto &Nu = vinfo.GetDim(5); + const auto &Nv = vinfo.GetDim(4); + // loop over all components + for (int t = 0; t < Nt; ++t) { + for (int u = 0; u < Nu; ++u) { + for (int v = 0; v < Nv; ++v) { + // Get the correct record + std::string comp_name; + if (is_scalar) { + comp_name = openPMD::MeshRecordComponent::SCALAR; + } else if (vinfo.is_vector) { + if (v == 0) { + comp_name = "x"; + } else if (v == 1) { + comp_name = "y"; + } else if (v == 2) { + comp_name = "z"; + } else { + PARTHENON_THROW("Expected v index doesn't match vector expectation."); + } + } else { + comp_name = vinfo.component_labels[idx_component]; + } + PARTHENON_REQUIRE_THROWS(mesh_record.contains(comp_name), + "Missing component in mesh record of restart file."); + auto mesh_comp = mesh_record[comp_name]; + + openPMD::Offset chunk_offset = { + pmb->loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), + pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), + pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; + openPMD::Extent chunk_extent = { + static_cast(pmb->block_size.nx(X3DIR)), + static_cast(pmb->block_size.nx(X2DIR)), + static_cast(pmb->block_size.nx(X1DIR))}; + mesh_comp.loadChunkRaw(&data_vec[comp_offset], chunk_extent, chunk_extent); + // TODO(pgrete) check if output utils machinery can be used for non-cell + // centered fields, which might not be that straightforward as a global mesh is + // stored rather than individual blocks. + comp_offset += pmb->block_size.nx(X1DIR) * pmb->block_size.nx(X2DIR) * + pmb->block_size.nx(X3DIR); + idx_component += 1; + } + } + } // loop over components + } // loop over blocks + + // Now actually read the registered chunks form disk + it->seriesFlush(); +} } // namespace parthenon diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index 46a5658eb031..b1a41699a609 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -46,7 +46,7 @@ class RestartReaderOPMD : public RestartReader { // fills internal data for given pointer void ReadBlocks(const std::string &name, IndexRange range, const OutputUtils::VarInfo &info, std::vector &dataVec, - int file_output_format_version) const override; + int file_output_format_version, Mesh *pmesh) const override; void ReadSwarmVar(const std::string &swarmname, const std::string &varname, const std::size_t count, const std::size_t offset, const Metadata &m, diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index ed35c2afcfbf..edb1e4c2dcce 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -326,7 +326,7 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { } // Read relevant data from the hdf file, this works for dense and sparse variables try { - resfile.ReadBlocks(label, myBlocks, v_info, tmp, file_output_format_ver); + resfile.ReadBlocks(label, myBlocks, v_info, tmp, file_output_format_ver, &rm); } catch (std::exception &ex) { std::cout << "[" << Globals::my_rank << "] WARNING: Failed to read variable " << label << " from restart file:" << std::endl From 19863d73fe9efc8d831f5d5479b0c7ef447c51df Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 12 Apr 2024 16:55:53 +0200 Subject: [PATCH 020/125] Fix loc level --- src/outputs/restart_opmd.cpp | 36 +++++++++++++++++++++++++++++++----- src/parthenon_manager.cpp | 3 ++- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index e7bbbaec533f..c5d98fb0479b 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -91,18 +91,42 @@ std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, return 0; } -void RestartReaderOPMD::ReadParams(const std::string &name, Params &p) {} +void RestartReaderOPMD::ReadParams(const std::string &name, Params &p) { +#if 0 + // views and vecs of scalar types + ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); + ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); + ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); + ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); + ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); + ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); + ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); + + // strings + ReadFromHDF5AllParamsOfType(prefix, group); + ReadFromHDF5AllParamsOfType>(prefix, group); + + template +void call_my_func(my_list ) +{ + (myFunc(), ...); +} +#endif +} void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block_range, const OutputUtils::VarInfo &vinfo, std::vector &data_vec, int file_output_format_version, Mesh *pm) const { for (auto &pmb : pm->block_list) { // TODO(pgrete) check if we should skip the suffix for level 0 - const auto level = pmb->loc.level() - pm->GetRootLevel(); + // TODO(pgrete) ask LR why this is not mirrored from writing + // const auto level = pmb->loc.level() - pm->GetRootLevel(); + const auto level = pmb->loc.level(); const std::string &mesh_record_name = var_name + "_lvl" + std::to_string(level); PARTHENON_REQUIRE_THROWS(it->meshes.contains(mesh_record_name), - "Missing mesh record in restart file."); + "Missing mesh record '" + mesh_record_name + + "' in restart file."); auto mesh_record = it->meshes[mesh_record_name]; int64_t comp_offset = 0; // offset data_vector to store component data @@ -134,7 +158,9 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block comp_name = vinfo.component_labels[idx_component]; } PARTHENON_REQUIRE_THROWS(mesh_record.contains(comp_name), - "Missing component in mesh record of restart file."); + "Missing component'" + comp_name + + "' in mesh record '" + mesh_record_name + + "' of restart file."); auto mesh_comp = mesh_record[comp_name]; openPMD::Offset chunk_offset = { @@ -145,7 +171,7 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block static_cast(pmb->block_size.nx(X3DIR)), static_cast(pmb->block_size.nx(X2DIR)), static_cast(pmb->block_size.nx(X1DIR))}; - mesh_comp.loadChunkRaw(&data_vec[comp_offset], chunk_extent, chunk_extent); + mesh_comp.loadChunkRaw(&data_vec[comp_offset], chunk_offset, chunk_extent); // TODO(pgrete) check if output utils machinery can be used for non-cell // centered fields, which might not be that straightforward as a global mesh is // stored rather than individual blocks. diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index edb1e4c2dcce..d77ae9377b15 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -257,7 +257,8 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { // Currently supports versions 3 and 4. const auto file_output_format_ver = resfile.GetOutputFormatVersion(); - if (file_output_format_ver < HDF5::OUTPUT_VERSION_FORMAT - 1) { + // TODO(pgrete) figure out what to do about versions of different outputs + if (false && file_output_format_ver < HDF5::OUTPUT_VERSION_FORMAT - 1) { std::stringstream msg; msg << "File format version " << file_output_format_ver << " not supported. " << "Current format is " << HDF5::OUTPUT_VERSION_FORMAT << std::endl; From e2b2bd19420277b677fc9d3f6e88434d90696c06 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 15 Apr 2024 12:17:34 +0200 Subject: [PATCH 021/125] Make Series persistent and fix rootlevel typo --- src/outputs/openpmd.cpp | 2 +- src/outputs/restart_opmd.cpp | 9 ++++----- src/outputs/restart_opmd.hpp | 4 ++++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 7e5b88f022a8..27dc4d6627a7 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -159,7 +159,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // restart info, write always it.setAttribute("NBNew", pm->nbnew); it.setAttribute("NBDel", pm->nbdel); - it.setAttribute("RootLevel", pm->GetLegacyTreeRootLevel()); + it.setAttribute("RootLevel", pm->GetRootLevel()); it.setAttribute("Refine", pm->adaptive ? 1 : 0); it.setAttribute("Multilevel", pm->multilevel ? 1 : 0); diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index c5d98fb0479b..d14c5710ee39 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -6,6 +6,7 @@ //! \file restart_opmd.cpp // \brief Restarts a simulation from an OpenPMD output with ADIOS2 backend +#include #include #include #include @@ -24,8 +25,8 @@ namespace parthenon { //---------------------------------------------------------------------------------------- //! \fn void RestartReader::RestartReader(const std::string filename) // \brief Opens the restart file and stores appropriate file handle in fh_ -RestartReaderOPMD::RestartReaderOPMD(const char *filename) : filename_(filename) { - auto series = openPMD::Series(filename, openPMD::Access::READ_ONLY, MPI_COMM_WORLD); +RestartReaderOPMD::RestartReaderOPMD(const char *filename) + : filename_(filename), series(filename, openPMD::Access::READ_ONLY, MPI_COMM_WORLD) { PARTHENON_REQUIRE_THROWS( series.iterations.size() == 1, "Parthenon restarts should only contain one iteration/timestep."); @@ -119,9 +120,7 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block int file_output_format_version, Mesh *pm) const { for (auto &pmb : pm->block_list) { // TODO(pgrete) check if we should skip the suffix for level 0 - // TODO(pgrete) ask LR why this is not mirrored from writing - // const auto level = pmb->loc.level() - pm->GetRootLevel(); - const auto level = pmb->loc.level(); + const auto level = pmb->loc.level() - pm->GetRootLevel(); const std::string &mesh_record_name = var_name + "_lvl" + std::to_string(level); PARTHENON_REQUIRE_THROWS(it->meshes.contains(mesh_record_name), diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index b1a41699a609..47913cf460fd 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -14,6 +14,7 @@ #include "basic_types.hpp" #include "openPMD/Iteration.hpp" +#include "openPMD/Series.hpp" #include "outputs/restart.hpp" #include "mesh/domain.hpp" @@ -73,6 +74,9 @@ class RestartReaderOPMD : public RestartReader { private: const std::string filename_; + openPMD::Series series; + // Iteration is a pointer because it cannot be default constructed (it depends on the + // Series). std::unique_ptr it; }; From f9373e8a4fe0073a64d46134b1e4a3f6821455d7 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 15 Apr 2024 15:50:23 +0200 Subject: [PATCH 022/125] WIP Read/Write Params --- src/interface/params.hpp | 6 ++++ src/outputs/openpmd.cpp | 44 +++++++++++++++++++++++++++ src/outputs/outputs.hpp | 1 + src/outputs/restart_opmd.cpp | 58 ++++++++++++++++++++++-------------- src/parthenon_manager.cpp | 3 +- 5 files changed, 89 insertions(+), 23 deletions(-) diff --git a/src/interface/params.hpp b/src/interface/params.hpp index 4b2f4a09d0cd..e99f8561d0ae 100644 --- a/src/interface/params.hpp +++ b/src/interface/params.hpp @@ -118,6 +118,12 @@ class Params { return it->second; } + const Mutability &GetMutability(const std::string &key) const { + auto const it = myMutable_.find(key); + PARTHENON_REQUIRE_THROWS(it != myMutable_.end(), "Key " + key + " doesn't exist"); + return it->second; + } + std::vector GetKeys() const { std::vector keys; for (auto &x : myParams_) { diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 27dc4d6627a7..69f7885c026e 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -37,16 +38,19 @@ #include "defs.hpp" #include "driver/driver.hpp" #include "globals.hpp" +#include "interface/state_descriptor.hpp" #include "interface/variable_state.hpp" #include "mesh/mesh.hpp" #include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/IO/Access.hpp" +#include "openPMD/Iteration.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/Series.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" #include "outputs/output_utils.hpp" #include "outputs/outputs.hpp" +#include "outputs/parthenon_hdf5.hpp" // needd for VALId_VEC_TYPES -> move #include "parthenon_array_generic.hpp" #include "utils/error_checking.hpp" #include "utils/instrument.hpp" @@ -60,6 +64,32 @@ namespace parthenon { using namespace OutputUtils; +template +void WriteAllParamsOfType(std::shared_ptr pkg, openPMD::Iteration *it) { + const std::string prefix = "Params/" + pkg->label() + "/"; + const auto ¶ms = pkg->AllParams(); + for (const auto &key : params.GetKeys()) { + const auto type = params.GetType(key); + if (type == std::type_index(typeid(T))) { + // auto typed_ptr = dynamic_cast *>((p.second).get()); + it->setAttribute(prefix + key, params.Get(key)); + } + } +} + +template +void WriteAllParamsOfMultipleTypes(std::shared_ptr pkg, + openPMD::Iteration *it) { + ([&] { WriteAllParamsOfType(pkg, it); }(), ...); +} + +template +void WriteAllParams(std::shared_ptr pkg, openPMD::Iteration *it) { + WriteAllParamsOfMultipleTypes>(pkg, it); + // TODO(pgrete) check why this doens't work, i.e., which type is causing problems + // WriteAllParamsOfMultipleTypes(pkg, it); +} + //---------------------------------------------------------------------------------------- //! \fn void OpenPMDOutput:::WriteOutputFile(Mesh *pm) // \brief Expose mesh and all Cell variables for processing with Ascent @@ -125,6 +155,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, it.setDt(-1.0); } { // FIXME move this to dump params + PARTHENON_INSTRUMENT_REGION("Dump Params"); const auto view_d = Kokkos::View("blub", 5, 3); // Map a view onto a host allocation (so that we can call deep_copy) @@ -133,6 +164,19 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, view_h(host_vec.data(), view_d.extent_int(0), view_d.extent_int(1)); Kokkos::deep_copy(view_h, view_d); it.setAttribute("blub", host_vec); + + for (const auto &[key, pkg] : pm->packages.AllPackages()) { + // WriteAllParams(pkg, &it); // check why this (vector of bool) doesn't work + WriteAllParams(pkg, &it); + WriteAllParams(pkg, &it); + WriteAllParams(pkg, &it); + WriteAllParams(pkg, &it); + WriteAllParams(pkg, &it); + WriteAllParams(pkg, &it); + WriteAllParams(pkg, &it); + WriteAllParamsOfType(pkg, &it); + // WriteAllParamsOfType>(pkg, &it); + } } // Then our own { diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index 59cb03610374..972a35b36490 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -32,6 +32,7 @@ #include "interface/mesh_data.hpp" #include "io_wrapper.hpp" #include "kokkos_abstraction.hpp" +#include "openPMD/Iteration.hpp" #include "parthenon_arrays.hpp" #include "utils/error_checking.hpp" diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index d14c5710ee39..30a2aafc5ed8 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -92,28 +92,42 @@ std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, return 0; } -void RestartReaderOPMD::ReadParams(const std::string &name, Params &p) { -#if 0 - // views and vecs of scalar types - ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); - ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); - ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); - ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); - ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); - ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); - ReadFromHDF5AllParamsOfTypeOrVec(prefix, group); - - // strings - ReadFromHDF5AllParamsOfType(prefix, group); - ReadFromHDF5AllParamsOfType>(prefix, group); - - template -void call_my_func(my_list ) -{ - (myFunc(), ...); +template +void ReadAllParamsOfType(const std::string &pkg_name, openPMD::Iteration *it, + Params ¶ms) { + for (const auto &key : params.GetKeys()) { + const auto type = params.GetType(key); + auto mutability = params.GetMutability(key); + if (type == std::type_index(typeid(T)) && mutability == Params::Mutability::Restart) { + auto val = it->getAttribute("Params/" + pkg_name + "/" + key).get(); + params.Update(key, val); + } + } +} + +template +void ReadAllParamsOfMultipleTypes(const std::string &pkg_name, openPMD::Iteration *it, + Params &p) { + ([&] { ReadAllParamsOfType(pkg_name, it, p); }(), ...); } -#endif + +template +void ReadAllParams(const std::string &pkg_name, openPMD::Iteration *it, Params &p) { + ReadAllParamsOfMultipleTypes>(pkg_name, it, p); + // TODO(pgrete) check why this doens't work, i.e., which type is causing problems + // ReadAllParamsOfMultipleTypes(pkg, it); +} +void RestartReaderOPMD::ReadParams(const std::string &pkg_name, Params &p) { + ReadAllParams(pkg_name, it, p); + ReadAllParams(pkg_name, it, p); + ReadAllParams(pkg_name, it, p); + ReadAllParams(pkg_name, it, p); + ReadAllParams(pkg_name, it, p); + ReadAllParams(pkg_name, it, p); + ReadAllParams(pkg_name, it, p); + ReadAllParamsOfType(pkg_name, it, p); } + void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block_range, const OutputUtils::VarInfo &vinfo, std::vector &data_vec, @@ -172,8 +186,8 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block static_cast(pmb->block_size.nx(X1DIR))}; mesh_comp.loadChunkRaw(&data_vec[comp_offset], chunk_offset, chunk_extent); // TODO(pgrete) check if output utils machinery can be used for non-cell - // centered fields, which might not be that straightforward as a global mesh is - // stored rather than individual blocks. + // centered fields, which might not be that straightforward as a global mesh + // is stored rather than individual blocks. comp_offset += pmb->block_size.nx(X1DIR) * pmb->block_size.nx(X2DIR) * pmb->block_size.nx(X3DIR); idx_component += 1; diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index d77ae9377b15..71e108b012f1 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -353,7 +353,8 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { // Double note that this also needs to be update in case // we update the HDF5 infrastructure! - if (file_output_format_ver >= HDF5::OUTPUT_VERSION_FORMAT - 1) { + // TODO(pgrete) figure out what to do about versions of different outputs + if (true || file_output_format_ver >= HDF5::OUTPUT_VERSION_FORMAT - 1) { OutputUtils::PackOrUnpackVar( v_info, resfile.hasGhost, index, [&](auto index, int topo, int t, int u, int v, int k, int j, int i) { From 96a3f4cc41f81c524680ebe979c37023da8680a5 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 15 Apr 2024 19:02:14 +0200 Subject: [PATCH 023/125] Make ReadParams private member --- src/outputs/restart_opmd.cpp | 29 ++++++++++++++--------------- src/outputs/restart_opmd.hpp | 7 +++++++ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 30a2aafc5ed8..23ba83bbf4af 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -93,8 +93,7 @@ std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, } template -void ReadAllParamsOfType(const std::string &pkg_name, openPMD::Iteration *it, - Params ¶ms) { +void RestartReaderOPMD::ReadAllParamsOfType(const std::string &pkg_name, Params ¶ms) { for (const auto &key : params.GetKeys()) { const auto type = params.GetType(key); auto mutability = params.GetMutability(key); @@ -106,26 +105,26 @@ void ReadAllParamsOfType(const std::string &pkg_name, openPMD::Iteration *it, } template -void ReadAllParamsOfMultipleTypes(const std::string &pkg_name, openPMD::Iteration *it, - Params &p) { - ([&] { ReadAllParamsOfType(pkg_name, it, p); }(), ...); +void RestartReaderOPMD::ReadAllParamsOfMultipleTypes(const std::string &pkg_name, + Params &p) { + ([&] { ReadAllParamsOfType(pkg_name, p); }(), ...); } template -void ReadAllParams(const std::string &pkg_name, openPMD::Iteration *it, Params &p) { - ReadAllParamsOfMultipleTypes>(pkg_name, it, p); +void RestartReaderOPMD::ReadAllParams(const std::string &pkg_name, Params &p) { + ReadAllParamsOfMultipleTypes>(pkg_name, p); // TODO(pgrete) check why this doens't work, i.e., which type is causing problems // ReadAllParamsOfMultipleTypes(pkg, it); } void RestartReaderOPMD::ReadParams(const std::string &pkg_name, Params &p) { - ReadAllParams(pkg_name, it, p); - ReadAllParams(pkg_name, it, p); - ReadAllParams(pkg_name, it, p); - ReadAllParams(pkg_name, it, p); - ReadAllParams(pkg_name, it, p); - ReadAllParams(pkg_name, it, p); - ReadAllParams(pkg_name, it, p); - ReadAllParamsOfType(pkg_name, it, p); + ReadAllParams(pkg_name, p); + ReadAllParams(pkg_name, p); + ReadAllParams(pkg_name, p); + ReadAllParams(pkg_name, p); + ReadAllParams(pkg_name, p); + ReadAllParams(pkg_name, p); + ReadAllParams(pkg_name, p); + ReadAllParamsOfType(pkg_name, p); } void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block_range, diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index 47913cf460fd..f1c60d3efb7f 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -78,6 +78,13 @@ class RestartReaderOPMD : public RestartReader { // Iteration is a pointer because it cannot be default constructed (it depends on the // Series). std::unique_ptr it; + + template + void ReadAllParamsOfType(const std::string &pkg_name, Params ¶ms); + template + void ReadAllParamsOfMultipleTypes(const std::string &pkg_name, Params &p); + template + void ReadAllParams(const std::string &pkg_name, Params &p); }; } // namespace parthenon From 56976a8514c0c54dfb02fa03261a350876e51d9e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 18 Apr 2024 11:04:29 +0200 Subject: [PATCH 024/125] Fix root level in output --- src/outputs/openpmd.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 69f7885c026e..19c264034a73 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -203,14 +203,14 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // restart info, write always it.setAttribute("NBNew", pm->nbnew); it.setAttribute("NBDel", pm->nbdel); - it.setAttribute("RootLevel", pm->GetRootLevel()); + it.setAttribute("RootLevel", pm->GetLegacyTreeRootLevel()); it.setAttribute("Refine", pm->adaptive ? 1 : 0); it.setAttribute("Multilevel", pm->multilevel ? 1 : 0); it.setAttribute("BlocksPerPE", pm->GetNbList()); // Mesh block size - const auto base_block_size = pm->GetBlockSize(); + const auto base_block_size = pm->GetDefaultBlockSize(); it.setAttribute("MeshBlockSize", std::vector{base_block_size.nx(X1DIR), base_block_size.nx(X2DIR), base_block_size.nx(X3DIR)}); From 6199843c2241a1fcaadd0c682f3e3bb46f7be78f Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 22 Apr 2024 16:32:43 +0200 Subject: [PATCH 025/125] Move to mesh per record standard for writing --- src/outputs/openpmd.cpp | 76 +++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 19c264034a73..3b3b35b4eea2 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -316,40 +316,43 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, for (auto &pmb : pm->block_list) { // TODO(pgrete) check if we should skip the suffix for level 0 const auto level = pmb->loc.level() - pm->GetRootLevel(); - const std::string &mesh_record_name = var_name + "_lvl" + std::to_string(level); - - // Create the mesh_record for this variable at the given level (if it doesn't exist - // yet) - if (!it.meshes.contains(mesh_record_name)) { - auto mesh_record = it.meshes[mesh_record_name]; - - // These following attributes are shared across all components of the record. - - PARTHENON_REQUIRE_THROWS( - typeid(Coordinates_t) == typeid(UniformCartesian), - "OpenPMD in Parthenon currently only supports Cartesian coordinates."); - mesh_record.setGeometry(openPMD::Mesh::Geometry::cartesian); - auto &coords = pmb->coords; - // For uniform Cartesian, all dxN are const across the block so we just pick the - // first index. - Real dx1 = coords.CellWidth(0, 0, 0); - Real dx2 = coords.CellWidth(0, 0, 0); - Real dx3 = coords.CellWidth(0, 0, 0); - - // TODO(pgrete) check if this should be tied to the MemoryLayout - mesh_record.setDataOrder(openPMD::Mesh::DataOrder::C); - mesh_record.setGridSpacing(std::vector{dx3, dx2, dx1}); - mesh_record.setAxisLabels({"z", "y", "x"}); - mesh_record.setGridGlobalOffset({ - pm->mesh_size.xmin(X3DIR), - pm->mesh_size.xmin(X2DIR), - pm->mesh_size.xmin(X1DIR), - }); - - // TODO(pgrete) need unitDimension and timeOffset for this record? - } - auto mesh_record = it.meshes[mesh_record_name]; + for (const auto &comp_lbl : vinfo.component_labels) { + + const std::string &mesh_record_name = + var_name + "_" + comp_lbl + "_lvl" + std::to_string(level); + + // Create the mesh_record for this variable at the given level (if it doesn't + // exist yet) + if (!it.meshes.contains(mesh_record_name)) { + auto mesh_record = it.meshes[mesh_record_name]; + + // These following attributes are shared across all components of the record. + + PARTHENON_REQUIRE_THROWS( + typeid(Coordinates_t) == typeid(UniformCartesian), + "OpenPMD in Parthenon currently only supports Cartesian coordinates."); + mesh_record.setGeometry(openPMD::Mesh::Geometry::cartesian); + auto &coords = pmb->coords; + // For uniform Cartesian, all dxN are const across the block so we just pick the + // first index. + Real dx1 = coords.CellWidth(0, 0, 0); + Real dx2 = coords.CellWidth(0, 0, 0); + Real dx3 = coords.CellWidth(0, 0, 0); + + // TODO(pgrete) check if this should be tied to the MemoryLayout + mesh_record.setDataOrder(openPMD::Mesh::DataOrder::C); + mesh_record.setGridSpacing(std::vector{dx3, dx2, dx1}); + mesh_record.setAxisLabels({"z", "y", "x"}); + mesh_record.setGridGlobalOffset({ + pm->mesh_size.xmin(X3DIR), + pm->mesh_size.xmin(X2DIR), + pm->mesh_size.xmin(X1DIR), + }); + + // TODO(pgrete) need unitDimension and timeOffset for this record? + } + } // Now that the mesh record exists, actually write the data auto out_var = pmb->meshblock_data.Get()->GetVarPtr(var_name); @@ -396,8 +399,13 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, PARTHENON_THROW("Expected v index doesn't match vector expectation."); } } else { - comp_name = vinfo.component_labels[idx_component]; + comp_name = openPMD::MeshRecordComponent::SCALAR; + // comp_name = vinfo.component_labels[idx_component]; } + const std::string &mesh_record_name = + var_name + "_" + vinfo.component_labels[idx_component] + "_lvl" + + std::to_string(level); + auto mesh_record = it.meshes[mesh_record_name]; auto mesh_comp = mesh_record[comp_name]; // TODO(pgrete) needs to be updated for face and edges etc From 684b7ab63f2a74a98b6c60dfddd483dc16929f94 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 23 Apr 2024 10:17:43 +0200 Subject: [PATCH 026/125] Allow for 2D and 3D output. Fix single dataset reset. --- src/outputs/openpmd.cpp | 99 +++++++++++++++++++++++++++++------------ 1 file changed, 70 insertions(+), 29 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index 3b3b35b4eea2..efd0284db1bd 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -342,13 +342,51 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // TODO(pgrete) check if this should be tied to the MemoryLayout mesh_record.setDataOrder(openPMD::Mesh::DataOrder::C); - mesh_record.setGridSpacing(std::vector{dx3, dx2, dx1}); - mesh_record.setAxisLabels({"z", "y", "x"}); - mesh_record.setGridGlobalOffset({ - pm->mesh_size.xmin(X3DIR), - pm->mesh_size.xmin(X2DIR), - pm->mesh_size.xmin(X1DIR), - }); + + // TODO(pgrete) allwo for proper vectors/tensors + auto mesh_comp = mesh_record[openPMD::MeshRecordComponent::SCALAR]; + // TODO(pgrete) needs to be updated for face and edges etc + // Also this feels wrong for deep hierachies... + auto effective_nx = static_cast(std::pow(2, level)); + openPMD::Extent global_extent; + if (pm->ndim == 3) { + mesh_record.setGridSpacing(std::vector{dx3, dx2, dx1}); + mesh_record.setAxisLabels({"z", "y", "x"}); + mesh_record.setGridGlobalOffset({ + pm->mesh_size.xmin(X3DIR), + pm->mesh_size.xmin(X2DIR), + pm->mesh_size.xmin(X1DIR), + }); + // TODO(pgrete) needs to be updated for face and edges etc + mesh_comp.setPosition(std::vector{0.5, 0.5, 0.5}); + global_extent = { + static_cast(pm->mesh_size.nx(X3DIR)) * effective_nx, + static_cast(pm->mesh_size.nx(X2DIR)) * effective_nx, + static_cast(pm->mesh_size.nx(X1DIR)) * effective_nx, + }; + } else if (pm->ndim == 2) { + mesh_record.setGridSpacing(std::vector{dx2, dx1}); + mesh_record.setAxisLabels({"y", "x"}); + mesh_record.setGridGlobalOffset({ + pm->mesh_size.xmin(X2DIR), + pm->mesh_size.xmin(X1DIR), + }); + + // TODO(pgrete) needs to be updated for face and edges etc + mesh_comp.setPosition(std::vector{0.5, 0.5}); + global_extent = { + static_cast(pm->mesh_size.nx(X2DIR)) * effective_nx, + static_cast(pm->mesh_size.nx(X1DIR)) * effective_nx, + }; + + } else { + PARTHENON_THROW("1D output for openpmd not yet supported."); + } + // Handling this here to now re-reset dataset later when iterating through the + // blocks + auto const dataset = + openPMD::Dataset(openPMD::determineDatatype(), global_extent); + mesh_comp.resetDataset(dataset); // TODO(pgrete) need unitDimension and timeOffset for this record? } @@ -408,20 +446,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, auto mesh_record = it.meshes[mesh_record_name]; auto mesh_comp = mesh_record[comp_name]; - // TODO(pgrete) needs to be updated for face and edges etc - mesh_comp.setPosition(std::vector{0.5, 0.5, 0.5}); - // TODO(pgrete) needs to be updated for face and edges etc - // Also this feels wrong for deep hierachies... - auto effective_nx = static_cast(std::pow(2, level)); - openPMD::Extent global_extent = { - static_cast(pm->mesh_size.nx(X3DIR)) * effective_nx, - static_cast(pm->mesh_size.nx(X2DIR)) * effective_nx, - static_cast(pm->mesh_size.nx(X1DIR)) * effective_nx, - }; - auto const dataset = - openPMD::Dataset(openPMD::determineDatatype(), global_extent); - mesh_comp.resetDataset(dataset); - const auto comp_offset = tmp_offset; for (int k = kb.s; k <= kb.e; ++k) { for (int j = jb.s; j <= jb.e; ++j) { @@ -431,14 +455,31 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } } } - openPMD::Offset chunk_offset = { - pmb->loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), - pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), - pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; - openPMD::Extent chunk_extent = { - static_cast(pmb->block_size.nx(X3DIR)), - static_cast(pmb->block_size.nx(X2DIR)), - static_cast(pmb->block_size.nx(X1DIR))}; + openPMD::Offset chunk_offset; + openPMD::Extent chunk_extent; + if (pm->ndim == 3) { + chunk_offset = { + pmb->loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), + pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), + pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; + chunk_extent = {static_cast(pmb->block_size.nx(X3DIR)), + static_cast(pmb->block_size.nx(X2DIR)), + static_cast(pmb->block_size.nx(X1DIR))}; + } else if (pm->ndim == 2) { + chunk_offset = { + pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), + pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; + chunk_extent = {static_cast(pmb->block_size.nx(X2DIR)), + static_cast(pmb->block_size.nx(X1DIR))}; + } else { + PARTHENON_THROW("1D output for openpmd not yet supported."); + } + std::cout << "Block " << pmb->gid << " writes chunk of [" << chunk_extent[0] + << " " << chunk_extent[1] << " " + << "] with offset [" << chunk_offset[0] << " " << chunk_offset[1] + << "] and logical locs [" << pmb->loc.lx2() << " " + << pmb->loc.lx1() << "]\n"; + mesh_comp.storeChunkRaw(&tmp_data[comp_offset], chunk_offset, chunk_extent); idx_component += 1; } From 251c6ea7660fbcd2bd3c7539b95bc1e4e4e959b7 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 23 Apr 2024 17:42:00 +0200 Subject: [PATCH 027/125] Fix logical loc --- src/outputs/openpmd.cpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/outputs/openpmd.cpp b/src/outputs/openpmd.cpp index efd0284db1bd..d611bc7095b6 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/openpmd.cpp @@ -457,28 +457,24 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } openPMD::Offset chunk_offset; openPMD::Extent chunk_extent; + const auto loc = pm->Forest().GetLegacyTreeLocation(pmb->loc); if (pm->ndim == 3) { chunk_offset = { - pmb->loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), - pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), - pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; + loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), + loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), + loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; chunk_extent = {static_cast(pmb->block_size.nx(X3DIR)), static_cast(pmb->block_size.nx(X2DIR)), static_cast(pmb->block_size.nx(X1DIR))}; } else if (pm->ndim == 2) { chunk_offset = { - pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), - pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; + loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), + loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; chunk_extent = {static_cast(pmb->block_size.nx(X2DIR)), static_cast(pmb->block_size.nx(X1DIR))}; } else { PARTHENON_THROW("1D output for openpmd not yet supported."); } - std::cout << "Block " << pmb->gid << " writes chunk of [" << chunk_extent[0] - << " " << chunk_extent[1] << " " - << "] with offset [" << chunk_offset[0] << " " << chunk_offset[1] - << "] and logical locs [" << pmb->loc.lx2() << " " - << pmb->loc.lx1() << "]\n"; mesh_comp.storeChunkRaw(&tmp_data[comp_offset], chunk_offset, chunk_extent); idx_component += 1; From 03f80c716db8e395e96781e87aad4459b0b27557 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 24 Apr 2024 09:38:25 +0200 Subject: [PATCH 028/125] Rename opmd files --- src/CMakeLists.txt | 2 +- src/outputs/{openpmd.cpp => parthenon_opmd.cpp} | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) rename src/outputs/{openpmd.cpp => parthenon_opmd.cpp} (99%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 140ea9733f47..81f7c4c00bcf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -182,7 +182,6 @@ add_library(parthenon outputs/history.cpp outputs/io_wrapper.cpp outputs/io_wrapper.hpp - outputs/openpmd.cpp outputs/output_utils.cpp outputs/output_utils.hpp outputs/outputs.cpp @@ -194,6 +193,7 @@ add_library(parthenon outputs/parthenon_hdf5_types.hpp outputs/parthenon_xdmf.cpp outputs/parthenon_hdf5.hpp + outputs/parthenon_opmd.cpp outputs/parthenon_xdmf.hpp outputs/restart.hpp outputs/restart_hdf5.cpp diff --git a/src/outputs/openpmd.cpp b/src/outputs/parthenon_opmd.cpp similarity index 99% rename from src/outputs/openpmd.cpp rename to src/outputs/parthenon_opmd.cpp index d611bc7095b6..cc6fd1fd03b0 100644 --- a/src/outputs/openpmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -14,7 +14,7 @@ // license in this material to reproduce, prepare derivative works, distribute copies to // the public, perform publicly and display publicly, and to permit others to do so. //======================================================================================== -//! \file openpmd.cpp +//! \file parthenon_openpmd.cpp // \brief Output for OpenPMD https://www.openpmd.org/ (supporting various backends) #include @@ -90,6 +90,13 @@ void WriteAllParams(std::shared_ptr pkg, openPMD::Iteration *it // WriteAllParamsOfMultipleTypes(pkg, it); } +namespace OpenPMDUtils { + + auto GetMeshRecordAndComponentNames() { + + } +} + //---------------------------------------------------------------------------------------- //! \fn void OpenPMDOutput:::WriteOutputFile(Mesh *pm) // \brief Expose mesh and all Cell variables for processing with Ascent @@ -422,7 +429,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, for (int t = 0; t < Nt; ++t) { for (int u = 0; u < Nu; ++u) { for (int v = 0; v < Nv; ++v) { - // Get the correct record std::string comp_name; if (is_scalar) { comp_name = openPMD::MeshRecordComponent::SCALAR; From 890fffe334a0c142425a7bb8e278ad291349460e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 24 Apr 2024 10:14:23 +0200 Subject: [PATCH 029/125] Separate common calls to chunks and names --- src/outputs/parthenon_opmd.cpp | 126 ++++++++++++++++++--------------- 1 file changed, 67 insertions(+), 59 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index cc6fd1fd03b0..ae1b1f40bcb2 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include // Parthenon headers @@ -41,6 +42,7 @@ #include "interface/state_descriptor.hpp" #include "interface/variable_state.hpp" #include "mesh/mesh.hpp" +#include "mesh/meshblock.hpp" #include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/IO/Access.hpp" @@ -92,10 +94,60 @@ void WriteAllParams(std::shared_ptr pkg, openPMD::Iteration *it namespace OpenPMDUtils { - auto GetMeshRecordAndComponentNames() { +// Construct OpenPMD Mesh "record" name and comonnent identifier. +// - comp_idx is a flattended index over all components of the vectors and tensors, i.e., +// the typical v,u,t indices. +// - level is the current effective level of the Mesh record +auto GetMeshRecordAndComponentNames(const VarInfo &vinfo, const int comp_idx, + const int level) { + std::string comp_name; + if (vinfo.is_vector) { + if (comp_idx == 0) { + comp_name = "x"; + } else if (comp_idx == 1) { + comp_name = "y"; + } else if (comp_idx == 2) { + comp_name = "z"; + } else { + PARTHENON_THROW("Expected component index doesn't match vector expectation."); + } + // Current unclear how to properly handle other vectors and tensors, so everything + // that not's a proper vector is a a scalar for now. + } else { + comp_name = openPMD::MeshRecordComponent::SCALAR; + } + // TODO(pgrete) need to make sure that var names are allowed within standard + const std::string &mesh_record_name = vinfo.label + "_" + + vinfo.component_labels[comp_idx] + "_lvl" + + std::to_string(level); + return std::make_tuple(mesh_record_name, comp_name); +} +// Calculate logical location on effective mesh (i.e., a mesh with size that matches full +// coverage at given resolution on a particular level) +// TODO(pgrete) needs to be updated to properly work with Forests +auto GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb) { + openPMD::Offset chunk_offset; + openPMD::Extent chunk_extent; + const auto loc = pm->Forest().GetLegacyTreeLocation(pmb->loc); + if (pm->ndim == 3) { + chunk_offset = {loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), + loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), + loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; + chunk_extent = {static_cast(pmb->block_size.nx(X3DIR)), + static_cast(pmb->block_size.nx(X2DIR)), + static_cast(pmb->block_size.nx(X1DIR))}; + } else if (pm->ndim == 2) { + chunk_offset = {loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), + loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; + chunk_extent = {static_cast(pmb->block_size.nx(X2DIR)), + static_cast(pmb->block_size.nx(X1DIR))}; + } else { + PARTHENON_THROW("1D output for openpmd not yet supported."); } + return std::make_tuple(chunk_offset, chunk_extent); } +} // namespace OpenPMDUtils //---------------------------------------------------------------------------------------- //! \fn void OpenPMDOutput:::WriteOutputFile(Mesh *pm) @@ -308,8 +360,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, memset(tmp_data.data(), 0, tmp_data.size() * sizeof(OutT)); uint64_t tmp_offset = 0; - const bool is_scalar = - vinfo.GetDim(4) == 1 && vinfo.GetDim(5) == 1 && vinfo.GetDim(6) == 1; if (vinfo.is_vector) { // sanity check PARTHENON_REQUIRE_THROWS( @@ -318,21 +368,18 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, "dimensionality of the simulation.") } - // TODO(pgrete) need to make sure that var names are allowed within standard - const std::string var_name = vinfo.label; for (auto &pmb : pm->block_list) { // TODO(pgrete) check if we should skip the suffix for level 0 const auto level = pmb->loc.level() - pm->GetRootLevel(); - for (const auto &comp_lbl : vinfo.component_labels) { - - const std::string &mesh_record_name = - var_name + "_" + comp_lbl + "_lvl" + std::to_string(level); + for (int comp_idx = 0; comp_idx < vinfo.component_labels.size(); comp_idx++) { + const auto [record_name, comp_name] = + OpenPMDUtils::GetMeshRecordAndComponentNames(vinfo, comp_idx, level); // Create the mesh_record for this variable at the given level (if it doesn't // exist yet) - if (!it.meshes.contains(mesh_record_name)) { - auto mesh_record = it.meshes[mesh_record_name]; + if (!it.meshes.contains(record_name)) { + auto mesh_record = it.meshes[record_name]; // These following attributes are shared across all components of the record. @@ -350,8 +397,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // TODO(pgrete) check if this should be tied to the MemoryLayout mesh_record.setDataOrder(openPMD::Mesh::DataOrder::C); - // TODO(pgrete) allwo for proper vectors/tensors - auto mesh_comp = mesh_record[openPMD::MeshRecordComponent::SCALAR]; + auto mesh_comp = mesh_record[comp_name]; // TODO(pgrete) needs to be updated for face and edges etc // Also this feels wrong for deep hierachies... auto effective_nx = static_cast(std::pow(2, level)); @@ -400,7 +446,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } // Now that the mesh record exists, actually write the data - auto out_var = pmb->meshblock_data.Get()->GetVarPtr(var_name); + auto out_var = pmb->meshblock_data.Get()->GetVarPtr(vinfo.label); PARTHENON_REQUIRE_THROWS(out_var->metadata().Where() == MetadataFlag(Metadata::Cell), "Currently only cell centered vars are supported."); @@ -421,7 +467,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, Kokkos::deep_copy(component_buffer_view, data); #endif auto out_var_h = out_var->data.GetHostMirrorAndCopy(); - int idx_component = 0; + int comp_idx = 0; const auto &Nt = out_var->GetDim(6); const auto &Nu = out_var->GetDim(5); const auto &Nv = out_var->GetDim(4); @@ -429,28 +475,9 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, for (int t = 0; t < Nt; ++t) { for (int u = 0; u < Nu; ++u) { for (int v = 0; v < Nv; ++v) { - std::string comp_name; - if (is_scalar) { - comp_name = openPMD::MeshRecordComponent::SCALAR; - } else if (vinfo.is_vector) { - if (v == 0) { - comp_name = "x"; - } else if (v == 1) { - comp_name = "y"; - } else if (v == 2) { - comp_name = "z"; - } else { - PARTHENON_THROW("Expected v index doesn't match vector expectation."); - } - } else { - comp_name = openPMD::MeshRecordComponent::SCALAR; - // comp_name = vinfo.component_labels[idx_component]; - } - const std::string &mesh_record_name = - var_name + "_" + vinfo.component_labels[idx_component] + "_lvl" + - std::to_string(level); - auto mesh_record = it.meshes[mesh_record_name]; - auto mesh_comp = mesh_record[comp_name]; + const auto [record_name, comp_name] = + OpenPMDUtils::GetMeshRecordAndComponentNames(vinfo, comp_idx, level); + auto mesh_comp = it.meshes[record_name][comp_name]; const auto comp_offset = tmp_offset; for (int k = kb.s; k <= kb.e; ++k) { @@ -461,29 +488,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } } } - openPMD::Offset chunk_offset; - openPMD::Extent chunk_extent; - const auto loc = pm->Forest().GetLegacyTreeLocation(pmb->loc); - if (pm->ndim == 3) { - chunk_offset = { - loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), - loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), - loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; - chunk_extent = {static_cast(pmb->block_size.nx(X3DIR)), - static_cast(pmb->block_size.nx(X2DIR)), - static_cast(pmb->block_size.nx(X1DIR))}; - } else if (pm->ndim == 2) { - chunk_offset = { - loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), - loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; - chunk_extent = {static_cast(pmb->block_size.nx(X2DIR)), - static_cast(pmb->block_size.nx(X1DIR))}; - } else { - PARTHENON_THROW("1D output for openpmd not yet supported."); - } - + const auto [chunk_offset, chunk_extent] = + OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb); mesh_comp.storeChunkRaw(&tmp_data[comp_offset], chunk_offset, chunk_extent); - idx_component += 1; + comp_idx += 1; } } } // loop over components From 04359d37d462f49c894bef1ea651949972497156 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 24 Apr 2024 10:33:51 +0200 Subject: [PATCH 030/125] Reuse shared chunk and name for restarts --- src/CMakeLists.txt | 1 + src/outputs/parthenon_opmd.cpp | 20 ++++++--------- src/outputs/parthenon_opmd.hpp | 37 +++++++++++++++++++++++++++ src/outputs/restart_opmd.cpp | 46 ++++++++++------------------------ 4 files changed, 59 insertions(+), 45 deletions(-) create mode 100644 src/outputs/parthenon_opmd.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 81f7c4c00bcf..0b61ea712035 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -194,6 +194,7 @@ add_library(parthenon outputs/parthenon_xdmf.cpp outputs/parthenon_hdf5.hpp outputs/parthenon_opmd.cpp + outputs/parthenon_opmd.hpp outputs/parthenon_xdmf.hpp outputs/restart.hpp outputs/restart_hdf5.cpp diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index ae1b1f40bcb2..c7fb88453ecc 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -94,12 +94,9 @@ void WriteAllParams(std::shared_ptr pkg, openPMD::Iteration *it namespace OpenPMDUtils { -// Construct OpenPMD Mesh "record" name and comonnent identifier. -// - comp_idx is a flattended index over all components of the vectors and tensors, i.e., -// the typical v,u,t indices. -// - level is the current effective level of the Mesh record -auto GetMeshRecordAndComponentNames(const VarInfo &vinfo, const int comp_idx, - const int level) { +std::tuple GetMeshRecordAndComponentNames(const VarInfo &vinfo, + const int comp_idx, + const int level) { std::string comp_name; if (vinfo.is_vector) { if (comp_idx == 0) { @@ -120,13 +117,12 @@ auto GetMeshRecordAndComponentNames(const VarInfo &vinfo, const int comp_idx, const std::string &mesh_record_name = vinfo.label + "_" + vinfo.component_labels[comp_idx] + "_lvl" + std::to_string(level); - return std::make_tuple(mesh_record_name, comp_name); + // return std::make_tuple(mesh_record_name, comp_name); + return {mesh_record_name, comp_name}; } -// Calculate logical location on effective mesh (i.e., a mesh with size that matches full -// coverage at given resolution on a particular level) -// TODO(pgrete) needs to be updated to properly work with Forests -auto GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb) { +std::tuple +GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb) { openPMD::Offset chunk_offset; openPMD::Extent chunk_extent; const auto loc = pm->Forest().GetLegacyTreeLocation(pmb->loc); @@ -145,7 +141,7 @@ auto GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb) { } else { PARTHENON_THROW("1D output for openpmd not yet supported."); } - return std::make_tuple(chunk_offset, chunk_extent); + return {chunk_offset, chunk_extent}; } } // namespace OpenPMDUtils diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp new file mode 100644 index 000000000000..31c606ef4e61 --- /dev/null +++ b/src/outputs/parthenon_opmd.hpp @@ -0,0 +1,37 @@ +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2024 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +#ifndef OUTPUTS_PARTHENON_OPMD_HPP_ +#define OUTPUTS_PARTHENON_OPMD_HPP_ +//! \file restart_opmd.hpp +// \brief Provides support for restarting from OpenPMD output + +#include + +#include "mesh/meshblock.hpp" +#include "openPMD/Dataset.hpp" +#include "outputs/output_utils.hpp" + +namespace parthenon { + +namespace OpenPMDUtils { + +// Construct OpenPMD Mesh "record" name and comonnent identifier. +// - comp_idx is a flattended index over all components of the vectors and tensors, i.e., +// the typical v,u,t indices. +// - level is the current effective level of the Mesh record +std::tuple +GetMeshRecordAndComponentNames(const OutputUtils::VarInfo &vinfo, const int comp_idx, + const int level); + +// Calculate logical location on effective mesh (i.e., a mesh with size that matches full +// coverage at given resolution on a particular level) +// TODO(pgrete) needs to be updated to properly work with Forests +std::tuple +GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb); + +} // namespace OpenPMDUtils +} // namespace parthenon +#endif // OUTPUTS_PARTHENON_OPMD_HPP_ diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 23ba83bbf4af..66f4ea555507 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -16,6 +16,7 @@ #include "interface/params.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" +#include "outputs/parthenon_opmd.hpp" #include "outputs/restart.hpp" #include "outputs/restart_opmd.hpp" #include "utils/error_checking.hpp" @@ -134,15 +135,9 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block for (auto &pmb : pm->block_list) { // TODO(pgrete) check if we should skip the suffix for level 0 const auto level = pmb->loc.level() - pm->GetRootLevel(); - const std::string &mesh_record_name = var_name + "_lvl" + std::to_string(level); - - PARTHENON_REQUIRE_THROWS(it->meshes.contains(mesh_record_name), - "Missing mesh record '" + mesh_record_name + - "' in restart file."); - auto mesh_record = it->meshes[mesh_record_name]; int64_t comp_offset = 0; // offset data_vector to store component data - int idx_component = 0; // used in label for non-vector variables + int comp_idx = 0; // used in label for non-vector variables const bool is_scalar = vinfo.GetDim(4) == 1 && vinfo.GetDim(5) == 1 && vinfo.GetDim(6) == 1; const auto &Nt = vinfo.GetDim(6); @@ -153,43 +148,28 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block for (int u = 0; u < Nu; ++u) { for (int v = 0; v < Nv; ++v) { // Get the correct record - std::string comp_name; - if (is_scalar) { - comp_name = openPMD::MeshRecordComponent::SCALAR; - } else if (vinfo.is_vector) { - if (v == 0) { - comp_name = "x"; - } else if (v == 1) { - comp_name = "y"; - } else if (v == 2) { - comp_name = "z"; - } else { - PARTHENON_THROW("Expected v index doesn't match vector expectation."); - } - } else { - comp_name = vinfo.component_labels[idx_component]; - } + const auto [record_name, comp_name] = + OpenPMDUtils::GetMeshRecordAndComponentNames(vinfo, comp_idx, level); + + PARTHENON_REQUIRE_THROWS(it->meshes.contains(record_name), + "Missing mesh record '" + record_name + + "' in restart file."); + auto mesh_record = it->meshes[record_name]; PARTHENON_REQUIRE_THROWS(mesh_record.contains(comp_name), "Missing component'" + comp_name + - "' in mesh record '" + mesh_record_name + + "' in mesh record '" + record_name + "' of restart file."); auto mesh_comp = mesh_record[comp_name]; - openPMD::Offset chunk_offset = { - pmb->loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), - pmb->loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), - pmb->loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; - openPMD::Extent chunk_extent = { - static_cast(pmb->block_size.nx(X3DIR)), - static_cast(pmb->block_size.nx(X2DIR)), - static_cast(pmb->block_size.nx(X1DIR))}; + const auto [chunk_offset, chunk_extent] = + OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb); mesh_comp.loadChunkRaw(&data_vec[comp_offset], chunk_offset, chunk_extent); // TODO(pgrete) check if output utils machinery can be used for non-cell // centered fields, which might not be that straightforward as a global mesh // is stored rather than individual blocks. comp_offset += pmb->block_size.nx(X1DIR) * pmb->block_size.nx(X2DIR) * pmb->block_size.nx(X3DIR); - idx_component += 1; + comp_idx += 1; } } } // loop over components From 2b89659d605a5c600b976fb3e3b3a0aade180e75 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 24 Apr 2024 12:14:22 +0200 Subject: [PATCH 031/125] Add regression test --- src/outputs/parthenon_opmd.cpp | 5 +- tst/regression/CMakeLists.txt | 15 +++ .../test_suites/restart_opmd/__init__.py | 0 .../restart_opmd/parthinput.restart | 60 ++++++++++++ .../restart_opmd/parthinput_override.restart | 9 ++ .../test_suites/restart_opmd/restart_opmd.py | 97 +++++++++++++++++++ 6 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 tst/regression/test_suites/restart_opmd/__init__.py create mode 100644 tst/regression/test_suites/restart_opmd/parthinput.restart create mode 100644 tst/regression/test_suites/restart_opmd/parthinput_override.restart create mode 100644 tst/regression/test_suites/restart_opmd/restart_opmd.py diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index c7fb88453ecc..5423c4f3757e 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -164,7 +164,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // but an interation) This just describes the pattern of the filename. The correct file // will be accessed through the iteration idx below. The file suffix maps to the chosen // backend. - Series series = Series("opmd.%05T.bp", Access::CREATE); + // TODO(pgrete) add final and now logic + Series series = + Series(output_params.file_basename + "." + output_params.file_id + ".%05T.bp", + Access::CREATE); // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? // TODO(pgrete) Should we update for restart or only set this once? Or make it per diff --git a/tst/regression/CMakeLists.txt b/tst/regression/CMakeLists.txt index 96a54da6e427..78b1089acf1f 100644 --- a/tst/regression/CMakeLists.txt +++ b/tst/regression/CMakeLists.txt @@ -130,6 +130,21 @@ if (ENABLE_HDF5) endif() +if (PARTHENON_ENABLE_OPENPMD) + + # h5py is needed for restart and hdf5 test + list(APPEND REQUIRED_PYTHON_MODULES openpmd_api) + + # Restart + list(APPEND TEST_DIRS restart_opmd) + list(APPEND TEST_PROCS ${NUM_MPI_PROC_TESTING}) + list(APPEND TEST_ARGS "--driver ${PROJECT_BINARY_DIR}/example/advection/advection-example \ + --driver_input ${CMAKE_CURRENT_SOURCE_DIR}/test_suites/restart_opmd/parthinput.restart \ + --num_steps 2") + list(APPEND EXTRA_TEST_LABELS "") + + endif() + # Any external modules that are required by python can be added to REQUIRED_PYTHON_MODULES # list variable, before including TestSetup.cmake. list(APPEND REQUIRED_PYTHON_MODULES numpy) diff --git a/tst/regression/test_suites/restart_opmd/__init__.py b/tst/regression/test_suites/restart_opmd/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tst/regression/test_suites/restart_opmd/parthinput.restart b/tst/regression/test_suites/restart_opmd/parthinput.restart new file mode 100644 index 000000000000..a30112475502 --- /dev/null +++ b/tst/regression/test_suites/restart_opmd/parthinput.restart @@ -0,0 +1,60 @@ +# ======================================================================================== +# Parthenon performance portable AMR framework +# Copyright(C) 2024 The Parthenon collaboration +# Licensed under the 3-clause BSD License, see LICENSE file for details +# ======================================================================================== + + +problem_id = advection + + +refinement = adaptive +numlevel = 3 + +nx1 = 64 +x1min = -0.5 +x1max = 0.5 +ix1_bc = periodic +ox1_bc = periodic + +nx2 = 64 +x2min = -0.5 +x2max = 0.5 +ix2_bc = periodic +ox2_bc = periodic + +nx3 = 1 +x3min = -0.5 +x3max = 0.5 +ix3_bc = periodic +ox3_bc = periodic + + +nx1 = 16 +nx2 = 16 +nx3 = 1 + + +nlim = -1 +tlim = 0.1 +integrator = rk2 +ncycle_out_mesh = -10000 + + +cfl = 0.45 +vx = 1.0 +vy = 1.0 +vz = 1.0 +profile = hard_sphere + +refine_tol = 0.3 # control the package specific refinement tagging function +derefine_tol = 0.03 +compute_error = false +num_vars = 1 # number of variables +vec_size = 1 # size of each variable +fill_derived = false # whether to fill one-copy test vars + + +file_type = openpmd +dt = 0.050 + diff --git a/tst/regression/test_suites/restart_opmd/parthinput_override.restart b/tst/regression/test_suites/restart_opmd/parthinput_override.restart new file mode 100644 index 000000000000..c5b368aebcbd --- /dev/null +++ b/tst/regression/test_suites/restart_opmd/parthinput_override.restart @@ -0,0 +1,9 @@ +# ======================================================================================== +# Parthenon performance portable AMR framework +# Copyright(C) 2024 The Parthenon collaboration +# Licensed under the 3-clause BSD License, see LICENSE file for details +# ======================================================================================== + +# Testing to override parameters in a restart file from an input file + +problem_id=silver diff --git a/tst/regression/test_suites/restart_opmd/restart_opmd.py b/tst/regression/test_suites/restart_opmd/restart_opmd.py new file mode 100644 index 000000000000..50aabe99b6ab --- /dev/null +++ b/tst/regression/test_suites/restart_opmd/restart_opmd.py @@ -0,0 +1,97 @@ +# ======================================================================================== +# Parthenon performance portable AMR framework +# Copyright(C) 2024 The Parthenon collaboration +# Licensed under the 3-clause BSD License, see LICENSE file for details +# ======================================================================================== + +# Modules +import sys +import utils.test_case + + +# To prevent littering up imported folders with .pyc files or __pycache_ folder +sys.dont_write_bytecode = True + + +class TestCase(utils.test_case.TestCaseAbs): + def Prepare(self, parameters, step): + # enable coverage testing on pass where restart + # files are both read and written + parameters.coverage_status = "both" + + # run baseline (to the very end) + if step == 1: + parameters.driver_cmd_line_args = ["parthenon/job/problem_id=gold"] + # restart from an early snapshot + elif step == 2: + parameters.driver_cmd_line_args = [ + "-r", + "gold.out1.00001.bp", + "-i", + f"{parameters.parthenon_path}/tst/regression/test_suites/restart_opmd/parthinput_override.restart", + ] + + return parameters + + def Analyse(self, parameters): + try: + import openpmd_api as ompd + except ModuleNotFoundError: + print("Couldn't find required openpmd_api module to compare test results.") + return False + success = True + + def compare_attributes(series_a, series_b): + all_equal = True + for attr in series_a.attributes: + if series_b.contains_attribute(attr): + attr_a = series_a.get_attribute(attr) + attr_b = series_b.get_attribute(attr) + if attr_a != attr_b: + print(f"Mismatch in attribute '{attr}'. " + f"'{attr_a}' versus '{attr_b}'\n" + ) + all_equal = False + else: + print(f"Missing attribute '{attr}' in second file.") + all_equal = False + return all_equal + + + + + def compare_files(name): + series_gold = opmd.Series("gold.out1.%T.bp/", opmd.Access.read_only) + series_silver = opmd.Series("silver.out1.%T.bp/", opmd.Access.read_only) + delta = compare( + [ + "gold.out0.%s.rhdf" % name, + "silver.out0.%s.rhdf" % name, + ], + one=True, + ) + + if delta != 0: + print( + "ERROR: Found difference between gold and silver output '%s'." + % name + ) + return False + + return True + + # comapre a few files throughout the simulations + success &= compare_files("00002") + success &= compare_files("00005") + success &= compare_files("00009") + success &= compare_files("final") + + found_line = False + for line in parameters.stdouts[1].decode("utf-8").split("\n"): + if "Terminating on wall-time limit" in line: + found_line = True + if not found_line: + print("ERROR: wall-time limit based termination not triggered.") + success = False + + return success From 804e60ddcf7d0f5220d8c138cf22b6755a662a97 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 24 Apr 2024 18:29:16 +0200 Subject: [PATCH 032/125] Somewhat make restarts working --- src/outputs/restart_opmd.cpp | 6 +- .../restart_opmd/parthinput.restart | 2 +- .../test_suites/restart_opmd/restart_opmd.py | 97 ++++++++++++------- 3 files changed, 65 insertions(+), 40 deletions(-) diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 66f4ea555507..953d1205e0ae 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -132,14 +132,12 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block const OutputUtils::VarInfo &vinfo, std::vector &data_vec, int file_output_format_version, Mesh *pm) const { + int64_t comp_offset = 0; // offset data_vector to store component data for (auto &pmb : pm->block_list) { // TODO(pgrete) check if we should skip the suffix for level 0 const auto level = pmb->loc.level() - pm->GetRootLevel(); - int64_t comp_offset = 0; // offset data_vector to store component data - int comp_idx = 0; // used in label for non-vector variables - const bool is_scalar = - vinfo.GetDim(4) == 1 && vinfo.GetDim(5) == 1 && vinfo.GetDim(6) == 1; + int comp_idx = 0; // used in label for non-vector variables const auto &Nt = vinfo.GetDim(6); const auto &Nu = vinfo.GetDim(5); const auto &Nv = vinfo.GetDim(4); diff --git a/tst/regression/test_suites/restart_opmd/parthinput.restart b/tst/regression/test_suites/restart_opmd/parthinput.restart index a30112475502..768453160a7b 100644 --- a/tst/regression/test_suites/restart_opmd/parthinput.restart +++ b/tst/regression/test_suites/restart_opmd/parthinput.restart @@ -36,7 +36,7 @@ nx3 = 1 nlim = -1 -tlim = 0.1 +tlim = 0.2 integrator = rk2 ncycle_out_mesh = -10000 diff --git a/tst/regression/test_suites/restart_opmd/restart_opmd.py b/tst/regression/test_suites/restart_opmd/restart_opmd.py index 50aabe99b6ab..89156f003fd6 100644 --- a/tst/regression/test_suites/restart_opmd/restart_opmd.py +++ b/tst/regression/test_suites/restart_opmd/restart_opmd.py @@ -35,63 +35,90 @@ def Prepare(self, parameters, step): def Analyse(self, parameters): try: - import openpmd_api as ompd + import openpmd_api as opmd except ModuleNotFoundError: print("Couldn't find required openpmd_api module to compare test results.") return False success = True def compare_attributes(series_a, series_b): + skip_attributes = [ + "iterationFormat", # Stores the file name format. Expected to differ. + "WallTime", + "InputFile", # Is updated during runtime, e.g., startime and thus differs + ] all_equal = True for attr in series_a.attributes: if series_b.contains_attribute(attr): attr_a = series_a.get_attribute(attr) attr_b = series_b.get_attribute(attr) - if attr_a != attr_b: - print(f"Mismatch in attribute '{attr}'. " - f"'{attr_a}' versus '{attr_b}'\n" - ) + if attr not in skip_attributes and attr_a != attr_b: + print( + f"Mismatch in attribute '{attr}'. " + f"'{attr_a}' versus '{attr_b}'\n" + ) all_equal = False else: print(f"Missing attribute '{attr}' in second file.") all_equal = False return all_equal - + # need series in order to flush + def compare_data(it_a, it_b, series_a, series_b): + all_equal = True + for mesh_name, mesh_a in it_a.meshes.items(): + if mesh_name not in it_b.meshes: + print(f"Missing mesh '{mesh_name}' in second file.") + all_equal = False + continue + mesh_b = it_b.meshes[mesh_name] + + for comp_name, comp_a in mesh_a.items(): + if comp_name not in mesh_b: + print( + f"Missing component '{comp_name}' in mesh '{mesh_name}' of second file." + ) + all_equal = False + continue + comp_b = mesh_b[comp_name] + data_a = comp_a.load_chunk() + series_a.flush() + data_b = comp_b.load_chunk() + series_b.flush() + + if (data_a != data_b).any(): + print( + f"Data of component '{comp_name}' in mesh '{mesh_name}' does not match." + ) + all_equal = False + continue + return all_equal - def compare_files(name): + def compare_files(idx_it): + all_good = True series_gold = opmd.Series("gold.out1.%T.bp/", opmd.Access.read_only) series_silver = opmd.Series("silver.out1.%T.bp/", opmd.Access.read_only) - delta = compare( - [ - "gold.out0.%s.rhdf" % name, - "silver.out0.%s.rhdf" % name, - ], - one=True, - ) - - if delta != 0: - print( - "ERROR: Found difference between gold and silver output '%s'." - % name - ) - return False - - return True + + # PG: yes, this is inefficient but keeps the logic simple + all_good &= compare_attributes(series_gold, series_silver) + all_good &= compare_attributes(series_silver, series_gold) + + it_gold = series_gold.iterations[idx_it] + it_silver = series_silver.iterations[idx_it] + all_good &= compare_attributes(it_gold, it_silver) + all_good &= compare_attributes(it_silver, it_gold) + + all_good &= compare_data(it_silver, it_gold, series_gold, series_silver) + all_good &= compare_data(it_gold, it_silver, series_gold, series_silver) + + return all_good # comapre a few files throughout the simulations - success &= compare_files("00002") - success &= compare_files("00005") - success &= compare_files("00009") - success &= compare_files("final") - - found_line = False - for line in parameters.stdouts[1].decode("utf-8").split("\n"): - if "Terminating on wall-time limit" in line: - found_line = True - if not found_line: - print("ERROR: wall-time limit based termination not triggered.") - success = False + success &= compare_files(1) + success &= compare_files(2) + success &= compare_files(3) + success &= compare_files(4) + # success &= compare_files("final") return success From a436f5523a9efe9de967714dc0cf2aa569eae801 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 26 Apr 2024 10:54:12 +0200 Subject: [PATCH 033/125] Fix order of arguments for correct flush --- .../test_suites/restart_opmd/restart_opmd.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tst/regression/test_suites/restart_opmd/restart_opmd.py b/tst/regression/test_suites/restart_opmd/restart_opmd.py index 89156f003fd6..f1a6117053cd 100644 --- a/tst/regression/test_suites/restart_opmd/restart_opmd.py +++ b/tst/regression/test_suites/restart_opmd/restart_opmd.py @@ -7,6 +7,7 @@ # Modules import sys import utils.test_case +import numpy as np # To prevent littering up imported folders with .pyc files or __pycache_ folder @@ -86,9 +87,12 @@ def compare_data(it_a, it_b, series_a, series_b): data_b = comp_b.load_chunk() series_b.flush() - if (data_a != data_b).any(): + try: + np.testing.assert_array_max_ulp(data_a, data_b) + except AssertionError as err: print( - f"Data of component '{comp_name}' in mesh '{mesh_name}' does not match." + f"Data of component '{comp_name}' in mesh '{mesh_name}' does not match:\n" + f"{err}\n" ) all_equal = False continue @@ -109,7 +113,7 @@ def compare_files(idx_it): all_good &= compare_attributes(it_gold, it_silver) all_good &= compare_attributes(it_silver, it_gold) - all_good &= compare_data(it_silver, it_gold, series_gold, series_silver) + all_good &= compare_data(it_silver, it_gold, series_silver, series_gold) all_good &= compare_data(it_gold, it_silver, series_gold, series_silver) return all_good From 28d725d097c43151fd547e0c38530f2432bb94bf Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 26 Apr 2024 11:29:45 +0200 Subject: [PATCH 034/125] Fix handling of output variable names --- src/outputs/outputs.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index 031664c92895..302778e694d1 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -212,8 +212,13 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { // set output variable and optional data format string used in formatted writes if ((op.file_type != "hst") && (op.file_type != "rst") && (op.file_type != "ascent") && (op.file_type != "histogram")) { - op.variables = pin->GetOrAddVector(pib->block_name, "variables", - std::vector()); + // differentiating here whether a block exists or not to not add an empty + // parameter to the input file (which might interfere with restarts) + if (pin->DoesParameterExist(pib->block_name, "variables")) { + op.variables = pin->GetVector(pib->block_name, "variables"); + } else { + op.variables = std::vector(); + } // JMM: If the requested var isn't present for a given swarm, // it is simply not output. op.swarms.clear(); // Not sure this is needed From a039ea1d0a8f750a799526197851266a66f9e617 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 26 Apr 2024 15:06:41 +0200 Subject: [PATCH 035/125] Fix reading chunks for sparsely populated output files --- src/outputs/restart_opmd.cpp | 2 + .../test_suites/restart_opmd/restart_opmd.py | 43 ++++++++++++++++--- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 953d1205e0ae..2f114c6a4287 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -36,6 +36,8 @@ RestartReaderOPMD::RestartReaderOPMD(const char *filename) idx = i.first; } it = std::make_unique(series.iterations[idx]); + // Explicitly open (important for parallel execution) + it->open(); } int RestartReaderOPMD::GetOutputFormatVersion() const { diff --git a/tst/regression/test_suites/restart_opmd/restart_opmd.py b/tst/regression/test_suites/restart_opmd/restart_opmd.py index f1a6117053cd..3548418f61d4 100644 --- a/tst/regression/test_suites/restart_opmd/restart_opmd.py +++ b/tst/regression/test_suites/restart_opmd/restart_opmd.py @@ -25,9 +25,13 @@ def Prepare(self, parameters, step): parameters.driver_cmd_line_args = ["parthenon/job/problem_id=gold"] # restart from an early snapshot elif step == 2: + # TODO(pgrete or someone else) ideally we want to restart from a later snapshot + # BUT results are not bitwise identical for AMR runs. PG thinks this is + # related to not storing the deref counter (and similar) and also thinks + # it's worth fixing. parameters.driver_cmd_line_args = [ "-r", - "gold.out1.00001.bp", + "gold.out1.00000.bp", "-i", f"{parameters.parthenon_path}/tst/regression/test_suites/restart_opmd/parthinput_override.restart", ] @@ -82,10 +86,39 @@ def compare_data(it_a, it_b, series_a, series_b): all_equal = False continue comp_b = mesh_b[comp_name] - data_a = comp_a.load_chunk() - series_a.flush() - data_b = comp_b.load_chunk() - series_b.flush() + + if comp_a.shape != comp_b.shape: + print( + f"Mismatch is mech record component shapes of " + " compontent '{comp_name}' in mesh '{mesh_name}': " + f"{comp_a.shape} versus {comp_b.shape}\n" + ) + all_equal = False + continue + + # Given that the shapes are guaranteed to match (follow the check above) + # we can load chunks from both files. + # Note that we have to go over chunks as data might be sparse on disk so + # loading the entire record will contain gargabe in sparse places. + data_a = np.empty(comp_a.shape) + data_a[:] = np.nan + data_b = np.copy(data_a) + for chunk in comp_a.available_chunks(): + # Following OpenPMD-viewer `chunk_to_slice` here + # https://github.com/openPMD/openPMD-viewer/blob/6eccb608893d2c9b8d158d950c3f0451898a80f6/openpmd_viewer/openpmd_timeseries/data_reader/io_reader/utilities.py#L14 + stops = [a + b for a, b in zip(chunk.offset, chunk.extent)] + indices_per_dim = zip(chunk.offset, stops) + sl = tuple( + map(lambda s: slice(s[0], s[1], None), indices_per_dim) + ) + + tmp = comp_a[sl] + series_a.flush() + data_a[sl] = tmp + + tmp = comp_b[sl] + series_b.flush() + data_b[sl] = tmp try: np.testing.assert_array_max_ulp(data_a, data_b) From 7476641aeb88892b81818969b031ad280ac81142 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 3 May 2024 17:04:08 +0200 Subject: [PATCH 036/125] Dont tell anyone I spent days on this... --- src/outputs/parthenon_opmd.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 5423c4f3757e..9b0adbd5f7d4 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -167,7 +167,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // TODO(pgrete) add final and now logic Series series = Series(output_params.file_basename + "." + output_params.file_id + ".%05T.bp", - Access::CREATE); + Access::CREATE, MPI_COMM_WORLD); // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? // TODO(pgrete) Should we update for restart or only set this once? Or make it per From 39d4b99f02b353369bbbed92df4cf037ad0632c6 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 3 May 2024 17:37:20 +0200 Subject: [PATCH 037/125] Temp disable dumping Views from device --- CMakeLists.txt | 2 +- src/outputs/outputs.hpp | 1 - src/outputs/parthenon_opmd.cpp | 27 +++++++++++++++++---------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a902e0e785e..a9a343008d4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,7 +31,7 @@ include(CTest) # Compile time constants # Compile Options -option(PARTHENON_SINGLE_PRENSION "Run in single precision" OFF) +option(PARTHENON_SINGLE_PRECISION "Run in single precision" OFF) option(PARTHENON_DISABLE_MPI "MPI is enabled by default if found, set this to True to disable MPI" OFF) option(PARTHENON_ENABLE_HOST_COMM_BUFFERS "CUDA/HIP Only: Allocate communication buffers on host (may be slower)" OFF) option(PARTHENON_DISABLE_HDF5 "HDF5 is enabled by default if found, set this to True to disable HDF5" OFF) diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index e0a578a8e249..14a3df4eb66f 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -214,7 +214,6 @@ class OpenPMDOutput : public OutputType { explicit OpenPMDOutput(const OutputParameters &oparams) : OutputType(oparams) {} void WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, const SignalHandler::OutputSignal signal) override; - }; #ifdef ENABLE_HDF5 diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 9b0adbd5f7d4..9b669d70de8a 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -177,7 +177,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, series.setComment("Hello world!"); series.setMachine("bla"); series.setSoftware("Parthenon + Downstream info"); - series.setDate("2024-02-29"); + series.setDate("2024-02-29 17:48:42 +0100"); // TODO(pgrete) Units? @@ -213,15 +213,19 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, it.setDt(-1.0); } { // FIXME move this to dump params - PARTHENON_INSTRUMENT_REGION("Dump Params"); - const auto view_d = - Kokkos::View("blub", 5, 3); - // Map a view onto a host allocation (so that we can call deep_copy) - auto host_vec = std::vector(view_d.size()); - Kokkos::View> - view_h(host_vec.data(), view_d.extent_int(0), view_d.extent_int(1)); - Kokkos::deep_copy(view_h, view_d); - it.setAttribute("blub", host_vec); + + // TODO(pgrete) Make this test piece work on devices + if constexpr (false) { + PARTHENON_INSTRUMENT_REGION("Dump Params"); + const auto view_d = + Kokkos::View("blub", 5, 3); + // Map a view onto a host allocation (so that we can call deep_copy) + auto host_vec = std::vector(view_d.size()); + Kokkos::View> + view_h(host_vec.data(), view_d.extent_int(0), view_d.extent_int(1)); + Kokkos::deep_copy(view_h, view_d); + it.setAttribute("blub", host_vec); + } for (const auto &[key, pkg] : pm->packages.AllPackages()) { // WriteAllParams(pkg, &it); // check why this (vector of bool) doesn't work @@ -438,6 +442,9 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // blocks auto const dataset = openPMD::Dataset(openPMD::determineDatatype(), global_extent); + // TODO(pgrete) check whether this should/need to be a collective so that the + // mesh generation should be done across all ranks prior to writing data, rather + // than in-situ for the local blocks only mesh_comp.resetDataset(dataset); // TODO(pgrete) need unitDimension and timeOffset for this record? From 4241198d9fb02352bf4f00ed04262a283d67ad08 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 12 Jun 2024 13:47:22 +0200 Subject: [PATCH 038/125] Dump deref cnt in opmd restart --- src/outputs/parthenon_opmd.cpp | 5 +++++ src/outputs/restart_opmd.cpp | 2 ++ src/outputs/restart_opmd.hpp | 7 ++++--- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 9b669d70de8a..02b761363174 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -315,6 +315,11 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, std::vector id_local = OutputUtils::ComputeIDsAndFlags(pm); auto id_global = FlattendedLocalToGlobal(pm, id_local); it.setAttribute("loc.level-gid-lid-cnghost-gflag", id_global); + + // derefinement count + std::vector derefcnt_local = OutputUtils::ComputeDerefinementCount(pm); + auto derefcnt_global = FlattendedLocalToGlobal(pm, derefcnt_local); + it.setAttribute("derefinement_count", derefcnt_global); } // TODO(pgrete) check var name standard compatiblity diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 2f114c6a4287..0bd3ba5d3768 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -72,6 +72,8 @@ RestartReaderOPMD::MeshInfo RestartReaderOPMD::GetMeshInfo() const { mesh_info.lx123 = it->getAttribute("loc.lx123").get>(); mesh_info.level_gid_lid_cnghost_gflag = it->getAttribute("loc.level-gid-lid-cnghost-gflag").get>(); + mesh_info.derefinement_count = + it->getAttribute("derefinement_count").get>(); return mesh_info; } diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index f1c60d3efb7f..e8fad77b6efc 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -41,6 +41,9 @@ class RestartReaderOPMD : public RestartReader { // Return output format version number. Return -1 if not existent. [[nodiscard]] int GetOutputFormatVersion() const override; + // Current not supported + [[nodiscard]] int HasGhost() const override { return 0; }; + public: // Gets data for all blocks on current rank. // Assumes blocks are contiguous @@ -69,11 +72,9 @@ class RestartReaderOPMD : public RestartReader { // perhaps belongs in a destructor? void Close(); - // Does file have ghost cells? - int hasGhost; - private: const std::string filename_; + openPMD::Series series; // Iteration is a pointer because it cannot be default constructed (it depends on the // Series). From 60a38b2a9ffde83363f0a5153d98daa4ea08a608 Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Tue, 18 Jun 2024 14:31:49 -0400 Subject: [PATCH 039/125] install openpmd in macOS CI --- .github/workflows/ci-macos.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index a1666b5a9520..45edb719c746 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -28,10 +28,14 @@ jobs: cache-dependency-path: '**/requirements.txt' - run: pip install -r requirements.txt - - name: Install dependencies + - name: Install dependencies (Homebrew) run: | brew install openmpi hdf5-mpi adios2 || true + - name: Install OpenPMD + run: | + openPMD_USE_MPI=ON python3 -m pip install openpmd-api --no-binary openpmd-api + - name: Configure run: cmake -B build -DCMAKE_BUILD_TYPE=Release From 28020db6c13541b4e8d133f6cb49dbf0ea41d9e4 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 25 Jun 2024 13:16:04 +0200 Subject: [PATCH 040/125] Remove extraneous popRegion --- src/outputs/parthenon_opmd.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 02b761363174..29aabf310705 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -298,7 +298,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } it.setAttribute("BoundaryConditions", boundary_condition_str); - Kokkos::Profiling::popRegion(); // write Info } // Info section Kokkos::Profiling::popRegion(); // write Attributes From af4b966a8025e0588dedda9b684c07ec2e88b2fc Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 11 Jul 2024 10:14:57 +0200 Subject: [PATCH 041/125] Fix formatting --- src/outputs/parthenon_opmd.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 29aabf310705..55f19155053d 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -298,7 +298,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } it.setAttribute("BoundaryConditions", boundary_condition_str); - } // Info section + } // Info section Kokkos::Profiling::popRegion(); // write Attributes @@ -505,10 +505,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } } } // loop over components - } // out_var->IsAllocated() - } // loop over blocks + } // out_var->IsAllocated() + } // loop over blocks it.seriesFlush(); - } // loop over vars + } // loop over vars Kokkos::Profiling::popRegion(); // write all variable data // The iteration can be closed in order to help free up resources. From 0e015d10aa8486f9a00a8232f6e9a0c89a35789e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 11 Jul 2024 11:16:47 +0200 Subject: [PATCH 042/125] Make format clang16 compatible --- src/outputs/parthenon_opmd.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 55f19155053d..33cb1c3110cc 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -505,10 +505,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } } } // loop over components - } // out_var->IsAllocated() - } // loop over blocks + } // out_var->IsAllocated() + } // loop over blocks it.seriesFlush(); - } // loop over vars + } // loop over vars Kokkos::Profiling::popRegion(); // write all variable data // The iteration can be closed in order to help free up resources. From 5135aea1966b4d5e78dcf64f959d4f2a61cbe723 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 12 Jul 2024 08:51:13 +0200 Subject: [PATCH 043/125] Fix default backend_config parsing --- src/outputs/outputs.cpp | 5 ++++- src/outputs/outputs.hpp | 9 ++++++++- src/outputs/parthenon_opmd.cpp | 7 ++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index c63b9f19a4b5..2d449d4d940a 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -269,7 +269,10 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { pnew_type = new AscentOutput(op); } else if (op.file_type == "openpmd") { #ifdef PARTHENON_ENABLE_OPENPMD - pnew_type = new OpenPMDOutput(op); + const auto backend_config = + pin->GetOrAddString(op.block_name, "backend_config", "{}"); + + pnew_type = new OpenPMDOutput(op, backend_config); #else msg << "### FATAL ERROR in Outputs constructor" << std::endl << "Executable not configured for OpenPMD outputs, but OpenPMD file format " diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index 7dc2607cd3fc..e4d6738579e4 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "Kokkos_ScatterView.hpp" @@ -177,9 +178,15 @@ class AscentOutput : public OutputType { class OpenPMDOutput : public OutputType { public: - explicit OpenPMDOutput(const OutputParameters &oparams) : OutputType(oparams) {} + explicit OpenPMDOutput(const OutputParameters &oparams, + std::string backend_config) + : OutputType(oparams), backend_config_(std::move(backend_config)) {} void WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, const SignalHandler::OutputSignal signal) override; + + private: + // path to file containing config passed to backend + std::string backend_config_; }; #ifdef ENABLE_HDF5 diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 33cb1c3110cc..7883347a8f22 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -165,9 +165,14 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // will be accessed through the iteration idx below. The file suffix maps to the chosen // backend. // TODO(pgrete) add final and now logic + if (backend_config_ != "{}") { + // the prepending @ indicates that the config is a file to be read and parsed + backend_config_.insert(0, "@"); + } + Series series = Series(output_params.file_basename + "." + output_params.file_id + ".%05T.bp", - Access::CREATE, MPI_COMM_WORLD); + Access::CREATE, MPI_COMM_WORLD, backend_config_); // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? // TODO(pgrete) Should we update for restart or only set this once? Or make it per From b344291a4cfc4ff2bc064f207fc6a268a57ff91d Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 12 Jul 2024 09:04:39 +0200 Subject: [PATCH 044/125] another attempt --- src/outputs/outputs.cpp | 2 +- src/outputs/parthenon_opmd.cpp | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index 2d449d4d940a..dad81a799175 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -270,7 +270,7 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { } else if (op.file_type == "openpmd") { #ifdef PARTHENON_ENABLE_OPENPMD const auto backend_config = - pin->GetOrAddString(op.block_name, "backend_config", "{}"); + pin->GetOrAddString(op.block_name, "backend_config", "default"); pnew_type = new OpenPMDOutput(op, backend_config); #else diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 7883347a8f22..61372fb48db0 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -165,14 +165,13 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // will be accessed through the iteration idx below. The file suffix maps to the chosen // backend. // TODO(pgrete) add final and now logic - if (backend_config_ != "{}") { - // the prepending @ indicates that the config is a file to be read and parsed - backend_config_.insert(0, "@"); - } + // Prepending @ indicates that the config is a file to be read and parsed. + std::string backend_config = + backend_config_ == "default" ? "{}" : "@" + backend_config_; Series series = Series(output_params.file_basename + "." + output_params.file_id + ".%05T.bp", - Access::CREATE, MPI_COMM_WORLD, backend_config_); + Access::CREATE, MPI_COMM_WORLD, backend_config); // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? // TODO(pgrete) Should we update for restart or only set this once? Or make it per From 7486915ccc8e00bca984154a5c8ece1865e304d6 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 12 Jul 2024 10:06:15 +0200 Subject: [PATCH 045/125] Bump OpenPMD version --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a90fe04a3ee..f6642ccb2451 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -204,7 +204,8 @@ if (PARTHENON_ENABLE_OPENPMD) set(openPMD_USE_PYTHON OFF) FetchContent_Declare(openPMD GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git" - GIT_TAG "0.15.2") + # we need newer than the latest 0.15.2 release to support writing attriutes from a subset of ranks + GIT_TAG "bda3544") # develop as of 2024-07-12 FetchContent_MakeAvailable(openPMD) install(TARGETS openPMD EXPORT parthenonTargets) endif() From 0e3f758490ee362184714dd92aed09277614f07e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 25 Jul 2024 20:23:53 +0200 Subject: [PATCH 046/125] pmd: Write scalar particle data --- src/outputs/parthenon_opmd.cpp | 61 ++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 61372fb48db0..7f1956250b9e 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -515,6 +515,67 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } // loop over vars Kokkos::Profiling::popRegion(); // write all variable data + // -------------------------------------------------------------------------------- // + // WRITING PARTICLE DATA // + // -------------------------------------------------------------------------------- // + + Kokkos::Profiling::pushRegion("write particle data"); + // TODO(pgrete) as above, first wrt differentiating between restart_ (last arg) + AllSwarmInfo all_swarm_info(pm->block_list, output_params.swarms, true); + for (auto &[swname, swinfo] : all_swarm_info.all_info) { + openPMD::ParticleSpecies swm = it.particles[swname]; + // These indicate particles/meshblock and location in global index + // space where each meshblock starts + swm.setAttribute("counts", swinfo.counts); + swm.setAttribute("offsets", swinfo.offsets); + + if (swinfo.global_count == 0) { + continue; + } + + // TODO(pgrete) dedup code (figure out why FillHostBuffer cannot be called from within + // auto lambda) + auto &int_vars = std::get>(swinfo.vars); + for (auto &[vname, swmvarvec] : int_vars) { + const auto &vinfo = swinfo.var_info.at(vname); + PARTHENON_REQUIRE_THROWS(vinfo.tensor_rank == 0, + "Only scalar particles supported at the moment."); + auto host_data = swinfo.FillHostBuffer(vname, swmvarvec); + openPMD::RecordComponent rc = swm[vname][openPMD::MeshRecordComponent::SCALAR]; + + auto const dataset = openPMD::Dataset(openPMD::determineDatatype(host_data.data()), + {swinfo.global_count}); + rc.resetDataset(dataset); + rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); + // Flush because the host buffer is temporary + it.seriesFlush(); + } + auto &real_vars = std::get>(swinfo.vars); + for (auto &[vname, swmvarvec] : real_vars) { + const auto &vinfo = swinfo.var_info.at(vname); + PARTHENON_REQUIRE_THROWS(vinfo.tensor_rank == 0, + "Only scalar particles supported at the moment."); + auto host_data = swinfo.FillHostBuffer(vname, swmvarvec); + openPMD::RecordComponent rc = swm[vname][openPMD::MeshRecordComponent::SCALAR]; + + auto const dataset = openPMD::Dataset(openPMD::determineDatatype(host_data.data()), + {swinfo.global_count}); + rc.resetDataset(dataset); + rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); + // Flush because the host buffer is temporary + it.seriesFlush(); + } + + // From the HDF5 output: + // If swarm does not contain an "id" object, generate a sequential + // one for vis. + // BUT PG: this may break things in unpredicable ways + // I'm in favor of enforcing a global id somehow. We shold discuss. + PARTHENON_REQUIRE_THROWS(swinfo.var_info.count("id") != 0, + "Particles should always carry a unique, persistent id!"); + } + Kokkos::Profiling::popRegion(); // write particle data + // The iteration can be closed in order to help free up resources. // The iteration's content will be flushed automatically. // An iteration once closed cannot (yet) be reopened. From e39ef6135ae5ef8a9baaddb3b36f3c94fe77a4cf Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 25 Jul 2024 21:29:58 +0200 Subject: [PATCH 047/125] Code dedup --- src/outputs/output_utils.hpp | 2 +- src/outputs/parthenon_opmd.cpp | 57 +++++++++++++++------------------- 2 files changed, 26 insertions(+), 33 deletions(-) diff --git a/src/outputs/output_utils.hpp b/src/outputs/output_utils.hpp index e1b411ccf480..88fd7629fd37 100644 --- a/src/outputs/output_utils.hpp +++ b/src/outputs/output_utils.hpp @@ -233,7 +233,7 @@ struct SwarmInfo { // Copies swarmvar to host in prep for output template std::vector FillHostBuffer(const std::string vname, - ParticleVariableVector &swmvarvec) { + const ParticleVariableVector &swmvarvec) const { const auto &vinfo = var_info.at(vname); std::vector host_data(count_on_rank * vinfo.nvar); std::size_t ivec = 0; diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 7f1956250b9e..8e104f8a5453 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -48,6 +48,7 @@ #include "openPMD/IO/Access.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Mesh.hpp" +#include "openPMD/ParticleSpecies.hpp" #include "openPMD/Series.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" #include "outputs/output_utils.hpp" @@ -94,6 +95,28 @@ void WriteAllParams(std::shared_ptr pkg, openPMD::Iteration *it namespace OpenPMDUtils { +template +void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, + openPMD::Iteration it) { + auto &vars_of_type_T = std::get>(swinfo.vars); + for (const auto &[vname, swmvarvec] : vars_of_type_T) { + const auto &vinfo = swinfo.var_info.at(vname); + if (vinfo.tensor_rank != 0) { + continue; + } + PARTHENON_REQUIRE_THROWS(vinfo.tensor_rank == 0, + "Only scalar particles supported at the moment."); + auto host_data = swinfo.FillHostBuffer(vname, swmvarvec); + openPMD::RecordComponent rc = swm[vname][openPMD::MeshRecordComponent::SCALAR]; + + auto const dataset = openPMD::Dataset(openPMD::determineDatatype(host_data.data()), + {swinfo.global_count}); + rc.resetDataset(dataset); + rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); + // Flush because the host buffer is temporary + it.seriesFlush(); + } +} std::tuple GetMeshRecordAndComponentNames(const VarInfo &vinfo, const int comp_idx, const int level) { @@ -533,38 +556,8 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, continue; } - // TODO(pgrete) dedup code (figure out why FillHostBuffer cannot be called from within - // auto lambda) - auto &int_vars = std::get>(swinfo.vars); - for (auto &[vname, swmvarvec] : int_vars) { - const auto &vinfo = swinfo.var_info.at(vname); - PARTHENON_REQUIRE_THROWS(vinfo.tensor_rank == 0, - "Only scalar particles supported at the moment."); - auto host_data = swinfo.FillHostBuffer(vname, swmvarvec); - openPMD::RecordComponent rc = swm[vname][openPMD::MeshRecordComponent::SCALAR]; - - auto const dataset = openPMD::Dataset(openPMD::determineDatatype(host_data.data()), - {swinfo.global_count}); - rc.resetDataset(dataset); - rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); - // Flush because the host buffer is temporary - it.seriesFlush(); - } - auto &real_vars = std::get>(swinfo.vars); - for (auto &[vname, swmvarvec] : real_vars) { - const auto &vinfo = swinfo.var_info.at(vname); - PARTHENON_REQUIRE_THROWS(vinfo.tensor_rank == 0, - "Only scalar particles supported at the moment."); - auto host_data = swinfo.FillHostBuffer(vname, swmvarvec); - openPMD::RecordComponent rc = swm[vname][openPMD::MeshRecordComponent::SCALAR]; - - auto const dataset = openPMD::Dataset(openPMD::determineDatatype(host_data.data()), - {swinfo.global_count}); - rc.resetDataset(dataset); - rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); - // Flush because the host buffer is temporary - it.seriesFlush(); - } + OpenPMDUtils::WriteSwarmVar(swinfo, swm, it); + OpenPMDUtils::WriteSwarmVar(swinfo, swm, it); // From the HDF5 output: // If swarm does not contain an "id" object, generate a sequential From b938511d246616616afec86e3dd26827b200f563 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 25 Jul 2024 22:00:50 +0200 Subject: [PATCH 048/125] Allow writing non-scalar particles --- src/outputs/output_utils.hpp | 1 + src/outputs/parthenon_opmd.cpp | 24 ++++++++++++++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/outputs/output_utils.hpp b/src/outputs/output_utils.hpp index 88fd7629fd37..8041ec96df02 100644 --- a/src/outputs/output_utils.hpp +++ b/src/outputs/output_utils.hpp @@ -242,6 +242,7 @@ struct SwarmInfo { for (int n4 = 0; n4 < vinfo.GetN(4); ++n4) { for (int n3 = 0; n3 < vinfo.GetN(3); ++n3) { for (int n2 = 0; n2 < vinfo.GetN(2); ++n2) { + // TODO(pgrete) understand what's doing on with the blocks here... std::size_t block_idx = 0; for (auto &swmvar : swmvarvec) { // Copied extra times. JMM: If we defrag, unneeded? diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 8e104f8a5453..d24deab9e8a6 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -101,18 +101,26 @@ void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, auto &vars_of_type_T = std::get>(swinfo.vars); for (const auto &[vname, swmvarvec] : vars_of_type_T) { const auto &vinfo = swinfo.var_info.at(vname); - if (vinfo.tensor_rank != 0) { - continue; - } - PARTHENON_REQUIRE_THROWS(vinfo.tensor_rank == 0, - "Only scalar particles supported at the moment."); auto host_data = swinfo.FillHostBuffer(vname, swmvarvec); - openPMD::RecordComponent rc = swm[vname][openPMD::MeshRecordComponent::SCALAR]; auto const dataset = openPMD::Dataset(openPMD::determineDatatype(host_data.data()), {swinfo.global_count}); - rc.resetDataset(dataset); - rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); + // TODO(pgrete) ask OpenPMD group if this is the right approach of if our non-scalar + // partices should be a multi-D `dataset` if is scalar + if (vinfo.tensor_rank == 0) { + openPMD::RecordComponent rc = swm[vname][openPMD::MeshRecordComponent::SCALAR]; + rc.resetDataset(dataset); + rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); + + // else flatten components + } else { + for (auto n = 0; n < vinfo.nvar; n++) { + openPMD::RecordComponent rc = swm[vname][std::to_string(n)]; + rc.resetDataset(dataset); + rc.storeChunkRaw(&host_data[n * swinfo.count_on_rank], + {swinfo.offsets[Globals::my_rank]}, {swinfo.count_on_rank}); + } + } // Flush because the host buffer is temporary it.seriesFlush(); } From 08d6b41add94f7a43622331c3465a6f82b765af7 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 25 Jul 2024 22:16:46 +0200 Subject: [PATCH 049/125] Make positions standard compliant --- src/outputs/parthenon_opmd.cpp | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index d24deab9e8a6..8ea920b55604 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -40,6 +40,7 @@ #include "driver/driver.hpp" #include "globals.hpp" #include "interface/state_descriptor.hpp" +#include "interface/swarm_default_names.hpp" #include "interface/variable_state.hpp" #include "mesh/mesh.hpp" #include "mesh/meshblock.hpp" @@ -108,10 +109,33 @@ void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, // TODO(pgrete) ask OpenPMD group if this is the right approach of if our non-scalar // partices should be a multi-D `dataset` if is scalar if (vinfo.tensor_rank == 0) { - openPMD::RecordComponent rc = swm[vname][openPMD::MeshRecordComponent::SCALAR]; + // special sauce to align "positions" with standard + std::string mesh_record; + std::string mesh_record_component; + if (vname == swarm_position::x::name()) { + mesh_record = "position"; + mesh_record_component = "x"; + } else if (vname == swarm_position::y::name()) { + mesh_record = "position"; + mesh_record_component = "y"; + } else if (vname == swarm_position::z::name()) { + mesh_record = "position"; + mesh_record_component = "z"; + } else { + mesh_record = vname; + mesh_record_component = openPMD::MeshRecordComponent::SCALAR; + } + openPMD::RecordComponent rc = swm[mesh_record][mesh_record_component]; rc.resetDataset(dataset); rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); + // if positional, add offsets + if (mesh_record_component != openPMD::MeshRecordComponent::SCALAR) { + auto rc_offset = swm["positionOffset"][mesh_record_component]; + rc_offset.resetDataset(dataset); + rc_offset.makeConstant(0.0); + } + // else flatten components } else { for (auto n = 0; n < vinfo.nvar; n++) { From bf74c7e5f0cc2519a3d98e313f273cb8484389da Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 25 Jul 2024 23:26:22 +0200 Subject: [PATCH 050/125] Allow for particles restarts (serial works) --- src/outputs/parthenon_opmd.cpp | 26 ++++++++-------- src/outputs/restart_opmd.cpp | 21 +++++++++++-- src/outputs/restart_opmd.hpp | 57 ++++++++++++++++++++++++++++++++-- 3 files changed, 87 insertions(+), 17 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 8ea920b55604..0e4e391cb7df 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -110,28 +110,28 @@ void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, // partices should be a multi-D `dataset` if is scalar if (vinfo.tensor_rank == 0) { // special sauce to align "positions" with standard - std::string mesh_record; - std::string mesh_record_component; + std::string particle_record; + std::string particle_record_component; if (vname == swarm_position::x::name()) { - mesh_record = "position"; - mesh_record_component = "x"; + particle_record = "position"; + particle_record_component = "x"; } else if (vname == swarm_position::y::name()) { - mesh_record = "position"; - mesh_record_component = "y"; + particle_record = "position"; + particle_record_component = "y"; } else if (vname == swarm_position::z::name()) { - mesh_record = "position"; - mesh_record_component = "z"; + particle_record = "position"; + particle_record_component = "z"; } else { - mesh_record = vname; - mesh_record_component = openPMD::MeshRecordComponent::SCALAR; + particle_record = vname; + particle_record_component = openPMD::MeshRecordComponent::SCALAR; } - openPMD::RecordComponent rc = swm[mesh_record][mesh_record_component]; + openPMD::RecordComponent rc = swm[particle_record][particle_record_component]; rc.resetDataset(dataset); rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); // if positional, add offsets - if (mesh_record_component != openPMD::MeshRecordComponent::SCALAR) { - auto rc_offset = swm["positionOffset"][mesh_record_component]; + if (particle_record_component != openPMD::MeshRecordComponent::SCALAR) { + auto rc_offset = swm["positionOffset"][particle_record_component]; rc_offset.resetDataset(dataset); rc_offset.makeConstant(0.0); } diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 0bd3ba5d3768..862dfa001b9e 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -6,6 +6,7 @@ //! \file restart_opmd.cpp // \brief Restarts a simulation from an OpenPMD output with ADIOS2 backend +#include #include #include #include @@ -93,8 +94,24 @@ std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, const IndexRange &range, std::vector &counts, std::vector &offsets) { - // TODO(pgrete) needs impl - return 0; + // datasets + auto counts_dset = + it->particles[swarm].getAttribute("counts").get>(); + auto offsets_dset = + it->particles[swarm].getAttribute("offsets").get>(); + + // Read data for requested blocks in range + counts.resize(range.e - range.s + 1); + offsets.resize(range.e - range.s + 1); + + std::copy(counts_dset.begin() + range.s, counts_dset.begin() + range.e + 1, + counts.begin()); + std::copy(offsets_dset.begin() + range.s, offsets_dset.begin() + range.e + 1, + offsets.begin()); + + // Compute total count rank + std::size_t total_count_on_rank = std::accumulate(counts.begin(), counts.end(), 0); + return total_count_on_rank; } template diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index e8fad77b6efc..497de7766eef 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -13,6 +13,7 @@ #include #include "basic_types.hpp" +#include "interface/swarm_default_names.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" #include "outputs/restart.hpp" @@ -52,12 +53,64 @@ class RestartReaderOPMD : public RestartReader { const OutputUtils::VarInfo &info, std::vector &dataVec, int file_output_format_version, Mesh *pmesh) const override; + // Gets the data from a swarm var on current rank. Assumes all + // blocks are contiguous. Fills dataVec based on shape from swarmvar + // metadata. + template + void ReadSwarmVar(const std::string &swarmname, const std::string &varname, + const std::size_t count, const std::size_t offset, const Metadata &m, + std::vector &data_vec) { + openPMD::ParticleSpecies swm = it->particles[swarmname]; + + const auto &shape = m.Shape(); + const int rank = shape.size(); + std::size_t nvar = 1; + for (int i = 0; i < rank; ++i) { + nvar *= shape[rank - 1 - i]; + } + std::size_t total_count = nvar * count; + if (data_vec.size() < total_count) { // greedy re-alloc + data_vec.resize(total_count); + } + + std::string particle_record; + std::string particle_record_component; + for (auto n = 0; n < nvar; n++) { + if (varname == swarm_position::x::name()) { + particle_record = "position"; + particle_record_component = "x"; + } else if (varname == swarm_position::y::name()) { + particle_record = "position"; + particle_record_component = "y"; + } else if (varname == swarm_position::z::name()) { + particle_record = "position"; + particle_record_component = "z"; + } else { + particle_record = varname; + particle_record_component = + rank == 0 ? openPMD::MeshRecordComponent::SCALAR : std::to_string(n); + } + + openPMD::RecordComponent rc = swm[particle_record][particle_record_component]; + rc.loadChunkRaw(&data_vec[n * count], {offset}, {count}); + } + + // Now actually read the registered chunks form disk + it->seriesFlush(); + } + void ReadSwarmVar(const std::string &swarmname, const std::string &varname, const std::size_t count, const std::size_t offset, const Metadata &m, - std::vector &dataVec) override{}; + std::vector &dataVec) override { + + ReadSwarmVar<>(swarmname, varname, count, offset, m, dataVec); + }; void ReadSwarmVar(const std::string &swarmname, const std::string &varname, const std::size_t count, const std::size_t offset, const Metadata &m, - std::vector &dataVec) override{}; + std::vector &dataVec) override { + + ReadSwarmVar<>(swarmname, varname, count, offset, m, dataVec); + }; // Gets the counts and offsets for MPI ranks for the meshblocks set // by the indexrange. Returns the total count on this rank. From 3b37c26f11a6a2ab5b17a04428a782612a0ec129 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 26 Jul 2024 14:37:49 +0200 Subject: [PATCH 051/125] Support particle restarts in parallel --- src/outputs/output_utils.cpp | 2 ++ src/outputs/output_utils.hpp | 4 ++-- src/outputs/parthenon_opmd.cpp | 18 +++++++++++++----- src/utils/mpi_types.hpp | 6 ++++++ 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/outputs/output_utils.cpp b/src/outputs/output_utils.cpp index effa915df82c..70776dcea93c 100644 --- a/src/outputs/output_utils.cpp +++ b/src/outputs/output_utils.cpp @@ -285,6 +285,8 @@ std::vector FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_loca } // explicit template instantiation +template std::vector +FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_local); template std::vector FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_local); template std::vector FlattendedLocalToGlobal(Mesh *pm, diff --git a/src/outputs/output_utils.hpp b/src/outputs/output_utils.hpp index 8041ec96df02..ecd331cddd9e 100644 --- a/src/outputs/output_utils.hpp +++ b/src/outputs/output_utils.hpp @@ -210,8 +210,8 @@ struct SwarmInfo { std::size_t count_on_rank = 0; // per-meshblock std::size_t global_offset; // global std::size_t global_count; // global - std::vector counts; // per-meshblock - std::vector offsets; // global + std::vector counts; // on local meshblocks + std::vector offsets; // global offset for local meshblocks // std::vector> masks; // used for reading swarms without defrag std::vector max_indices; // JMM: If we defrag, unneeded? void AddOffsets(const SP_Swarm &swarm); // sets above metadata diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 0e4e391cb7df..1cbd31cdd5da 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -127,7 +127,10 @@ void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, } openPMD::RecordComponent rc = swm[particle_record][particle_record_component]; rc.resetDataset(dataset); - rc.storeChunk(host_data, {swinfo.offsets[Globals::my_rank]}, {host_data.size()}); + // only write if there's sth to write (otherwise the host_data nullptr is caught) + if (swinfo.count_on_rank != 0) { + rc.storeChunk(host_data, {swinfo.global_offset}, {host_data.size()}); + } // if positional, add offsets if (particle_record_component != openPMD::MeshRecordComponent::SCALAR) { @@ -141,8 +144,11 @@ void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, for (auto n = 0; n < vinfo.nvar; n++) { openPMD::RecordComponent rc = swm[vname][std::to_string(n)]; rc.resetDataset(dataset); - rc.storeChunkRaw(&host_data[n * swinfo.count_on_rank], - {swinfo.offsets[Globals::my_rank]}, {swinfo.count_on_rank}); + // only write if there's sth to write (otherwise the host_data nullptr is caught) + if (swinfo.count_on_rank != 0) { + rc.storeChunkRaw(&host_data[n * swinfo.count_on_rank], {swinfo.global_offset}, + {swinfo.count_on_rank}); + } } } // Flush because the host buffer is temporary @@ -581,8 +587,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, openPMD::ParticleSpecies swm = it.particles[swname]; // These indicate particles/meshblock and location in global index // space where each meshblock starts - swm.setAttribute("counts", swinfo.counts); - swm.setAttribute("offsets", swinfo.offsets); + auto counts_global = FlattendedLocalToGlobal(pm, swinfo.counts); + swm.setAttribute("counts", counts_global); + auto offsets_global = FlattendedLocalToGlobal(pm, swinfo.offsets); + swm.setAttribute("offsets", offsets_global); if (swinfo.global_count == 0) { continue; diff --git a/src/utils/mpi_types.hpp b/src/utils/mpi_types.hpp index 164ab66f1115..6f5e117504ad 100644 --- a/src/utils/mpi_types.hpp +++ b/src/utils/mpi_types.hpp @@ -53,6 +53,12 @@ inline MPI_Datatype MPITypeMap::type() { return MPI_CXX_BOOL; } +template <> +inline MPI_Datatype MPITypeMap::type() { + // TODO(pgrete) do we need special checks here wrt to conflicts on MacOS? + return MPI_UINT64_T; +} + } // namespace parthenon #endif From 5f95466b8e8d75b33267250439f4d84bac50a7c4 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 6 Aug 2024 12:35:36 +0200 Subject: [PATCH 052/125] Add now prefix to pmd outputs --- src/outputs/output_utils.hpp | 2 +- src/outputs/outputs.hpp | 3 +-- src/outputs/parthenon_opmd.cpp | 19 +++++++------------ src/outputs/parthenon_opmd.hpp | 2 ++ src/outputs/restart_opmd.cpp | 4 ++-- src/outputs/restart_opmd.hpp | 2 -- 6 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/outputs/output_utils.hpp b/src/outputs/output_utils.hpp index ecd331cddd9e..8d8b88d897bb 100644 --- a/src/outputs/output_utils.hpp +++ b/src/outputs/output_utils.hpp @@ -211,7 +211,7 @@ struct SwarmInfo { std::size_t global_offset; // global std::size_t global_count; // global std::vector counts; // on local meshblocks - std::vector offsets; // global offset for local meshblocks + std::vector offsets; // global offset for local meshblocks // std::vector> masks; // used for reading swarms without defrag std::vector max_indices; // JMM: If we defrag, unneeded? void AddOffsets(const SP_Swarm &swarm); // sets above metadata diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index e4d6738579e4..58f5646e4e67 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -178,8 +178,7 @@ class AscentOutput : public OutputType { class OpenPMDOutput : public OutputType { public: - explicit OpenPMDOutput(const OutputParameters &oparams, - std::string backend_config) + explicit OpenPMDOutput(const OutputParameters &oparams, std::string backend_config) : OutputType(oparams), backend_config_(std::move(backend_config)) {} void WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, const SignalHandler::OutputSignal signal) override; diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 1cbd31cdd5da..646b4124b10d 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -17,23 +17,17 @@ //! \file parthenon_openpmd.cpp // \brief Output for OpenPMD https://www.openpmd.org/ (supporting various backends) -#include #include #include #include #include -#include -#include #include #include -#include #include -#include #include #include // Parthenon headers -#include "Kokkos_Core_fwd.hpp" #include "basic_types.hpp" #include "coordinates/coordinates.hpp" #include "defs.hpp" @@ -51,10 +45,9 @@ #include "openPMD/Mesh.hpp" #include "openPMD/ParticleSpecies.hpp" #include "openPMD/Series.hpp" -#include "openPMD/backend/MeshRecordComponent.hpp" #include "outputs/output_utils.hpp" #include "outputs/outputs.hpp" -#include "outputs/parthenon_hdf5.hpp" // needd for VALId_VEC_TYPES -> move +#include "outputs/parthenon_opmd.hpp" #include "parthenon_array_generic.hpp" #include "utils/error_checking.hpp" #include "utils/instrument.hpp" @@ -230,9 +223,12 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, std::string backend_config = backend_config_ == "default" ? "{}" : "@" + backend_config_; - Series series = - Series(output_params.file_basename + "." + output_params.file_id + ".%05T.bp", - Access::CREATE, MPI_COMM_WORLD, backend_config); + auto filename = output_params.file_basename + "." + output_params.file_id; + if (signal == SignalHandler::OutputSignal::now) { + filename.append(".now"); + } + filename.append(".%05T.bp"); + Series series = Series(filename, Access::CREATE, MPI_COMM_WORLD, backend_config); // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? // TODO(pgrete) Should we update for restart or only set this once? Or make it per @@ -278,7 +274,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, it.setDt(-1.0); } { // FIXME move this to dump params - // TODO(pgrete) Make this test piece work on devices if constexpr (false) { PARTHENON_INSTRUMENT_REGION("Dump Params"); diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index 31c606ef4e61..94a9161ec424 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -9,6 +9,8 @@ // \brief Provides support for restarting from OpenPMD output #include +#include +#include #include "mesh/meshblock.hpp" #include "openPMD/Dataset.hpp" diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 862dfa001b9e..b6767eaa3714 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -6,8 +6,8 @@ //! \file restart_opmd.cpp // \brief Restarts a simulation from an OpenPMD output with ADIOS2 backend +#include #include -#include #include #include #include @@ -32,7 +32,7 @@ RestartReaderOPMD::RestartReaderOPMD(const char *filename) PARTHENON_REQUIRE_THROWS( series.iterations.size() == 1, "Parthenon restarts should only contain one iteration/timestep."); - unsigned long idx; + std::uint64_t idx; for (const auto &i : series.iterations) { idx = i.first; } diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index 497de7766eef..4353bf59e233 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -102,13 +102,11 @@ class RestartReaderOPMD : public RestartReader { void ReadSwarmVar(const std::string &swarmname, const std::string &varname, const std::size_t count, const std::size_t offset, const Metadata &m, std::vector &dataVec) override { - ReadSwarmVar<>(swarmname, varname, count, offset, m, dataVec); }; void ReadSwarmVar(const std::string &swarmname, const std::string &varname, const std::size_t count, const std::size_t offset, const Metadata &m, std::vector &dataVec) override { - ReadSwarmVar<>(swarmname, varname, count, offset, m, dataVec); }; From 681159446354213cfd2713ba52b89edaa112ffad Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 7 Aug 2024 10:42:16 +0200 Subject: [PATCH 053/125] Make linter happy --- src/outputs/parthenon_opmd.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 646b4124b10d..8e1268ef4413 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -17,6 +17,7 @@ //! \file parthenon_openpmd.cpp // \brief Output for OpenPMD https://www.openpmd.org/ (supporting various backends) +#include #include #include #include @@ -25,6 +26,7 @@ #include #include #include +#include #include // Parthenon headers @@ -524,7 +526,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, if (out_var->IsAllocated()) { // TODO(pgrete) check if we can work with a direct copy from a subview to not // duplicate the memory footprint here -#if 0 +#if 0 // Pick a subview of the active cells of this component auto const data = Kokkos::subview( var->data, 0, 0, icomp, std::make_pair(kb.s, kb.e + 1), From b2d75253575774ace1eefe480eb46d8fa5b61210 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 4 Sep 2024 11:12:10 +0200 Subject: [PATCH 054/125] Bump OPMD version and add delim --- CMakeLists.txt | 2 +- src/outputs/parthenon_opmd.cpp | 3 ++- src/outputs/parthenon_opmd.hpp | 7 +++++++ src/outputs/restart_opmd.cpp | 11 ++++++++++- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eda4bee63126..5c4435b04c5a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,7 +205,7 @@ if (PARTHENON_ENABLE_OPENPMD) FetchContent_Declare(openPMD GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git" # we need newer than the latest 0.15.2 release to support writing attriutes from a subset of ranks - GIT_TAG "bda3544") # develop as of 2024-07-12 + GIT_TAG "1c7d7ff") # develop as of 2024-09-02 FetchContent_MakeAvailable(openPMD) install(TARGETS openPMD EXPORT parthenonTargets) endif() diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 8e1268ef4413..2526728b9c8d 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -65,7 +65,8 @@ using namespace OutputUtils; template void WriteAllParamsOfType(std::shared_ptr pkg, openPMD::Iteration *it) { - const std::string prefix = "Params/" + pkg->label() + "/"; + using OpenPMDUtils::delim; + const std::string prefix = "Params" + delim + pkg->label() + delim; const auto ¶ms = pkg->AllParams(); for (const auto &key : params.GetKeys()) { const auto type = params.GetType(key); diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index 94a9161ec424..2c7035dd13e9 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -20,6 +20,13 @@ namespace parthenon { namespace OpenPMDUtils { +// Deliminter to separate packages and parameters in attributes. +// More or less a workaround as the OpenPMD API does currently not expose +// access to non-standard groups (such as "Params" versus the standard "meshes"). +// TODO(pgrete & reviewer) (agree on delim and add check for package name and keys) OR +// better use of opmd-api +inline static const std::string delim = "+"; + // Construct OpenPMD Mesh "record" name and comonnent identifier. // - comp_idx is a flattended index over all components of the vectors and tensors, i.e., // the typical v,u,t indices. diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index b6767eaa3714..8fd939f2ae84 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -116,11 +116,20 @@ std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, template void RestartReaderOPMD::ReadAllParamsOfType(const std::string &pkg_name, Params ¶ms) { + using OpenPMDUtils::delim; for (const auto &key : params.GetKeys()) { const auto type = params.GetType(key); auto mutability = params.GetMutability(key); if (type == std::type_index(typeid(T)) && mutability == Params::Mutability::Restart) { - auto val = it->getAttribute("Params/" + pkg_name + "/" + key).get(); + auto attrs = it->attributes(); + for (const auto & attr : attrs) { + std::cout << "Contains attribute: " << attr << std::endl; + } + std::cout << "Reading '" + << "Params" + delim + pkg_name + delim + key << "' with type: " << typeid(T).name() + << std::endl; + + auto val = it->getAttribute("Params" + delim + pkg_name + delim + key).get(); params.Update(key, val); } } From 3283635ad1d509d78723e3412beede61eb466cb0 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 26 Sep 2024 17:40:33 +0200 Subject: [PATCH 055/125] Change delim. Is this stupid? --- src/outputs/parthenon_opmd.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index 2c7035dd13e9..c67c7366a1a8 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -25,7 +25,7 @@ namespace OpenPMDUtils { // access to non-standard groups (such as "Params" versus the standard "meshes"). // TODO(pgrete & reviewer) (agree on delim and add check for package name and keys) OR // better use of opmd-api -inline static const std::string delim = "+"; +inline static const std::string delim = "🤝"; // Construct OpenPMD Mesh "record" name and comonnent identifier. // - comp_idx is a flattended index over all components of the vectors and tensors, i.e., From 3f95fc4686b98ba4b23e30565356949f6ab907d0 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 7 Oct 2024 15:55:05 +0200 Subject: [PATCH 056/125] Make params IO test more flexible --- tst/unit/test_unit_params.cpp | 67 ++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 633d7f856163..8d69b62ab2b8 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -25,6 +25,7 @@ #include "interface/params.hpp" #include "kokkos_abstraction.hpp" #include "outputs/parthenon_hdf5.hpp" +#include "outputs/restart_hdf5.hpp" using parthenon::Params; using parthenon::Real; @@ -136,9 +137,19 @@ TEST_CASE("when hasKey is called", "[hasKey]") { } } -#ifdef ENABLE_HDF5 - -TEST_CASE("A set of params can be dumped to file", "[params][output]") { +#if defined(ENABLE_HDF5) && defined(PARTHENON_ENABLE_OPENPMD) +using parthenon::RestartReaderHDF5; +using OutputTypes = std::tuple; +#elif defined(ENABLE_HDF5) +using parthenon::RestartReaderHDF5; +using OutputTypes = std::tuple; +#elif defined(PARTHENON_ENABLE_OPENPMD) +#else +using OutputTypes = std::tuple<>; +#endif + +TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][output]", + OutputTypes) { GIVEN("A params object with a few kinds of objects") { Params params; const auto restart = Params::Mutability::Restart; @@ -171,35 +182,42 @@ TEST_CASE("A set of params can be dumped to file", "[params][output]") { } params.Add("hostarr2d", hostarr, restart); - THEN("We can output to hdf5") { + THEN("We can output") { const std::string filename = "params_test.h5"; const std::string groupname = "params"; const std::string prefix = "test_pkg"; - using namespace parthenon::HDF5; - { + if constexpr (std::is_same_v) { + using namespace parthenon::HDF5; + H5F file = H5F::FromHIDCheck( H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)); auto group = MakeGroup(file, groupname); params.WriteAllToHDF5(prefix, group); + } else { + FAIL("This logic is flawed. I should not be here."); } - AND_THEN("We can directly read the relevant data from the hdf5 file") { - H5F file = - H5F::FromHIDCheck(H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT)); - const H5O obj = H5O::FromHIDCheck(H5Oopen(file, groupname.c_str(), H5P_DEFAULT)); - + AND_THEN("We can directly read the relevant data from the file") { Real in_scalar; - HDF5ReadAttribute(obj, prefix + "/scalar", in_scalar); - REQUIRE(std::abs(scalar - in_scalar) <= 1e-10); - std::vector in_vector; - HDF5ReadAttribute(obj, prefix + "/vector", in_vector); + parthenon::ParArray2D in_arr2d("myarr", 1, 1); + if constexpr (std::is_same_v) { + + H5F file = + H5F::FromHIDCheck(H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT)); + const H5O obj = + H5O::FromHIDCheck(H5Oopen(file, groupname.c_str(), H5P_DEFAULT)); + + HDF5ReadAttribute(obj, prefix + "/scalar", in_scalar); + HDF5ReadAttribute(obj, prefix + "/vector", in_vector); + HDF5ReadAttribute(obj, prefix + "/arr2d", in_arr2d); + } + REQUIRE(scalar == in_scalar); + for (int i = 0; i < vector.size(); ++i) { REQUIRE(in_vector[i] == vector[i]); } // deliberately the wrong size - parthenon::ParArray2D in_arr2d("myarr", 1, 1); - HDF5ReadAttribute(obj, prefix + "/arr2d", in_arr2d); REQUIRE(in_arr2d.extent_int(0) == arr2d.extent_int(0)); REQUIRE(in_arr2d.extent_int(1) == arr2d.extent_int(1)); int nwrong = 1; @@ -232,14 +250,17 @@ TEST_CASE("A set of params can be dumped to file", "[params][output]") { parthenon::HostArray2D test_hostarr("hostarr2d", 1, 1); rparams.Add("hostarr2d", test_hostarr, restart); - H5F file = - H5F::FromHIDCheck(H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT)); - const H5G obj = H5G::FromHIDCheck(H5Oopen(file, groupname.c_str(), H5P_DEFAULT)); - rparams.ReadFromRestart(prefix, obj); + if constexpr (std::is_same_v) { + H5F file = + H5F::FromHIDCheck(H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT)); + const H5G obj = + H5G::FromHIDCheck(H5Oopen(file, groupname.c_str(), H5P_DEFAULT)); + rparams.ReadFromRestart(prefix, obj); + } AND_THEN("The values for the restartable params are updated to match the file") { auto test_scalar = rparams.Get("scalar"); - REQUIRE(std::abs(test_scalar - scalar) <= 1e-10); + REQUIRE(test_scalar == scalar); auto test_bool = rparams.Get("boolscalar"); REQUIRE(test_bool == boolscalar); @@ -264,5 +285,3 @@ TEST_CASE("A set of params can be dumped to file", "[params][output]") { } } } - -#endif // ENABLE_HDF5 From 1141ff3efa50386830433c7af1d56c2dfa11930a Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 7 Oct 2024 17:15:15 +0200 Subject: [PATCH 057/125] Add test case for opmd params IO --- src/outputs/parthenon_opmd.cpp | 52 +++++++++++++++++++++------------- src/outputs/parthenon_opmd.hpp | 4 +++ src/outputs/restart_opmd.cpp | 13 ++++++--- tst/unit/test_unit_params.cpp | 26 +++++++++++++++-- 4 files changed, 69 insertions(+), 26 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 2526728b9c8d..c11ad5517838 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -63,11 +63,11 @@ namespace parthenon { using namespace OutputUtils; +namespace OpenPMDUtils { + template -void WriteAllParamsOfType(std::shared_ptr pkg, openPMD::Iteration *it) { - using OpenPMDUtils::delim; - const std::string prefix = "Params" + delim + pkg->label() + delim; - const auto ¶ms = pkg->AllParams(); +void WriteAllParamsOfType(const Params ¶ms, const std::string &prefix, + openPMD::Iteration *it) { for (const auto &key : params.GetKeys()) { const auto type = params.GetType(key); if (type == std::type_index(typeid(T))) { @@ -78,19 +78,33 @@ void WriteAllParamsOfType(std::shared_ptr pkg, openPMD::Iterati } template -void WriteAllParamsOfMultipleTypes(std::shared_ptr pkg, +void WriteAllParamsOfMultipleTypes(const Params ¶ms, const std::string &prefix, openPMD::Iteration *it) { - ([&] { WriteAllParamsOfType(pkg, it); }(), ...); + ([&] { WriteAllParamsOfType(params, prefix, it); }(), ...); } template -void WriteAllParams(std::shared_ptr pkg, openPMD::Iteration *it) { - WriteAllParamsOfMultipleTypes>(pkg, it); +void WriteAllParams(const Params ¶ms, const std::string &prefix, + openPMD::Iteration *it) { + WriteAllParamsOfMultipleTypes>(params, prefix, it); // TODO(pgrete) check why this doens't work, i.e., which type is causing problems // WriteAllParamsOfMultipleTypes(pkg, it); } -namespace OpenPMDUtils { +void WriteAllParams(const Params ¶ms, const std::string &prefix, + openPMD::Iteration *it) { + // WriteAllParams(params, prefix, it); // check why this (vector of bool) doesn't + // work + WriteAllParams(params, prefix, it); + WriteAllParams(params, prefix, it); + WriteAllParams(params, prefix, it); + WriteAllParams(params, prefix, it); + WriteAllParams(params, prefix, it); + WriteAllParams(params, prefix, it); + WriteAllParams(params, prefix, it); + WriteAllParamsOfType(params, prefix, it); + // WriteAllParamsOfType>(params,prefix, it); +} template void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, @@ -231,7 +245,11 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, filename.append(".now"); } filename.append(".%05T.bp"); - Series series = Series(filename, Access::CREATE, MPI_COMM_WORLD, backend_config); + Series series = Series(filename, Access::CREATE, +#ifdef MPI_PARALLEL + MPI_COMM_WORLD, +#endif + backend_config); // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? // TODO(pgrete) Should we update for restart or only set this once? Or make it per @@ -291,16 +309,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } for (const auto &[key, pkg] : pm->packages.AllPackages()) { - // WriteAllParams(pkg, &it); // check why this (vector of bool) doesn't work - WriteAllParams(pkg, &it); - WriteAllParams(pkg, &it); - WriteAllParams(pkg, &it); - WriteAllParams(pkg, &it); - WriteAllParams(pkg, &it); - WriteAllParams(pkg, &it); - WriteAllParams(pkg, &it); - WriteAllParamsOfType(pkg, &it); - // WriteAllParamsOfType>(pkg, &it); + using OpenPMDUtils::delim; + const std::string prefix = "Params" + delim + pkg->label() + delim; + const auto ¶ms = pkg->AllParams(); + OpenPMDUtils::WriteAllParams(params, prefix, &it); } } // Then our own diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index c67c7366a1a8..078934f90088 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -14,12 +14,16 @@ #include "mesh/meshblock.hpp" #include "openPMD/Dataset.hpp" +#include "openPMD/Iteration.hpp" #include "outputs/output_utils.hpp" namespace parthenon { namespace OpenPMDUtils { +void WriteAllParams(const Params ¶ms, const std::string &prefix, + openPMD::Iteration *it); + // Deliminter to separate packages and parameters in attributes. // More or less a workaround as the OpenPMD API does currently not expose // access to non-standard groups (such as "Params" versus the standard "meshes"). diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 8fd939f2ae84..1696d5535fc5 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -28,7 +28,12 @@ namespace parthenon { //! \fn void RestartReader::RestartReader(const std::string filename) // \brief Opens the restart file and stores appropriate file handle in fh_ RestartReaderOPMD::RestartReaderOPMD(const char *filename) - : filename_(filename), series(filename, openPMD::Access::READ_ONLY, MPI_COMM_WORLD) { + : filename_(filename), series(filename, openPMD::Access::READ_ONLY +#ifdef MPI_PARALLEL + , + MPI_COMM_WORLD +#endif + ) { PARTHENON_REQUIRE_THROWS( series.iterations.size() == 1, "Parthenon restarts should only contain one iteration/timestep."); @@ -122,12 +127,12 @@ void RestartReaderOPMD::ReadAllParamsOfType(const std::string &pkg_name, Params auto mutability = params.GetMutability(key); if (type == std::type_index(typeid(T)) && mutability == Params::Mutability::Restart) { auto attrs = it->attributes(); - for (const auto & attr : attrs) { + for (const auto &attr : attrs) { std::cout << "Contains attribute: " << attr << std::endl; } std::cout << "Reading '" - << "Params" + delim + pkg_name + delim + key << "' with type: " << typeid(T).name() - << std::endl; + << "Params" + delim + pkg_name + delim + key + << "' with type: " << typeid(T).name() << std::endl; auto val = it->getAttribute("Params" + delim + pkg_name + delim + key).get(); params.Update(key, val); diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 8d69b62ab2b8..7def5ba46aa3 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -24,8 +24,11 @@ #include "config.hpp" #include "interface/params.hpp" #include "kokkos_abstraction.hpp" +#include "openPMD/Series.hpp" #include "outputs/parthenon_hdf5.hpp" +#include "outputs/parthenon_opmd.hpp" #include "outputs/restart_hdf5.hpp" +#include "outputs/restart_opmd.hpp" using parthenon::Params; using parthenon::Real; @@ -139,11 +142,14 @@ TEST_CASE("when hasKey is called", "[hasKey]") { #if defined(ENABLE_HDF5) && defined(PARTHENON_ENABLE_OPENPMD) using parthenon::RestartReaderHDF5; -using OutputTypes = std::tuple; +using parthenon::RestartReaderOPMD; +using OutputTypes = std::tuple; #elif defined(ENABLE_HDF5) using parthenon::RestartReaderHDF5; using OutputTypes = std::tuple; #elif defined(PARTHENON_ENABLE_OPENPMD) +using parthenon::RestartReaderOPMD; +using OutputTypes = std::tuple; #else using OutputTypes = std::tuple<>; #endif @@ -183,16 +189,23 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu params.Add("hostarr2d", hostarr, restart); THEN("We can output") { - const std::string filename = "params_test.h5"; + std::string filename; const std::string groupname = "params"; const std::string prefix = "test_pkg"; if constexpr (std::is_same_v) { using namespace parthenon::HDF5; + filename = "params_test.h5"; H5F file = H5F::FromHIDCheck( H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)); auto group = MakeGroup(file, groupname); params.WriteAllToHDF5(prefix, group); + } else if constexpr (std::is_same_v) { + filename = ("params_test.%05T.bp"); + auto series = openPMD::Series(filename, openPMD::Access::CREATE); + series.setIterationEncoding(openPMD::IterationEncoding::fileBased); + auto it = series.iterations[0]; + parthenon::OpenPMDUtils::WriteAllParams(params, prefix, &it); } else { FAIL("This logic is flawed. I should not be here."); } @@ -210,6 +223,15 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu HDF5ReadAttribute(obj, prefix + "/scalar", in_scalar); HDF5ReadAttribute(obj, prefix + "/vector", in_vector); HDF5ReadAttribute(obj, prefix + "/arr2d", in_arr2d); + } else if constexpr (std::is_same_v) { + auto series = openPMD::Series(filename, openPMD::Access::READ_ONLY); + auto it = std::make_unique(series.iterations[0]); + // Explicitly open (important for parallel execution) + it->open(); + in_scalar = it->getAttribute(prefix + "/scalar").get(); + in_vector = it->getAttribute(prefix + "/vector").get>(); + in_arr2d = + it->getAttribute(prefix + "/arr2d").get>(); } REQUIRE(scalar == in_scalar); From e6e4d0bcb4b5274eb3a416d79c38df6a0115ff0e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 7 Oct 2024 18:31:55 +0200 Subject: [PATCH 058/125] Reading/writing non-ParArray Params works --- src/outputs/parthenon_opmd.cpp | 8 +++++++- src/outputs/parthenon_opmd.hpp | 4 +--- tst/unit/test_unit_params.cpp | 16 +++++++++------- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index c11ad5517838..bb0986d9b549 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -72,7 +72,13 @@ void WriteAllParamsOfType(const Params ¶ms, const std::string &prefix, const auto type = params.GetType(key); if (type == std::type_index(typeid(T))) { // auto typed_ptr = dynamic_cast *>((p.second).get()); - it->setAttribute(prefix + key, params.Get(key)); + auto full_path = prefix + delim + key; + // The '/' is kind of a reserved character in the OpenPMD standard, which results + // in attribute keys with said character not being exposed. + // Thus we replace it. + std::replace(full_path.begin(), full_path.end(), '/', delim[0]); + + it->setAttribute(full_path, params.Get(key)); } } } diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index 078934f90088..23c3390c47f7 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -27,9 +27,7 @@ void WriteAllParams(const Params ¶ms, const std::string &prefix, // Deliminter to separate packages and parameters in attributes. // More or less a workaround as the OpenPMD API does currently not expose // access to non-standard groups (such as "Params" versus the standard "meshes"). -// TODO(pgrete & reviewer) (agree on delim and add check for package name and keys) OR -// better use of opmd-api -inline static const std::string delim = "🤝"; +inline static const std::string delim = "~"; // Construct OpenPMD Mesh "record" name and comonnent identifier. // - comp_idx is a flattended index over all components of the vectors and tensors, i.e., diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 7def5ba46aa3..e220e3d98e5c 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -212,7 +212,9 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu AND_THEN("We can directly read the relevant data from the file") { Real in_scalar; std::vector in_vector; + // deliberately the wrong size parthenon::ParArray2D in_arr2d("myarr", 1, 1); + if constexpr (std::is_same_v) { H5F file = @@ -226,12 +228,13 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu } else if constexpr (std::is_same_v) { auto series = openPMD::Series(filename, openPMD::Access::READ_ONLY); auto it = std::make_unique(series.iterations[0]); - // Explicitly open (important for parallel execution) - it->open(); - in_scalar = it->getAttribute(prefix + "/scalar").get(); - in_vector = it->getAttribute(prefix + "/vector").get>(); - in_arr2d = - it->getAttribute(prefix + "/arr2d").get>(); + // Note that we're explicitly using `delim` here which tests the character + // replacement of '/' in the WriteAllParams function. + using parthenon::OpenPMDUtils::delim; + in_scalar = it->getAttribute(prefix + delim + "scalar").get(); + in_vector = it->getAttribute(prefix + delim + "vector").get>(); + in_arr2d = it->getAttribute(prefix + delim + "arr2d") + .get>(); } REQUIRE(scalar == in_scalar); @@ -239,7 +242,6 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu REQUIRE(in_vector[i] == vector[i]); } - // deliberately the wrong size REQUIRE(in_arr2d.extent_int(0) == arr2d.extent_int(0)); REQUIRE(in_arr2d.extent_int(1) == arr2d.extent_int(1)); int nwrong = 1; From 8230f945bdd1ea84004a78545259ac2f16fd693e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 8 Oct 2024 11:01:11 +0200 Subject: [PATCH 059/125] Allow writing ParArray and Views to Params --- src/outputs/parthenon_opmd.cpp | 49 ++++++++++++++++++++++++---------- tst/unit/test_unit_params.cpp | 6 +++-- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index bb0986d9b549..cb3caba32b33 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -65,6 +66,25 @@ using namespace OutputUtils; namespace OpenPMDUtils { +template +auto GetFlatHostVecFromView(T view) { + // Take a view and return a flattendned (1D) std::vector that can then + // easily be passed to OpenPMD. + // Note, this function is not optimial as multiple (unnecessary) copies may be done. + // PG didn't come up with a smarter way but thinks that it's not a + // performance issue as this is only called for outputs (thus not that often) + // and for mostly small amounts of data. + // With a C++20 span we could probably direct reuse the host mirror data pointer. + auto view_h = Kokkos::create_mirror_view_and_copy(HostMemSpace(), view); + + using base_t = typename std::remove_pointer::type; + auto host_vec = std::vector(view_h.size()); + for (auto i = 0; i < view_h.size(); i++) { + host_vec[i] = view_h.data()[i]; + } + return host_vec; +} + template void WriteAllParamsOfType(const Params ¶ms, const std::string &prefix, openPMD::Iteration *it) { @@ -78,7 +98,17 @@ void WriteAllParamsOfType(const Params ¶ms, const std::string &prefix, // Thus we replace it. std::replace(full_path.begin(), full_path.end(), '/', delim[0]); - it->setAttribute(full_path, params.Get(key)); + if constexpr (implements::value) { + const auto &view = params.Get(key); + auto host_vec = GetFlatHostVecFromView(view); + it->setAttribute(full_path, host_vec); + } else if constexpr (is_specialization_of::value) { + const auto &view = params.Get(key).KokkosView(); + auto host_vec = GetFlatHostVecFromView(view); + it->setAttribute(full_path, host_vec); + } else { + it->setAttribute(full_path, params.Get(key)); + } } } } @@ -110,6 +140,8 @@ void WriteAllParams(const Params ¶ms, const std::string &prefix, WriteAllParams(params, prefix, it); WriteAllParamsOfType(params, prefix, it); // WriteAllParamsOfType>(params,prefix, it); + WriteAllParamsOfType>(params, prefix, it); + WriteAllParamsOfType>(params, prefix, it); } template @@ -300,19 +332,8 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, it.setTime(-1.0); it.setDt(-1.0); } - { // FIXME move this to dump params - // TODO(pgrete) Make this test piece work on devices - if constexpr (false) { - PARTHENON_INSTRUMENT_REGION("Dump Params"); - const auto view_d = - Kokkos::View("blub", 5, 3); - // Map a view onto a host allocation (so that we can call deep_copy) - auto host_vec = std::vector(view_d.size()); - Kokkos::View> - view_h(host_vec.data(), view_d.extent_int(0), view_d.extent_int(1)); - Kokkos::deep_copy(view_h, view_d); - it.setAttribute("blub", host_vec); - } + { + PARTHENON_INSTRUMENT_REGION("Dump Params"); for (const auto &[key, pkg] : pm->packages.AllPackages()) { using OpenPMDUtils::delim; diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index e220e3d98e5c..d0171c8bf629 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -233,8 +233,10 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu using parthenon::OpenPMDUtils::delim; in_scalar = it->getAttribute(prefix + delim + "scalar").get(); in_vector = it->getAttribute(prefix + delim + "vector").get>(); - in_arr2d = it->getAttribute(prefix + delim + "arr2d") - .get>(); + // Note that we also change the type here as ParArrays (or View in general) are + // downcasted to flattened vector. + // in_arr2d = it->getAttribute(prefix + delim + + // "arr2d").get>(); } REQUIRE(scalar == in_scalar); From e64ae7e7f72b24d79fa715062ed2106e785b2a92 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 8 Oct 2024 11:50:55 +0200 Subject: [PATCH 060/125] Read ParArray/View from opmd raw --- src/outputs/parthenon_opmd.cpp | 41 ++++++++++++++++++++++++++-------- tst/unit/test_unit_params.cpp | 24 ++++++++++++++++++-- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index cb3caba32b33..cbec6f3f7338 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -68,15 +68,36 @@ namespace OpenPMDUtils { template auto GetFlatHostVecFromView(T view) { - // Take a view and return a flattendned (1D) std::vector that can then - // easily be passed to OpenPMD. - // Note, this function is not optimial as multiple (unnecessary) copies may be done. - // PG didn't come up with a smarter way but thinks that it's not a - // performance issue as this is only called for outputs (thus not that often) - // and for mostly small amounts of data. - // With a C++20 span we could probably direct reuse the host mirror data pointer. + // Take a view and return a vector containing rank and dims and a flattened (1D) + // std::vector that can then easily be passed to OpenPMD. + // Note, this function is not + // optimial as multiple (unnecessary) copies may be done. PG didn't come up with a + // smarter way but thinks that it's not a performance issue as this is only called for + // outputs (thus not that often) and for mostly small amounts of data. With a C++20 span + // we could probably direct reuse the host mirror data pointer. auto view_h = Kokkos::create_mirror_view_and_copy(HostMemSpace(), view); + using base_t = typename std::remove_pointer::type; + auto host_vec = std::vector(view_h.size()); + for (auto i = 0; i < view_h.size(); i++) { + host_vec[i] = view_h.data()[i]; + } + // cpplint demands compile constants be all caps + constexpr auto RANK = static_cast(T::rank); + std::vector rank_and_dims(RANK + 1); + rank_and_dims[0] = RANK; + for (size_t d = 0; d < RANK; ++d) { + rank_and_dims[1 + d] = view.extent_int(d); + } + return std::make_tuple(rank_and_dims, host_vec); +} + +template +auto CopyFlatHostVecToView(const Vec &vec, T view) { + // Copy flat std::vector to view (doens't do any checks at the moment). + // As above, potentially extra copies are being made. + auto view_h = Kokkos::create_mirror_view(HostMemSpace(), view); + using base_t = typename std::remove_pointer::type; auto host_vec = std::vector(view_h.size()); for (auto i = 0; i < view_h.size(); i++) { @@ -100,11 +121,13 @@ void WriteAllParamsOfType(const Params ¶ms, const std::string &prefix, if constexpr (implements::value) { const auto &view = params.Get(key); - auto host_vec = GetFlatHostVecFromView(view); + auto [rank_and_dims, host_vec] = GetFlatHostVecFromView(view); + it->setAttribute(full_path + ".rankdims", rank_and_dims); it->setAttribute(full_path, host_vec); } else if constexpr (is_specialization_of::value) { const auto &view = params.Get(key).KokkosView(); - auto host_vec = GetFlatHostVecFromView(view); + auto [rank_and_dims, host_vec] = GetFlatHostVecFromView(view); + it->setAttribute(full_path + ".rankdims", rank_and_dims); it->setAttribute(full_path, host_vec); } else { it->setAttribute(full_path, params.Get(key)); diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index d0171c8bf629..3582d5513699 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -16,6 +16,7 @@ //======================================================================================== #include +#include #include #include @@ -29,6 +30,7 @@ #include "outputs/parthenon_opmd.hpp" #include "outputs/restart_hdf5.hpp" #include "outputs/restart_opmd.hpp" +#include "parthenon_array_generic.hpp" using parthenon::Params; using parthenon::Real; @@ -235,8 +237,26 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu in_vector = it->getAttribute(prefix + delim + "vector").get>(); // Note that we also change the type here as ParArrays (or View in general) are // downcasted to flattened vector. - // in_arr2d = it->getAttribute(prefix + delim + - // "arr2d").get>(); + auto rank_and_dims = it->getAttribute(prefix + delim + "arr2d.rankdims") + .get>(); + // Resize view. + using ViewType = + typename std::remove_reference::type; + ViewType::array_layout layout; + for (int d = 0; d < rank_and_dims[0]; ++d) { + layout.dimension[d] = rank_and_dims[1 + d]; + } + auto &in_arr2d_view = in_arr2d.KokkosView(); + in_arr2d_view = ViewType(in_arr2d_view.label(), layout); + auto view_h = + Kokkos::create_mirror_view(parthenon::HostMemSpace(), in_arr2d_view); + + auto in_arr2d_data = + it->getAttribute(prefix + delim + "arr2d").get>(); + for (auto i = 0; i < view_h.size(); i++) { + view_h.data()[i] = in_arr2d_data[i]; + } + Kokkos::deep_copy(in_arr2d_view, view_h); } REQUIRE(scalar == in_scalar); From b7897113d6981dacba5e02b1af5cb29c5d42a153 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 8 Oct 2024 12:39:02 +0200 Subject: [PATCH 061/125] Make basic parsing of Params work --- src/outputs/parthenon_opmd.cpp | 10 ++++---- src/outputs/restart_opmd.cpp | 45 ++++++++++++++++++++-------------- tst/unit/test_unit_params.cpp | 18 +++++++++----- 3 files changed, 44 insertions(+), 29 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index cbec6f3f7338..decb303a262b 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -150,8 +150,10 @@ void WriteAllParams(const Params ¶ms, const std::string &prefix, // WriteAllParamsOfMultipleTypes(pkg, it); } -void WriteAllParams(const Params ¶ms, const std::string &prefix, +void WriteAllParams(const Params ¶ms, const std::string &pkg_name, openPMD::Iteration *it) { + using OpenPMDUtils::delim; + const std::string prefix = "Params" + delim + pkg_name; // WriteAllParams(params, prefix, it); // check why this (vector of bool) doesn't // work WriteAllParams(params, prefix, it); @@ -358,11 +360,9 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, { PARTHENON_INSTRUMENT_REGION("Dump Params"); - for (const auto &[key, pkg] : pm->packages.AllPackages()) { - using OpenPMDUtils::delim; - const std::string prefix = "Params" + delim + pkg->label() + delim; + for (const auto &[pkg_name, pkg] : pm->packages.AllPackages()) { const auto ¶ms = pkg->AllParams(); - OpenPMDUtils::WriteAllParams(params, prefix, &it); + OpenPMDUtils::WriteAllParams(params, pkg_name, &it); } } // Then our own diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 1696d5535fc5..494b1c4fae16 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -27,13 +27,21 @@ namespace parthenon { //---------------------------------------------------------------------------------------- //! \fn void RestartReader::RestartReader(const std::string filename) // \brief Opens the restart file and stores appropriate file handle in fh_ -RestartReaderOPMD::RestartReaderOPMD(const char *filename) - : filename_(filename), series(filename, openPMD::Access::READ_ONLY +RestartReaderOPMD::RestartReaderOPMD(const char *filename) : filename_(filename) { + // This silly logic is required as the unit tests may or may not define MPI_PARALLEL but + // are always run in serial. #ifdef MPI_PARALLEL - , - MPI_COMM_WORLD + int mpi_initialized; + PARTHENON_MPI_CHECK(MPI_Initialized(&mpi_initialized)); + if (mpi_initialized) { + series = openPMD::Series(filename, openPMD::Access::READ_ONLY, MPI_COMM_WORLD); + } else { + series = openPMD::Series(filename, openPMD::Access::READ_ONLY); + } +#else + series = openPMD::Series(filename, openPMD::Access::READ_ONLY); + #endif - ) { PARTHENON_REQUIRE_THROWS( series.iterations.size() == 1, "Parthenon restarts should only contain one iteration/timestep."); @@ -120,9 +128,9 @@ std::size_t RestartReaderOPMD::GetSwarmCounts(const std::string &swarm, } template -void RestartReaderOPMD::ReadAllParamsOfType(const std::string &pkg_name, Params ¶ms) { - using OpenPMDUtils::delim; +void RestartReaderOPMD::ReadAllParamsOfType(const std::string &prefix, Params ¶ms) { for (const auto &key : params.GetKeys()) { + using OpenPMDUtils::delim; const auto type = params.GetType(key); auto mutability = params.GetMutability(key); if (type == std::type_index(typeid(T)) && mutability == Params::Mutability::Restart) { @@ -130,11 +138,10 @@ void RestartReaderOPMD::ReadAllParamsOfType(const std::string &pkg_name, Params for (const auto &attr : attrs) { std::cout << "Contains attribute: " << attr << std::endl; } - std::cout << "Reading '" - << "Params" + delim + pkg_name + delim + key + std::cout << "Reading '" << prefix + delim + key << "' with type: " << typeid(T).name() << std::endl; - auto val = it->getAttribute("Params" + delim + pkg_name + delim + key).get(); + auto val = it->getAttribute(prefix + delim + key).get(); params.Update(key, val); } } @@ -153,14 +160,16 @@ void RestartReaderOPMD::ReadAllParams(const std::string &pkg_name, Params &p) { // ReadAllParamsOfMultipleTypes(pkg, it); } void RestartReaderOPMD::ReadParams(const std::string &pkg_name, Params &p) { - ReadAllParams(pkg_name, p); - ReadAllParams(pkg_name, p); - ReadAllParams(pkg_name, p); - ReadAllParams(pkg_name, p); - ReadAllParams(pkg_name, p); - ReadAllParams(pkg_name, p); - ReadAllParams(pkg_name, p); - ReadAllParamsOfType(pkg_name, p); + using OpenPMDUtils::delim; + const auto prefix = "Params" + delim + pkg_name; + ReadAllParams(prefix, p); + ReadAllParams(prefix, p); + ReadAllParams(prefix, p); + ReadAllParams(prefix, p); + ReadAllParams(prefix, p); + ReadAllParams(prefix, p); + ReadAllParams(prefix, p); + ReadAllParamsOfType(prefix, p); } void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block_range, diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 3582d5513699..052231aca29a 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -192,7 +192,7 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu THEN("We can output") { std::string filename; - const std::string groupname = "params"; + const std::string groupname = "Params"; const std::string prefix = "test_pkg"; if constexpr (std::is_same_v) { using namespace parthenon::HDF5; @@ -233,12 +233,15 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu // Note that we're explicitly using `delim` here which tests the character // replacement of '/' in the WriteAllParams function. using parthenon::OpenPMDUtils::delim; - in_scalar = it->getAttribute(prefix + delim + "scalar").get(); - in_vector = it->getAttribute(prefix + delim + "vector").get>(); + in_scalar = + it->getAttribute(groupname + delim + prefix + delim + "scalar").get(); + in_vector = it->getAttribute(groupname + delim + prefix + delim + "vector") + .get>(); // Note that we also change the type here as ParArrays (or View in general) are // downcasted to flattened vector. - auto rank_and_dims = it->getAttribute(prefix + delim + "arr2d.rankdims") - .get>(); + auto rank_and_dims = + it->getAttribute(groupname + delim + prefix + delim + "arr2d.rankdims") + .get>(); // Resize view. using ViewType = typename std::remove_reference::type; @@ -277,7 +280,7 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu REQUIRE(nwrong == 0); } - AND_THEN("We can restart a params object from the HDF5 file") { + AND_THEN("We can restart a params object from the file") { Params rparams; // init the params object to restart into @@ -302,6 +305,9 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu const H5G obj = H5G::FromHIDCheck(H5Oopen(file, groupname.c_str(), H5P_DEFAULT)); rparams.ReadFromRestart(prefix, obj); + } else if constexpr (std::is_same_v) { + auto resfile = RestartReaderOPMD(filename.c_str()); + resfile.ReadParams(prefix, rparams); } AND_THEN("The values for the restartable params are updated to match the file") { From 466ecd2fdb7a0108623854e202d0a3137842763c Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 8 Oct 2024 14:37:10 +0200 Subject: [PATCH 062/125] Restore view from opmd params --- src/outputs/parthenon_opmd.cpp | 21 +++++++++++++-------- src/outputs/restart_opmd.cpp | 24 ++++++++++++++++++++---- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index decb303a262b..03b6fffbeb95 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -92,18 +92,24 @@ auto GetFlatHostVecFromView(T view) { return std::make_tuple(rank_and_dims, host_vec); } -template -auto CopyFlatHostVecToView(const Vec &vec, T view) { - // Copy flat std::vector to view (doens't do any checks at the moment). - // As above, potentially extra copies are being made. +template +auto RestoreViewAttribute(const std::string &full_path, T &view, openPMD::Iteration *it) { + auto rank_and_dims = + it->getAttribute(full_path + ".rankdims").get>(); + // Resize view. + typename T::array_layout layout; + for (int d = 0; d < rank_and_dims[0]; ++d) { + layout.dimension[d] = rank_and_dims[1 + d]; + } + Kokkos::resize(Kokkos::WithoutInitializing, view, layout); auto view_h = Kokkos::create_mirror_view(HostMemSpace(), view); using base_t = typename std::remove_pointer::type; - auto host_vec = std::vector(view_h.size()); + auto flat_data = it->getAttribute(full_path).get>(); for (auto i = 0; i < view_h.size(); i++) { - host_vec[i] = view_h.data()[i]; + view_h.data()[i] = flat_data[i]; } - return host_vec; + Kokkos::deep_copy(view, view_h); } template @@ -112,7 +118,6 @@ void WriteAllParamsOfType(const Params ¶ms, const std::string &prefix, for (const auto &key : params.GetKeys()) { const auto type = params.GetType(key); if (type == std::type_index(typeid(T))) { - // auto typed_ptr = dynamic_cast *>((p.second).get()); auto full_path = prefix + delim + key; // The '/' is kind of a reserved character in the OpenPMD standard, which results // in attribute keys with said character not being exposed. diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 494b1c4fae16..532ad28ecc2c 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -134,14 +134,30 @@ void RestartReaderOPMD::ReadAllParamsOfType(const std::string &prefix, Params &p const auto type = params.GetType(key); auto mutability = params.GetMutability(key); if (type == std::type_index(typeid(T)) && mutability == Params::Mutability::Restart) { + auto full_path = prefix + delim + key; + // The '/' is kind of a reserved character in the OpenPMD standard, which results + // in attribute keys with said character not being exposed. + // Thus we replace it. + std::replace(full_path.begin(), full_path.end(), '/', delim[0]); + auto attrs = it->attributes(); for (const auto &attr : attrs) { std::cout << "Contains attribute: " << attr << std::endl; } - std::cout << "Reading '" << prefix + delim + key - << "' with type: " << typeid(T).name() << std::endl; - - auto val = it->getAttribute(prefix + delim + key).get(); + std::cout << "Reading '" << full_path << "' with type: " << typeid(T).name() + << std::endl; + + T val; + if constexpr (implements::value) { + val = params.Get(key); + RestoreViewAttribute(full_path, val, it); + } else if constexpr (is_specialization_of::value) { + val = params.Get(key); + auto &view = val.KokkosView(); + RestoreViewAttribute(full_path, view, it); + } else { + val = it->getAttribute(full_path).get(); + } params.Update(key, val); } } From 3aa4c78e85376ec160eda6c9ab78650f084c0bb5 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 8 Oct 2024 16:30:34 +0200 Subject: [PATCH 063/125] Fix manual pararray reading --- src/outputs/parthenon_opmd.cpp | 3 ++- src/outputs/parthenon_opmd.hpp | 3 +++ src/outputs/restart_opmd.cpp | 2 ++ tst/unit/test_unit_params.cpp | 48 +++++++++++++++++++++++----------- 4 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 03b6fffbeb95..9b062b52cf38 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -93,7 +93,7 @@ auto GetFlatHostVecFromView(T view) { } template -auto RestoreViewAttribute(const std::string &full_path, T &view, openPMD::Iteration *it) { +void RestoreViewAttribute(const std::string &full_path, T &view, openPMD::Iteration *it) { auto rank_and_dims = it->getAttribute(full_path + ".rankdims").get>(); // Resize view. @@ -172,6 +172,7 @@ void WriteAllParams(const Params ¶ms, const std::string &pkg_name, // WriteAllParamsOfType>(params,prefix, it); WriteAllParamsOfType>(params, prefix, it); WriteAllParamsOfType>(params, prefix, it); + WriteAllParamsOfType>(params, prefix, it); } template diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index 23c3390c47f7..9e0c04e99a4c 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -21,6 +21,9 @@ namespace parthenon { namespace OpenPMDUtils { +template +void RestoreViewAttribute(const std::string &full_path, T &view, openPMD::Iteration *it); + void WriteAllParams(const Params ¶ms, const std::string &prefix, openPMD::Iteration *it); diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 532ad28ecc2c..f34aaaebb640 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -186,6 +186,8 @@ void RestartReaderOPMD::ReadParams(const std::string &pkg_name, Params &p) { ReadAllParams(prefix, p); ReadAllParams(prefix, p); ReadAllParamsOfType(prefix, p); + // ReadAllParamsOfType>(prefix, p); + // ReadAllParamsOfType>(prefix, p); } void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block_range, diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 052231aca29a..b164a7516565 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -182,13 +182,13 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu Kokkos::deep_copy(arr2d, arr2d_h); params.Add("arr2d", arr2d); - parthenon::HostArray2D hostarr("hostarr2d", 2, 3); + parthenon::HostArray2D hostarr2d("hostarr2d", 2, 3); for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { - hostarr(i, j) = 2 * i + j + 1; + hostarr2d(i, j) = 2 * i + j + 1; } } - params.Add("hostarr2d", hostarr, restart); + params.Add("hostarr2d", hostarr2d, restart); THEN("We can output") { std::string filename; @@ -216,6 +216,7 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu std::vector in_vector; // deliberately the wrong size parthenon::ParArray2D in_arr2d("myarr", 1, 1); + parthenon::HostArray2D in_hostarr2d("hostarr2d", 2, 3); if constexpr (std::is_same_v) { @@ -227,20 +228,28 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu HDF5ReadAttribute(obj, prefix + "/scalar", in_scalar); HDF5ReadAttribute(obj, prefix + "/vector", in_vector); HDF5ReadAttribute(obj, prefix + "/arr2d", in_arr2d); + HDF5ReadAttribute(obj, prefix + "/hostarr2d", in_hostarr2d); } else if constexpr (std::is_same_v) { auto series = openPMD::Series(filename, openPMD::Access::READ_ONLY); - auto it = std::make_unique(series.iterations[0]); + auto it = series.iterations[0]; // Note that we're explicitly using `delim` here which tests the character // replacement of '/' in the WriteAllParams function. using parthenon::OpenPMDUtils::delim; in_scalar = - it->getAttribute(groupname + delim + prefix + delim + "scalar").get(); - in_vector = it->getAttribute(groupname + delim + prefix + delim + "vector") + it.getAttribute(groupname + delim + prefix + delim + "scalar").get(); + in_vector = it.getAttribute(groupname + delim + prefix + delim + "vector") .get>(); - // Note that we also change the type here as ParArrays (or View in general) are - // downcasted to flattened vector. + + // auto &tmp_view = in_arr2d.KokkosView(); + // using ViewType = + // typename std::remove_reference::type; + // parthenon::OpenPMDUtils::RestoreViewAttribute( + // groupname + delim + prefix + delim + "arr2d", tmp_view, &it); + + // Note that we also change the type here as ParArrays (or View in general) + // are downcasted to flattened vector. auto rank_and_dims = - it->getAttribute(groupname + delim + prefix + delim + "arr2d.rankdims") + it.getAttribute(groupname + delim + prefix + delim + "arr2d.rankdims") .get>(); // Resize view. using ViewType = @@ -255,7 +264,8 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu Kokkos::create_mirror_view(parthenon::HostMemSpace(), in_arr2d_view); auto in_arr2d_data = - it->getAttribute(prefix + delim + "arr2d").get>(); + it.getAttribute(groupname + delim + prefix + delim + "arr2d") + .get>(); for (auto i = 0; i < view_h.size(); i++) { view_h.data()[i] = in_arr2d_data[i]; } @@ -278,6 +288,14 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu }, nwrong); REQUIRE(nwrong == 0); + + REQUIRE(in_hostarr2d.extent_int(0) == hostarr2d.extent_int(0)); + REQUIRE(in_hostarr2d.extent_int(1) == hostarr2d.extent_int(1)); + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 3; ++j) { + REQUIRE(hostarr2d(i, j) == in_hostarr2d(i, j)); + } + } } AND_THEN("We can restart a params object from the file") { @@ -318,11 +336,11 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu REQUIRE(test_bool == boolscalar); auto test_hostarr = params.Get>("hostarr2d"); - REQUIRE(test_hostarr.extent_int(0) == hostarr.extent_int(0)); - REQUIRE(test_hostarr.extent_int(1) == hostarr.extent_int(1)); - for (int i = 0; i < hostarr.extent_int(0); ++i) { - for (int j = 0; j < hostarr.extent_int(1); ++j) { - REQUIRE(test_hostarr(i, j) == hostarr(i, j)); + REQUIRE(test_hostarr.extent_int(0) == hostarr2d.extent_int(0)); + REQUIRE(test_hostarr.extent_int(1) == hostarr2d.extent_int(1)); + for (int i = 0; i < hostarr2d.extent_int(0); ++i) { + for (int j = 0; j < hostarr2d.extent_int(1); ++j) { + REQUIRE(test_hostarr(i, j) == hostarr2d(i, j)); } } } From 4c1c70fb830f9efc19962edf3ca5218763e39604 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 8 Oct 2024 22:38:59 +0200 Subject: [PATCH 064/125] Make linter happy? --- src/outputs/output_utils.cpp | 1 + src/outputs/restart_opmd.cpp | 3 ++- tst/unit/test_unit_params.cpp | 6 +++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/outputs/output_utils.cpp b/src/outputs/output_utils.cpp index 927853d2f7f7..3b7de06d89a0 100644 --- a/src/outputs/output_utils.cpp +++ b/src/outputs/output_utils.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index f34aaaebb640..e7f51e437fb6 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -233,7 +234,7 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block } } } // loop over components - } // loop over blocks + } // loop over blocks // Now actually read the registered chunks form disk it->seriesFlush(); diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index b164a7516565..8fad599eb81b 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -1,6 +1,6 @@ //======================================================================================== -// Athena++ astrophysical MHD code -// Copyright(C) 2014 James M. Stone and other code contributors +// Parthenon performance portable AMR framework +// Copyright(C) 2020-2024 The Parthenon collaboration // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== // (C) (or copyright) 2020-2021. Triad National Security, LLC. All rights reserved. @@ -16,6 +16,7 @@ //======================================================================================== #include +#include #include #include @@ -219,7 +220,6 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu parthenon::HostArray2D in_hostarr2d("hostarr2d", 2, 3); if constexpr (std::is_same_v) { - H5F file = H5F::FromHIDCheck(H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT)); const H5O obj = From 8a1850b2de73d55624a96e4598a04bd5d85ed95b Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 9 Oct 2024 22:48:17 +0200 Subject: [PATCH 065/125] Make restoring HostViews possible --- src/outputs/parthenon_opmd.cpp | 26 +++----------------------- src/outputs/restart_opmd.cpp | 4 ++-- src/outputs/restart_opmd.hpp | 21 +++++++++++++++++++++ tst/unit/test_unit_params.cpp | 4 ++-- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 9b062b52cf38..ddd59c59c2fb 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -92,26 +92,6 @@ auto GetFlatHostVecFromView(T view) { return std::make_tuple(rank_and_dims, host_vec); } -template -void RestoreViewAttribute(const std::string &full_path, T &view, openPMD::Iteration *it) { - auto rank_and_dims = - it->getAttribute(full_path + ".rankdims").get>(); - // Resize view. - typename T::array_layout layout; - for (int d = 0; d < rank_and_dims[0]; ++d) { - layout.dimension[d] = rank_and_dims[1 + d]; - } - Kokkos::resize(Kokkos::WithoutInitializing, view, layout); - auto view_h = Kokkos::create_mirror_view(HostMemSpace(), view); - - using base_t = typename std::remove_pointer::type; - auto flat_data = it->getAttribute(full_path).get>(); - for (auto i = 0; i < view_h.size(); i++) { - view_h.data()[i] = flat_data[i]; - } - Kokkos::deep_copy(view, view_h); -} - template void WriteAllParamsOfType(const Params ¶ms, const std::string &prefix, openPMD::Iteration *it) { @@ -636,10 +616,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } } } // loop over components - } // out_var->IsAllocated() - } // loop over blocks + } // out_var->IsAllocated() + } // loop over blocks it.seriesFlush(); - } // loop over vars + } // loop over vars Kokkos::Profiling::popRegion(); // write all variable data // -------------------------------------------------------------------------------- // diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index e7f51e437fb6..532137cc2019 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -155,7 +155,7 @@ void RestartReaderOPMD::ReadAllParamsOfType(const std::string &prefix, Params &p } else if constexpr (is_specialization_of::value) { val = params.Get(key); auto &view = val.KokkosView(); - RestoreViewAttribute(full_path, view, it); + RestoreViewAttribute(full_path, view); } else { val = it->getAttribute(full_path).get(); } @@ -188,7 +188,7 @@ void RestartReaderOPMD::ReadParams(const std::string &pkg_name, Params &p) { ReadAllParams(prefix, p); ReadAllParamsOfType(prefix, p); // ReadAllParamsOfType>(prefix, p); - // ReadAllParamsOfType>(prefix, p); + ReadAllParamsOfType>(prefix, p); } void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block_range, diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index 4353bf59e233..f986e84b8b97 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -119,6 +119,27 @@ class RestartReaderOPMD : public RestartReader { void ReadParams(const std::string &name, Params &p) override; + template + void RestoreViewAttribute(const std::string &full_path, T &view) { + auto rank_and_dims = + it->getAttribute(full_path + ".rankdims").get>(); + // Resize view. + typename T::array_layout layout; + for (int d = 0; d < rank_and_dims[0]; ++d) { + layout.dimension[d] = rank_and_dims[1 + d]; + } + // Cannot use Kokkos::resize here as it's ambiguous at this point. + // Also, resize() interally also just create a new view. + view = T(Kokkos::view_alloc(Kokkos::WithoutInitializing, view.label()), layout); + auto view_h = Kokkos::create_mirror_view(HostMemSpace(), view); + + using base_t = typename std::remove_pointer::type; + auto flat_data = it->getAttribute(full_path).get>(); + for (auto i = 0; i < view_h.size(); i++) { + view_h.data()[i] = flat_data[i]; + } + Kokkos::deep_copy(view, view_h); + } // closes out the restart file // perhaps belongs in a destructor? void Close(); diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 8fad599eb81b..33b1fdef122e 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -293,7 +293,7 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu REQUIRE(in_hostarr2d.extent_int(1) == hostarr2d.extent_int(1)); for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { - REQUIRE(hostarr2d(i, j) == in_hostarr2d(i, j)); + REQUIRE((hostarr2d(i, j) == in_hostarr2d(i, j) || true)); } } } @@ -335,7 +335,7 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu auto test_bool = rparams.Get("boolscalar"); REQUIRE(test_bool == boolscalar); - auto test_hostarr = params.Get>("hostarr2d"); + auto test_hostarr = rparams.Get>("hostarr2d"); REQUIRE(test_hostarr.extent_int(0) == hostarr2d.extent_int(0)); REQUIRE(test_hostarr.extent_int(1) == hostarr2d.extent_int(1)); for (int i = 0; i < hostarr2d.extent_int(0); ++i) { From 1f02ffaf102632dfb68559c406dc88422c7000ba Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 9 Oct 2024 23:00:41 +0200 Subject: [PATCH 066/125] Make reading host arrays work using the direct interface in the unit test --- tst/unit/test_unit_params.cpp | 44 +++++++++++------------------------ 1 file changed, 13 insertions(+), 31 deletions(-) diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 33b1fdef122e..9cd9a08542d9 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -235,41 +235,23 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu // Note that we're explicitly using `delim` here which tests the character // replacement of '/' in the WriteAllParams function. using parthenon::OpenPMDUtils::delim; + in_scalar = it.getAttribute(groupname + delim + prefix + delim + "scalar").get(); + in_vector = it.getAttribute(groupname + delim + prefix + delim + "vector") .get>(); - // auto &tmp_view = in_arr2d.KokkosView(); - // using ViewType = - // typename std::remove_reference::type; - // parthenon::OpenPMDUtils::RestoreViewAttribute( - // groupname + delim + prefix + delim + "arr2d", tmp_view, &it); - - // Note that we also change the type here as ParArrays (or View in general) - // are downcasted to flattened vector. - auto rank_and_dims = - it.getAttribute(groupname + delim + prefix + delim + "arr2d.rankdims") - .get>(); - // Resize view. - using ViewType = - typename std::remove_reference::type; - ViewType::array_layout layout; - for (int d = 0; d < rank_and_dims[0]; ++d) { - layout.dimension[d] = rank_and_dims[1 + d]; - } - auto &in_arr2d_view = in_arr2d.KokkosView(); - in_arr2d_view = ViewType(in_arr2d_view.label(), layout); - auto view_h = - Kokkos::create_mirror_view(parthenon::HostMemSpace(), in_arr2d_view); - - auto in_arr2d_data = - it.getAttribute(groupname + delim + prefix + delim + "arr2d") - .get>(); - for (auto i = 0; i < view_h.size(); i++) { - view_h.data()[i] = in_arr2d_data[i]; - } - Kokkos::deep_copy(in_arr2d_view, view_h); + // Technically, we're not reading "directly" here but the restart reader ctor + // literally just opens the file. + auto resfile = RestartReaderOPMD(filename.c_str()); + auto &in_arr2d_v = in_arr2d.KokkosView(); + resfile.RestoreViewAttribute(groupname + delim + prefix + delim + "arr2d", + in_arr2d_v); + + auto &in_hostarr2d_v = in_hostarr2d.KokkosView(); + resfile.RestoreViewAttribute(groupname + delim + prefix + delim + "hostarr2d", + in_hostarr2d_v); } REQUIRE(scalar == in_scalar); @@ -293,7 +275,7 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu REQUIRE(in_hostarr2d.extent_int(1) == hostarr2d.extent_int(1)); for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { - REQUIRE((hostarr2d(i, j) == in_hostarr2d(i, j) || true)); + REQUIRE(hostarr2d(i, j) == in_hostarr2d(i, j)); } } } From d6565c5b97f85574ad599fe9362bc559cdb8040e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 9 Oct 2024 23:44:50 +0200 Subject: [PATCH 067/125] Expands types for read/write to all vec --- src/CMakeLists.txt | 1 + src/outputs/output_attr.hpp | 45 ++++++++++++++++++++++++++++++++++ src/outputs/parthenon_hdf5.hpp | 22 +---------------- src/outputs/parthenon_opmd.cpp | 15 ++++-------- src/outputs/restart_opmd.cpp | 10 +++----- tst/unit/test_unit_params.cpp | 15 ++++++++++++ 6 files changed, 71 insertions(+), 37 deletions(-) create mode 100644 src/outputs/output_attr.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6b4c5fc54108..c2a530664674 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -188,6 +188,7 @@ add_library(parthenon outputs/history.cpp outputs/io_wrapper.cpp outputs/io_wrapper.hpp + outputs/output_attr.hpp outputs/output_utils.cpp outputs/output_utils.hpp outputs/outputs.cpp diff --git a/src/outputs/output_attr.hpp b/src/outputs/output_attr.hpp new file mode 100644 index 000000000000..20e471c4c415 --- /dev/null +++ b/src/outputs/output_attr.hpp @@ -0,0 +1,45 @@ +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2023-2024 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +// (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== + +#ifndef OUTPUTS_OUTPUT_ATTR_HPP_ +#define OUTPUTS_OUTPUT_ATTR_HPP_ + +// JMM: This could probably be done with template magic but I think +// using a macro is honestly the simplest and cleanest solution here. +// Template solution would be to define a variatic class to conain the +// list of types and then a hierarchy of structs/functions to turn +// that into function calls. Preprocessor seems easier, given we're +// not manipulating this list in any way. +// The following types are the ones we allow to be stored as attributes in outputs +// (specifically within Params). +#define PARTHENON_ATTR_VALID_VEC_TYPES(T) \ + T, std::vector, ParArray1D, ParArray2D, ParArray3D, HostArray1D, \ + HostArray2D, HostArray3D, Kokkos::View, Kokkos::View, \ + ParArrayND, ParArrayHost +// JMM: This is the list of template specializations we +// "pre-instantiate" We only pre-instantiate device memory, not host +// memory. The reason is that when building with the Kokkos serial +// backend, DevMemSpace and HostMemSpace are the same and so this +// resolves to the same type in the macro, which causes problems. +#define PARTHENON_ATTR_FOREACH_VECTOR_TYPE(T) \ + PARTHENON_ATTR_APPLY(T); \ + PARTHENON_ATTR_APPLY(Kokkos::View); \ + PARTHENON_ATTR_APPLY(Kokkos::View); \ + PARTHENON_ATTR_APPLY(Kokkos::View); \ + PARTHENON_ATTR_APPLY(device_view_t) + +#endif // OUTPUTS_OUTPUT_ATTR_HPP_ \ No newline at end of file diff --git a/src/outputs/parthenon_hdf5.hpp b/src/outputs/parthenon_hdf5.hpp index 72deaab90681..b2c809c2fb0d 100644 --- a/src/outputs/parthenon_hdf5.hpp +++ b/src/outputs/parthenon_hdf5.hpp @@ -20,29 +20,9 @@ #include "config.hpp" #include "kokkos_abstraction.hpp" +#include "output_attr.hpp" #include "parthenon_arrays.hpp" -// JMM: This could probably be done with template magic but I think -// using a macro is honestly the simplest and cleanest solution here. -// Template solution would be to define a variatic class to conain the -// list of types and then a hierarchy of structs/functions to turn -// that into function calls. Preprocessor seems easier, given we're -// not manipulating this list in any way. -#define PARTHENON_ATTR_VALID_VEC_TYPES(T) \ - T, std::vector, ParArray1D, ParArray2D, ParArray3D, HostArray1D, \ - HostArray2D, HostArray3D, Kokkos::View, Kokkos::View, \ - ParArrayND, ParArrayHost -// JMM: This is the list of template specializations we -// "pre-instantiate" We only pre-instantiate device memory, not host -// memory. The reason is that when building with the Kokkos serial -// backend, DevMemSpace and HostMemSpace are the same and so this -// resolves to the same type in the macro, which causes problems. -#define PARTHENON_ATTR_FOREACH_VECTOR_TYPE(T) \ - PARTHENON_ATTR_APPLY(T); \ - PARTHENON_ATTR_APPLY(Kokkos::View); \ - PARTHENON_ATTR_APPLY(Kokkos::View); \ - PARTHENON_ATTR_APPLY(Kokkos::View); \ - PARTHENON_ATTR_APPLY(device_view_t) // Only proceed if HDF5 output enabled #ifdef ENABLE_HDF5 diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index ddd59c59c2fb..f2b588e84e20 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -48,6 +48,7 @@ #include "openPMD/Mesh.hpp" #include "openPMD/ParticleSpecies.hpp" #include "openPMD/Series.hpp" +#include "outputs/output_attr.hpp" #include "outputs/output_utils.hpp" #include "outputs/outputs.hpp" #include "outputs/parthenon_opmd.hpp" @@ -130,17 +131,16 @@ void WriteAllParamsOfMultipleTypes(const Params ¶ms, const std::string &pref template void WriteAllParams(const Params ¶ms, const std::string &prefix, openPMD::Iteration *it) { - WriteAllParamsOfMultipleTypes>(params, prefix, it); - // TODO(pgrete) check why this doens't work, i.e., which type is causing problems - // WriteAllParamsOfMultipleTypes(pkg, it); + WriteAllParamsOfMultipleTypes(params, prefix, it); } void WriteAllParams(const Params ¶ms, const std::string &pkg_name, openPMD::Iteration *it) { using OpenPMDUtils::delim; const std::string prefix = "Params" + delim + pkg_name; - // WriteAllParams(params, prefix, it); // check why this (vector of bool) doesn't - // work + // check why this (vector of bool) doesn't work + // WriteAllParams(params, prefix, it); + WriteAllParamsOfType(params, prefix, it); WriteAllParams(params, prefix, it); WriteAllParams(params, prefix, it); WriteAllParams(params, prefix, it); @@ -148,11 +148,6 @@ void WriteAllParams(const Params ¶ms, const std::string &pkg_name, WriteAllParams(params, prefix, it); WriteAllParams(params, prefix, it); WriteAllParams(params, prefix, it); - WriteAllParamsOfType(params, prefix, it); - // WriteAllParamsOfType>(params,prefix, it); - WriteAllParamsOfType>(params, prefix, it); - WriteAllParamsOfType>(params, prefix, it); - WriteAllParamsOfType>(params, prefix, it); } template diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 532137cc2019..249f1d7266b8 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -18,6 +18,7 @@ #include "interface/params.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" +#include "outputs/output_attr.hpp" #include "outputs/parthenon_opmd.hpp" #include "outputs/restart.hpp" #include "outputs/restart_opmd.hpp" @@ -151,7 +152,7 @@ void RestartReaderOPMD::ReadAllParamsOfType(const std::string &prefix, Params &p T val; if constexpr (implements::value) { val = params.Get(key); - RestoreViewAttribute(full_path, val, it); + RestoreViewAttribute(full_path, val); } else if constexpr (is_specialization_of::value) { val = params.Get(key); auto &view = val.KokkosView(); @@ -172,9 +173,7 @@ void RestartReaderOPMD::ReadAllParamsOfMultipleTypes(const std::string &pkg_name template void RestartReaderOPMD::ReadAllParams(const std::string &pkg_name, Params &p) { - ReadAllParamsOfMultipleTypes>(pkg_name, p); - // TODO(pgrete) check why this doens't work, i.e., which type is causing problems - // ReadAllParamsOfMultipleTypes(pkg, it); + ReadAllParamsOfMultipleTypes(pkg_name, p); } void RestartReaderOPMD::ReadParams(const std::string &pkg_name, Params &p) { using OpenPMDUtils::delim; @@ -186,9 +185,8 @@ void RestartReaderOPMD::ReadParams(const std::string &pkg_name, Params &p) { ReadAllParams(prefix, p); ReadAllParams(prefix, p); ReadAllParams(prefix, p); + // TODO(pgrete) same as for the writing. fix vec of bool ReadAllParamsOfType(prefix, p); - // ReadAllParamsOfType>(prefix, p); - ReadAllParamsOfType>(prefix, p); } void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block_range, diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 9cd9a08542d9..994f17727387 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -183,6 +183,16 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu Kokkos::deep_copy(arr2d, arr2d_h); params.Add("arr2d", arr2d); + // "Vectors" of bools have some special sauce under the hood so let's try the logic + // with a plain view + Kokkos::View bool1d("boolview", 10); + auto bool1d_h = Kokkos::create_mirror_view(bool1d); + for (int i = 0; i < 10; ++i) { + bool1d_h(i) = i % 2; + } + Kokkos::deep_copy(bool1d, bool1d_h); + params.Add("bool1d", bool1d); + parthenon::HostArray2D hostarr2d("hostarr2d", 2, 3); for (int i = 0; i < 2; ++i) { for (int j = 0; j < 3; ++j) { @@ -218,6 +228,7 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu // deliberately the wrong size parthenon::ParArray2D in_arr2d("myarr", 1, 1); parthenon::HostArray2D in_hostarr2d("hostarr2d", 2, 3); + Kokkos::View in_bool1d("in_bool1d", 5); if constexpr (std::is_same_v) { H5F file = @@ -229,6 +240,7 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu HDF5ReadAttribute(obj, prefix + "/vector", in_vector); HDF5ReadAttribute(obj, prefix + "/arr2d", in_arr2d); HDF5ReadAttribute(obj, prefix + "/hostarr2d", in_hostarr2d); + HDF5ReadAttribute(obj, prefix + "/bool1d", in_bool1d); } else if constexpr (std::is_same_v) { auto series = openPMD::Series(filename, openPMD::Access::READ_ONLY); auto it = series.iterations[0]; @@ -252,6 +264,9 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu auto &in_hostarr2d_v = in_hostarr2d.KokkosView(); resfile.RestoreViewAttribute(groupname + delim + prefix + delim + "hostarr2d", in_hostarr2d_v); + // TODO(pgrete) make this work and also add checks for correctness below + // resfile.RestoreViewAttribute(groupname + delim + prefix + delim + "bool1d", + // in_bool1d); } REQUIRE(scalar == in_scalar); From 6065213f8042b61b0154225fd2715ef5b4ff206c Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 9 Oct 2024 23:50:45 +0200 Subject: [PATCH 068/125] Remove debug info --- src/outputs/restart_opmd.cpp | 11 ++--------- src/outputs/restart_opmd.hpp | 4 ++-- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 249f1d7266b8..c83eb1f6201c 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -142,13 +142,6 @@ void RestartReaderOPMD::ReadAllParamsOfType(const std::string &prefix, Params &p // Thus we replace it. std::replace(full_path.begin(), full_path.end(), '/', delim[0]); - auto attrs = it->attributes(); - for (const auto &attr : attrs) { - std::cout << "Contains attribute: " << attr << std::endl; - } - std::cout << "Reading '" << full_path << "' with type: " << typeid(T).name() - << std::endl; - T val; if constexpr (implements::value) { val = params.Get(key); @@ -166,9 +159,9 @@ void RestartReaderOPMD::ReadAllParamsOfType(const std::string &prefix, Params &p } template -void RestartReaderOPMD::ReadAllParamsOfMultipleTypes(const std::string &pkg_name, +void RestartReaderOPMD::ReadAllParamsOfMultipleTypes(const std::string &prefix, Params &p) { - ([&] { ReadAllParamsOfType(pkg_name, p); }(), ...); + ([&] { ReadAllParamsOfType(prefix, p); }(), ...); } template diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index f986e84b8b97..5f79a7662012 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -153,9 +153,9 @@ class RestartReaderOPMD : public RestartReader { std::unique_ptr it; template - void ReadAllParamsOfType(const std::string &pkg_name, Params ¶ms); + void ReadAllParamsOfType(const std::string &prefix, Params ¶ms); template - void ReadAllParamsOfMultipleTypes(const std::string &pkg_name, Params &p); + void ReadAllParamsOfMultipleTypes(const std::string &prefix, Params &p); template void ReadAllParams(const std::string &pkg_name, Params &p); }; From 503a5b61c2ff7c35b65f3369b6d7c4236eccc6c9 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Sat, 9 Nov 2024 16:32:59 +0100 Subject: [PATCH 069/125] Fix bc interface from PR 1177 --- src/outputs/parthenon_opmd.cpp | 15 +++++++++------ src/outputs/restart_opmd.cpp | 3 --- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index f2b588e84e20..201ae74f9fec 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -398,12 +398,15 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, pm->mesh_size.nx(X3DIR)}); // Boundary conditions - std::vector boundary_condition_str(BOUNDARY_NFACES); - for (size_t i = 0; i < boundary_condition_str.size(); i++) { - boundary_condition_str[i] = GetBoundaryString(pm->mesh_bcs[i]); - } - - it.setAttribute("BoundaryConditions", boundary_condition_str); + auto arr_to_vec = [](const auto &arr) { + std::vector vec(BOUNDARY_NFACES); + for (int i = 0; i < BOUNDARY_NFACES; i++) { + vec[i] = arr.at(i); + } + return vec; + }; + it.setAttribute("BoundaryConditions", arr_to_vec(pm->mesh_bc_names)); + it.setAttribute("SwarmBoundaryConditions", arr_to_vec(pm->mesh_swarm_bc_names)); } // Info section Kokkos::Profiling::popRegion(); // write Attributes diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index c83eb1f6201c..11f9943124a1 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -77,9 +77,6 @@ RestartReaderOPMD::MeshInfo RestartReaderOPMD::GetMeshInfo() const { mesh_info.nbtotal = it->getAttribute("NumMeshBlocks").get(); mesh_info.root_level = it->getAttribute("RootLevel").get(); - mesh_info.bound_cond = - it->getAttribute("BoundaryConditions").get>(); - mesh_info.block_size = it->getAttribute("MeshBlockSize").get>(); mesh_info.includes_ghost = it->getAttribute("IncludesGhost").get(); mesh_info.n_ghost = it->getAttribute("NGhost").get(); From 4769a632b40f9344db7d8bb44ecfa83de8c74f5d Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 29 Nov 2024 12:12:13 +0100 Subject: [PATCH 070/125] Fix output numbering for triggered opmd outputs --- src/outputs/output_attr.hpp | 4 +++- src/outputs/parthenon_opmd.cpp | 18 ++++++++++-------- tst/unit/test_unit_params.cpp | 4 ++-- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/outputs/output_attr.hpp b/src/outputs/output_attr.hpp index 20e471c4c415..8607157e09a2 100644 --- a/src/outputs/output_attr.hpp +++ b/src/outputs/output_attr.hpp @@ -18,6 +18,8 @@ #ifndef OUTPUTS_OUTPUT_ATTR_HPP_ #define OUTPUTS_OUTPUT_ATTR_HPP_ +#include + // JMM: This could probably be done with template magic but I think // using a macro is honestly the simplest and cleanest solution here. // Template solution would be to define a variatic class to conain the @@ -42,4 +44,4 @@ PARTHENON_ATTR_APPLY(Kokkos::View); \ PARTHENON_ATTR_APPLY(device_view_t) -#endif // OUTPUTS_OUTPUT_ATTR_HPP_ \ No newline at end of file +#endif // OUTPUTS_OUTPUT_ATTR_HPP_ diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 201ae74f9fec..c0ee2aa434b2 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -614,10 +614,10 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } } } // loop over components - } // out_var->IsAllocated() - } // loop over blocks + } // out_var->IsAllocated() + } // loop over blocks it.seriesFlush(); - } // loop over vars + } // loop over vars Kokkos::Profiling::popRegion(); // write all variable data // -------------------------------------------------------------------------------- // @@ -660,11 +660,13 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, series.close(); #endif // ifndef PARTHENON_ENABLE_OPENPMD - // advance output parameters - output_params.file_number++; - output_params.next_time += output_params.dt; - pin->SetInteger(output_params.block_name, "file_number", output_params.file_number); - pin->SetReal(output_params.block_name, "next_time", output_params.next_time); + // advance output parameters if this is not a triggered (now or final) output + if (signal == SignalHandler::OutputSignal::none) { + output_params.file_number++; + output_params.next_time += output_params.dt; + pin->SetInteger(output_params.block_name, "file_number", output_params.file_number); + pin->SetReal(output_params.block_name, "next_time", output_params.next_time); + } } } // namespace parthenon diff --git a/tst/unit/test_unit_params.cpp b/tst/unit/test_unit_params.cpp index 994f17727387..aae19bd1916c 100644 --- a/tst/unit/test_unit_params.cpp +++ b/tst/unit/test_unit_params.cpp @@ -264,9 +264,9 @@ TEMPLATE_LIST_TEST_CASE("A set of params can be dumped to file", "[params][outpu auto &in_hostarr2d_v = in_hostarr2d.KokkosView(); resfile.RestoreViewAttribute(groupname + delim + prefix + delim + "hostarr2d", in_hostarr2d_v); - // TODO(pgrete) make this work and also add checks for correctness below + // TODO(pgrete) make this work and also add checks for correctness below // resfile.RestoreViewAttribute(groupname + delim + prefix + delim + "bool1d", - // in_bool1d); + // in_bool1d); } REQUIRE(scalar == in_scalar); From 543dbf9eb741ce53b5cdaecad40c8c33055cd1ac Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 29 Nov 2024 12:27:42 +0100 Subject: [PATCH 071/125] Make formatter happy --- src/outputs/restart_opmd.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 11f9943124a1..384e6847e812 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -222,7 +222,7 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block } } } // loop over components - } // loop over blocks + } // loop over blocks // Now actually read the registered chunks form disk it->seriesFlush(); From 449cd5e91296251b11edbb0d33699073248bf592 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 23 May 2025 16:06:01 +0200 Subject: [PATCH 072/125] Bump opmd to 0.16.1 stable version (not requiring work around any more) --- CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b742b21df5ee..bb41cddf3358 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -215,8 +215,7 @@ if (PARTHENON_ENABLE_OPENPMD) set(openPMD_USE_PYTHON OFF) FetchContent_Declare(openPMD GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git" - # we need newer than the latest 0.15.2 release to support writing attriutes from a subset of ranks - GIT_TAG "1c7d7ff") # develop as of 2024-09-02 + GIT_TAG "3a60e77") # Release 0.16.1 FetchContent_MakeAvailable(openPMD) install(TARGETS openPMD EXPORT parthenonTargets) endif() From 7f813c0805117c88026f75a6e1045c7f1755c995 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 23 May 2025 16:06:16 +0200 Subject: [PATCH 073/125] AMR restarts work and are bitwise identical now --- tst/regression/test_suites/restart_opmd/restart_opmd.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tst/regression/test_suites/restart_opmd/restart_opmd.py b/tst/regression/test_suites/restart_opmd/restart_opmd.py index 3548418f61d4..33c9d88aedf9 100644 --- a/tst/regression/test_suites/restart_opmd/restart_opmd.py +++ b/tst/regression/test_suites/restart_opmd/restart_opmd.py @@ -25,13 +25,9 @@ def Prepare(self, parameters, step): parameters.driver_cmd_line_args = ["parthenon/job/problem_id=gold"] # restart from an early snapshot elif step == 2: - # TODO(pgrete or someone else) ideally we want to restart from a later snapshot - # BUT results are not bitwise identical for AMR runs. PG thinks this is - # related to not storing the deref counter (and similar) and also thinks - # it's worth fixing. parameters.driver_cmd_line_args = [ "-r", - "gold.out1.00000.bp", + "gold.out1.00002.bp", "-i", f"{parameters.parthenon_path}/tst/regression/test_suites/restart_opmd/parthinput_override.restart", ] @@ -99,7 +95,7 @@ def compare_data(it_a, it_b, series_a, series_b): # Given that the shapes are guaranteed to match (follow the check above) # we can load chunks from both files. # Note that we have to go over chunks as data might be sparse on disk so - # loading the entire record will contain gargabe in sparse places. + # loading the entire record will contain garbage in sparse places. data_a = np.empty(comp_a.shape) data_a[:] = np.nan data_b = np.copy(data_a) @@ -152,7 +148,6 @@ def compare_files(idx_it): return all_good # comapre a few files throughout the simulations - success &= compare_files(1) success &= compare_files(2) success &= compare_files(3) success &= compare_files(4) From 3689e6cfa51a9f2d8d0334916b4be7b395e6ff8a Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 23 May 2025 16:22:21 +0200 Subject: [PATCH 074/125] Cross test opmd and rst outputs --- tst/regression/CMakeLists.txt | 2 +- .../restart_opmd/parthinput.restart | 4 +++ .../test_suites/restart_opmd/restart_opmd.py | 33 ++++++++++++++----- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/tst/regression/CMakeLists.txt b/tst/regression/CMakeLists.txt index 79ddd662f3e7..e535419a1978 100644 --- a/tst/regression/CMakeLists.txt +++ b/tst/regression/CMakeLists.txt @@ -162,7 +162,7 @@ if (PARTHENON_ENABLE_OPENPMD) list(APPEND TEST_PROCS ${NUM_MPI_PROC_TESTING}) list(APPEND TEST_ARGS "--driver ${PROJECT_BINARY_DIR}/example/advection/advection-example \ --driver_input ${CMAKE_CURRENT_SOURCE_DIR}/test_suites/restart_opmd/parthinput.restart \ - --num_steps 2") + --num_steps 3") list(APPEND EXTRA_TEST_LABELS "") endif() diff --git a/tst/regression/test_suites/restart_opmd/parthinput.restart b/tst/regression/test_suites/restart_opmd/parthinput.restart index 768453160a7b..06f482585edb 100644 --- a/tst/regression/test_suites/restart_opmd/parthinput.restart +++ b/tst/regression/test_suites/restart_opmd/parthinput.restart @@ -58,3 +58,7 @@ fill_derived = false # whether to fill one-copy test vars file_type = openpmd dt = 0.050 + +file_type = rst +dt = 0.050 + diff --git a/tst/regression/test_suites/restart_opmd/restart_opmd.py b/tst/regression/test_suites/restart_opmd/restart_opmd.py index 33c9d88aedf9..d91029e0f0e2 100644 --- a/tst/regression/test_suites/restart_opmd/restart_opmd.py +++ b/tst/regression/test_suites/restart_opmd/restart_opmd.py @@ -14,6 +14,13 @@ sys.dont_write_bytecode = True +# The test case uses an AMR simulation (with blocks being created and destroyed) as basline. +# The initial run will run to completion (writing hdf5 rst and opmd output with the same cadence). +# Then the simulation is restarted from the first, non-initial condition opmd output and again run to completion. +# Finally, the simulation is restarted again but using the hdf5 output generated from the previous restart. +# For testing all resulting pmd snapshots are compared against each other at the same simulation time. +# If they agree, restarting from opmd works and also the info contained in the opmd restart matches +# the info in the hdf5 rst files (and vice versa). class TestCase(utils.test_case.TestCaseAbs): def Prepare(self, parameters, step): # enable coverage testing on pass where restart @@ -23,14 +30,21 @@ def Prepare(self, parameters, step): # run baseline (to the very end) if step == 1: parameters.driver_cmd_line_args = ["parthenon/job/problem_id=gold"] - # restart from an early snapshot + # restart from an early openpmd snapshot elif step == 2: parameters.driver_cmd_line_args = [ "-r", - "gold.out1.00002.bp", + "gold.out1.00001.bp", "-i", f"{parameters.parthenon_path}/tst/regression/test_suites/restart_opmd/parthinput_override.restart", ] + # restart from an hdf5 snapshot produced from the restarted opmd one + elif step == 3: + parameters.driver_cmd_line_args = [ + "-r", + "silver.out2.00002.rhdf", + "parthenon/job/problem_id=bronze", + ] return parameters @@ -128,10 +142,10 @@ def compare_data(it_a, it_b, series_a, series_b): return all_equal - def compare_files(idx_it): + def compare_files(idx_it, name_a, name_b): all_good = True - series_gold = opmd.Series("gold.out1.%T.bp/", opmd.Access.read_only) - series_silver = opmd.Series("silver.out1.%T.bp/", opmd.Access.read_only) + series_gold = opmd.Series(f"{name_a}.out1.%T.bp/", opmd.Access.read_only) + series_silver = opmd.Series(f"{name_b}.out1.%T.bp/", opmd.Access.read_only) # PG: yes, this is inefficient but keeps the logic simple all_good &= compare_attributes(series_gold, series_silver) @@ -148,9 +162,12 @@ def compare_files(idx_it): return all_good # comapre a few files throughout the simulations - success &= compare_files(2) - success &= compare_files(3) - success &= compare_files(4) + success &= compare_files(2, "gold", "silver") + # bronze outputs only exists from dump 3 on + success &= compare_files(3, "gold", "silver") + success &= compare_files(3, "silver", "bronze") + success &= compare_files(4, "gold", "silver") + success &= compare_files(4, "silver", "bronze") # success &= compare_files("final") return success From aea0519266784e33bba074fa7938877d578e88a8 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 23 May 2025 17:08:01 +0200 Subject: [PATCH 075/125] Fix merge issues --- src/outputs/parthenon_opmd.cpp | 2 +- src/outputs/restart_opmd.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index c0ee2aa434b2..cf293949037c 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -37,7 +37,6 @@ #include "driver/driver.hpp" #include "globals.hpp" #include "interface/state_descriptor.hpp" -#include "interface/swarm_default_names.hpp" #include "interface/variable_state.hpp" #include "mesh/mesh.hpp" #include "mesh/meshblock.hpp" @@ -52,6 +51,7 @@ #include "outputs/output_utils.hpp" #include "outputs/outputs.hpp" #include "outputs/parthenon_opmd.hpp" +#include "pack/swarm_default_names.hpp" #include "parthenon_array_generic.hpp" #include "utils/error_checking.hpp" #include "utils/instrument.hpp" diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index 5f79a7662012..41b3acecb0b6 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -13,10 +13,10 @@ #include #include "basic_types.hpp" -#include "interface/swarm_default_names.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" #include "outputs/restart.hpp" +#include "pack/swarm_default_names.hpp" #include "mesh/domain.hpp" From 57b719a66d5797d0189f1c8d78eb749c8cd1b76b Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 23 May 2025 17:52:02 +0200 Subject: [PATCH 076/125] Fix compile issue with std::string dtor of vectypes --- src/outputs/restart_opmd.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 384e6847e812..18217518bcaf 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -174,7 +174,9 @@ void RestartReaderOPMD::ReadParams(const std::string &pkg_name, Params &p) { ReadAllParams(prefix, p); ReadAllParams(prefix, p); ReadAllParams(prefix, p); - ReadAllParams(prefix, p); + // TODO(pgrete) figure out why std::string does not work on hip/device (sth is calling a + // dtor on device) ReadAllParams(prefix, p); + ReadAllParamsOfType(prefix, p); // TODO(pgrete) same as for the writing. fix vec of bool ReadAllParamsOfType(prefix, p); } From 9bed0ce3019c9490ea1bdf4b9a45ae71fe7b23fe Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 4 Jun 2025 16:09:11 +0200 Subject: [PATCH 077/125] Allow embedded and system opmd build w and wo MPI --- CMakeLists.txt | 63 ++++++++++++++++++++++++---------- src/outputs/parthenon_opmd.cpp | 3 -- src/outputs/restart_opmd.hpp | 1 + 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f807363dfe97..6269378fe6f2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,7 +36,8 @@ option(PARTHENON_DISABLE_MPI "MPI is enabled by default if found, set this to Tr option(PARTHENON_ENABLE_HOST_COMM_BUFFERS "CUDA/HIP Only: Allocate communication buffers on host (may be slower)" OFF) option(PARTHENON_DISABLE_HDF5 "HDF5 is enabled by default if found, set this to True to disable HDF5" OFF) option(PARTHENON_DISABLE_HDF5_COMPRESSION "HDF5 compression is enabled by default, set this to True to disable compression in HDF5 output/restart files" OFF) -option(PARTHENON_ENABLE_OPENPMD "OpenPMD is enabled by default if found, set this to True to disable OpenPMD" ON) +option(PARTHENON_DISABLE_OPENPMD "OpenPMD is enabled by default if found, set this to ON to disable OpenPMD" OFF) +option(PARTHENON_USE_SYSTEM_OPENPMD "OpenPMD API is downloaded and installed automatically by default. Set this to ON to find and use a version installed on the system." OFF) option(PARTHENON_DISABLE_SPARSE "Sparse capability is enabled by default, set this to True to compile-time disable all sparse capability" OFF) option(PARTHENON_ENABLE_ASCENT "Enable Ascent for in situ visualization and analysis" OFF) option(PARTHENON_LINT_DEFAULT "Linting is turned off by default, use the \"lint\" target or set \ @@ -202,23 +203,49 @@ if (NOT PARTHENON_DISABLE_HDF5) install(TARGETS HDF5_C EXPORT parthenonTargets) endif() -if (PARTHENON_ENABLE_OPENPMD) -#TODO(pgrete) add logic for serial/parallel -#TODO(pgrete) add logic for internal/external build - include(FetchContent) - set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) - set(openPMD_BUILD_CLI_TOOLS OFF) - set(openPMD_BUILD_EXAMPLES OFF) - set(openPMD_BUILD_TESTING OFF) - set(openPMD_BUILD_SHARED_LIBS OFF) # precedence over BUILD_SHARED_LIBS if needed - set(openPMD_INSTALL OFF) # or instead use: - # set(openPMD_INSTALL ${BUILD_SHARED_LIBS}) # only install if used as a shared library - set(openPMD_USE_PYTHON OFF) - FetchContent_Declare(openPMD - GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git" - GIT_TAG "3a60e77") # Release 0.16.1 - FetchContent_MakeAvailable(openPMD) - install(TARGETS openPMD EXPORT parthenonTargets) +set(PARTHENON_ENABLE_OPENPMD OFF) +if (NOT PARTHENON_DISABLE_OPENPMD) + if (PARTHENON_USE_SYSTEM_OPENPMD) + # we want to enforce the use of ADIOS2 with this backend + set(PARTHENON_OPENPMD_COMPONENTS "ADIOS2") + if (ENABLE_MPI) + list(APPEND PARTHENON_OPENPMD_COMPONENTS "MPI") + else() + list(APPEND PARTHENON_OPENPMD_COMPONENTS "NOMPI") + endif() + find_package(openPMD 0.16.1 CONFIG COMPONENTS ${PARTHENON_OPENPMD_COMPONENTS}) + if (NOT openPMD_FOUND) + message(FATAL_ERROR "OpenPMD API requested to be used from the environment but it was " + "not found. Either append the path to the installed OpenPMD lib via the " + "openPMD_DIR or CMAKE_PREFIX_PATH, or use the version shipped with Parthenon by keeping " + "PARTHENON_USE_SYSTEM_OPENPMD=OFF") + endif() + else() + include(FetchContent) + set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) + set(openPMD_BUILD_CLI_TOOLS OFF) + set(openPMD_BUILD_EXAMPLES OFF) + set(openPMD_BUILD_TESTING OFF) + set(openPMD_BUILD_SHARED_LIBS OFF) + set(openPMD_INSTALL OFF) + set(openPMD_USE_ADIOS2 ON) # we definitely want ADIOS2 + if( NOT Python3_Interpreter_FOUND) + find_package(Python3 REQUIRED COMPONENTS Interpreter) + endif() + # Why not build the python interface for easier testing when Python is available + if(Python3_Interpreter_FOUND) + set(openPMD_USE_PYTHON ON) + else() + set(openPMD_USE_PYTHON OFF) + endif() + FetchContent_Declare(openPMD + GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git" + GIT_TAG "3a60e77") # Release 0.16.1 + FetchContent_MakeAvailable(openPMD) + install(TARGETS openPMD EXPORT parthenonTargets) + endif() + + set(PARTHENON_ENABLE_OPENPMD ON) endif() # Kokkos recommendatation resulting in not using default GNU extensions diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index cf293949037c..b8f579954ef7 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -307,9 +307,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // TODO(pgrete) Units? - // TODO(pgrete) We probably want this for params! - series.setAttribute("bla", true); - // In line with existing outputs, we write one file per iteration/snapshot series.setIterationEncoding(openPMD::IterationEncoding::fileBased); diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index 41b3acecb0b6..587f6f65a23e 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -85,6 +85,7 @@ class RestartReaderOPMD : public RestartReader { } else if (varname == swarm_position::z::name()) { particle_record = "position"; particle_record_component = "z"; + // TODO(pgrete) before merge add default id field } else { particle_record = varname; particle_record_component = From 9ae2fdeff294395572de30847eb95fa2a1c09e70 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 5 Jun 2025 10:41:09 +0200 Subject: [PATCH 078/125] Try opmd build in CI --- .github/workflows/ci-short.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-short.yml b/.github/workflows/ci-short.yml index 7d3f40800b9e..739a737d4ec8 100644 --- a/.github/workflows/ci-short.yml +++ b/.github/workflows/ci-short.yml @@ -22,7 +22,7 @@ jobs: style: runs-on: [self-hosted, A100] container: - image: ghcr.io/parthenon-hpc-lab/cuda11.6-mpi-hdf5-ascent + image: ghcr.io/parthenon-hpc-lab/cuda12.8-mpi-hdf5-ascent # map to local user id on CI machine to allow writing to build cache options: --user 1001 --cap-add CAP_SYS_PTRACE --shm-size="8g" --ulimit memlock=134217728 steps: @@ -30,7 +30,7 @@ jobs: with: submodules: 'true' - name: cpplint - run: python ./tst/style/cpplint.py --counting=detailed --recursive src example tst + run: python3 ./tst/style/cpplint.py --counting=detailed --recursive src example tst - name: copyright run: | cmake -DCMAKE_CXX_FLAGS=-Werror -Bbuild-copyright-check @@ -47,7 +47,7 @@ jobs: device: ['cuda', 'host'] runs-on: [self-hosted, A100] container: - image: ghcr.io/parthenon-hpc-lab/cuda11.6-mpi-hdf5-ascent + image: ghcr.io/parthenon-hpc-lab/cuda12.8-mpi-hdf5-ascent # map to local user id on CI machine to allow writing to build cache options: --user 1001 --cap-add CAP_SYS_PTRACE --shm-size="8g" --ulimit memlock=134217728 steps: @@ -79,7 +79,7 @@ jobs: device: ['cuda', 'host'] runs-on: [self-hosted, A100] container: - image: ghcr.io/parthenon-hpc-lab/cuda11.6-mpi-hdf5-ascent + image: ghcr.io/parthenon-hpc-lab/cuda12.8-mpi-hdf5-ascent # map to local user id on CI machine to allow writing to build cache options: --user 1001 --cap-add CAP_SYS_PTRACE --shm-size="8g" --ulimit memlock=134217728 steps: @@ -99,6 +99,7 @@ jobs: # Pick GPU with most available memory export CUDA_VISIBLE_DEVICES=$(nvidia-smi --query-gpu=memory.free,index --format=csv,nounits,noheader | sort -nr | head -1 | awk '{ print $NF }') ctest -R regression_mpi_test:output_hdf5 + ctest -R regression_mpi_test:restart_opmd # Test example with swarms - name: particle-leapfrog run: | From 0ac42c675d8f7564efc4aa6c08a5dc84b4192a2b Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 5 Jun 2025 12:22:15 +0200 Subject: [PATCH 079/125] Provide path to openpmd_api --- .github/workflows/ci-short.yml | 3 +++ tst/regression/CMakeLists.txt | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-short.yml b/.github/workflows/ci-short.yml index 739a737d4ec8..6f3515d314f6 100644 --- a/.github/workflows/ci-short.yml +++ b/.github/workflows/ci-short.yml @@ -17,6 +17,9 @@ env: OMPI_MCA_btl_smcuda_use_cuda_ipc: 0 # https://github.com/open-mpi/ompi/issues/4948#issuecomment-395468231 OMPI_MCA_btl_vader_single_copy_mechanism: none + # not great, but does the job for now + # (openpmd_api is installed here rather than in dist-packages) + PYTHONPATH: /usr/local/lib/python3.12/site-packages jobs: style: diff --git a/tst/regression/CMakeLists.txt b/tst/regression/CMakeLists.txt index fdeb0f96bc8f..45559cd3b3f9 100644 --- a/tst/regression/CMakeLists.txt +++ b/tst/regression/CMakeLists.txt @@ -160,8 +160,8 @@ endif() if (PARTHENON_ENABLE_OPENPMD) - # h5py is needed for restart and hdf5 test - list(APPEND REQUIRED_PYTHON_MODULES openpmd_api) + # h5py is also needed for the current test + list(APPEND REQUIRED_PYTHON_MODULES openpmd_api h5py) # Restart list(APPEND TEST_DIRS restart_opmd) From d230db9b40fa3ef0939978d5df04dc1322e75f01 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 5 Jun 2025 14:13:54 +0200 Subject: [PATCH 080/125] Update CI containers to be used --- .github/workflows/ci-extended.yml | 2 +- .github/workflows/ci-short.yml | 6 ++---- cmake/machinecfg/CI.cmake | 4 +++- cmake/machinecfg/GitHubActions.cmake | 4 +++- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-extended.yml b/.github/workflows/ci-extended.yml index c673aeb6dd60..8476c6b51f4d 100644 --- a/.github/workflows/ci-extended.yml +++ b/.github/workflows/ci-extended.yml @@ -131,7 +131,7 @@ jobs: parallel: ['serial', 'mpi'] runs-on: [self-hosted, navi1030] container: - image: ghcr.io/parthenon-hpc-lab/rocm5.4.3-mpi-hdf5 + image: ghcr.io/parthenon-hpc-lab/rocm6.2-mpi-hdf5 # Map to local user id on CI machine to allow writing to build cache and # forward device handles to access AMD GPU within container options: --user 1000 -w /home/ci --device /dev/kfd --device /dev/dri --security-opt seccomp=unconfined diff --git a/.github/workflows/ci-short.yml b/.github/workflows/ci-short.yml index 6f3515d314f6..48d5ea47a66d 100644 --- a/.github/workflows/ci-short.yml +++ b/.github/workflows/ci-short.yml @@ -17,9 +17,6 @@ env: OMPI_MCA_btl_smcuda_use_cuda_ipc: 0 # https://github.com/open-mpi/ompi/issues/4948#issuecomment-395468231 OMPI_MCA_btl_vader_single_copy_mechanism: none - # not great, but does the job for now - # (openpmd_api is installed here rather than in dist-packages) - PYTHONPATH: /usr/local/lib/python3.12/site-packages jobs: style: @@ -141,7 +138,7 @@ jobs: integration-amdgpu: runs-on: [self-hosted, navi1030] container: - image: ghcr.io/parthenon-hpc-lab/rocm5.4.3-mpi-hdf5 + image: ghcr.io/parthenon-hpc-lab/rocm6.2-mpi-hdf5 # Map to local user id on CI machine to allow writing to build cache and # forward device handles to access AMD GPU within container options: --user 1000 -w /home/ci --device /dev/kfd --device /dev/dri --security-opt seccomp=unconfined @@ -165,6 +162,7 @@ jobs: cmake --build build -t advection-example cd build ctest -R regression_mpi_test:output_hdf5 + ctest -R regression_mpi_test:restart_opmd # Test example with swarms - name: particle-leapfrog run: | diff --git a/cmake/machinecfg/CI.cmake b/cmake/machinecfg/CI.cmake index d9b7bb6df013..16c0dc0c6cf0 100644 --- a/cmake/machinecfg/CI.cmake +++ b/cmake/machinecfg/CI.cmake @@ -1,6 +1,6 @@ #======================================================================================== # Parthenon performance portable AMR framework -# Copyright(C) 2020 The Parthenon collaboration +# Copyright(C) 2020-2025 The Parthenon collaboration # Licensed under the 3-clause BSD License, see LICENSE file for details #======================================================================================== # (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved. @@ -38,3 +38,5 @@ else() set(HDF5_ROOT /usr/local/hdf5/serial CACHE STRING "HDF5 path") set(PARTHENON_DISABLE_MPI ON CACHE BOOL "Disable MPI") endif() + +set(PARTHENON_USE_SYSTEM_OPENPMD ON CACHE BOOL "Use API in container") diff --git a/cmake/machinecfg/GitHubActions.cmake b/cmake/machinecfg/GitHubActions.cmake index b524483d996c..ad5bb586c5f1 100644 --- a/cmake/machinecfg/GitHubActions.cmake +++ b/cmake/machinecfg/GitHubActions.cmake @@ -1,6 +1,6 @@ #======================================================================================== # Parthenon performance portable AMR framework -# Copyright(C) 2021 The Parthenon collaboration +# Copyright(C) 2021-2025 The Parthenon collaboration # Licensed under the 3-clause BSD License, see LICENSE file for details #======================================================================================== # (C) (or copyright) 2021. Triad National Security, LLC. All rights reserved. @@ -50,4 +50,6 @@ else() set(PARTHENON_DISABLE_MPI ON CACHE BOOL "Disable MPI") endif() +set(PARTHENON_USE_SYSTEM_OPENPMD ON CACHE BOOL "Use API in container") + set(CMAKE_CXX_FLAGS "${MACHINE_CXX_FLAGS}" CACHE STRING "Default flags for this config") From 41c94c5fd193d00fc4484ac4104e6168aec46fd3 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 5 Jun 2025 17:48:27 +0200 Subject: [PATCH 081/125] Disable read/write of string views (not supported by Kokkos) --- .github/workflows/ci-short.yml | 3 ++- src/outputs/parthenon_opmd.cpp | 5 ++++- src/outputs/restart_opmd.cpp | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-short.yml b/.github/workflows/ci-short.yml index 48d5ea47a66d..5066645bc806 100644 --- a/.github/workflows/ci-short.yml +++ b/.github/workflows/ci-short.yml @@ -32,8 +32,9 @@ jobs: - name: cpplint run: python3 ./tst/style/cpplint.py --counting=detailed --recursive src example tst - name: copyright + # using variant mpi because openpmd lib is only installed with mpi support (no serial) run: | - cmake -DCMAKE_CXX_FLAGS=-Werror -Bbuild-copyright-check + cmake -DCMAKE_CXX_FLAGS=-Werror -Bbuild-copyright-check -DMACHINE_VARIANT=mpi cmake --build build-copyright-check -t check-copyright - uses: actions/upload-artifact@v4 with: diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index b8f579954ef7..959c7a0b73b6 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -147,7 +147,10 @@ void WriteAllParams(const Params ¶ms, const std::string &pkg_name, WriteAllParams(params, prefix, it); WriteAllParams(params, prefix, it); WriteAllParams(params, prefix, it); - WriteAllParams(params, prefix, it); + + // strings + WriteAllParamsOfType(params, prefix, it); + WriteAllParamsOfType>(params, prefix, it); } template diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 18217518bcaf..df135fc690aa 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -174,9 +174,10 @@ void RestartReaderOPMD::ReadParams(const std::string &pkg_name, Params &p) { ReadAllParams(prefix, p); ReadAllParams(prefix, p); ReadAllParams(prefix, p); - // TODO(pgrete) figure out why std::string does not work on hip/device (sth is calling a - // dtor on device) ReadAllParams(prefix, p); + + // strings (not supported in Kokkos Views) ReadAllParamsOfType(prefix, p); + ReadAllParamsOfType>(prefix, p); // TODO(pgrete) same as for the writing. fix vec of bool ReadAllParamsOfType(prefix, p); } From 8e089bb288353fc2d923fc94bcd5e95bdb3f950d Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 18 Jun 2025 17:22:39 -0400 Subject: [PATCH 082/125] Add opmd dump/load of new uint64 ids --- src/outputs/parthenon_opmd.cpp | 9 +++++++-- src/outputs/restart_opmd.hpp | 11 ++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 4b4fe57a8917..e4f186e14e1d 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -178,6 +178,9 @@ void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, } else if (vname == swarm_position::z::name()) { particle_record = "position"; particle_record_component = "z"; + } else if (vname == swarm_position::id::name()) { + particle_record = "id"; + particle_record_component = openPMD::MeshRecordComponent::SCALAR; } else { particle_record = vname; particle_record_component = openPMD::MeshRecordComponent::SCALAR; @@ -626,7 +629,8 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, Kokkos::Profiling::pushRegion("write particle data"); // TODO(pgrete) as above, first wrt differentiating between restart_ (last arg) - AllSwarmInfo all_swarm_info(pm->block_list, output_params.swarms, true); + AllSwarmInfo all_swarm_info(pm->block_list, output_params.swarms, + DumpOutputMode::RESTART); for (auto &[swname, swinfo] : all_swarm_info.all_info) { openPMD::ParticleSpecies swm = it.particles[swname]; // These indicate particles/meshblock and location in global index @@ -641,6 +645,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } OpenPMDUtils::WriteSwarmVar(swinfo, swm, it); + OpenPMDUtils::WriteSwarmVar(swinfo, swm, it); OpenPMDUtils::WriteSwarmVar(swinfo, swm, it); // From the HDF5 output: @@ -648,7 +653,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // one for vis. // BUT PG: this may break things in unpredicable ways // I'm in favor of enforcing a global id somehow. We shold discuss. - PARTHENON_REQUIRE_THROWS(swinfo.var_info.count("id") != 0, + PARTHENON_REQUIRE_THROWS(swinfo.var_info.count(swarm_position::id::name()) != 0, "Particles should always carry a unique, persistent id!"); } Kokkos::Profiling::popRegion(); // write particle data diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index 587f6f65a23e..ab250b790c9f 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -8,6 +8,7 @@ //! \file restart_opmd.hpp // \brief Provides support for restarting from OpenPMD output +#include #include #include #include @@ -15,6 +16,7 @@ #include "basic_types.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" +#include "openPMD/backend/MeshRecordComponent.hpp" #include "outputs/restart.hpp" #include "pack/swarm_default_names.hpp" @@ -85,7 +87,9 @@ class RestartReaderOPMD : public RestartReader { } else if (varname == swarm_position::z::name()) { particle_record = "position"; particle_record_component = "z"; - // TODO(pgrete) before merge add default id field + } else if (varname == swarm_position::id::name()) { + particle_record = "id"; + particle_record_component = openPMD::MeshRecordComponent::SCALAR; } else { particle_record = varname; particle_record_component = @@ -110,6 +114,11 @@ class RestartReaderOPMD : public RestartReader { std::vector &dataVec) override { ReadSwarmVar<>(swarmname, varname, count, offset, m, dataVec); }; + void ReadSwarmVar(const std::string &swarmname, const std::string &varname, + const std::size_t count, const std::size_t offset, const Metadata &m, + std::vector &dataVec) override { + ReadSwarmVar<>(swarmname, varname, count, offset, m, dataVec); + }; // Gets the counts and offsets for MPI ranks for the meshblocks set // by the indexrange. Returns the total count on this rank. From 1b8b9041a0337cc2756e8ac3561e7df143fe07dc Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 19 Jun 2025 17:38:19 -0400 Subject: [PATCH 083/125] Add opmd write support for all field types --- src/outputs/output_utils.cpp | 2 + src/outputs/parthenon_opmd.cpp | 262 ++++++++++++++++++--------------- src/outputs/parthenon_opmd.hpp | 8 +- src/outputs/restart_opmd.cpp | 5 +- 4 files changed, 158 insertions(+), 119 deletions(-) diff --git a/src/outputs/output_utils.cpp b/src/outputs/output_utils.cpp index 7cfafcebf066..33409c1f86c7 100644 --- a/src/outputs/output_utils.cpp +++ b/src/outputs/output_utils.cpp @@ -40,6 +40,8 @@ namespace parthenon { namespace OutputUtils { +// This function returns the max dimensions over all topological elements of the given +// variable, i.e., it returns nx1+1, nx2+1, nx3+1 for a face centered variable. Triple_t VarInfo::GetNumKJI(const IndexDomain domain) const { int nx3 = 1, nx2 = 1, nx1 = 1; // TODO(JMM): I know that this could be done by hand, but I'd rather diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index e4f186e14e1d..d1f677e59db9 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -215,9 +215,9 @@ void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, it.seriesFlush(); } } -std::tuple GetMeshRecordAndComponentNames(const VarInfo &vinfo, - const int comp_idx, - const int level) { +std::tuple +GetMeshRecordAndComponentNames(const VarInfo &vinfo, const TopologicalElement te, + const int comp_idx, const int level) { std::string comp_name; if (vinfo.is_vector) { if (comp_idx == 0) { @@ -234,16 +234,39 @@ std::tuple GetMeshRecordAndComponentNames(const VarInf } else { comp_name = openPMD::MeshRecordComponent::SCALAR; } + + // Default for cell centered fields is an empty string + // to maintain backwards compatiblity with first iteration of + // OpenPMD outputs. + std::string te_str = ""; + if (te == TopologicalElement::F1) { + te_str = "F1_"; + } else if (te == TopologicalElement::F2) { + te_str = "F2_"; + } else if (te == TopologicalElement::F3) { + te_str = "F3_"; + } else if (te == TopologicalElement::E1) { + te_str = "E1_"; + } else if (te == TopologicalElement::E2) { + te_str = "E2_"; + } else if (te == TopologicalElement::E3) { + te_str = "E3_"; + } else if (te == TopologicalElement::NN) { + te_str = "NN_"; + } else { + PARTHENON_REQUIRE_THROWS(te == TopologicalElement::CC, + "Outputs for this type of TE not implemented.") + } // TODO(pgrete) need to make sure that var names are allowed within standard - const std::string &mesh_record_name = vinfo.label + "_" + + const std::string &mesh_record_name = vinfo.label + "_" + te_str + vinfo.component_labels[comp_idx] + "_lvl" + std::to_string(level); - // return std::make_tuple(mesh_record_name, comp_name); return {mesh_record_name, comp_name}; } std::tuple -GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb) { +GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb, + const TopologicalElement te) { openPMD::Offset chunk_offset; openPMD::Extent chunk_extent; const auto loc = pm->Forest().GetLegacyTreeLocation(pmb->loc); @@ -251,9 +274,10 @@ GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb) { chunk_offset = {loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; - chunk_extent = {static_cast(pmb->block_size.nx(X3DIR)), - static_cast(pmb->block_size.nx(X2DIR)), - static_cast(pmb->block_size.nx(X1DIR))}; + chunk_extent = { + static_cast(pmb->block_size.nx(X3DIR) + TopologicalOffsetK(te)), + static_cast(pmb->block_size.nx(X2DIR) + TopologicalOffsetJ(te)), + static_cast(pmb->block_size.nx(X1DIR) + TopologicalOffsetI(te))}; } else if (pm->ndim == 2) { chunk_offset = {loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; @@ -467,10 +491,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, Real; // typename std::conditional::type; std::vector tmp_data(var_size_max * num_blocks_local); - // TODO(pgrete) This needs to be in the loop for non-cell-centered vars - auto ib = bounds.GetBoundsI(IndexDomain::interior); - auto jb = bounds.GetBoundsJ(IndexDomain::interior); - auto kb = bounds.GetBoundsK(IndexDomain::interior); // for each variable we write for (auto &vinfo : all_vars_info) { PARTHENON_INSTRUMENT_REGION("Write variable loop") @@ -490,88 +510,92 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, for (auto &pmb : pm->block_list) { // TODO(pgrete) check if we should skip the suffix for level 0 const auto level = pmb->loc.level() - pm->GetRootLevel(); - - for (int comp_idx = 0; comp_idx < vinfo.component_labels.size(); comp_idx++) { - const auto [record_name, comp_name] = - OpenPMDUtils::GetMeshRecordAndComponentNames(vinfo, comp_idx, level); - - // Create the mesh_record for this variable at the given level (if it doesn't - // exist yet) - if (!it.meshes.contains(record_name)) { - auto mesh_record = it.meshes[record_name]; - - // These following attributes are shared across all components of the record. - - PARTHENON_REQUIRE_THROWS( - typeid(Coordinates_t) == typeid(UniformCartesian), - "OpenPMD in Parthenon currently only supports Cartesian coordinates."); - mesh_record.setGeometry(openPMD::Mesh::Geometry::cartesian); - auto &coords = pmb->coords; - // For uniform Cartesian, all dxN are const across the block so we just pick the - // first index. - Real dx1 = coords.CellWidth(0, 0, 0); - Real dx2 = coords.CellWidth(0, 0, 0); - Real dx3 = coords.CellWidth(0, 0, 0); - - // TODO(pgrete) check if this should be tied to the MemoryLayout - mesh_record.setDataOrder(openPMD::Mesh::DataOrder::C); - - auto mesh_comp = mesh_record[comp_name]; - // TODO(pgrete) needs to be updated for face and edges etc - // Also this feels wrong for deep hierachies... - auto effective_nx = static_cast(std::pow(2, level)); - openPMD::Extent global_extent; - if (pm->ndim == 3) { - mesh_record.setGridSpacing(std::vector{dx3, dx2, dx1}); - mesh_record.setAxisLabels({"z", "y", "x"}); - mesh_record.setGridGlobalOffset({ - pm->mesh_size.xmin(X3DIR), - pm->mesh_size.xmin(X2DIR), - pm->mesh_size.xmin(X1DIR), - }); - // TODO(pgrete) needs to be updated for face and edges etc - mesh_comp.setPosition(std::vector{0.5, 0.5, 0.5}); - global_extent = { - static_cast(pm->mesh_size.nx(X3DIR)) * effective_nx, - static_cast(pm->mesh_size.nx(X2DIR)) * effective_nx, - static_cast(pm->mesh_size.nx(X1DIR)) * effective_nx, - }; - } else if (pm->ndim == 2) { - mesh_record.setGridSpacing(std::vector{dx2, dx1}); - mesh_record.setAxisLabels({"y", "x"}); - mesh_record.setGridGlobalOffset({ - pm->mesh_size.xmin(X2DIR), - pm->mesh_size.xmin(X1DIR), - }); - - // TODO(pgrete) needs to be updated for face and edges etc - mesh_comp.setPosition(std::vector{0.5, 0.5}); - global_extent = { - static_cast(pm->mesh_size.nx(X2DIR)) * effective_nx, - static_cast(pm->mesh_size.nx(X1DIR)) * effective_nx, - }; - - } else { - PARTHENON_THROW("1D output for openpmd not yet supported."); + for (const auto &te : vinfo.topological_elements) { + for (int comp_idx = 0; comp_idx < vinfo.component_labels.size(); comp_idx++) { + const auto [record_name, comp_name] = + OpenPMDUtils::GetMeshRecordAndComponentNames(vinfo, te, comp_idx, level); + + // Create the mesh_record for this variable at the given level (if it doesn't + // exist yet) + if (!it.meshes.contains(record_name)) { + auto mesh_record = it.meshes[record_name]; + + // These following attributes are shared across all components of the record. + + PARTHENON_REQUIRE_THROWS( + typeid(Coordinates_t) == typeid(UniformCartesian), + "OpenPMD in Parthenon currently only supports Cartesian coordinates."); + mesh_record.setGeometry(openPMD::Mesh::Geometry::cartesian); + auto &coords = pmb->coords; + // For uniform Cartesian, all dxN are const across the block so we just pick + // the first index. + Real dx1 = coords.CellWidth(0, 0, 0); + Real dx2 = coords.CellWidth(0, 0, 0); + Real dx3 = coords.CellWidth(0, 0, 0); + + // TODO(pgrete) check if this should be tied to the MemoryLayout + mesh_record.setDataOrder(openPMD::Mesh::DataOrder::C); + + auto mesh_comp = mesh_record[comp_name]; + // TODO(pgrete) This feels wrong for deep hierachies... Check with OPMD people + auto effective_nx = static_cast(std::pow(2, level)); + openPMD::Extent global_extent; + if (pm->ndim == 3) { + mesh_record.setGridSpacing(std::vector{dx3, dx2, dx1}); + mesh_record.setAxisLabels({"z", "y", "x"}); + mesh_record.setGridGlobalOffset({ + pm->mesh_size.xmin(X3DIR), + pm->mesh_size.xmin(X2DIR), + pm->mesh_size.xmin(X1DIR), + }); + mesh_comp.setPosition(std::vector{ + 0.5 - 0.5 * TopologicalOffsetK(te), 0.5 - 0.5 * TopologicalOffsetJ(te), + 0.5 - 0.5 * TopologicalOffsetI(te)}); + global_extent = { + static_cast(pm->mesh_size.nx(X3DIR)) * effective_nx + + TopologicalOffsetK(te), + static_cast(pm->mesh_size.nx(X2DIR)) * effective_nx + + TopologicalOffsetJ(te), + static_cast(pm->mesh_size.nx(X1DIR)) * effective_nx + + TopologicalOffsetI(te), + }; + } else if (pm->ndim == 2) { + mesh_record.setGridSpacing(std::vector{dx2, dx1}); + mesh_record.setAxisLabels({"y", "x"}); + mesh_record.setGridGlobalOffset({ + pm->mesh_size.xmin(X2DIR), + pm->mesh_size.xmin(X1DIR), + }); + + mesh_comp.setPosition( + std::vector{0.5 - 0.5 * TopologicalOffsetJ(te), + 0.5 - 0.5 * TopologicalOffsetI(te)}); + global_extent = { + static_cast(pm->mesh_size.nx(X2DIR)) * effective_nx + + TopologicalOffsetJ(te), + static_cast(pm->mesh_size.nx(X1DIR)) * effective_nx + + TopologicalOffsetI(te), + }; + + } else { + PARTHENON_THROW("1D output for openpmd not yet supported."); + } + // Handling this here to now re-reset dataset later when iterating through the + // blocks + auto const dataset = + openPMD::Dataset(openPMD::determineDatatype(), global_extent); + // TODO(pgrete) check whether this should/need to be a collective so that the + // mesh generation should be done across all ranks prior to writing data, + // rather than in-situ for the local blocks only + mesh_comp.resetDataset(dataset); + + // TODO(pgrete) need unitDimension and timeOffset for this record? } - // Handling this here to now re-reset dataset later when iterating through the - // blocks - auto const dataset = - openPMD::Dataset(openPMD::determineDatatype(), global_extent); - // TODO(pgrete) check whether this should/need to be a collective so that the - // mesh generation should be done across all ranks prior to writing data, rather - // than in-situ for the local blocks only - mesh_comp.resetDataset(dataset); - - // TODO(pgrete) need unitDimension and timeOffset for this record? } } // Now that the mesh record exists, actually write the data auto out_var = pmb->meshblock_data.Get()->GetVarPtr(vinfo.label); - PARTHENON_REQUIRE_THROWS(out_var->metadata().Where() == - MetadataFlag(Metadata::Cell), - "Currently only cell centered vars are supported."); if (out_var->IsAllocated()) { // TODO(pgrete) check if we can work with a direct copy from a subview to not @@ -589,36 +613,44 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, Kokkos::deep_copy(component_buffer_view, data); #endif auto out_var_h = out_var->data.GetHostMirrorAndCopy(); - int comp_idx = 0; - const auto &Nt = out_var->GetDim(6); - const auto &Nu = out_var->GetDim(5); - const auto &Nv = out_var->GetDim(4); - // loop over all components - for (int t = 0; t < Nt; ++t) { - for (int u = 0; u < Nu; ++u) { - for (int v = 0; v < Nv; ++v) { - const auto [record_name, comp_name] = - OpenPMDUtils::GetMeshRecordAndComponentNames(vinfo, comp_idx, level); - auto mesh_comp = it.meshes[record_name][comp_name]; - - const auto comp_offset = tmp_offset; - for (int k = kb.s; k <= kb.e; ++k) { - for (int j = jb.s; j <= jb.e; ++j) { - for (int i = ib.s; i <= ib.e; ++i) { - tmp_data[tmp_offset] = static_cast(out_var_h(t, u, v, k, j, i)); - tmp_offset++; + for (const auto &te : vinfo.topological_elements) { + auto ib = bounds.GetBoundsI(IndexDomain::interior, te); + auto jb = bounds.GetBoundsJ(IndexDomain::interior, te); + auto kb = bounds.GetBoundsK(IndexDomain::interior, te); + int comp_idx = 0; + const auto &Nt = out_var->GetDim(6); + const auto &Nu = out_var->GetDim(5); + const auto &Nv = out_var->GetDim(4); + // loop over all components + for (int t = 0; t < Nt; ++t) { + for (int u = 0; u < Nu; ++u) { + for (int v = 0; v < Nv; ++v) { + const auto [record_name, comp_name] = + OpenPMDUtils::GetMeshRecordAndComponentNames(vinfo, te, comp_idx, + level); + auto mesh_comp = it.meshes[record_name][comp_name]; + + const auto comp_offset = tmp_offset; + for (int k = kb.s; k <= kb.e; ++k) { + for (int j = jb.s; j <= jb.e; ++j) { + for (int i = ib.s; i <= ib.e; ++i) { + tmp_data[tmp_offset] = static_cast( + out_var_h(static_cast(te) % 3, t, u, v, k, j, i)); + tmp_offset++; + } } } + const auto [chunk_offset, chunk_extent] = + OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb, te); + mesh_comp.storeChunkRaw(&tmp_data[comp_offset], chunk_offset, + chunk_extent); + comp_idx += 1; } - const auto [chunk_offset, chunk_extent] = - OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb); - mesh_comp.storeChunkRaw(&tmp_data[comp_offset], chunk_offset, chunk_extent); - comp_idx += 1; } - } - } // loop over components - } // out_var->IsAllocated() - } // loop over blocks + } // loop over components + } // loop over topological elements + } // out_var->IsAllocated() + } // loop over blocks it.seriesFlush(); } // loop over vars Kokkos::Profiling::popRegion(); // write all variable data diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index 9e0c04e99a4c..2665bf9c2f85 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -12,6 +12,7 @@ #include #include +#include "basic_types.hpp" #include "mesh/meshblock.hpp" #include "openPMD/Dataset.hpp" #include "openPMD/Iteration.hpp" @@ -33,18 +34,21 @@ void WriteAllParams(const Params ¶ms, const std::string &prefix, inline static const std::string delim = "~"; // Construct OpenPMD Mesh "record" name and comonnent identifier. +// - te is the TopologicalElement (which is used as part of the variable name record) // - comp_idx is a flattended index over all components of the vectors and tensors, i.e., // the typical v,u,t indices. // - level is the current effective level of the Mesh record std::tuple -GetMeshRecordAndComponentNames(const OutputUtils::VarInfo &vinfo, const int comp_idx, +GetMeshRecordAndComponentNames(const OutputUtils::VarInfo &vinfo, + const TopologicalElement te, const int comp_idx, const int level); // Calculate logical location on effective mesh (i.e., a mesh with size that matches full // coverage at given resolution on a particular level) // TODO(pgrete) needs to be updated to properly work with Forests std::tuple -GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb); +GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb, + const TopologicalElement te); } // namespace OpenPMDUtils } // namespace parthenon diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index df135fc690aa..0859cf46874b 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -201,7 +201,8 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block for (int v = 0; v < Nv; ++v) { // Get the correct record const auto [record_name, comp_name] = - OpenPMDUtils::GetMeshRecordAndComponentNames(vinfo, comp_idx, level); + OpenPMDUtils::GetMeshRecordAndComponentNames(vinfo, TopologicalElement::CC, + comp_idx, level); PARTHENON_REQUIRE_THROWS(it->meshes.contains(record_name), "Missing mesh record '" + record_name + @@ -214,7 +215,7 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block auto mesh_comp = mesh_record[comp_name]; const auto [chunk_offset, chunk_extent] = - OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb); + OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb, TopologicalElement::CC); mesh_comp.loadChunkRaw(&data_vec[comp_offset], chunk_offset, chunk_extent); // TODO(pgrete) check if output utils machinery can be used for non-cell // centered fields, which might not be that straightforward as a global mesh From a1768b0364ef739a1ecd35bac15bfe911782eeb3 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 20 Jun 2025 10:35:12 -0400 Subject: [PATCH 084/125] Fix 2D offsets --- src/outputs/parthenon_opmd.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index d1f677e59db9..863c2dd3e29b 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -281,8 +281,9 @@ GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb, } else if (pm->ndim == 2) { chunk_offset = {loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; - chunk_extent = {static_cast(pmb->block_size.nx(X2DIR)), - static_cast(pmb->block_size.nx(X1DIR))}; + chunk_extent = { + static_cast(pmb->block_size.nx(X2DIR) + TopologicalOffsetJ(te)), + static_cast(pmb->block_size.nx(X1DIR) + TopologicalOffsetI(te))}; } else { PARTHENON_THROW("1D output for openpmd not yet supported."); } From 3d480167e1ff97725aa36e0725f63151831b6cc5 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 2 Jul 2025 18:09:39 +0200 Subject: [PATCH 085/125] First stab at opmd restart for non-CC fields --- src/outputs/output_utils.cpp | 23 ++++++++--- src/outputs/output_utils.hpp | 23 ++++++++--- src/outputs/parthenon_hdf5.cpp | 2 +- src/outputs/restart.hpp | 6 +++ src/outputs/restart_hdf5.hpp | 5 +++ src/outputs/restart_opmd.cpp | 71 +++++++++++++++++----------------- src/outputs/restart_opmd.hpp | 6 ++- src/parthenon_manager.cpp | 7 ++-- 8 files changed, 91 insertions(+), 52 deletions(-) diff --git a/src/outputs/output_utils.cpp b/src/outputs/output_utils.cpp index 33409c1f86c7..5321477299ae 100644 --- a/src/outputs/output_utils.cpp +++ b/src/outputs/output_utils.cpp @@ -35,6 +35,7 @@ #include "mesh/meshblock.hpp" #include "outputs/output_utils.hpp" #include "parameter_input.hpp" +#include "utils/error_checking.hpp" #include "utils/mpi_types.hpp" namespace parthenon { @@ -42,7 +43,7 @@ namespace OutputUtils { // This function returns the max dimensions over all topological elements of the given // variable, i.e., it returns nx1+1, nx2+1, nx3+1 for a face centered variable. -Triple_t VarInfo::GetNumKJI(const IndexDomain domain) const { +Triple_t VarInfo::GetPaddedNumKJI(const IndexDomain domain) const { int nx3 = 1, nx2 = 1, nx1 = 1; // TODO(JMM): I know that this could be done by hand, but I'd rather // rely on the loop bounds machinery and this should be cheap. @@ -96,13 +97,23 @@ int VarInfo::TensorSize() const { } } -int VarInfo::FillSize(const IndexDomain domain) const { +int VarInfo::FillSize(const IndexDomain domain, const bool is_padded) const { if (where == MetadataFlag({Metadata::None})) { return Size(); - } else { - auto [n3, n2, n1] = GetNumKJI(domain); + } + if (is_padded) { + auto [n3, n2, n1] = GetPaddedNumKJI(domain); return ntop_elems * TensorSize() * n3 * n2 * n1; } + // Use raw info from topological elements (including some safety checks) + auto ncells = cellbounds.GetTotal(domain, topological_elements.at(0)); + for (auto el_idx = 1; el_idx < ntop_elems; el_idx++) { + PARTHENON_REQUIRE_THROWS( + ncells == cellbounds.GetTotal(domain, topological_elements.at(el_idx)), + "All topological elements in a given output variable should have the same total " + "number of cells."); + } + return ntop_elems * TensorSize() * ncells; } // number of elements of data that describe variable shape @@ -116,7 +127,7 @@ int VarInfo::GetNDim() const { std::vector VarInfo::GetPaddedShape(IndexDomain domain) const { std::vector out = GetRawShape(); if (where != MetadataFlag({Metadata::None})) { - auto [nx3, nx2, nx1] = GetNumKJI(domain); + auto [nx3, nx2, nx1] = GetPaddedNumKJI(domain); out[0] = nx3; out[1] = nx2; out[2] = nx1; @@ -126,7 +137,7 @@ std::vector VarInfo::GetPaddedShape(IndexDomain domain) const { std::vector VarInfo::GetPaddedShapeReversed(IndexDomain domain) const { std::vector out(rnx_.begin(), rnx_.end()); if (where != MetadataFlag({Metadata::None})) { - auto [nx3, nx2, nx1] = GetNumKJI(domain); + auto [nx3, nx2, nx1] = GetPaddedNumKJI(domain); out[VNDIM - 3] = nx3; out[VNDIM - 2] = nx2; out[VNDIM - 1] = nx1; diff --git a/src/outputs/output_utils.hpp b/src/outputs/output_utils.hpp index 8a4234ddea9c..b4371ddf2e8a 100644 --- a/src/outputs/output_utils.hpp +++ b/src/outputs/output_utils.hpp @@ -72,14 +72,16 @@ struct VarInfo { // whether or not topological element matters. bool element_matters; - Triple_t GetNumKJI(const IndexDomain domain) const; + Triple_t GetPaddedNumKJI(const IndexDomain domain) const; Triple_t GetPaddedBoundsKJI(const IndexDomain domain) const; int Size() const; // Includes topological element shape int TensorSize() const; - // Size of region that needs to be filled with 0s if not allocated - int FillSize(const IndexDomain domain) const; + // Size of region that needs to be filled with 0s if not allocated. + // is_padded is set to true by default as it's the assumption in the original (HDF5) + // output files. + int FillSize(const IndexDomain domain, const bool is_padded = true) const; // number of elements of data that describe variable shape int GetNDim() const; @@ -93,7 +95,7 @@ struct VarInfo { // For nx1,nx2,nx3 find max storage required in each direction // accross topological elements. Unused indices will be written but // empty. - auto [nx3, nx2, nx1] = GetNumKJI(domain); + auto [nx3, nx2, nx1] = GetPaddedNumKJI(domain); // fill topological element, if relevant if (element_matters) { data[0] = ntop_elems; @@ -188,6 +190,8 @@ struct VarInfo { private: // TODO(JMM): Probably nx_ and rnx_ both not necessary... but it was // easiest for me to reason about it this way. + // Note, nx_ is usually initialized to the view dimensions (i.e., padded for face and + // edge centered fields). std::array nx_; std::vector rnx_; }; @@ -314,16 +318,19 @@ std::vector FlattenBlockInfo(Mesh *pm, int shape, Function_t f) { // mirror must be provided because copying done externally template -void PackOrUnpackVar(const VarInfo &info, bool do_ghosts, idx_t &idx, Function_t f) { +void PackOrUnpackVar(const VarInfo &info, bool do_ghosts, bool is_padded, idx_t &idx, + Function_t f) { const IndexDomain domain = (do_ghosts ? IndexDomain::entire : IndexDomain::interior); // shape as written to or read from. contains additional padding // in orthogonal directions. // e.g., Face1-centered var is shape (N1+1)x(N2+1)x(N3+1) // format is // topological_elems x tensor_elems x block_elems + // If variable is written without padding, we'll cut the indices below. const auto shape = info.GetPaddedShapeReversed(domain); // TODO(JMM): Should I hide this inside VarInfo? auto [kb, jb, ib] = info.GetPaddedBoundsKJI(domain); + // Adjust padded indices for variables not tied to the mesh if (info.where == MetadataFlag({Metadata::None})) { kb.s = 0; kb.e = std::max(0, shape[4] - 1); @@ -333,6 +340,12 @@ void PackOrUnpackVar(const VarInfo &info, bool do_ghosts, idx_t &idx, Function_t ib.e = std::max(0, shape[6] - 1); } for (int topo = 0; topo < shape[0]; ++topo) { + // Adjust padded indices for variables not written with padding + if (!is_padded) { + kb = info.cellbounds.GetBoundsK(domain, info.topological_elements.at(topo)); + jb = info.cellbounds.GetBoundsJ(domain, info.topological_elements.at(topo)); + ib = info.cellbounds.GetBoundsI(domain, info.topological_elements.at(topo)); + } for (int t = 0; t < shape[1]; ++t) { for (int u = 0; u < shape[2]; ++u) { for (int v = 0; v < shape[3]; ++v) { diff --git a/src/outputs/parthenon_hdf5.cpp b/src/outputs/parthenon_hdf5.cpp index 3e55cc07ec59..4a2053c6136f 100644 --- a/src/outputs/parthenon_hdf5.cpp +++ b/src/outputs/parthenon_hdf5.cpp @@ -375,7 +375,7 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm if (v->IsAllocated() && (var_name == v->label())) { auto v_h = v->data.GetHostMirrorAndCopy(); OutputUtils::PackOrUnpackVar( - vinfo, output_params.include_ghost_zones, index, + vinfo, output_params.include_ghost_zones, true, index, [&](auto index, int topo, int t, int u, int v, int k, int j, int i) { tmpData[index] = static_cast(v_h(topo, t, u, v, k, j, i)); }); diff --git a/src/outputs/restart.hpp b/src/outputs/restart.hpp index 53eaa37377e2..812533648429 100644 --- a/src/outputs/restart.hpp +++ b/src/outputs/restart.hpp @@ -108,6 +108,12 @@ class RestartReader { const OutputUtils::VarInfo &info, std::vector &dataVec, int file_output_format_version, Mesh *pmesh) const = 0; + // The PackOrUnpack logic requires knowledge of how data is stored and being read into + // the buffer. For HDF5 data is padded if needed (i.e., a face centered field has tims + // nx#+1 in all dimensions) or OpenPMD it's not (i.e., a face centered field has dims + // nx1+1, nx2, nx3 in case of the F1 field). + [[nodiscard]] virtual bool BlockdataIsPadded() const = 0; + // Gets the data from a swarm var on current rank. Assumes all // blocks are contiguous. Fills dataVec based on shape from swarmvar // metadata. diff --git a/src/outputs/restart_hdf5.hpp b/src/outputs/restart_hdf5.hpp index c0d8bfd2e988..6f2d93ecd7da 100644 --- a/src/outputs/restart_hdf5.hpp +++ b/src/outputs/restart_hdf5.hpp @@ -117,6 +117,11 @@ class RestartReaderHDF5 : public RestartReader { const OutputUtils::VarInfo &info, std::vector &dataVec, int file_output_format_version, Mesh *pmesh) const override; + // The PackOrUnpack logic requires knowledge of how data is stored and being read into + // the buffer. For HDF5 data is padded if needed (i.e., a face centered field has tims + // nx#+1 in all dimensions). + [[nodiscard]] bool BlockdataIsPadded() const override { return true; }; + // Gets the data from a swarm var on current rank. Assumes all // blocks are contiguous. Fills dataVec based on shape from swarmvar // metadata. diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 0859cf46874b..c936665e2816 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -1,6 +1,6 @@ //======================================================================================== // Parthenon performance portable AMR framework -// Copyright(C) 2024 The Parthenon collaboration +// Copyright(C) 2024-2025 The Parthenon collaboration // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== //! \file restart_opmd.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -191,42 +192,40 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block // TODO(pgrete) check if we should skip the suffix for level 0 const auto level = pmb->loc.level() - pm->GetRootLevel(); - int comp_idx = 0; // used in label for non-vector variables - const auto &Nt = vinfo.GetDim(6); - const auto &Nu = vinfo.GetDim(5); - const auto &Nv = vinfo.GetDim(4); - // loop over all components - for (int t = 0; t < Nt; ++t) { - for (int u = 0; u < Nu; ++u) { - for (int v = 0; v < Nv; ++v) { - // Get the correct record - const auto [record_name, comp_name] = - OpenPMDUtils::GetMeshRecordAndComponentNames(vinfo, TopologicalElement::CC, - comp_idx, level); - - PARTHENON_REQUIRE_THROWS(it->meshes.contains(record_name), - "Missing mesh record '" + record_name + - "' in restart file."); - auto mesh_record = it->meshes[record_name]; - PARTHENON_REQUIRE_THROWS(mesh_record.contains(comp_name), - "Missing component'" + comp_name + - "' in mesh record '" + record_name + - "' of restart file."); - auto mesh_comp = mesh_record[comp_name]; - - const auto [chunk_offset, chunk_extent] = - OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb, TopologicalElement::CC); - mesh_comp.loadChunkRaw(&data_vec[comp_offset], chunk_offset, chunk_extent); - // TODO(pgrete) check if output utils machinery can be used for non-cell - // centered fields, which might not be that straightforward as a global mesh - // is stored rather than individual blocks. - comp_offset += pmb->block_size.nx(X1DIR) * pmb->block_size.nx(X2DIR) * - pmb->block_size.nx(X3DIR); - comp_idx += 1; + for (const auto &te : vinfo.topological_elements) { + int comp_idx = 0; // used in label for non-vector variables + const auto &Nt = vinfo.GetDim(6); + const auto &Nu = vinfo.GetDim(5); + const auto &Nv = vinfo.GetDim(4); + // loop over all components + for (int t = 0; t < Nt; ++t) { + for (int u = 0; u < Nu; ++u) { + for (int v = 0; v < Nv; ++v) { + // Get the correct record + const auto [record_name, comp_name] = + OpenPMDUtils::GetMeshRecordAndComponentNames(vinfo, te, comp_idx, level); + + PARTHENON_REQUIRE_THROWS(it->meshes.contains(record_name), + "Missing mesh record '" + record_name + + "' in restart file."); + auto mesh_record = it->meshes[record_name]; + PARTHENON_REQUIRE_THROWS(mesh_record.contains(comp_name), + "Missing component'" + comp_name + + "' in mesh record '" + record_name + + "' of restart file."); + auto mesh_comp = mesh_record[comp_name]; + + const auto [chunk_offset, chunk_extent] = + OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb, te); + mesh_comp.loadChunkRaw(&data_vec[comp_offset], chunk_offset, chunk_extent); + comp_offset += std::accumulate(chunk_extent.cbegin(), chunk_extent.cend(), 1, + std::multiplies{}); + comp_idx += 1; + } } - } - } // loop over components - } // loop over blocks + } // loop over components + } // loop over topological elements + } // loop over blocks // Now actually read the registered chunks form disk it->seriesFlush(); diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index ab250b790c9f..37982a7c09bb 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -16,7 +16,6 @@ #include "basic_types.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" -#include "openPMD/backend/MeshRecordComponent.hpp" #include "outputs/restart.hpp" #include "pack/swarm_default_names.hpp" @@ -55,6 +54,11 @@ class RestartReaderOPMD : public RestartReader { const OutputUtils::VarInfo &info, std::vector &dataVec, int file_output_format_version, Mesh *pmesh) const override; + // The PackOrUnpack logic requires knowledge of how data is stored and being read into + // the buffer. OpenPMD is dense (i.e., a face centered field has dims + // nx1+1, nx2, nx3 in case of the F1 field). + [[nodiscard]] bool BlockdataIsPadded() const override { return false; }; + // Gets the data from a swarm var on current rank. Assumes all // blocks are contiguous. Fills dataVec based on shape from swarmvar // metadata. diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index 1edd5b496aab..254a69b07317 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -315,7 +315,8 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { } max_fillsize = - std::max(max_fillsize, static_cast(v_info.FillSize(theDomain))); + std::max(max_fillsize, static_cast(v_info.FillSize(theDomain), + resfile.BlockdataIsPadded())); } // make sure we have all sparse variables that are in the restart file @@ -326,7 +327,7 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { std::vector tmp(static_cast(nb) * max_fillsize); for (const auto &v_info : all_vars_info) { const auto vlen = v_info.num_components * v_info.ntop_elems; - const auto fill_size = v_info.FillSize(theDomain); + const auto fill_size = v_info.FillSize(theDomain, resfile.BlockdataIsPadded()); const auto &label = v_info.label; if (Globals::my_rank == 0) { @@ -367,7 +368,7 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { // TODO(pgrete) figure out what to do about versions of different outputs if (true || file_output_format_ver >= HDF5::OUTPUT_VERSION_FORMAT - 1) { OutputUtils::PackOrUnpackVar( - v_info, resfile.HasGhost() != 0, index, + v_info, resfile.HasGhost() != 0, resfile.BlockdataIsPadded(), index, [&](auto index, int topo, int t, int u, int v, int k, int j, int i) { v_h(topo, t, u, v, k, j, i) = tmp[index]; }); From 301f25ba4fa945a383439f3da8fae2169a187361 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 2 Jul 2025 21:36:58 +0200 Subject: [PATCH 086/125] First attempt at adding/restoring opmd sparseinfo --- src/outputs/output_utils.cpp | 2 ++ src/outputs/parthenon_hdf5.cpp | 3 +- src/outputs/parthenon_opmd.cpp | 55 ++++++++++++++++++++++++++++++++-- src/outputs/restart_opmd.cpp | 21 +++++++++++-- 4 files changed, 75 insertions(+), 6 deletions(-) diff --git a/src/outputs/output_utils.cpp b/src/outputs/output_utils.cpp index 5321477299ae..c08d27238a99 100644 --- a/src/outputs/output_utils.cpp +++ b/src/outputs/output_utils.cpp @@ -341,6 +341,8 @@ std::vector FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_loca // explicit template instantiation template std::vector FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_local); +template std::vector +FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_local); template std::vector FlattendedLocalToGlobal(Mesh *pm, const std::vector &data_local); template std::vector FlattendedLocalToGlobal(Mesh *pm, diff --git a/src/outputs/parthenon_hdf5.cpp b/src/outputs/parthenon_hdf5.cpp index 4a2053c6136f..5f193d59cb55 100644 --- a/src/outputs/parthenon_hdf5.cpp +++ b/src/outputs/parthenon_hdf5.cpp @@ -291,7 +291,8 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm // The dataset SparseInfo itself is a 2D array of bools. The first index is the // global block index and the second index is the sparse field (same order as the // SparseFields attribute). SparseInfo[b][v] is true if the sparse field with index - // v is allocated on the block with index b, otherwise the value is false + // v is allocated on the block with index b, otherwise the value is false. + // If the logic here is ever updated, ensure to update the OpenPMD logic, too. std::vector sparse_names; std::unordered_map sparse_field_idx; diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 863c2dd3e29b..0745123e5fd6 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -477,6 +477,39 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, auto all_vars_info = GetAllVarsInfo( GetVarsToWrite(pm->block_list.front(), true, output_params.variables), bounds); + // Mirroring the SparseInfo handling in HDF5 here. + // Could probably made easier by just sequentially filling vectors, but better be safe + // than sorry. + // + // We need to add information about the sparse variables to the output file, namely: + // 1) Which variables are sparse + // 2) Is a sparse id of a particular sparse variable allocated on a given block + // + // This information is stored in the dataset called "SparseInfo". The data set + // contains an attribute "SparseFields" that is a vector of strings with the names + // of the sparse fields (field name with sparse id, i.e. "bar_28", "bar_7", foo_1", + // "foo_145"). The field names are in alphabetical order, which is the same order + // they show up in all_unique_vars (because it's a sorted set). + // + // The dataset SparseInfo itself is a 2D array of bools. The first index is the + // global block index and the second index is the sparse field (same order as the + // SparseFields attribute). SparseInfo[b][v] is true if the sparse field with index + // v is allocated on the block with index b, otherwise the value is false. + // If the logic here is ever updated, ensure to update the HDF5 logic, too. + std::vector sparse_names; + std::unordered_map sparse_field_idx; + for (auto &vinfo : all_vars_info) { + if (vinfo.is_sparse) { + sparse_field_idx.insert({vinfo.label, sparse_names.size()}); + sparse_names.push_back(vinfo.label); + } + } + auto num_sparse = sparse_names.size(); + // Note, we're using int8_t here to circument the global reduction of a bool vector, + // which would require much more boilerplate. + std::vector sparse_allocated(num_blocks_local * num_sparse); + std::vector sparse_dealloc_count(num_blocks_local * num_sparse); + // We're currently writing (flushing) one var at a time. This saves host memory but // results more smaller write. Might be updated in the future. // Allocate space for largest size variable @@ -508,7 +541,9 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, "dimensionality of the simulation.") } - for (auto &pmb : pm->block_list) { + // for each local mesh block + for (size_t b_idx = 0; b_idx < num_blocks_local; ++b_idx) { + const auto &pmb = pm->block_list[b_idx]; // TODO(pgrete) check if we should skip the suffix for level 0 const auto level = pmb->loc.level() - pm->GetRootLevel(); for (const auto &te : vinfo.topological_elements) { @@ -651,11 +686,27 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } // loop over components } // loop over topological elements } // out_var->IsAllocated() - } // loop over blocks + if (vinfo.is_sparse) { + auto sparse_idx = sparse_field_idx.at(vinfo.label); + sparse_allocated.at(b_idx * num_sparse + sparse_idx) = + static_cast(out_var->IsAllocated()); + sparse_dealloc_count.at(b_idx * num_sparse + sparse_idx) = out_var->dealloc_count; + } + } // loop over blocks it.seriesFlush(); } // loop over vars Kokkos::Profiling::popRegion(); // write all variable data + // -------------------------------------------------------------------------------- // + // WRITING Sparse metadata // + // -------------------------------------------------------------------------------- // + auto sparse_allocated_global = FlattendedLocalToGlobal(pm, sparse_allocated); + it.setAttribute("SparseInfo", sparse_allocated_global); + it.setAttribute("SparseFields", sparse_names); + auto sparse_dealloc_count_global = + FlattendedLocalToGlobal(pm, sparse_dealloc_count); + it.setAttribute("SparseDeallocCount", sparse_dealloc_count_global); + // -------------------------------------------------------------------------------- // // WRITING PARTICLE DATA // // -------------------------------------------------------------------------------- // diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index c936665e2816..cbd57925ac2c 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include @@ -67,8 +67,23 @@ int RestartReaderOPMD::GetOutputFormatVersion() const { } RestartReaderOPMD::SparseInfo RestartReaderOPMD::GetSparseInfo() const { - // TODO(pgrete) needs impl - return {}; + SparseInfo info; + // Only read if data exists. Otherwise return default constructed. + if (it->containsAttribute("SparseInfo")) { + auto sinfo_vec = it->getAttribute("SparseInfo").get>(); + info.labels = it->getAttribute("SparseFields").get>(); + info.num_sparse = static_cast(info.labels.size()); + info.num_blocks = sinfo_vec.size() / info.num_sparse; + info.dealloc_count = it->getAttribute("SparseDeallocCount").get>(); + + // copy "vector" data to bool pointer for compatiblity between output backends + info.allocated.reset(new bool[sinfo_vec.size()]); + for (int i = 0; i < sinfo_vec.size(); i++) { + info.allocated[i] = sinfo_vec.at(i); + } + } + + return info; } RestartReaderOPMD::MeshInfo RestartReaderOPMD::GetMeshInfo() const { From 0d5d4d3c51292e058c483356d8e8f0d993948fac Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 2 Jul 2025 23:29:23 +0200 Subject: [PATCH 087/125] Fix misplaced comma --- src/parthenon_manager.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index 254a69b07317..1c6881fac0be 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -314,9 +314,8 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { " is marked as sparse in restart file"); } - max_fillsize = - std::max(max_fillsize, static_cast(v_info.FillSize(theDomain), - resfile.BlockdataIsPadded())); + max_fillsize = std::max(max_fillsize, static_cast(v_info.FillSize( + theDomain, resfile.BlockdataIsPadded()))); } // make sure we have all sparse variables that are in the restart file From 207d24b251241d49c7e06b5320aee6b483ba6459 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 3 Jul 2025 12:18:43 +0200 Subject: [PATCH 088/125] Add missing int8_t to mpi type map --- src/outputs/parthenon_opmd.cpp | 1 + src/utils/mpi_types.hpp | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 0745123e5fd6..feec8b11743b 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include diff --git a/src/utils/mpi_types.hpp b/src/utils/mpi_types.hpp index 6f5e117504ad..6fc8404c1422 100644 --- a/src/utils/mpi_types.hpp +++ b/src/utils/mpi_types.hpp @@ -43,6 +43,11 @@ inline MPI_Datatype MPITypeMap::type() { return MPI_INT64_T; } +template <> +inline MPI_Datatype MPITypeMap::type() { + return MPI_INT8_T; +} + template <> inline MPI_Datatype MPITypeMap::type() { return MPI_INT; From 4fe5d632fa3f5b4ad0717a2ededfc09192777646 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 3 Jul 2025 12:23:02 +0200 Subject: [PATCH 089/125] Only write SparseInfo when sparse fields are present --- src/outputs/parthenon_opmd.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index feec8b11743b..2d7beb52c6df 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -701,12 +701,14 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // -------------------------------------------------------------------------------- // // WRITING Sparse metadata // // -------------------------------------------------------------------------------- // - auto sparse_allocated_global = FlattendedLocalToGlobal(pm, sparse_allocated); - it.setAttribute("SparseInfo", sparse_allocated_global); - it.setAttribute("SparseFields", sparse_names); - auto sparse_dealloc_count_global = - FlattendedLocalToGlobal(pm, sparse_dealloc_count); - it.setAttribute("SparseDeallocCount", sparse_dealloc_count_global); + if (num_sparse > 0) { + auto sparse_allocated_global = FlattendedLocalToGlobal(pm, sparse_allocated); + it.setAttribute("SparseInfo", sparse_allocated_global); + it.setAttribute("SparseFields", sparse_names); + auto sparse_dealloc_count_global = + FlattendedLocalToGlobal(pm, sparse_dealloc_count); + it.setAttribute("SparseDeallocCount", sparse_dealloc_count_global); + } // -------------------------------------------------------------------------------- // // WRITING PARTICLE DATA // From be682038160f470e79de4c801d9e0caca333dbb8 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 24 Jul 2025 11:51:51 +0200 Subject: [PATCH 090/125] Fix doc build --- .github/workflows/docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index bc8ee82056ec..0eb78d3af7f1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -36,6 +36,7 @@ jobs: mkdir -p build && cd build cmake -DPARTHENON_DISABLE_HDF5=ON \ -DPARTHENON_DISABLE_MPI=ON \ + -DPARTHENON_DISABLE_OPENPMD=ON \ -DREGRESSION_GOLD_STANDARD_SYNC=OFF \ -DBUILD_TESTING=OFF \ .. From 5ce430456d1558d37091912117bfd9e00cdf8f4f Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Mon, 28 Jul 2025 17:48:36 -0400 Subject: [PATCH 091/125] Add mutability maybe restart and made it so we can read old restart files with next time and next n --- src/interface/params.cpp | 25 +++++++++++++++---- src/interface/params.hpp | 19 +++++++++++++-- src/outputs/outputs.cpp | 21 ++++++++++++++++ src/outputs/outputs_package.cpp | 40 ++++++++++++++++--------------- src/outputs/parthenon_hdf5.cpp | 2 ++ src/outputs/parthenon_hdf5.hpp | 6 +++++ src/outputs/restart_hdf5.cpp | 2 ++ src/parameter_input.cpp | 22 +++++++++++++++++ src/parameter_input.hpp | 1 + tst/unit/test_parameter_input.cpp | 22 +++++++++++++++++ 10 files changed, 135 insertions(+), 25 deletions(-) diff --git a/src/interface/params.cpp b/src/interface/params.cpp index 0832af4bec0b..9052d7e282a7 100644 --- a/src/interface/params.cpp +++ b/src/interface/params.cpp @@ -11,10 +11,12 @@ // the public, perform publicly and display publicly, and to permit others to do so. //======================================================================================== +#include #include #include "utils/error_checking.hpp" +#include "globals.hpp" #include "kokkos_abstraction.hpp" #include "parthenon_arrays.hpp" @@ -55,13 +57,28 @@ template void Params::ReadFromHDF5AllParamsOfType(const std::string &prefix, const HDF5::H5G &group) { for (auto &[key, pparam] : myParams_) { - auto mutability = myMutable_.at(key); if (std::type_index(pparam->type()) == std::type_index(typeid(T)) && - mutability == Mutability::Restart) { + IsRestartable(key)) { + auto mutability = myMutable_.at(key); auto typed_ptr = std::any_cast(pparam.get()); auto &val = *typed_ptr; - HDF5::HDF5ReadAttribute(group, prefix + "/" + key, val); - Update(key, val); + std::string fullpath = prefix + "/" + key; + if (mutability == Mutability::Restart) { + HDF5::HDF5ReadAttribute(group, fullpath, val); + Update(key, val); + } else if (mutability == Mutability::MaybeRestart) { + try { + HDF5::HDF5ReadAttribute(group, fullpath, val); + Update(key, val); + } catch (std::runtime_error e) { + if (Globals::my_rank == 0) { + std::stringstream ss; + ss << "Failed to load parameter " << fullpath + << " from the restart file! Using default value." << std::endl; + PARTHENON_WARN(ss); + } + } + } } } } diff --git a/src/interface/params.hpp b/src/interface/params.hpp index 0dca36e1b264..8210d99dc6d6 100644 --- a/src/interface/params.hpp +++ b/src/interface/params.hpp @@ -39,7 +39,12 @@ class Params { // Immutable is default. Mutable is it can be updated at runtime. // Restart is a subset of mutable. Param not only can be updated at // runtime, but should be read from the restart file upon restart. - enum class Mutability : int { Immutable = 0, Mutable = 1, Restart = 2 }; + enum class Mutability : int { + Immutable = 0, + Mutable = 1, + Restart = 2, + MaybeRestart = 3 + }; Params() {} @@ -66,7 +71,7 @@ class Params { void Update(const std::string &key, T value) { PARTHENON_REQUIRE_THROWS((hasKey(key)), "Key " + key + "missing."); // immutable casts to false all others cast to true - PARTHENON_REQUIRE_THROWS(static_cast(myMutable_.at(key)), + PARTHENON_REQUIRE_THROWS(IsMutable(key), "Parameter " + key + " must be marked as mutable"); PARTHENON_REQUIRE_THROWS(std::type_index(myParams_.at(key)->type()) == std::type_index(typeid(T)), @@ -130,6 +135,16 @@ class Params { return keys; } + auto GetMutability(const std::string &key) const { return myMutable_.at(key); } + bool IsMutable(const std::string &key) const { + return static_cast(myMutable_.at(key)); + } + bool IsRestartable(const std::string &key) const { + auto mutability = myMutable_.at(key); + return (mutability == Mutability::Restart) || + (mutability == Mutability::MaybeRestart); + } + // void Params:: void list() { std::cout << std::endl << "Items are:" << std::endl; diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index 6b612a15eda7..9996de5bd575 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -138,6 +138,27 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { (*pactive)[iinput] = true; } + // JMM: Backwards compatibility hack. Don't allow this unless + // we're restarting from a legacy file format. + if (Globals::is_restart) { // should this be pmesh->is_restart? + bool next_time_exists = pin->DoesParameterExist(op.block_name, "next_time"); + bool next_n_exists = pin->DoesParameterExist(op.block_name, "next_n"); + if (next_time_exists) { + Real next_time = pin->GetReal(op.block_name, "next_time"); + (*plast_times)[iinput] = next_time - dt; + pin->RemoveParameter(op.block_name, "next_time"); + } + if (next_n_exists) { + int next_n = pin->GetInteger(op.block_name, "next_n"); + (*plast_ns)[iinput] = next_n - dn; + pin->RemoveParameter(op.block_name, "next_n"); + } + if (next_time_exists || next_n_exists) { + (*pfile_numbers)[iinput] = pin->GetOrAddInteger(op.block_name, "file_number", 0); + pin->RemoveParameter(op.block_name, "file_number"); + } + } + PARTHENON_REQUIRE_THROWS(!(dt >= 0.0 && dn >= 0), "dt and dn are enabled for the same output block, which " "is not supported. Please set at most one value >= 0."); diff --git a/src/outputs/outputs_package.cpp b/src/outputs/outputs_package.cpp index 7518eb09185c..d202e0c277d0 100644 --- a/src/outputs/outputs_package.cpp +++ b/src/outputs/outputs_package.cpp @@ -25,6 +25,7 @@ #include #include +#include "globals.hpp" #include "interface/state_descriptor.hpp" #include "outputs/outputs_package.hpp" #include "parameter_input.hpp" @@ -53,21 +54,6 @@ std::shared_ptr Initialize(ParameterInput *pin) { std::string outn = pib->block_name.substr(16); // 6 because counting starts at 0! std::string block_name = pib->block_name; - if (pin->DoesParameterExist(block_name, "next_time")) { - std::stringstream msg; - msg << "You have used the next_time parameter in the " << block_name - << " output block. This parameter is deprecated. Instead change" - << " the output cadence with dt." << std::endl; - PARTHENON_THROW(msg); - } - if (pin->DoesParameterExist(block_name, "next_n")) { - std::stringstream msg; - msg << "You have used the next_n parameter in the " << block_name - << " output block. This parameter is deprecated. Instead change" - << " the output cadence with dn." << std::endl; - PARTHENON_THROW(msg); - } - // these are used for book-keeping block_names.push_back(block_name); block_numbers.push_back(atoi(outn.c_str())); @@ -81,14 +67,30 @@ std::shared_ptr Initialize(ParameterInput *pin) { // is that we want to ensure a first output is performed. last_times.push_back(std::numeric_limits::lowest()); last_ns.push_back(std::numeric_limits::lowest()); + + bool next_time_exists = pin->DoesParameterExist(block_name, "next_time"); + bool next_n_exists = pin->DoesParameterExist(block_name, "next_n"); + if (next_time_exists || next_n_exists) { + std::stringstream msg; + msg << "You have used the next_time or next_n parameter in the " << block_name + << " output block. This parameter is deprecated. Instead change" + << " the output cadence with dt or dn." << std::endl; + if (Globals::is_restart) { + if (Globals::my_rank == 0) { + PARTHENON_WARN(msg); + } + } else { + PARTHENON_THROW(msg); + } + } } } pkg->AddParam("block_names", block_names); pkg->AddParam("block_numbers", block_numbers); - pkg->AddParam("active", active, Params::Mutability::Restart); - pkg->AddParam("file_numbers", file_numbers, Params::Mutability::Restart); - pkg->AddParam("last_times", last_times, Params::Mutability::Restart); - pkg->AddParam("last_ns", last_ns, Params::Mutability::Restart); + pkg->AddParam("active", active, Params::Mutability::MaybeRestart); + pkg->AddParam("file_numbers", file_numbers, Params::Mutability::MaybeRestart); + pkg->AddParam("last_times", last_times, Params::Mutability::MaybeRestart); + pkg->AddParam("last_ns", last_ns, Params::Mutability::MaybeRestart); return pkg; } diff --git a/src/outputs/parthenon_hdf5.cpp b/src/outputs/parthenon_hdf5.cpp index 2d3d6bfcea38..09b8879b8d73 100644 --- a/src/outputs/parthenon_hdf5.cpp +++ b/src/outputs/parthenon_hdf5.cpp @@ -73,6 +73,8 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm const SignalHandler::OutputSignal signal) { using namespace HDF5; using namespace OutputUtils; + // modify HDF5 error handling to throw an error + H5Eset_auto(H5E_DEFAULT, aborting_error_handler, NULL); if constexpr (WRITE_SINGLE_PRECISION) { Kokkos::Profiling::pushRegion("PHDF5::WriteOutputFileSinglePrec"); diff --git a/src/outputs/parthenon_hdf5.hpp b/src/outputs/parthenon_hdf5.hpp index e99c58b14815..6dc3ffcb69c3 100644 --- a/src/outputs/parthenon_hdf5.hpp +++ b/src/outputs/parthenon_hdf5.hpp @@ -67,6 +67,12 @@ namespace parthenon { namespace HDF5 { +inline herr_t aborting_error_handler(hid_t stack, void *client_data) { + H5Eprint2(stack, stderr); + PARTHENON_THROW("HDF5 error detected! Erroring out\n"); + return -1; +} + hid_t GenerateFileAccessProps(); H5G MakeGroup(hid_t file, const std::string &name); diff --git a/src/outputs/restart_hdf5.cpp b/src/outputs/restart_hdf5.cpp index 8078fbd5fa9a..5e5066baf18b 100644 --- a/src/outputs/restart_hdf5.cpp +++ b/src/outputs/restart_hdf5.cpp @@ -50,6 +50,8 @@ RestartReaderHDF5::RestartReaderHDF5(const char *filename) : filename_(filename) << "is required for restarts" << std::endl; PARTHENON_FAIL(msg); #else // HDF5 enabled + // modify HDF5 error handling to throw an error + H5Eset_auto(H5E_DEFAULT, HDF5::aborting_error_handler, NULL); // Open the HDF file in read only mode fh_ = H5F::FromHIDCheck(H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT)); params_group_ = H5G::FromHIDCheck(H5Oopen(fh_, "Params", H5P_DEFAULT)); diff --git a/src/parameter_input.cpp b/src/parameter_input.cpp index 89388f1f2b63..0afcec18ad5d 100644 --- a/src/parameter_input.cpp +++ b/src/parameter_input.cpp @@ -934,6 +934,28 @@ std::string ParameterInput::SetString(const std::string &block, const std::strin return value; } +void ParameterInput::RemoveParameter(const std::string &block, const std::string &name) { + InputBlock *pb = GetPtrToBlock(block); + InputLine *plast = pb->pline; + bool deleted = false; + for (InputLine *pl = pb->pline; pl != nullptr; pl = pl->pnext) { + if (name.compare(pl->param_name) == 0) { + plast->pnext = pl->pnext; + delete pl; + deleted = true; + break; + } + plast = pl; + } + if (deleted) { + auto key = std::make_pair(block, name); + auto it = queries_.find(key); + if (it != queries_.end()) { + queries_.erase(it); + } + } +} + void ParameterInput::CheckRequired(const std::string &block, const std::string &name) { bool missing = true; if (DoesParameterExist(block, name)) { diff --git a/src/parameter_input.hpp b/src/parameter_input.hpp index eb6fc27addee..902a09ae5d67 100644 --- a/src/parameter_input.hpp +++ b/src/parameter_input.hpp @@ -275,6 +275,7 @@ class ParameterInput { SetQueryDependency_(block, name, ref); return ret; } + void RemoveParameter(const std::string &block, const std::string &name); void CheckRequired(const std::string &block, const std::string &name); void CheckDesired(const std::string &block, const std::string &name); void CheckOrphans() const; diff --git a/tst/unit/test_parameter_input.cpp b/tst/unit/test_parameter_input.cpp index d401269d9b03..648513188cca 100644 --- a/tst/unit/test_parameter_input.cpp +++ b/tst/unit/test_parameter_input.cpp @@ -176,3 +176,25 @@ TEST_CASE("Parameter inputs can be hashed and hashing provides useful sanity che } } } + +TEST_CASE("Test deleting parameters from ParameterInput", "[ParameterInput]") { + GIVEN("A ParameterInput object already populated") { + ParameterInput in; + std::stringstream ss; + ss << "" << std::endl + << "var1 = 0 # comment" << std::endl + << "" << std::endl + << "var2 = 2" + << std::endl + + std::istringstream s(ss.str()); + in.LoadFromStream(s); + + THEN("block1/var1 exists") { REQUIRE(in.DoesParameterExist("block1", "var1")); } + + WHEN("We delete a parameter") { + in.RemoveParameter("block1", "var1"); + THEN("It no longer exists") { REQUIRE(!in.DoesParameterExist("block1", "var1")); } + } + } +} From ce7ef95328dcbd3087dbc28746304c2cfece1a02 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 29 Jul 2025 10:56:56 +0200 Subject: [PATCH 092/125] Just warn for default restart Params --- src/interface/params.cpp | 22 ++++++++-------------- src/interface/params.hpp | 12 +----------- src/outputs/outputs_package.cpp | 11 +++++++---- tst/unit/test_parameter_input.cpp | 5 ++--- 4 files changed, 18 insertions(+), 32 deletions(-) diff --git a/src/interface/params.cpp b/src/interface/params.cpp index 9052d7e282a7..e1ef14afbf23 100644 --- a/src/interface/params.cpp +++ b/src/interface/params.cpp @@ -58,25 +58,19 @@ void Params::ReadFromHDF5AllParamsOfType(const std::string &prefix, const HDF5::H5G &group) { for (auto &[key, pparam] : myParams_) { if (std::type_index(pparam->type()) == std::type_index(typeid(T)) && - IsRestartable(key)) { - auto mutability = myMutable_.at(key); + GetMutability(key) == Mutability::Restart) { auto typed_ptr = std::any_cast(pparam.get()); auto &val = *typed_ptr; std::string fullpath = prefix + "/" + key; - if (mutability == Mutability::Restart) { + try { HDF5::HDF5ReadAttribute(group, fullpath, val); Update(key, val); - } else if (mutability == Mutability::MaybeRestart) { - try { - HDF5::HDF5ReadAttribute(group, fullpath, val); - Update(key, val); - } catch (std::runtime_error e) { - if (Globals::my_rank == 0) { - std::stringstream ss; - ss << "Failed to load parameter " << fullpath - << " from the restart file! Using default value." << std::endl; - PARTHENON_WARN(ss); - } + } catch (std::runtime_error e) { + if (Globals::my_rank == 0) { + std::stringstream ss; + ss << "Failed to load parameter " << fullpath + << " from the restart file! Using default value." << std::endl; + PARTHENON_WARN(ss); } } } diff --git a/src/interface/params.hpp b/src/interface/params.hpp index 8210d99dc6d6..1a19de82e7bf 100644 --- a/src/interface/params.hpp +++ b/src/interface/params.hpp @@ -39,12 +39,7 @@ class Params { // Immutable is default. Mutable is it can be updated at runtime. // Restart is a subset of mutable. Param not only can be updated at // runtime, but should be read from the restart file upon restart. - enum class Mutability : int { - Immutable = 0, - Mutable = 1, - Restart = 2, - MaybeRestart = 3 - }; + enum class Mutability : int { Immutable = 0, Mutable = 1, Restart = 2 }; Params() {} @@ -139,11 +134,6 @@ class Params { bool IsMutable(const std::string &key) const { return static_cast(myMutable_.at(key)); } - bool IsRestartable(const std::string &key) const { - auto mutability = myMutable_.at(key); - return (mutability == Mutability::Restart) || - (mutability == Mutability::MaybeRestart); - } // void Params:: void list() { diff --git a/src/outputs/outputs_package.cpp b/src/outputs/outputs_package.cpp index d202e0c277d0..d1590b85a7a5 100644 --- a/src/outputs/outputs_package.cpp +++ b/src/outputs/outputs_package.cpp @@ -77,6 +77,9 @@ std::shared_ptr Initialize(ParameterInput *pin) { << " the output cadence with dt or dn." << std::endl; if (Globals::is_restart) { if (Globals::my_rank == 0) { + msg << "The parameters will automatically be updated internally and the " + "warning should not be shown for subsequent " + "restarts.\n"; PARTHENON_WARN(msg); } } else { @@ -87,10 +90,10 @@ std::shared_ptr Initialize(ParameterInput *pin) { } pkg->AddParam("block_names", block_names); pkg->AddParam("block_numbers", block_numbers); - pkg->AddParam("active", active, Params::Mutability::MaybeRestart); - pkg->AddParam("file_numbers", file_numbers, Params::Mutability::MaybeRestart); - pkg->AddParam("last_times", last_times, Params::Mutability::MaybeRestart); - pkg->AddParam("last_ns", last_ns, Params::Mutability::MaybeRestart); + pkg->AddParam("active", active, Params::Mutability::Restart); + pkg->AddParam("file_numbers", file_numbers, Params::Mutability::Restart); + pkg->AddParam("last_times", last_times, Params::Mutability::Restart); + pkg->AddParam("last_ns", last_ns, Params::Mutability::Restart); return pkg; } diff --git a/tst/unit/test_parameter_input.cpp b/tst/unit/test_parameter_input.cpp index 648513188cca..3e901c0aaebf 100644 --- a/tst/unit/test_parameter_input.cpp +++ b/tst/unit/test_parameter_input.cpp @@ -184,10 +184,9 @@ TEST_CASE("Test deleting parameters from ParameterInput", "[ParameterInput]") { ss << "" << std::endl << "var1 = 0 # comment" << std::endl << "" << std::endl - << "var2 = 2" - << std::endl + << "var2 = 2" << std::endl; - std::istringstream s(ss.str()); + std::istringstream s(ss.str()); in.LoadFromStream(s); THEN("block1/var1 exists") { REQUIRE(in.DoesParameterExist("block1", "var1")); } From 80e3ef7b270c8360b1dadf44e7cf08a9ab017743 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 29 Jul 2025 14:20:22 +0200 Subject: [PATCH 093/125] Update restart logic. Fixes #1235 --- src/argument_parser.hpp | 6 +++--- src/globals.cpp | 5 ++--- src/globals.hpp | 1 - src/outputs/outputs.cpp | 2 +- src/outputs/outputs_package.cpp | 4 ++-- src/outputs/outputs_package.hpp | 2 +- src/parthenon_manager.cpp | 12 +++++------- src/parthenon_manager.hpp | 1 - 8 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/argument_parser.hpp b/src/argument_parser.hpp index de7027f92f99..1003d0986736 100644 --- a/src/argument_parser.hpp +++ b/src/argument_parser.hpp @@ -51,12 +51,12 @@ class ArgParse { break; case 'r': // -r invalid = invalid_arg(); - res_flag = 1; + is_restart = true; restart_filename = argv[++i]; break; case 'a': // -a invalid = invalid_arg(); - res_flag = 1; + is_restart = true; analysis_flag = true; restart_filename = argv[++i]; break; @@ -135,7 +135,7 @@ class ArgParse { char *prundir = nullptr; char *params_regex = nullptr; bool analysis_flag = false; - int res_flag = 0; + bool is_restart = false; int param_flag = 0; int mesh_flag = 0; int wtlim = 0; diff --git a/src/globals.cpp b/src/globals.cpp index 110f66f75d64..2796509b7edc 100644 --- a/src/globals.cpp +++ b/src/globals.cpp @@ -32,9 +32,8 @@ namespace Globals { int nghost; // all of these global variables are set at the start of main(): -int my_rank; // MPI rank of this process -int nranks; // total number of MPI ranks -bool is_restart; // Whether this simulation is restarted from a checkpoint file +int my_rank; // MPI rank of this process +int nranks; // total number of MPI ranks // sparse configuration values that are needed in various places SparseConfig sparse_config; diff --git a/src/globals.hpp b/src/globals.hpp index e6451dadab7c..f2f18fa4d3a9 100644 --- a/src/globals.hpp +++ b/src/globals.hpp @@ -38,7 +38,6 @@ struct SparseConfig { }; extern int my_rank, nranks, nghost; -extern bool is_restart; extern SparseConfig sparse_config; diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index 9996de5bd575..4035c8b1425c 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -140,7 +140,7 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { // JMM: Backwards compatibility hack. Don't allow this unless // we're restarting from a legacy file format. - if (Globals::is_restart) { // should this be pmesh->is_restart? + if (pm->is_restart) { bool next_time_exists = pin->DoesParameterExist(op.block_name, "next_time"); bool next_n_exists = pin->DoesParameterExist(op.block_name, "next_n"); if (next_time_exists) { diff --git a/src/outputs/outputs_package.cpp b/src/outputs/outputs_package.cpp index d1590b85a7a5..b51bd7163b07 100644 --- a/src/outputs/outputs_package.cpp +++ b/src/outputs/outputs_package.cpp @@ -34,7 +34,7 @@ namespace parthenon { namespace OutputsPackage { -std::shared_ptr Initialize(ParameterInput *pin) { +std::shared_ptr Initialize(ParameterInput *pin, const bool is_restart) { auto pkg = std::make_shared("Outputs"); std::string basename = pin->GetOrAddString("parthenon/job", "problem_id", "parthenon", @@ -75,7 +75,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { msg << "You have used the next_time or next_n parameter in the " << block_name << " output block. This parameter is deprecated. Instead change" << " the output cadence with dt or dn." << std::endl; - if (Globals::is_restart) { + if (is_restart) { if (Globals::my_rank == 0) { msg << "The parameters will automatically be updated internally and the " "warning should not be shown for subsequent " diff --git a/src/outputs/outputs_package.hpp b/src/outputs/outputs_package.hpp index 852fc3b54ad7..2b58e0ec7f7d 100644 --- a/src/outputs/outputs_package.hpp +++ b/src/outputs/outputs_package.hpp @@ -31,7 +31,7 @@ class StateDescriptor; namespace OutputsPackage { -std::shared_ptr Initialize(ParameterInput *pin); +std::shared_ptr Initialize(ParameterInput *pin, bool is_restart); } // namespace OutputsPackage } // namespace parthenon diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index 2ff3486d24c1..3c46ce617ff7 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -72,8 +72,6 @@ ParthenonStatus ParthenonManager::ParthenonInitEnv(int argc, char *argv[]) { Globals::nranks = 1; #endif // MPI_PARALLEL - Globals::is_restart = IsRestart(); - Kokkos::initialize(argc, argv); // pgrete: This is a hack to disable allocation tracking until the Kokkos @@ -98,7 +96,7 @@ ParthenonStatus ParthenonManager::ParthenonInitEnv(int argc, char *argv[]) { // Populate the ParameterInput object. // If restart, then ParameterInput in the restart file takes precedence. - if (arg.res_flag != 0) { + if (arg.is_restart) { // Read input from restart file if (fs::path(arg.restart_filename).extension() == ".rhdf") { restartReader = std::make_unique(arg.restart_filename); @@ -115,7 +113,7 @@ ParthenonStatus ParthenonManager::ParthenonInitEnv(int argc, char *argv[]) { // If an input file was provided if (arg.input_filename != nullptr) { // Modify info read from restart file - if (arg.res_flag != 0) { + if (arg.is_restart) { IOWrapper infile; infile.Open(arg.input_filename, IOWrapper::FileMode::read); pinput->LoadFromFile(infile); @@ -186,10 +184,10 @@ void ParthenonManager::ParthenonInitPackagesAndMesh( auto packages = ProcessPackages(pinput); // always add the Refinement package packages.Add(Refinement::Initialize(pinput.get())); - packages.Add(OutputsPackage::Initialize(pinput.get())); + packages.Add(OutputsPackage::Initialize(pinput.get(), arg.is_restart)); if (forest_def) { pmesh = std::make_unique(pinput.get(), app_input.get(), packages, *forest_def); - } else if (arg.res_flag == 0) { + } else if (!arg.is_restart) { pmesh = std::make_unique(pinput.get(), app_input.get(), packages, arg.mesh_flag); } else { @@ -234,7 +232,7 @@ void ParthenonManager::ParthenonInitPackagesAndMesh( pinput->SetString("parthenon/job", "output_params_block_regex", arg.params_regex); } - pmesh->Initialize(!IsRestart(), pinput.get(), app_input.get()); + pmesh->Initialize(!arg.is_restart, pinput.get(), app_input.get()); ChangeRunDir(arg.prundir); } diff --git a/src/parthenon_manager.hpp b/src/parthenon_manager.hpp index 2f05f671b1b0..1c9d54bcc2e4 100644 --- a/src/parthenon_manager.hpp +++ b/src/parthenon_manager.hpp @@ -44,7 +44,6 @@ class ParthenonManager { ParthenonInitPackagesAndMesh(std::optional forest_def = {}); ParthenonStatus ParthenonFinalize(); - bool IsRestart() { return (arg.restart_filename == nullptr ? false : true); } static Packages_t ProcessPackagesDefault(std::unique_ptr &pin); void RestartPackages(Mesh &rm, RestartReader &resfile); From f1339627fe7fe468cfeaf05f1e8f26fd35a2f4d4 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 29 Jul 2025 15:43:20 +0200 Subject: [PATCH 094/125] Only warn about default non-input params --- src/parameter_input.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/parameter_input.hpp b/src/parameter_input.hpp index 902a09ae5d67..cdcaaf835b42 100644 --- a/src/parameter_input.hpp +++ b/src/parameter_input.hpp @@ -457,11 +457,12 @@ class ParameterInput { // the new default and move on. record.default_value = defval.value(); record.default_value_str = record.ToString(defval.value()); - } else { + } else if (record.origin_type != QueryRecord::OriginType::Input) { // JMM: Forbid setting a default value after requesting but // allow requesting without a default if a default has // already been set. I know this is unpleasantly stateful, // but we do this in a few places in the code. + // PG: but only trigger if input does not contain the info std::stringstream msg; msg << "Input parameter " << block << "/" << name << " called previously without a default value and now called with one." From fbfd3568ea7f0889244f978035da323a0cb6a49c Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 29 Jul 2025 16:03:44 +0200 Subject: [PATCH 095/125] Ensure last output is not in the future --- src/outputs/outputs.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index 4035c8b1425c..c3cd7bca5e6e 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -145,12 +145,13 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { bool next_n_exists = pin->DoesParameterExist(op.block_name, "next_n"); if (next_time_exists) { Real next_time = pin->GetReal(op.block_name, "next_time"); - (*plast_times)[iinput] = next_time - dt; + (*plast_times)[iinput] = dt < 0 ? 0.0 : next_time - dt; pin->RemoveParameter(op.block_name, "next_time"); } if (next_n_exists) { int next_n = pin->GetInteger(op.block_name, "next_n"); - (*plast_ns)[iinput] = next_n - dn; + + (*plast_ns)[iinput] = dn < 0 ? 0 : next_n - dn; pin->RemoveParameter(op.block_name, "next_n"); } if (next_time_exists || next_n_exists) { From c7f8ab5e2aaa22e58bcaa5ffb4e13107d91ed48d Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 30 Jul 2025 10:27:11 +0200 Subject: [PATCH 096/125] remove pmesh->is_restart for Globals --- doc/sphinx/src/interface/state.rst | 4 ++-- example/advection/advection_package.cpp | 2 +- src/argument_parser.hpp | 4 ++++ src/globals.cpp | 9 +++++---- src/globals.hpp | 5 +++-- src/mesh/mesh.cpp | 3 +-- src/mesh/mesh.hpp | 1 - src/outputs/outputs.cpp | 6 +++--- src/outputs/outputs_package.cpp | 4 ++-- src/outputs/outputs_package.hpp | 2 +- src/parthenon_manager.cpp | 4 +++- 11 files changed, 25 insertions(+), 19 deletions(-) diff --git a/doc/sphinx/src/interface/state.rst b/doc/sphinx/src/interface/state.rst index 9bd5ab92e0c0..0e17247f8147 100644 --- a/doc/sphinx/src/interface/state.rst +++ b/doc/sphinx/src/interface/state.rst @@ -121,8 +121,8 @@ several useful features and functions. has been generated, (2) problem generators are called, and (3) comms are executed, but before any time evolution. This work is done both on first initialization and on restart. If you would like to avoid doing the - work upon restart, you can check for the const ``is_restart`` member - field of the ``Mesh`` object. It is worth making a clear distinction + work upon restart, you can check the ``Globals::is_restart`` variable. + It is worth making a clear distinction between ``UserWorkBeforeLoopMesh`` and ``ApplicationInput``s ``PostInitialization``. ``PostInitialization`` is very much so tied to initialization, and will not be called upon restarts. ``PostInitialization`` diff --git a/example/advection/advection_package.cpp b/example/advection/advection_package.cpp index 8d02cab9d887..d8a0f444da49 100644 --- a/example/advection/advection_package.cpp +++ b/example/advection/advection_package.cpp @@ -249,7 +249,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { void AdvectionGreetings(Mesh *pmesh, ParameterInput *pin, parthenon::SimTime &tm) { if (parthenon::Globals::my_rank == 0) { std::cout << "Hello from the advection package in the advection example!\n" - << "This run is a restart: " << pmesh->is_restart << "\n" + << "This run is a restart: " << parthenon::Globals::is_restart << "\n" << std::endl; } } diff --git a/src/argument_parser.hpp b/src/argument_parser.hpp index 1003d0986736..477058b87c10 100644 --- a/src/argument_parser.hpp +++ b/src/argument_parser.hpp @@ -1,4 +1,8 @@ //======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2021-2025 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== // (C) (or copyright) 2021-2025. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los diff --git a/src/globals.cpp b/src/globals.cpp index 2796509b7edc..b7d6c77e6ee9 100644 --- a/src/globals.cpp +++ b/src/globals.cpp @@ -1,6 +1,6 @@ //======================================================================================== -// Athena++ astrophysical MHD code -// Copyright(C) 2014 James M. Stone and other code contributors +// Parthenon performance portable AMR framework +// Copyright(C) 2020-2025 The Parthenon collaboration // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== // (C) (or copyright) 2020-2025. Triad National Security, LLC. All rights reserved. @@ -32,8 +32,9 @@ namespace Globals { int nghost; // all of these global variables are set at the start of main(): -int my_rank; // MPI rank of this process -int nranks; // total number of MPI ranks +int my_rank; // MPI rank of this process +int nranks; // total number of MPI ranks +bool is_restart; // Whether this simulation is restarted from a checkpoint file // sparse configuration values that are needed in various places SparseConfig sparse_config; diff --git a/src/globals.hpp b/src/globals.hpp index f2f18fa4d3a9..6a07de0a21ea 100644 --- a/src/globals.hpp +++ b/src/globals.hpp @@ -1,6 +1,6 @@ //======================================================================================== -// Athena++ astrophysical MHD code -// Copyright(C) 2014 James M. Stone and other code contributors +// Parthenon performance portable AMR framework +// Copyright(C) 2020-2025 The Parthenon collaboration // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== // (C) (or copyright) 2020-2025. Triad National Security, LLC. All rights reserved. @@ -38,6 +38,7 @@ struct SparseConfig { }; extern int my_rank, nranks, nghost; +extern bool is_restart; extern SparseConfig sparse_config; diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index eef1420fa395..7b4b9a32afbc 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -64,7 +64,7 @@ namespace parthenon { Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages, base_constructor_selector_t) : // public members: - modified(true), is_restart(false), + modified(true), adaptive(pin->GetOrAddString("parthenon/mesh", "refinement", "none", std::vector{"none", "static", "adaptive"}, "mesh refinement mode") == "adaptive" @@ -246,7 +246,6 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages, Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr, Packages_t &packages, int mesh_test) : Mesh(pin, app_in, packages, hyper_rectangular_constructor_selector_t()) { - is_restart = true; std::stringstream msg; // mesh test diff --git a/src/mesh/mesh.hpp b/src/mesh/mesh.hpp index 11a96bad3baa..589e815d0ba6 100644 --- a/src/mesh/mesh.hpp +++ b/src/mesh/mesh.hpp @@ -121,7 +121,6 @@ class Mesh { // data bool modified; - bool is_restart; RegionSize mesh_size; RegionSize base_block_size; diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index c3cd7bca5e6e..e2aeaf3eea61 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -140,7 +140,7 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { // JMM: Backwards compatibility hack. Don't allow this unless // we're restarting from a legacy file format. - if (pm->is_restart) { + if (parthenon::Globals::is_restart) { bool next_time_exists = pin->DoesParameterExist(op.block_name, "next_time"); bool next_n_exists = pin->DoesParameterExist(op.block_name, "next_n"); if (next_time_exists) { @@ -185,7 +185,7 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { signaling number. However, I think the flag is less fraught. */ if (dt >= 0) { - // TODO(JMM): Should this be a check for pmesh->is_restart instead? + // TODO(JMM): Should this be a check for Globals::is_restart instead? if (op.last_time > std::numeric_limits::lowest()) { op.next_time = op.last_time + dt; } else { @@ -193,7 +193,7 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { } } if (dn >= 0) { - // TODO(JMM): Should this be a check for pmesh->is_restart instead? + // TODO(JMM): Should this be a check for Globals::is_restart instead? if (op.last_n > std::numeric_limits::lowest()) { op.next_n = op.last_n + dn; } else { diff --git a/src/outputs/outputs_package.cpp b/src/outputs/outputs_package.cpp index b51bd7163b07..801b7045f283 100644 --- a/src/outputs/outputs_package.cpp +++ b/src/outputs/outputs_package.cpp @@ -34,7 +34,7 @@ namespace parthenon { namespace OutputsPackage { -std::shared_ptr Initialize(ParameterInput *pin, const bool is_restart) { +std::shared_ptr Initialize(ParameterInput *pin) { auto pkg = std::make_shared("Outputs"); std::string basename = pin->GetOrAddString("parthenon/job", "problem_id", "parthenon", @@ -75,7 +75,7 @@ std::shared_ptr Initialize(ParameterInput *pin, const bool is_r msg << "You have used the next_time or next_n parameter in the " << block_name << " output block. This parameter is deprecated. Instead change" << " the output cadence with dt or dn." << std::endl; - if (is_restart) { + if (parthenon::Globals::is_restart) { if (Globals::my_rank == 0) { msg << "The parameters will automatically be updated internally and the " "warning should not be shown for subsequent " diff --git a/src/outputs/outputs_package.hpp b/src/outputs/outputs_package.hpp index 2b58e0ec7f7d..852fc3b54ad7 100644 --- a/src/outputs/outputs_package.hpp +++ b/src/outputs/outputs_package.hpp @@ -31,7 +31,7 @@ class StateDescriptor; namespace OutputsPackage { -std::shared_ptr Initialize(ParameterInput *pin, bool is_restart); +std::shared_ptr Initialize(ParameterInput *pin); } // namespace OutputsPackage } // namespace parthenon diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index 3c46ce617ff7..58a6b8876cce 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -89,6 +89,8 @@ ParthenonStatus ParthenonManager::ParthenonInitEnv(int argc, char *argv[]) { } else if (arg_status == ArgStatus::complete) { return ParthenonStatus::complete; } + // Now that the input is parsed we can pass the info to globals + Globals::is_restart = arg.is_restart; // Set up the signal handler SignalHandler::SignalHandlerInit(); @@ -184,7 +186,7 @@ void ParthenonManager::ParthenonInitPackagesAndMesh( auto packages = ProcessPackages(pinput); // always add the Refinement package packages.Add(Refinement::Initialize(pinput.get())); - packages.Add(OutputsPackage::Initialize(pinput.get(), arg.is_restart)); + packages.Add(OutputsPackage::Initialize(pinput.get())); if (forest_def) { pmesh = std::make_unique(pinput.get(), app_input.get(), packages, *forest_def); } else if (!arg.is_restart) { From dd94417e8e9291d2fd04b965191e6b64f5e6f4f1 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 30 Jul 2025 12:02:13 +0200 Subject: [PATCH 097/125] Allow writing legacy id fields for particles --- src/outputs/parthenon_hdf5.cpp | 8 +++++--- src/outputs/parthenon_xdmf.cpp | 7 ++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/outputs/parthenon_hdf5.cpp b/src/outputs/parthenon_hdf5.cpp index 09b8879b8d73..e3c272b597e0 100644 --- a/src/outputs/parthenon_hdf5.cpp +++ b/src/outputs/parthenon_hdf5.cpp @@ -585,9 +585,11 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm local_count, global_count, pl_xfer, H5P_DEFAULT); } - // If swarm does not contain the default id object, generate a sequential - // one for vis called "id" (to differentiate between the default one) - if (swinfo.var_info.count(swarm_position::id::name()) == 0) { + // If swarm does not contain the default id object nor a custom one called "id", + // generate a sequential one for vis called "id" (to differentiate between the default + // one) + if (swinfo.var_info.count(swarm_position::id::name()) == 0 && + swinfo.var_info.count("id") == 0) { std::vector ids(swinfo.global_count); std::iota(std::begin(ids), std::end(ids), swinfo.global_offset); local_offset[0] = swinfo.global_offset; diff --git a/src/outputs/parthenon_xdmf.cpp b/src/outputs/parthenon_xdmf.cpp index fc3a09ae5f5a..27be6b686cc9 100644 --- a/src/outputs/parthenon_xdmf.cpp +++ b/src/outputs/parthenon_xdmf.cpp @@ -249,9 +249,10 @@ void genXDMF(std::string hdfFile, Mesh *pm, SimTime *tm, IndexDomain domain, int ParticleVariableRef(pxdmf, varname, varinfo, swmname, hdfFile, swminfo.global_count); } - // Write info for default (auto-generated) "id"s in case there's not native id field - // for the given swarm. - if (swminfo.var_info.count(swarm_position::id::name()) == 0) { + // Write info for default (auto-generated) "id"s in case there's no native id field + // for the given swarm nor a custom "id" field. + if (swminfo.var_info.count(swarm_position::id::name()) == 0 && + swminfo.var_info.count("id") == 0) { auto swid = SwarmVarInfo(1, 1, 1, 1, 1, 0, "Int", false); ParticleVariableRef(pxdmf, "id", swid, swmname, hdfFile, swminfo.global_count); } From 9d90214a186d1660e99ca575aa685d73d9d1361a Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 30 Jul 2025 13:27:25 +0200 Subject: [PATCH 098/125] Allow empty restart swarms --- src/outputs/restart.hpp | 2 ++ src/outputs/restart_hdf5.hpp | 4 ++++ src/parthenon_manager.cpp | 8 +++++++- src/parthenon_manager.hpp | 1 + 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/outputs/restart.hpp b/src/outputs/restart.hpp index 4f41652fc2eb..022c4adf684d 100644 --- a/src/outputs/restart.hpp +++ b/src/outputs/restart.hpp @@ -130,6 +130,8 @@ class RestartReader { virtual void ReadParams(const std::string &name, Params &p) = 0; + [[nodiscard]] virtual bool VariableExists(const std::string &name) const = 0; + // closes out the restart file // perhaps belongs in a destructor? void Close(); diff --git a/src/outputs/restart_hdf5.hpp b/src/outputs/restart_hdf5.hpp index 482e17a361a6..4eaa9e9b80e5 100644 --- a/src/outputs/restart_hdf5.hpp +++ b/src/outputs/restart_hdf5.hpp @@ -228,6 +228,10 @@ class RestartReaderHDF5 : public RestartReader { void ReadParams(const std::string &name, Params &p) override; + [[nodiscard]] bool VariableExists(const std::string &name) const override { + // make sure dataset exists + return PARTHENON_HDF5_CHECK(H5Oexists_by_name(fh_, name.c_str(), H5P_DEFAULT)); + } // closes out the restart file // perhaps belongs in a destructor? void Close(); diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index 58a6b8876cce..c340ea045dde 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -392,8 +392,14 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { auto swarms = (mb.meshblock_data.Get()->GetSwarmData())->GetSwarmsByFlag(flags); for (auto &swarm : swarms) { auto swarmname = swarm->label(); + auto var_missing_on_disk = !resfile.VariableExists(swarmname); if (Globals::my_rank == 0) { - std::cout << "Swarm: " << swarmname << std::endl; + std::cout << "Swarm: " << swarmname + << (var_missing_on_disk ? " missing on disk\n" : "\n"); + } + if (var_missing_on_disk) { + // TODO(JMM/PG) Add failed load list of "fail/needs fix" list + continue; } std::vector counts, offsets; std::size_t count_on_rank = diff --git a/src/parthenon_manager.hpp b/src/parthenon_manager.hpp index 1c9d54bcc2e4..25995a0c3916 100644 --- a/src/parthenon_manager.hpp +++ b/src/parthenon_manager.hpp @@ -81,6 +81,7 @@ class ParthenonManager { Globals::my_rank, swarmname.c_str(), varname.c_str(), ex.what()) << std::endl; + // TODO(JMM/PG) Add failed load list of "fail/needs fix" list continue; } From 1988ed255d2aa61e8a950a53bb147c10d59be3be Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 30 Jul 2025 13:56:09 +0200 Subject: [PATCH 099/125] Unify missing var handling --- src/parthenon_manager.cpp | 19 ++++++++++++++----- src/parthenon_manager.hpp | 25 +++++++++++++++++-------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index c340ea045dde..c7f0f6eb2d03 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -334,17 +335,25 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { const auto fill_size = v_info.FillSize(theDomain); const auto &label = v_info.label; + auto var_missing_on_disk = !resfile.VariableExists(label); if (Globals::my_rank == 0) { - std::cout << "Var: " << label << ":" << vlen << std::endl; + std::cout << "Var: " << label << ":" << vlen + << (var_missing_on_disk ? " missing on disk\n" : "\n"); + } + if (var_missing_on_disk) { + // TODO(JMM/PG) Add failed load list of "fail/needs fix" list + continue; } // Read relevant data from the hdf file, this works for dense and sparse variables try { resfile.ReadBlocks(label, myBlocks, v_info, tmp, file_output_format_ver); + // Variable does exist but could not be read. So we definitely want to fail here. } catch (std::exception &ex) { - std::cout << "[" << Globals::my_rank << "] WARNING: Failed to read variable " - << label << " from restart file:" << std::endl - << ex.what() << std::endl; - continue; + std::stringstream msg; + msg << "[" << Globals::my_rank << "] WARNING: Failed to read variable " << label + << " from restart file:" << std::endl + << ex.what() << std::endl; + PARTHENON_THROW(msg); } size_t index = 0; diff --git a/src/parthenon_manager.hpp b/src/parthenon_manager.hpp index 25995a0c3916..6c8e0d4f01d3 100644 --- a/src/parthenon_manager.hpp +++ b/src/parthenon_manager.hpp @@ -30,6 +30,7 @@ #include "mesh/mesh.hpp" #include "outputs/restart.hpp" #include "parameter_input.hpp" +#include "utils/error_checking.hpp" #include "utils/utils.hpp" namespace parthenon { @@ -68,21 +69,29 @@ class ParthenonManager { std::vector dataVec; for (const auto &var : pswarm->GetVariableVector()) { const std::string &varname = var->label(); - std::cout << "SwarmVar: " << varname << std::endl; const auto &m = var->metadata(); auto arrdims = m.GetArrayDims(pswarm->GetBlockPointer(), false); + auto var_missing_on_disk = + !restartReader->VariableExists(swarmname + "/SwarmVars/" + varname); + if (Globals::my_rank == 0) { + std::cout << "SwarmVar: " << varname + << (var_missing_on_disk ? " missing on disk\n" : "\n"); + } + if (var_missing_on_disk) { + // TODO(JMM/PG) Add failed load list of "fail/needs fix" list + continue; + } + try { restartReader->ReadSwarmVar(swarmname, varname, count_on_rank, offset, m, dataVec); } catch (std::exception &ex) { - std::cout << StringPrintf("[%d] WARNING: Failed to read Swarm %s Variable %s " - "from restart file:\n%s", - Globals::my_rank, swarmname.c_str(), varname.c_str(), - ex.what()) - << std::endl; - // TODO(JMM/PG) Add failed load list of "fail/needs fix" list - continue; + // Variable does exist but could not be read. So we definitely want to fail here. + PARTHENON_THROW(StringPrintf("[%d] WARNING: Failed to read Swarm %s Variable %s " + "from restart file:\n%s", + Globals::my_rank, swarmname.c_str(), varname.c_str(), + ex.what())); } // Only safe because swarm starts completely defragged. From 38b1ec5dd566cfa96bfe1bb3a0c49a140ddadc00 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 30 Jul 2025 14:03:05 +0200 Subject: [PATCH 100/125] Add reminder todo to params --- src/interface/params.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/interface/params.cpp b/src/interface/params.cpp index e1ef14afbf23..5d4d2ad04ffb 100644 --- a/src/interface/params.cpp +++ b/src/interface/params.cpp @@ -66,6 +66,7 @@ void Params::ReadFromHDF5AllParamsOfType(const std::string &prefix, HDF5::HDF5ReadAttribute(group, fullpath, val); Update(key, val); } catch (std::runtime_error e) { + // TODO(JMM/PG) Add failed load list of "fail/needs fix" list if (Globals::my_rank == 0) { std::stringstream ss; ss << "Failed to load parameter " << fullpath From b3849d43dee118f672b62cd31c6c784b0691a3fb Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 30 Jul 2025 14:23:50 +0200 Subject: [PATCH 101/125] Add Changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 012f4369da00..b32abff6b74b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ - [[PR 1280]](https://github.com/parthenon-hpc-lab/parthenon/pull/1280) Print history file headers on restart ### Fixed (not changing behavior/API/variables/...) +- [[PR 1297]](https://github.com/parthenon-hpc-lab/parthenon/pull/1297) Backward compatibility fixes (`last_/next_` output package, restart with new `Restart` vars, `is_restart`) +- [[PR 1289]](https://github.com/parthenon-hpc-lab/parthenon/pull/1289) Fix a bug in 1214 - [[PR 1291]](https://github.com/parthenon-hpc-lab/parthenon/pull/1291) Fix provenance for downstream codes - [[PR 1289]](https://github.com/parthenon-hpc-lab/parthenon/pull/1289) Fix a bug in 1214 - [[PR 1214]](https://github.com/parthenon-hpc-lab/parthenon/pull/1214) Initialize MPI in catch2 to prevent errors when constructing Meshes @@ -37,6 +39,7 @@ ### Incompatibilities (i.e. breaking changes) - [[PR 1253]](https://github.com/parthenon-hpc-lab/parthenon/pull/1253) Add support for uint64 swarm variables and add default id +- [[PR 1297]](https://github.com/parthenon-hpc-lab/parthenon/pull/1297) `pmesh->is_restart` removed in favor of `Globals::is_restart` From 51f45b754764ec1bbade8438449816bf8c25614f Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 30 Jul 2025 10:10:01 -0400 Subject: [PATCH 102/125] change check orphans to default to false for restarts --- src/driver/driver.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/driver/driver.cpp b/src/driver/driver.cpp index e16bec44f760..c8587bb80d56 100644 --- a/src/driver/driver.cpp +++ b/src/driver/driver.cpp @@ -68,8 +68,9 @@ void Driver::DumpInputParameters() { void Driver::PreExecute() { bool check_orphans = pinput->GetOrAddBoolean( - "parthenon/job", "check_orphans", true, - "print a warning if any parameters are in the input deck but not used in the code"); + "parthenon/job", "check_orphans", !Globals::is_restart, + "Print a warning if any parameters are in the input deck but not used in the code. " + "Defaults to true for new runs and false for restarts"); // Output a text file of all parameters at this point // Optionally also dump to console DumpInputParameters(); From dcc025dec1dbb499533494de5ca0240a2832ed0a Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 30 Jul 2025 10:14:01 -0400 Subject: [PATCH 103/125] more careful logic --- src/driver/driver.cpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/driver/driver.cpp b/src/driver/driver.cpp index c8587bb80d56..906b9d7e50e5 100644 --- a/src/driver/driver.cpp +++ b/src/driver/driver.cpp @@ -67,17 +67,25 @@ void Driver::DumpInputParameters() { } void Driver::PreExecute() { - bool check_orphans = pinput->GetOrAddBoolean( - "parthenon/job", "check_orphans", !Globals::is_restart, - "Print a warning if any parameters are in the input deck but not used in the code. " - "Defaults to true for new runs and false for restarts"); + bool check_orphans = + pinput->GetOrAddBoolean("parthenon/job", "check_orphans", true, + "Print a warning if any parameters are in the input deck " + "but not used in the code. Only active for new runs."); + bool force_check_orphans = + pinput->GetOrAddBoolean("parthenon/job", "force_check_orphans", false, + "Print a warning if any parameters are in the input deck " + "but not used in the code. Forces this output on restart."); // Output a text file of all parameters at this point // Optionally also dump to console DumpInputParameters(); if (Globals::my_rank == 0) { - if (check_orphans) pinput->CheckOrphans(); - std::cout << "# Variables in use:\n" << *(pmesh->resolved_packages) << std::endl; + if ((Globals::is_restart && force_check_orphans) || + (!Globals::is_restart && check_orphans)) { + pinput->CheckOrphans(); + } + if (check_orphans && !Globals::is_restart) + std::cout << "# Variables in use:\n" << *(pmesh->resolved_packages) << std::endl; std::cout << std::endl; std::cout << "Setup complete, executing driver...\n" << std::endl; } From 04f0cab489e39416b116effb17a6f8e0465a769a Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 30 Jul 2025 10:17:24 -0400 Subject: [PATCH 104/125] oops added an if --- src/driver/driver.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/driver/driver.cpp b/src/driver/driver.cpp index 906b9d7e50e5..9fc984791616 100644 --- a/src/driver/driver.cpp +++ b/src/driver/driver.cpp @@ -71,21 +71,19 @@ void Driver::PreExecute() { pinput->GetOrAddBoolean("parthenon/job", "check_orphans", true, "Print a warning if any parameters are in the input deck " "but not used in the code. Only active for new runs."); - bool force_check_orphans = - pinput->GetOrAddBoolean("parthenon/job", "force_check_orphans", false, - "Print a warning if any parameters are in the input deck " - "but not used in the code. Forces this output on restart."); + bool force_check_orphans = pinput->GetOrAddBoolean( + "parthenon/job", "force_check_orphans", false, + "Print a warning if any parameters are in the input deck " + "but not used in the code. Forces this output even on restart."); // Output a text file of all parameters at this point // Optionally also dump to console DumpInputParameters(); if (Globals::my_rank == 0) { - if ((Globals::is_restart && force_check_orphans) || - (!Globals::is_restart && check_orphans)) { + if (force_check_orphans || (!Globals::is_restart && check_orphans)) { pinput->CheckOrphans(); } - if (check_orphans && !Globals::is_restart) - std::cout << "# Variables in use:\n" << *(pmesh->resolved_packages) << std::endl; + std::cout << "# Variables in use:\n" << *(pmesh->resolved_packages) << std::endl; std::cout << std::endl; std::cout << "Setup complete, executing driver...\n" << std::endl; } From 3ab53530b34f7ff414c0934dce7c864c6515b3d7 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 30 Jul 2025 10:31:35 -0400 Subject: [PATCH 105/125] ok try this --- src/driver/driver.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/driver/driver.cpp b/src/driver/driver.cpp index 9fc984791616..218fedc81507 100644 --- a/src/driver/driver.cpp +++ b/src/driver/driver.cpp @@ -67,20 +67,19 @@ void Driver::DumpInputParameters() { } void Driver::PreExecute() { - bool check_orphans = - pinput->GetOrAddBoolean("parthenon/job", "check_orphans", true, - "Print a warning if any parameters are in the input deck " - "but not used in the code. Only active for new runs."); - bool force_check_orphans = pinput->GetOrAddBoolean( - "parthenon/job", "force_check_orphans", false, - "Print a warning if any parameters are in the input deck " - "but not used in the code. Forces this output even on restart."); + std::string check_orphans = pinput->GetOrAddString( + "parthenon/job", "check_orphans", "initially", + std::vector{"always", "initially", "never"}, + "Print a warning if any parameters are in the input deck but not used in the code. " + "By default this check is performed for new runs, but can also be enabled for " + "restarts or completely disabled."); // Output a text file of all parameters at this point // Optionally also dump to console DumpInputParameters(); if (Globals::my_rank == 0) { - if (force_check_orphans || (!Globals::is_restart && check_orphans)) { + if ((check_orphans == "always") || + (!Globals::is_restart && (check_orphans == "initially"))) { pinput->CheckOrphans(); } std::cout << "# Variables in use:\n" << *(pmesh->resolved_packages) << std::endl; From 11a9884030b4ca888e696c2e277f969575aa4433 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 31 Jul 2025 12:04:27 +0200 Subject: [PATCH 106/125] Fix removing first elemn in linked list --- src/parameter_input.cpp | 7 ++++++- src/parthenon_manager.cpp | 1 + tst/unit/test_parameter_input.cpp | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/parameter_input.cpp b/src/parameter_input.cpp index 0afcec18ad5d..6c4ae5c7db19 100644 --- a/src/parameter_input.cpp +++ b/src/parameter_input.cpp @@ -940,7 +940,12 @@ void ParameterInput::RemoveParameter(const std::string &block, const std::string bool deleted = false; for (InputLine *pl = pb->pline; pl != nullptr; pl = pl->pnext) { if (name.compare(pl->param_name) == 0) { - plast->pnext = pl->pnext; + // if head of list + if (plast == pb->pline) { + pb->pline = pl->pnext; + } else { + plast->pnext = pl->pnext; + } delete pl; deleted = true; break; diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index c7f0f6eb2d03..f58258c4704a 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -345,6 +345,7 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { continue; } // Read relevant data from the hdf file, this works for dense and sparse variables + // because sparse variables are currently densely written for HDF5. try { resfile.ReadBlocks(label, myBlocks, v_info, tmp, file_output_format_ver); // Variable does exist but could not be read. So we definitely want to fail here. diff --git a/tst/unit/test_parameter_input.cpp b/tst/unit/test_parameter_input.cpp index 3e901c0aaebf..df9d7ddc4e0f 100644 --- a/tst/unit/test_parameter_input.cpp +++ b/tst/unit/test_parameter_input.cpp @@ -183,6 +183,7 @@ TEST_CASE("Test deleting parameters from ParameterInput", "[ParameterInput]") { std::stringstream ss; ss << "" << std::endl << "var1 = 0 # comment" << std::endl + << "var2 = 0 # comment" << std::endl << "" << std::endl << "var2 = 2" << std::endl; @@ -194,6 +195,7 @@ TEST_CASE("Test deleting parameters from ParameterInput", "[ParameterInput]") { WHEN("We delete a parameter") { in.RemoveParameter("block1", "var1"); THEN("It no longer exists") { REQUIRE(!in.DoesParameterExist("block1", "var1")); } + THEN("And others still do") { REQUIRE(in.DoesParameterExist("block1", "var2")); } } } } From 43fcf782d25c4050a8e49ddc1c57b330d8ddd04b Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 31 Jul 2025 12:17:15 +0200 Subject: [PATCH 107/125] Add missing guards --- CHANGELOG.md | 1 - src/outputs/restart_hdf5.hpp | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b32abff6b74b..59b6931022bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,6 @@ ### Fixed (not changing behavior/API/variables/...) - [[PR 1297]](https://github.com/parthenon-hpc-lab/parthenon/pull/1297) Backward compatibility fixes (`last_/next_` output package, restart with new `Restart` vars, `is_restart`) -- [[PR 1289]](https://github.com/parthenon-hpc-lab/parthenon/pull/1289) Fix a bug in 1214 - [[PR 1291]](https://github.com/parthenon-hpc-lab/parthenon/pull/1291) Fix provenance for downstream codes - [[PR 1289]](https://github.com/parthenon-hpc-lab/parthenon/pull/1289) Fix a bug in 1214 - [[PR 1214]](https://github.com/parthenon-hpc-lab/parthenon/pull/1214) Initialize MPI in catch2 to prevent errors when constructing Meshes diff --git a/src/outputs/restart_hdf5.hpp b/src/outputs/restart_hdf5.hpp index 4eaa9e9b80e5..3b1c8bb92c9c 100644 --- a/src/outputs/restart_hdf5.hpp +++ b/src/outputs/restart_hdf5.hpp @@ -229,8 +229,13 @@ class RestartReaderHDF5 : public RestartReader { void ReadParams(const std::string &name, Params &p) override; [[nodiscard]] bool VariableExists(const std::string &name) const override { +#ifdef ENABLE_HDF5 // make sure dataset exists return PARTHENON_HDF5_CHECK(H5Oexists_by_name(fh_, name.c_str(), H5P_DEFAULT)); +#else + PARTHENON_FAIL("Restart functionality is not available because HDF5 is disabled"); + return false; +#endif // ENABLE_HDF5 } // closes out the restart file // perhaps belongs in a destructor? From fcc4a5b54e173d7c512b311d9a37af40de237bb3 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 1 Aug 2025 09:40:33 +0200 Subject: [PATCH 108/125] Fix interface --- src/interface/params.hpp | 1 - src/outputs/parthenon_opmd.cpp | 2 +- src/outputs/restart_opmd.hpp | 6 +++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/interface/params.hpp b/src/interface/params.hpp index a2b14115ca3b..a8c7a55e2f8d 100644 --- a/src/interface/params.hpp +++ b/src/interface/params.hpp @@ -136,7 +136,6 @@ class Params { return keys; } - auto GetMutability(const std::string &key) const { return myMutable_.at(key); } bool IsMutable(const std::string &key) const { return static_cast(myMutable_.at(key)); } diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 567b86bd1379..ecde4032de00 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -1,6 +1,6 @@ //======================================================================================== // Parthenon performance portable AMR framework -// Copyright(C) 2024 The Parthenon collaboration +// Copyright(C) 2024-2025 The Parthenon collaboration // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== // (C) (or copyright) 2024. Triad National Security, LLC. All rights reserved. diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index 37982a7c09bb..0db1bc842743 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -1,6 +1,6 @@ //======================================================================================== // Parthenon performance portable AMR framework -// Copyright(C) 2024 The Parthenon collaboration +// Copyright(C) 2024-2025 The Parthenon collaboration // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== #ifndef OUTPUTS_RESTART_OPMD_HPP_ @@ -154,6 +154,10 @@ class RestartReaderOPMD : public RestartReader { } Kokkos::deep_copy(view, view_h); } + [[nodiscard]] bool VariableExists(const std::string &name) const override { + // TODO(pgrete) needs impl + return true; + } // closes out the restart file // perhaps belongs in a destructor? void Close(); From 72412bd2a4a9d1e73b4b1203000696002c639f99 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 1 Aug 2025 10:22:46 +0200 Subject: [PATCH 109/125] Check pin for opmd out and check vars skel --- src/outputs/parthenon_opmd.cpp | 6 +++++- src/outputs/restart.hpp | 4 +++- src/outputs/restart_hdf5.hpp | 8 ++++++-- src/outputs/restart_opmd.hpp | 3 ++- src/parthenon_manager.cpp | 6 ++++-- src/parthenon_manager.hpp | 4 ++-- 6 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index ecde4032de00..6626e88c29d0 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -303,6 +303,9 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, "compiled in. Skipping this output type."); } #else + // Check that the parameter input is safe to write (i.e., consistent across ranks) + OutputUtils::CheckParameterInputConsistent(pin); + using openPMD::Access; using openPMD::Series; @@ -746,7 +749,8 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // one for vis. // BUT PG: this may break things in unpredicable ways // I'm in favor of enforcing a global id somehow. We shold discuss. - PARTHENON_REQUIRE_THROWS(swinfo.var_info.count(swarm_position::id::name()) != 0, + PARTHENON_REQUIRE_THROWS(swinfo.var_info.count(swarm_position::id::name()) != 0 || + swinfo.var_info.count("id") != 0, "Particles should always carry a unique, persistent id!"); } Kokkos::Profiling::popRegion(); // write particle data diff --git a/src/outputs/restart.hpp b/src/outputs/restart.hpp index 0af3102ea2db..efa6acac46dd 100644 --- a/src/outputs/restart.hpp +++ b/src/outputs/restart.hpp @@ -136,7 +136,9 @@ class RestartReader { virtual void ReadParams(const std::string &name, Params &p) = 0; - [[nodiscard]] virtual bool VariableExists(const std::string &name) const = 0; + enum class DataType { Field, Swarm, SwarmVar }; + [[nodiscard]] virtual bool VariableExists(const std::string &name, + const DataType data_type) const = 0; // closes out the restart file // perhaps belongs in a destructor? diff --git a/src/outputs/restart_hdf5.hpp b/src/outputs/restart_hdf5.hpp index 2a4af886c0bc..577c385c1d60 100644 --- a/src/outputs/restart_hdf5.hpp +++ b/src/outputs/restart_hdf5.hpp @@ -233,9 +233,13 @@ class RestartReaderHDF5 : public RestartReader { void ReadParams(const std::string &name, Params &p) override; - [[nodiscard]] bool VariableExists(const std::string &name) const override { + [[nodiscard]] bool VariableExists(const std::string &name, + const DataType /*data_type*/) const override { #ifdef ENABLE_HDF5 - // make sure dataset exists + // Make sure dataset exists + // Our HDF5 output does not differentiate between fields and swarms. Everything is an + // object, so we can ignore the data_type. Note, we may eventually want to fix is as + // swarm and fields with the same name may cause issues. return PARTHENON_HDF5_CHECK(H5Oexists_by_name(fh_, name.c_str(), H5P_DEFAULT)); #else PARTHENON_FAIL("Restart functionality is not available because HDF5 is disabled"); diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index 0db1bc842743..dec4ded8a042 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -154,7 +154,8 @@ class RestartReaderOPMD : public RestartReader { } Kokkos::deep_copy(view, view_h); } - [[nodiscard]] bool VariableExists(const std::string &name) const override { + [[nodiscard]] bool VariableExists(const std::string &name, + const DataType data_type) const override { // TODO(pgrete) needs impl return true; } diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index 5de3877d355e..f9272f253686 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -339,7 +339,8 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { const auto fill_size = v_info.FillSize(theDomain, resfile.BlockdataIsPadded()); const auto &label = v_info.label; - auto var_missing_on_disk = !resfile.VariableExists(label); + auto var_missing_on_disk = + !resfile.VariableExists(label, RestartReader::DataType::Field); if (Globals::my_rank == 0) { std::cout << "Var: " << label << ":" << vlen << (var_missing_on_disk ? " missing on disk\n" : "\n"); @@ -407,7 +408,8 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { auto swarms = (mb.meshblock_data.Get()->GetSwarmData())->GetSwarmsByFlag(flags); for (auto &swarm : swarms) { auto swarmname = swarm->label(); - auto var_missing_on_disk = !resfile.VariableExists(swarmname); + auto var_missing_on_disk = + !resfile.VariableExists(swarmname, RestartReader::DataType::Swarm); if (Globals::my_rank == 0) { std::cout << "Swarm: " << swarmname << (var_missing_on_disk ? " missing on disk\n" : "\n"); diff --git a/src/parthenon_manager.hpp b/src/parthenon_manager.hpp index 6c8e0d4f01d3..d5851d146fd2 100644 --- a/src/parthenon_manager.hpp +++ b/src/parthenon_manager.hpp @@ -72,8 +72,8 @@ class ParthenonManager { const auto &m = var->metadata(); auto arrdims = m.GetArrayDims(pswarm->GetBlockPointer(), false); - auto var_missing_on_disk = - !restartReader->VariableExists(swarmname + "/SwarmVars/" + varname); + auto var_missing_on_disk = !restartReader->VariableExists( + swarmname + "/SwarmVars/" + varname, RestartReader::DataType::SwarmVar); if (Globals::my_rank == 0) { std::cout << "SwarmVar: " << varname << (var_missing_on_disk ? " missing on disk\n" : "\n"); From 73e445fa242d371edb5e04a63e976b14ba40050b Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 1 Aug 2025 16:23:00 +0200 Subject: [PATCH 110/125] Add option for single prec output in openpmd --- src/outputs/outputs.cpp | 24 +++++++++------ src/outputs/outputs.hpp | 3 ++ src/outputs/parthenon_opmd.cpp | 55 ++++++++++++++++++++++++++-------- 3 files changed, 61 insertions(+), 21 deletions(-) diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index 2e4ac2433e5e..c2a73c4252db 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -228,28 +228,34 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { // read single precision output option const bool is_hdf5_output = (op.file_type == "rst") || (op.file_type == "hdf5") || (op.file_type == "corehdf5"); + const bool is_openpmd_output = (op.file_type == "openpmd"); - if (is_hdf5_output) { + if (is_hdf5_output || is_openpmd_output) { op.single_precision_output = pin->GetOrAddBoolean(op.block_name, "single_precision_output", false); - op.sparse_seed_nans = - pin->GetOrAddBoolean(op.block_name, "sparse_seed_nans", false, - "write non-allocated sparse data as NaN"); - op.meshdata_name = pin->GetOrAddString(op.block_name, "meshdata_name", "base", - "which meshdata object to write from"); } else { op.single_precision_output = false; - op.sparse_seed_nans = false; - if (pin->DoesParameterExist(op.block_name, "single_precision_output")) { std::stringstream warn; warn << "Output option single_precision_output only applies to " "HDF5 outputs or restarts. Ignoring it for output block '" << op.block_name << "'"; - PARTHENON_WARN(warn); + if (Globals::my_rank == 0) { + PARTHENON_WARN(warn); + } } } + if (is_hdf5_output) { + op.sparse_seed_nans = + pin->GetOrAddBoolean(op.block_name, "sparse_seed_nans", false, + "write non-allocated sparse data as NaN"); + op.meshdata_name = pin->GetOrAddString(op.block_name, "meshdata_name", "base", + "which meshdata object to write from"); + } else { + op.sparse_seed_nans = false; + } + if (is_hdf5_output) { int default_compression_level = 5; #ifdef PARTHENON_DISABLE_HDF5_COMPRESSION diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index 8057cdc0baf0..fc8993b22a85 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -152,6 +152,9 @@ class OpenPMDOutput : public OutputType { : OutputType(oparams), backend_config_(std::move(backend_config)) {} void WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, const SignalHandler::OutputSignal signal) override; + template + void WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm, + const SignalHandler::OutputSignal signal); private: // path to file containing config passed to backend diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 6626e88c29d0..0b42eea1eded 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -294,7 +294,7 @@ GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb, //---------------------------------------------------------------------------------------- //! \fn void OpenPMDOutput:::WriteOutputFile(Mesh *pm) -// \brief Expose mesh and all Cell variables for processing with Ascent +// \brief Write output in OpenPMD format void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, const SignalHandler::OutputSignal signal) { #ifndef PARTHENON_ENABLE_OPENPMD @@ -303,6 +303,31 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, "compiled in. Skipping this output type."); } #else + if (output_params.single_precision_output) { + this->template WriteOutputFileImpl(pm, pin, tm, signal); + } else { + this->template WriteOutputFileImpl(pm, pin, tm, signal); + } +#endif // ifndef PARTHENON_ENABLE_OPENPMD +} + +//---------------------------------------------------------------------------------------- +//! \fn void OpenPMDOutput:::WriteOutputFile(Mesh *pm) +// \brief Write output in OpenPMD format +template +void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm, + const SignalHandler::OutputSignal signal) { +#ifndef PARTHENON_ENABLE_OPENPMD + if (Globals::my_rank == 0) { + PARTHENON_WARN("OpenPMD output requested by input file, but OpenPMD support not " + "compiled in. Skipping this output type."); + } +#else + if constexpr (WRITE_SINGLE_PRECISION) { + Kokkos::Profiling::pushRegion("OPMD::WriteOutputFileSinglePrec"); + } else { + Kokkos::Profiling::pushRegion("OPMD::WriteOutputFileRealPrec"); + } // Check that the parameter input is safe to write (i.e., consistent across ranks) OutputUtils::CheckParameterInputConsistent(pin); @@ -329,13 +354,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, MPI_COMM_WORLD, #endif backend_config); - if (signal == SignalHandler::OutputSignal::none) { - // After file has been opened with the current number, already advance output - // parameters so that for restarts the file is not immediatly overwritten again. - // Only applies to default time-based data dumps, so that writing "now" and "final" - // outputs does not change the desired output numbering. - UpdateNextOutput_(pm, tm); - } // TODO(pgrete) How to handle downstream info, e.g., on how/what defines a vector? // TODO(pgrete) Should we update for restart or only set this once? Or make it per // iteration? @@ -356,6 +374,14 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, auto it = series.iterations[output_params.file_number]; it.open(); // explicit open() is important when run in parallel + if (signal == SignalHandler::OutputSignal::none) { + // After file has been opened with the current number, already advance output + // parameters so that for restarts the file is not immediatly overwritten again. + // Only applies to default time-based data dumps, so that writing "now" and "final" + // outputs does not change the desired output numbering. + UpdateNextOutput_(pm, tm); + } + auto const &first_block = *(pm->block_list.front()); // TODO(?) in principle, we could abstract this to a more general WriteAttributes place @@ -529,10 +555,7 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, var_size_max = std::max(var_size_max, var_size); } - // TODO(pgrete) adjust for single prec output - // openPMD::Datatype dtype = openPMD::determineDatatype(); - using OutT = - Real; // typename std::conditional::type; + using OutT = typename std::conditional::type; std::vector tmp_data(var_size_max * num_blocks_local); // for each variable we write @@ -760,7 +783,15 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // An iteration once closed cannot (yet) be reopened. it.close(); series.close(); + Kokkos::Profiling::popRegion(); // WriteOutputFile???Prec #endif // ifndef PARTHENON_ENABLE_OPENPMD } +// explicit template instantiation +template void +OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm, + const SignalHandler::OutputSignal signal); +template void +OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm, + const SignalHandler::OutputSignal signal); } // namespace parthenon From d6936ae985c2ef6f8da6c672749398c165597f9f Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 1 Aug 2025 17:26:45 +0200 Subject: [PATCH 111/125] Allow coarsened opmd outputs --- src/outputs/outputs.cpp | 7 +++- src/outputs/outputs.hpp | 7 ++-- src/outputs/parthenon_opmd.cpp | 63 ++++++++++++++++++++++------------ src/outputs/parthenon_opmd.hpp | 2 +- src/outputs/restart_opmd.cpp | 4 ++- 5 files changed, 56 insertions(+), 27 deletions(-) diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index c2a73c4252db..fadd6670025d 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -359,8 +359,13 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { #ifdef PARTHENON_ENABLE_OPENPMD const auto backend_config = pin->GetOrAddString(op.block_name, "backend_config", "default"); + const auto coarsening_factor = + pin->GetOrAddInteger(op.block_name, "coarsening_factor", 1, + "Output data coarsened by given factor n. Every n^dim " + "data point is used, i.e., the data is not average. " + "Requires even number of cells in each block dimension."); - pnew_type = std::make_shared(op, backend_config); + pnew_type = std::make_shared(op, backend_config, coarsening_factor); #else msg << "### FATAL ERROR in Outputs constructor" << std::endl << "Executable not configured for OpenPMD outputs, but OpenPMD file format " diff --git a/src/outputs/outputs.hpp b/src/outputs/outputs.hpp index fc8993b22a85..99572c182519 100644 --- a/src/outputs/outputs.hpp +++ b/src/outputs/outputs.hpp @@ -148,8 +148,10 @@ class AscentOutput : public OutputType { class OpenPMDOutput : public OutputType { public: - explicit OpenPMDOutput(const OutputParameters &oparams, std::string backend_config) - : OutputType(oparams), backend_config_(std::move(backend_config)) {} + explicit OpenPMDOutput(const OutputParameters &oparams, std::string backend_config, + int coarsening_factor) + : OutputType(oparams), backend_config_(std::move(backend_config)), + coarsening_factor_(coarsening_factor) {} void WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, const SignalHandler::OutputSignal signal) override; template @@ -159,6 +161,7 @@ class OpenPMDOutput : public OutputType { private: // path to file containing config passed to backend std::string backend_config_; + int coarsening_factor_; }; #ifdef ENABLE_HDF5 diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 0b42eea1eded..d02972a594e7 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -267,24 +267,21 @@ GetMeshRecordAndComponentNames(const VarInfo &vinfo, const TopologicalElement te std::tuple GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb, - const TopologicalElement te) { + const TopologicalElement te, const int coarsening_factor) { openPMD::Offset chunk_offset; openPMD::Extent chunk_extent; const auto loc = pm->Forest().GetLegacyTreeLocation(pmb->loc); + uint64_t nx1_eff = pmb->block_size.nx(X1DIR) / coarsening_factor; + uint64_t nx2_eff = pmb->block_size.nx(X2DIR) / coarsening_factor; + uint64_t nx3_eff = pmb->block_size.nx(X3DIR) / coarsening_factor; if (pm->ndim == 3) { - chunk_offset = {loc.lx3() * static_cast(pmb->block_size.nx(X3DIR)), - loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), - loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; - chunk_extent = { - static_cast(pmb->block_size.nx(X3DIR) + TopologicalOffsetK(te)), - static_cast(pmb->block_size.nx(X2DIR) + TopologicalOffsetJ(te)), - static_cast(pmb->block_size.nx(X1DIR) + TopologicalOffsetI(te))}; + chunk_offset = {loc.lx3() * nx3_eff, loc.lx2() * nx2_eff, loc.lx1() * nx1_eff}; + chunk_extent = {nx3_eff + TopologicalOffsetK(te), nx2_eff + TopologicalOffsetJ(te), + nx1_eff + TopologicalOffsetI(te)}; } else if (pm->ndim == 2) { - chunk_offset = {loc.lx2() * static_cast(pmb->block_size.nx(X2DIR)), - loc.lx1() * static_cast(pmb->block_size.nx(X1DIR))}; - chunk_extent = { - static_cast(pmb->block_size.nx(X2DIR) + TopologicalOffsetJ(te)), - static_cast(pmb->block_size.nx(X1DIR) + TopologicalOffsetI(te))}; + chunk_offset = {loc.lx2() * nx2_eff, loc.lx1() * nx1_eff}; + chunk_extent = {static_cast(nx2_eff + TopologicalOffsetJ(te)), + static_cast(nx1_eff + TopologicalOffsetI(te))}; } else { PARTHENON_THROW("1D output for openpmd not yet supported."); } @@ -440,8 +437,12 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * it.setAttribute("Multilevel", pm->multilevel ? 1 : 0); it.setAttribute("BlocksPerPE", pm->GetNbList()); + // TODO(pgrete) Add safety check for supported coarsenign factors + // probably already in or before ctor + it.setAttribute("CoarseningFactor", coarsening_factor_); // Mesh block size + // TODO(pgrete) Check if we potentially can modify this to restart from coarse outs const auto base_block_size = pm->GetDefaultBlockSize(); it.setAttribute("MeshBlockSize", std::vector{base_block_size.nx(X1DIR), base_block_size.nx(X2DIR), @@ -549,6 +550,8 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * // We're currently writing (flushing) one var at a time. This saves host memory but // results more smaller write. Might be updated in the future. // Allocate space for largest size variable + // Could in principle be reduced for coarsended outputs, but lets better be safe than + // sorry given the edge cases with non cell centered vars. int var_size_max = 0; for (auto &vinfo : all_vars_info) { const auto var_size = vinfo.Size(); @@ -598,9 +601,9 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * auto &coords = pmb->coords; // For uniform Cartesian, all dxN are const across the block so we just pick // the first index. - Real dx1 = coords.CellWidth(0, 0, 0); - Real dx2 = coords.CellWidth(0, 0, 0); - Real dx3 = coords.CellWidth(0, 0, 0); + Real dx1 = coords.CellWidth(0, 0, 0) * coarsening_factor_; + Real dx2 = coords.CellWidth(0, 0, 0) * coarsening_factor_; + Real dx3 = coords.CellWidth(0, 0, 0) * coarsening_factor_; // TODO(pgrete) check if this should be tied to the MemoryLayout mesh_record.setDataOrder(openPMD::Mesh::DataOrder::C); @@ -621,11 +624,17 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * 0.5 - 0.5 * TopologicalOffsetK(te), 0.5 - 0.5 * TopologicalOffsetJ(te), 0.5 - 0.5 * TopologicalOffsetI(te)}); global_extent = { - static_cast(pm->mesh_size.nx(X3DIR)) * effective_nx + + static_cast(pm->mesh_size.nx(X3DIR) / + coarsening_factor_) * + effective_nx + TopologicalOffsetK(te), - static_cast(pm->mesh_size.nx(X2DIR)) * effective_nx + + static_cast(pm->mesh_size.nx(X2DIR) / + coarsening_factor_) * + effective_nx + TopologicalOffsetJ(te), - static_cast(pm->mesh_size.nx(X1DIR)) * effective_nx + + static_cast(pm->mesh_size.nx(X1DIR) / + coarsening_factor_) * + effective_nx + TopologicalOffsetI(te), }; } else if (pm->ndim == 2) { @@ -640,9 +649,13 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * std::vector{0.5 - 0.5 * TopologicalOffsetJ(te), 0.5 - 0.5 * TopologicalOffsetI(te)}); global_extent = { - static_cast(pm->mesh_size.nx(X2DIR)) * effective_nx + + static_cast(pm->mesh_size.nx(X2DIR) / + coarsening_factor_) * + effective_nx + TopologicalOffsetJ(te), - static_cast(pm->mesh_size.nx(X1DIR)) * effective_nx + + static_cast(pm->mesh_size.nx(X1DIR) / + coarsening_factor_) * + effective_nx + TopologicalOffsetI(te), }; @@ -703,6 +716,11 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * for (int k = kb.s; k <= kb.e; ++k) { for (int j = jb.s; j <= jb.e; ++j) { for (int i = ib.s; i <= ib.e; ++i) { + if (((i - ib.s) % coarsening_factor_ != 0) || + ((j - jb.s) % coarsening_factor_ != 0) || + ((k - kb.s) % coarsening_factor_ != 0)) { + continue; + } tmp_data[tmp_offset] = static_cast( out_var_h(static_cast(te) % 3, t, u, v, k, j, i)); tmp_offset++; @@ -710,7 +728,8 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * } } const auto [chunk_offset, chunk_extent] = - OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb, te); + OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb, te, + coarsening_factor_); mesh_comp.storeChunkRaw(&tmp_data[comp_offset], chunk_offset, chunk_extent); comp_idx += 1; diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index 2665bf9c2f85..44970532cf0d 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -48,7 +48,7 @@ GetMeshRecordAndComponentNames(const OutputUtils::VarInfo &vinfo, // TODO(pgrete) needs to be updated to properly work with Forests std::tuple GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb, - const TopologicalElement te); + const TopologicalElement te, const int coarsening_factor); } // namespace OpenPMDUtils } // namespace parthenon diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index cbd57925ac2c..4d91f3d0d398 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -230,8 +230,10 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block "' of restart file."); auto mesh_comp = mesh_record[comp_name]; + // Restarting from coarsened output not supported at the moment + const int coarsening_factor = 1; const auto [chunk_offset, chunk_extent] = - OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb, te); + OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb, te, coarsening_factor); mesh_comp.loadChunkRaw(&data_vec[comp_offset], chunk_offset, chunk_extent); comp_offset += std::accumulate(chunk_extent.cbegin(), chunk_extent.cend(), 1, std::multiplies{}); From cb8997bdcd1bf09714355eef0a81d0850c7c25a3 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 18 Aug 2025 23:37:46 +0200 Subject: [PATCH 112/125] Fix logic causing issues for restarts with varying output blocks --- CHANGELOG.md | 1 + src/outputs/output_parameters.hpp | 1 - src/outputs/outputs.cpp | 87 ++++++++++--------- src/outputs/outputs_package.cpp | 38 +++----- tst/regression/CMakeLists.txt | 2 +- .../test_suites/restart_fine/restart_fine.py | 30 ++++++- 6 files changed, 89 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e6ac37dd836..2719c41a6d6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - [[PR 1280]](https://github.com/parthenon-hpc-lab/parthenon/pull/1280) Print history file headers on restart ### Fixed (not changing behavior/API/variables/...) +- [[PR 1310]](https://github.com/parthenon-hpc-lab/parthenon/pull/1310) Fix logic causing issues for restarts with varying output blocks (introduced in #1266) - [[PR 1297]](https://github.com/parthenon-hpc-lab/parthenon/pull/1297) Backward compatibility fixes (`last_/next_` output package, restart with new `Restart` vars, `is_restart`) - [[PR 1303]](https://github.com/parthenon-hpc-lab/parthenon/pull/1303) Guard against block_list access when nmb==0 - [[PR 1291]](https://github.com/parthenon-hpc-lab/parthenon/pull/1291) Fix provenance for downstream codes diff --git a/src/outputs/output_parameters.hpp b/src/outputs/output_parameters.hpp index 644332ebea26..8c736a45bb19 100644 --- a/src/outputs/output_parameters.hpp +++ b/src/outputs/output_parameters.hpp @@ -41,7 +41,6 @@ struct OutputParameters { OutputParameters() = default; int block_number = 0; - int contiguous_block_index; std::string block_name; std::string file_basename; int file_number_width; diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index e2aeaf3eea61..f4abf2a86128 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -103,20 +103,26 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { // loop over "parthenon/output" blocks located in params auto pkg = pm->packages.Get("Outputs"); - const auto &block_names = pkg->Param>("block_names"); - const auto &block_numbers = pkg->Param>("block_numbers"); - auto *pactive = pkg->MutableParam>("active"); - auto *pfile_numbers = pkg->MutableParam>("file_numbers"); - auto *plast_times = pkg->MutableParam>("last_times"); - auto *plast_ns = pkg->MutableParam>("last_ns"); - for (int iinput = 0; iinput < block_names.size(); ++iinput) { + // loop over input block names. Find those that start with "parthenon/output", read + // parameters, and construct a vector of output types. + // PG: It could be discussed if we should work based on a vector that is initialized in + // the outpus packages (as before), but I don't think it's bad practice to work on + // `pinput` again here as we're actually processing (potentially even modifying) + // `pinput`. + for (InputBlock *pib = pin->pfirst_block; pib != nullptr; pib = pib->pnext) { + if (pib->block_name.compare(0, 16, "parthenon/output") != 0) { + continue; + } std::shared_ptr pnew_type; // the new output we will create bool restart = false; // we track restart outputs separately so we // need this temp variable to check OutputParameters op; // define temporary OutputParameters struct - op.block_name = block_names[iinput]; - op.block_number = block_numbers[iinput]; - op.contiguous_block_index = iinput; + op.block_name = pib->block_name; + const auto outn_str = pib->block_name.substr(16); // 16 because counting starts at 0! + op.block_number = atoi(outn_str.c_str()); + auto *pfile_number = pkg->MutableParam(outn_str + "/file_number"); + auto *plast_time = pkg->MutableParam(outn_str + "/last_time"); + auto *plast_n = pkg->MutableParam(outn_str + "/last_n"); Real dt = 0.0; // default value == 0 means that initial data is written by default int dn = -1; @@ -132,10 +138,7 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { } // if this output is "soft-disabled" (negative value) skip processing if (dt < 0.0 && dn < 0) { - (*pactive)[iinput] = false; continue; - } else { - (*pactive)[iinput] = true; } // JMM: Backwards compatibility hack. Don't allow this unless @@ -145,17 +148,17 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { bool next_n_exists = pin->DoesParameterExist(op.block_name, "next_n"); if (next_time_exists) { Real next_time = pin->GetReal(op.block_name, "next_time"); - (*plast_times)[iinput] = dt < 0 ? 0.0 : next_time - dt; + *plast_time = dt < 0 ? 0.0 : next_time - dt; pin->RemoveParameter(op.block_name, "next_time"); } if (next_n_exists) { int next_n = pin->GetInteger(op.block_name, "next_n"); - (*plast_ns)[iinput] = dn < 0 ? 0 : next_n - dn; + *plast_n = dn < 0 ? 0 : next_n - dn; pin->RemoveParameter(op.block_name, "next_n"); } if (next_time_exists || next_n_exists) { - (*pfile_numbers)[iinput] = pin->GetOrAddInteger(op.block_name, "file_number", 0); + *pfile_number = pin->GetOrAddInteger(op.block_name, "file_number", 0); pin->RemoveParameter(op.block_name, "file_number"); } } @@ -165,8 +168,8 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { "is not supported. Please set at most one value >= 0."); // set time of last output, time between outputs - op.last_time = (*plast_times)[iinput]; - op.last_n = (*plast_ns)[iinput]; + op.last_time = *plast_time; + op.last_n = *plast_n; if (tm != nullptr) { op.dt = dt; op.dn = dn; @@ -203,7 +206,7 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { } // set file number, basename, id, and format - op.file_number = std::max((*pfile_numbers)[iinput], 0); + op.file_number = std::max(*pfile_number, 0); op.file_basename = pin->GetOrAddString("parthenon/job", "problem_id", "parthenon", "prefix for output files"); op.file_number_width = pin->GetOrAddInteger(op.block_name, "file_number_width", 5); @@ -282,8 +285,8 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { if (op.file_type == "hst") { // Do not use GetOrAddVector because it will pollute the input parameters for // restarts - if (pin->DoesParameterExist(block_names[iinput], "packages")) { - op.packages = pin->GetVector(block_names[iinput], "packages"); + if (pin->DoesParameterExist(op.block_name, "packages")) { + op.packages = pin->GetVector(op.block_name, "packages"); } else { op.packages = std::vector(); } @@ -293,35 +296,39 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { if ((op.file_type != "hst") && (op.file_type != "rst") && (op.file_type != "corehdf5") && (op.file_type != "ascent") && (op.file_type != "histogram")) { - op.variables = pin->GetOrAddVector(block_names[iinput], "variables", - std::vector(), - "variables to output"); + // Do not use GetOrAddVector because it will pollute the input parameters for + // restarts + if (pin->DoesParameterExist(op.block_name, "variables")) { + op.variables = pin->GetVector(op.block_name, "variables"); + } else { + op.variables = std::vector(); + } // JMM: If the requested var isn't present for a given swarm, // it is simply not output. op.swarms.clear(); // Not sure this is needed - if (pin->DoesParameterExist(block_names[iinput], "swarms")) { - std::vector swarmnames = pin->GetVector( - block_names[iinput], "swarms", "swarms to output"); + if (pin->DoesParameterExist(op.block_name, "swarms")) { + std::vector swarmnames = + pin->GetVector(op.block_name, "swarms", "swarms to output"); std::size_t nswarms = swarmnames.size(); - if ((pin->DoesParameterExist(block_names[iinput], "swarm_variables")) && + if ((pin->DoesParameterExist(op.block_name, "swarm_variables")) && (nswarms > 1)) { std::stringstream msg; - msg << "The swarm_variables field is set in the block '" << block_names[iinput] + msg << "The swarm_variables field is set in the block '" << op.block_name << "' however, there are " << nswarms << " swarms." << " All swarms will be assumed to request the vars listed in " "swarm_variables."; PARTHENON_WARN(msg); } for (const auto &swname : swarmnames) { - if (pin->DoesParameterExist(block_names[iinput], "swarm_variables")) { + if (pin->DoesParameterExist(op.block_name, "swarm_variables")) { auto varnames = - pin->GetVector(block_names[iinput], "swarm_variables", + pin->GetVector(op.block_name, "swarm_variables", "swarm variables to output for all swarms"); op.swarms[swname].insert(varnames.begin(), varnames.end()); } - if (pin->DoesParameterExist(block_names[iinput], swname + "_variables")) { + if (pin->DoesParameterExist(op.block_name, swname + "_variables")) { auto varnames = pin->GetVector( - block_names[iinput], swname + "_variables", + op.block_name, swname + "_variables", "swarm variables to output for a specific swarm"); op.swarms[swname].insert(varnames.begin(), varnames.end()); } @@ -393,7 +400,6 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { output_types_.push_back(pnew_type); } } - // check there were no more than one restart file requested if (num_rst_outputs > 1) { msg << "### FATAL ERROR in Outputs constructor" << std::endl @@ -459,10 +465,11 @@ void Outputs::MakeOutputs(Mesh *pm, ParameterInput *pin, SimTime *tm, void OutputType::UpdateNextOutput_(Mesh *pm, SimTime *tm) { output_params.file_number++; auto pkg = pm->packages.Get("Outputs"); - auto *pfile_numbers = pkg->MutableParam>("file_numbers"); - auto *plast_times = pkg->MutableParam>("last_times"); - auto *plast_ns = pkg->MutableParam>("last_ns"); - (*pfile_numbers)[output_params.contiguous_block_index] = output_params.file_number; + const auto outn_str = std::to_string(output_params.block_number); + auto *pfile_number = pkg->MutableParam(outn_str + "/file_number"); + auto *plast_time = pkg->MutableParam(outn_str + "/last_time"); + auto *plast_n = pkg->MutableParam(outn_str + "/last_n"); + *pfile_number = output_params.file_number; if (tm != nullptr) { // JMM: Do NOT use the current time to update these, as that can // cause drift because timestep is not guaranteed to align with @@ -470,8 +477,8 @@ void OutputType::UpdateNextOutput_(Mesh *pm, SimTime *tm) { // time. output_params.last_n = output_params.next_n; output_params.last_time = output_params.next_time; - (*plast_ns)[output_params.contiguous_block_index] = output_params.last_n; - (*plast_times)[output_params.contiguous_block_index] = output_params.last_time; + *plast_n = output_params.last_n; + *plast_time = output_params.last_time; if (output_params.dt > 0.0) { output_params.next_time += output_params.dt; } diff --git a/src/outputs/outputs_package.cpp b/src/outputs/outputs_package.cpp index 801b7045f283..09b68a3a8c49 100644 --- a/src/outputs/outputs_package.cpp +++ b/src/outputs/outputs_package.cpp @@ -37,36 +37,22 @@ namespace OutputsPackage { std::shared_ptr Initialize(ParameterInput *pin) { auto pkg = std::make_shared("Outputs"); - std::string basename = pin->GetOrAddString("parthenon/job", "problem_id", "parthenon", - "prefix for output files"); - std::vector block_names; - std::vector block_numbers; - - std::vector active; - std::vector file_numbers; - std::vector last_times; - std::vector last_ns; - - // loop over input block names. Find those that start with "parthenon/output", read - // parameters, and construct singly linked list of OutputTypes. + // loop over input block names. Find those that start with "parthenon/output" and + // add/initialize `Params` for further processing (so that they're available to be read + // from restart files or are cleanly initialized). for (InputBlock *pib = pin->pfirst_block; pib != nullptr; pib = pib->pnext) { if (pib->block_name.compare(0, 16, "parthenon/output") == 0) { - std::string outn = pib->block_name.substr(16); // 6 because counting starts at 0! + std::string outn = pib->block_name.substr(16); // 16 because counting starts at 0! std::string block_name = pib->block_name; - // these are used for book-keeping - block_names.push_back(block_name); - block_numbers.push_back(atoi(outn.c_str())); - // These will be updated later or restarted from - active.push_back(false); - file_numbers.push_back(0); + int file_number = 0; // JMM: Limits to indicate these haven't been set yet. The reason // to set these to a "signal" number, rather than to start_time // is that we want to ensure a first output is performed. - last_times.push_back(std::numeric_limits::lowest()); - last_ns.push_back(std::numeric_limits::lowest()); + auto last_time = std::numeric_limits::lowest(); + auto last_n = std::numeric_limits::lowest(); bool next_time_exists = pin->DoesParameterExist(block_name, "next_time"); bool next_n_exists = pin->DoesParameterExist(block_name, "next_n"); @@ -86,14 +72,12 @@ std::shared_ptr Initialize(ParameterInput *pin) { PARTHENON_THROW(msg); } } + // It should be safe here to jsut use outn as output blocks are unique + pkg->AddParam(outn + "/file_number", file_number, Params::Mutability::Restart); + pkg->AddParam(outn + "/last_time", last_time, Params::Mutability::Restart); + pkg->AddParam(outn + "/last_n", last_n, Params::Mutability::Restart); } } - pkg->AddParam("block_names", block_names); - pkg->AddParam("block_numbers", block_numbers); - pkg->AddParam("active", active, Params::Mutability::Restart); - pkg->AddParam("file_numbers", file_numbers, Params::Mutability::Restart); - pkg->AddParam("last_times", last_times, Params::Mutability::Restart); - pkg->AddParam("last_ns", last_ns, Params::Mutability::Restart); return pkg; } diff --git a/tst/regression/CMakeLists.txt b/tst/regression/CMakeLists.txt index 36fff8e58d72..d8c3cce3224f 100644 --- a/tst/regression/CMakeLists.txt +++ b/tst/regression/CMakeLists.txt @@ -80,7 +80,7 @@ if (ENABLE_HDF5) list(APPEND TEST_PROCS ${NUM_MPI_PROC_TESTING}) list(APPEND TEST_ARGS "--driver ${PROJECT_BINARY_DIR}/example/fine_advection/fine_advection-example \ --driver_input ${CMAKE_CURRENT_SOURCE_DIR}/test_suites/restart_fine/parthinput.restart_fine \ - --num_steps 2") + --num_steps 3") list(APPEND EXTRA_TEST_LABELS "") # Calculate pi example diff --git a/tst/regression/test_suites/restart_fine/restart_fine.py b/tst/regression/test_suites/restart_fine/restart_fine.py index 4255e0fb3a12..affde4ac5fe9 100644 --- a/tst/regression/test_suites/restart_fine/restart_fine.py +++ b/tst/regression/test_suites/restart_fine/restart_fine.py @@ -33,12 +33,23 @@ def Prepare(self, parameters, step): # run baseline (to the very end) if step == 1: parameters.driver_cmd_line_args = ["parthenon/job/problem_id=gold"] - else: + elif step == 2: parameters.driver_cmd_line_args = [ "-r", "gold.out0.00004.rhdf", "parthenon/job/problem_id=silver", ] + # check that we can dynamically enable outputs + else: + parameters.driver_cmd_line_args = [ + "-r", + "gold.out0.00005.rhdf", + "parthenon/job/problem_id=bronze", + "parthenon/output1/file_type=hdf5", + "parthenon/output1/dt=0.05", + "parthenon/output1/last_time=0.25", + "parthenon/output1/variables=advection.C,advection.D,advection.phi_0,advection.phi_fine" + ] return parameters def Analyse(self, parameters): @@ -71,6 +82,23 @@ def compare_files(name, base="silver"): % name ) return False + delta = compare( + [ + "bronze.out1.%s.phdf" % name, + "{}.out0.{}.rhdf".format(base, name), + ], + # no need for metadata as the dynamically added output will cause + # different metadata and we're just interested in the right data + # being there. + one=True, check_metadata=False, + ) + + if delta != 0: + print( + "ERROR: Found difference between gold and bronze output '%s'." + % name + ) + return False return True From fa25d699a4a2460fd64b7e8babce55b8398f2f38 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 19 Aug 2025 11:09:20 +0200 Subject: [PATCH 113/125] Fix formatting and test case --- src/outputs/outputs_package.cpp | 2 +- .../test_suites/restart_fine/parthinput.restart_fine | 7 +++++++ tst/regression/test_suites/restart_fine/restart_fine.py | 9 +++++---- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/outputs/outputs_package.cpp b/src/outputs/outputs_package.cpp index 09b68a3a8c49..a7d0f763a28d 100644 --- a/src/outputs/outputs_package.cpp +++ b/src/outputs/outputs_package.cpp @@ -72,7 +72,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { PARTHENON_THROW(msg); } } - // It should be safe here to jsut use outn as output blocks are unique + // It should be safe here to just use outn as output blocks are unique pkg->AddParam(outn + "/file_number", file_number, Params::Mutability::Restart); pkg->AddParam(outn + "/last_time", last_time, Params::Mutability::Restart); pkg->AddParam(outn + "/last_n", last_n, Params::Mutability::Restart); diff --git a/tst/regression/test_suites/restart_fine/parthinput.restart_fine b/tst/regression/test_suites/restart_fine/parthinput.restart_fine index 02b207486835..0d1ffe4fa0f8 100644 --- a/tst/regression/test_suites/restart_fine/parthinput.restart_fine +++ b/tst/regression/test_suites/restart_fine/parthinput.restart_fine @@ -60,3 +60,10 @@ derefine_tol = 0.03 file_type = rst dt = 0.05 + +# Following output block is used for testing against dynamically +# added output block output1 + +file_type = hdf5 +dt = 0.25 +variables = advection.C diff --git a/tst/regression/test_suites/restart_fine/restart_fine.py b/tst/regression/test_suites/restart_fine/restart_fine.py index affde4ac5fe9..a70a173ccd5d 100644 --- a/tst/regression/test_suites/restart_fine/restart_fine.py +++ b/tst/regression/test_suites/restart_fine/restart_fine.py @@ -46,9 +46,9 @@ def Prepare(self, parameters, step): "gold.out0.00005.rhdf", "parthenon/job/problem_id=bronze", "parthenon/output1/file_type=hdf5", - "parthenon/output1/dt=0.05", + "parthenon/output1/dt=0.25", "parthenon/output1/last_time=0.25", - "parthenon/output1/variables=advection.C,advection.D,advection.phi_0,advection.phi_fine" + "parthenon/output1/variables=advection.C", ] return parameters @@ -85,12 +85,13 @@ def compare_files(name, base="silver"): delta = compare( [ "bronze.out1.%s.phdf" % name, - "{}.out0.{}.rhdf".format(base, name), + "{}.out2.{}.phdf".format(base, name), ], # no need for metadata as the dynamically added output will cause # different metadata and we're just interested in the right data # being there. - one=True, check_metadata=False, + one=True, + check_metadata=False, ) if delta != 0: From 6787b4259a8627b2caa638b53c27df02e858f4b2 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 19 Aug 2025 13:39:37 +0200 Subject: [PATCH 114/125] Fix/cleanup include logic and move format ver to impl --- src/CMakeLists.txt | 10 ++++--- src/outputs/parthenon_opmd.cpp | 30 ++------------------ src/outputs/parthenon_opmd.hpp | 6 ++-- src/outputs/restart.hpp | 2 +- src/outputs/restart_hdf5.cpp | 23 ++++++++-------- src/outputs/restart_hdf5.hpp | 2 +- src/outputs/restart_opmd.cpp | 8 +++--- src/outputs/restart_opmd.hpp | 7 +++-- src/parthenon_manager.cpp | 50 ++++++++++++---------------------- 9 files changed, 52 insertions(+), 86 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5a0588feaefd..c94cf67a01be 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -211,14 +211,10 @@ add_library(parthenon outputs/parthenon_hdf5_types.hpp outputs/parthenon_xdmf.cpp outputs/parthenon_hdf5.hpp - outputs/parthenon_opmd.cpp - outputs/parthenon_opmd.hpp outputs/parthenon_xdmf.hpp outputs/restart.hpp outputs/restart_hdf5.cpp outputs/restart_hdf5.hpp - outputs/restart_opmd.cpp - outputs/restart_opmd.hpp pack/block_selector.hpp pack/make_pack_descriptor.hpp @@ -363,6 +359,12 @@ if (ENABLE_HDF5) endif() if (PARTHENON_ENABLE_OPENPMD) + target_sources(parthenon PRIVATE + outputs/parthenon_opmd.cpp + outputs/parthenon_opmd.hpp + outputs/restart_opmd.cpp + outputs/restart_opmd.hpp + ) target_link_libraries(parthenon PUBLIC openPMD::openPMD) endif() diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index d02972a594e7..62f56f60c8a9 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -17,7 +17,6 @@ //! \file parthenon_openpmd.cpp // \brief Output for OpenPMD https://www.openpmd.org/ (supporting various backends) -#include #include #include #include @@ -31,6 +30,9 @@ #include #include +// OpenPMD headers +#include + // Parthenon headers #include "basic_types.hpp" #include "coordinates/coordinates.hpp" @@ -41,13 +43,6 @@ #include "interface/variable_state.hpp" #include "mesh/mesh.hpp" #include "mesh/meshblock.hpp" -#include "openPMD/Dataset.hpp" -#include "openPMD/Datatype.hpp" -#include "openPMD/IO/Access.hpp" -#include "openPMD/Iteration.hpp" -#include "openPMD/Mesh.hpp" -#include "openPMD/ParticleSpecies.hpp" -#include "openPMD/Series.hpp" #include "outputs/output_attr.hpp" #include "outputs/output_utils.hpp" #include "outputs/outputs.hpp" @@ -57,11 +52,6 @@ #include "utils/error_checking.hpp" #include "utils/instrument.hpp" -// OpenPMD headers -#ifdef PARTHENON_ENABLE_OPENPMD -#include -#endif // ifdef PARTHENON_ENABLE_OPENPMD - namespace parthenon { using namespace OutputUtils; @@ -294,18 +284,11 @@ GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb, // \brief Write output in OpenPMD format void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, const SignalHandler::OutputSignal signal) { -#ifndef PARTHENON_ENABLE_OPENPMD - if (Globals::my_rank == 0) { - PARTHENON_WARN("OpenPMD output requested by input file, but OpenPMD support not " - "compiled in. Skipping this output type."); - } -#else if (output_params.single_precision_output) { this->template WriteOutputFileImpl(pm, pin, tm, signal); } else { this->template WriteOutputFileImpl(pm, pin, tm, signal); } -#endif // ifndef PARTHENON_ENABLE_OPENPMD } //---------------------------------------------------------------------------------------- @@ -314,12 +297,6 @@ void OpenPMDOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, template void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm, const SignalHandler::OutputSignal signal) { -#ifndef PARTHENON_ENABLE_OPENPMD - if (Globals::my_rank == 0) { - PARTHENON_WARN("OpenPMD output requested by input file, but OpenPMD support not " - "compiled in. Skipping this output type."); - } -#else if constexpr (WRITE_SINGLE_PRECISION) { Kokkos::Profiling::pushRegion("OPMD::WriteOutputFileSinglePrec"); } else { @@ -803,7 +780,6 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * it.close(); series.close(); Kokkos::Profiling::popRegion(); // WriteOutputFile???Prec -#endif // ifndef PARTHENON_ENABLE_OPENPMD } // explicit template instantiation template void diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index 44970532cf0d..0b95b06f93fc 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -8,14 +8,15 @@ //! \file restart_opmd.hpp // \brief Provides support for restarting from OpenPMD output +// OpenPMD headers +#include + #include #include #include #include "basic_types.hpp" #include "mesh/meshblock.hpp" -#include "openPMD/Dataset.hpp" -#include "openPMD/Iteration.hpp" #include "outputs/output_utils.hpp" namespace parthenon { @@ -52,4 +53,5 @@ GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb, } // namespace OpenPMDUtils } // namespace parthenon + #endif // OUTPUTS_PARTHENON_OPMD_HPP_ diff --git a/src/outputs/restart.hpp b/src/outputs/restart.hpp index efa6acac46dd..797ed4da803b 100644 --- a/src/outputs/restart.hpp +++ b/src/outputs/restart.hpp @@ -106,7 +106,7 @@ class RestartReader { // fills internal data for given pointer virtual void ReadBlocks(const std::string &name, IndexRange range, const OutputUtils::VarInfo &info, std::vector &dataVec, - int file_output_format_version, Mesh *pmesh) const = 0; + Mesh *pmesh) const = 0; // The PackOrUnpack logic requires knowledge of how data is stored and being read into // the buffer. For HDF5 data is padded if needed (i.e., a face centered field has tims diff --git a/src/outputs/restart_hdf5.cpp b/src/outputs/restart_hdf5.cpp index 3edc6e4abc04..67780f54fd75 100644 --- a/src/outputs/restart_hdf5.cpp +++ b/src/outputs/restart_hdf5.cpp @@ -57,6 +57,15 @@ RestartReaderHDF5::RestartReaderHDF5(const char *filename) : filename_(filename) params_group_ = H5G::FromHIDCheck(H5Oopen(fh_, "Params", H5P_DEFAULT)); has_ghost = GetAttr("Info", "IncludesGhost"); + + // Currently supports versions 3 and 4. + const auto file_output_format_ver = GetOutputFormatVersion(); + if (file_output_format_ver < HDF5::OUTPUT_VERSION_FORMAT - 1) { + std::stringstream msg; + msg << "File format version " << file_output_format_ver << " not supported. " + << "Current format is " << HDF5::OUTPUT_VERSION_FORMAT << std::endl; + PARTHENON_THROW(msg) + } #endif // ENABLE_HDF5 } @@ -208,9 +217,7 @@ void RestartReaderHDF5::ReadParams(const std::string &name, Params &p) { } void RestartReaderHDF5::ReadBlocks(const std::string &name, IndexRange range, const OutputUtils::VarInfo &info, - std::vector &dataVec, - int file_output_format_version, - Mesh * /*pmesh*/) const { + std::vector &dataVec, Mesh * /*pmesh*/) const { #ifndef ENABLE_HDF5 PARTHENON_FAIL("Restart functionality is not available because HDF5 is disabled"); #else // HDF5 enabled @@ -228,15 +235,7 @@ void RestartReaderHDF5::ReadBlocks(const std::string &name, IndexRange range, count[0] = static_cast(range.e - range.s + 1); const IndexDomain domain = has_ghost != 0 ? IndexDomain::entire : IndexDomain::interior; - // Currently supports versions 3 and 4. - if (file_output_format_version >= HDF5::OUTPUT_VERSION_FORMAT - 1) { - total_dim = info.FillShape(domain, &(count[1])) + 1; - } else { - std::stringstream msg; - msg << "File format version " << file_output_format_version << " not supported. " - << "Current format is " << HDF5::OUTPUT_VERSION_FORMAT << std::endl; - PARTHENON_THROW(msg) - } + total_dim = info.FillShape(domain, &(count[1])) + 1; hsize_t total_count = 1; for (int i = 0; i < total_dim; ++i) { diff --git a/src/outputs/restart_hdf5.hpp b/src/outputs/restart_hdf5.hpp index 577c385c1d60..25ede5c138e0 100644 --- a/src/outputs/restart_hdf5.hpp +++ b/src/outputs/restart_hdf5.hpp @@ -115,7 +115,7 @@ class RestartReaderHDF5 : public RestartReader { // fills internal data for given pointer void ReadBlocks(const std::string &name, IndexRange range, const OutputUtils::VarInfo &info, std::vector &dataVec, - int file_output_format_version, Mesh *pmesh) const override; + Mesh *pmesh) const override; // The PackOrUnpack logic requires knowledge of how data is stored and being read into // the buffer. For HDF5 data is padded if needed (i.e., a face centered field has tims diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 4d91f3d0d398..345a544b398b 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -15,10 +15,11 @@ #include #include +// OpenPMD headers +#include + #include "basic_types.hpp" #include "interface/params.hpp" -#include "openPMD/Iteration.hpp" -#include "openPMD/Series.hpp" #include "outputs/output_attr.hpp" #include "outputs/parthenon_opmd.hpp" #include "outputs/restart.hpp" @@ -200,8 +201,7 @@ void RestartReaderOPMD::ReadParams(const std::string &pkg_name, Params &p) { void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block_range, const OutputUtils::VarInfo &vinfo, - std::vector &data_vec, - int file_output_format_version, Mesh *pm) const { + std::vector &data_vec, Mesh *pm) const { int64_t comp_offset = 0; // offset data_vector to store component data for (auto &pmb : pm->block_list) { // TODO(pgrete) check if we should skip the suffix for level 0 diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index dec4ded8a042..052c35a8bbdb 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -13,9 +13,10 @@ #include #include +// OpenPMD headers +#include + #include "basic_types.hpp" -#include "openPMD/Iteration.hpp" -#include "openPMD/Series.hpp" #include "outputs/restart.hpp" #include "pack/swarm_default_names.hpp" @@ -52,7 +53,7 @@ class RestartReaderOPMD : public RestartReader { // fills internal data for given pointer void ReadBlocks(const std::string &name, IndexRange range, const OutputUtils::VarInfo &info, std::vector &dataVec, - int file_output_format_version, Mesh *pmesh) const override; + Mesh *pmesh) const override; // The PackOrUnpack logic requires knowledge of how data is stored and being read into // the buffer. OpenPMD is dense (i.e., a face centered field has dims diff --git a/src/parthenon_manager.cpp b/src/parthenon_manager.cpp index f9272f253686..8908c9b9e604 100644 --- a/src/parthenon_manager.cpp +++ b/src/parthenon_manager.cpp @@ -28,8 +28,6 @@ #include #include -#include - #include "amr_criteria/amr_criteria.hpp" #include "amr_criteria/refinement_package.hpp" #include "config.hpp" @@ -41,7 +39,9 @@ #include "outputs/outputs_package.hpp" #include "outputs/restart.hpp" #include "outputs/restart_hdf5.hpp" +#ifdef PARTHENON_ENABLE_OPENPMD #include "outputs/restart_opmd.hpp" +#endif #include "utils/error_checking.hpp" #include "utils/utils.hpp" @@ -103,9 +103,18 @@ ParthenonStatus ParthenonManager::ParthenonInitEnv(int argc, char *argv[]) { if (arg.is_restart) { // Read input from restart file if (fs::path(arg.restart_filename).extension() == ".rhdf") { +#ifdef ENABLE_HDF5 restartReader = std::make_unique(arg.restart_filename); +#else // HDF5 disabled + PARTHENON_FAIL("Restart functionality is not available because HDF5 is disabled"); +#endif } else if (fs::path(arg.restart_filename).extension() == ".bp") { +#ifdef PARTHENON_ENABLE_OPENPMD restartReader = std::make_unique(arg.restart_filename); +#else + PARTHENON_FAIL("Trying to restart from OpenPMD file but OpenPMD support was not " + "compiled into Parthenon."); +#endif // ifdef PARTHENON_ENABLE_OPENPMD } else { PARTHENON_FAIL("Unsupported restart file format."); } @@ -263,9 +272,6 @@ ParthenonManager::ProcessPackagesDefault(std::unique_ptr &pin) { } void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { -#ifndef ENABLE_HDF5 - PARTHENON_FAIL("Restart functionality is not available because HDF5 is disabled"); -#else // HDF5 enabled // Restart packages with information for blocks in ids from the restart file // Assumption: blocks are contiguous in restart file, may have to revisit this. const IndexDomain theDomain = @@ -277,20 +283,9 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { int nbe = nbs + nb - 1; IndexRange myBlocks{nbs, nbe}; - // TODO(cleanup) why is this code here and not contained in the restart reader? std::cout << "Blocks assigned to rank " << Globals::my_rank << ": " << nbs << ":" << nbe << std::endl; - // Currently supports versions 3 and 4. - const auto file_output_format_ver = resfile.GetOutputFormatVersion(); - // TODO(pgrete) figure out what to do about versions of different outputs - if (false && file_output_format_ver < HDF5::OUTPUT_VERSION_FORMAT - 1) { - std::stringstream msg; - msg << "File format version " << file_output_format_ver << " not supported. " - << "Current format is " << HDF5::OUTPUT_VERSION_FORMAT << std::endl; - PARTHENON_THROW(msg) - } - // Get list of variables, they are the same for all blocks (since all blocks have the // same variable metadata) const auto indep_restart_vars = @@ -352,7 +347,7 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { // Read relevant data from the hdf file, this works for dense and sparse variables // because sparse variables are currently densely written for HDF5. try { - resfile.ReadBlocks(label, myBlocks, v_info, tmp, file_output_format_ver, &rm); + resfile.ReadBlocks(label, myBlocks, v_info, tmp, &rm); // Variable does exist but could not be read. So we definitely want to fail here. } catch (std::exception &ex) { std::stringstream msg; @@ -383,20 +378,12 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { auto v_h = v->data.GetHostMirror(); // Double note that this also needs to be update in case - // we update the HDF5 infrastructure! - // TODO(pgrete) figure out what to do about versions of different outputs - if (true || file_output_format_ver >= HDF5::OUTPUT_VERSION_FORMAT - 1) { - OutputUtils::PackOrUnpackVar( - v_info, resfile.HasGhost() != 0, resfile.BlockdataIsPadded(), index, - [&](auto index, int topo, int t, int u, int v, int k, int j, int i) { - v_h(topo, t, u, v, k, j, i) = tmp[index]; - }); - } else { - std::stringstream msg; - msg << "File format version " << file_output_format_ver << " not supported. " - << "Current format is " << HDF5::OUTPUT_VERSION_FORMAT << std::endl; - PARTHENON_THROW(msg) - } + // we update the OpenPMD/HDF5 infrastructure! + OutputUtils::PackOrUnpackVar( + v_info, resfile.HasGhost() != 0, resfile.BlockdataIsPadded(), index, + [&](auto index, int topo, int t, int u, int v, int k, int j, int i) { + v_h(topo, t, u, v, k, j, i) = tmp[index]; + }); v->data.DeepCopy(v_h); } @@ -446,7 +433,6 @@ void ParthenonManager::RestartPackages(Mesh &rm, RestartReader &resfile) { auto ¶ms = pkg->AllParams(); resfile.ReadParams(name, params); } -#endif // ifdef ENABLE_HDF5 } } // namespace parthenon From 0a5df77d1968d7e3065e5075e7398bebadc925f9 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 19 Aug 2025 13:51:23 +0200 Subject: [PATCH 115/125] Add missing try-catch for opmd params --- src/outputs/restart_opmd.cpp | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 345a544b398b..c6f4e5d62569 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -156,18 +156,28 @@ void RestartReaderOPMD::ReadAllParamsOfType(const std::string &prefix, Params &p // Thus we replace it. std::replace(full_path.begin(), full_path.end(), '/', delim[0]); - T val; - if constexpr (implements::value) { - val = params.Get(key); - RestoreViewAttribute(full_path, val); - } else if constexpr (is_specialization_of::value) { - val = params.Get(key); - auto &view = val.KokkosView(); - RestoreViewAttribute(full_path, view); - } else { - val = it->getAttribute(full_path).get(); + try { + T val; + if constexpr (implements::value) { + val = params.Get(key); + RestoreViewAttribute(full_path, val); + } else if constexpr (is_specialization_of::value) { + val = params.Get(key); + auto &view = val.KokkosView(); + RestoreViewAttribute(full_path, view); + } else { + val = it->getAttribute(full_path).get(); + } + params.Update(key, val); + } catch (std::runtime_error e) { + // TODO(JMM/PG) Add failed load list of "fail/needs fix" list + if (Globals::my_rank == 0) { + std::stringstream ss; + ss << "Failed to load parameter " << fullpath + << " from the restart file! Using default value." << std::endl; + PARTHENON_WARN(ss); + } } - params.Update(key, val); } } } From 3edc7af61895ced8a31adb8fdf4ef4eb5926efa2 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 19 Aug 2025 14:35:02 +0200 Subject: [PATCH 116/125] Fix opmd params try catch --- src/outputs/restart_opmd.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index c6f4e5d62569..709272830e40 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -169,11 +169,11 @@ void RestartReaderOPMD::ReadAllParamsOfType(const std::string &prefix, Params &p val = it->getAttribute(full_path).get(); } params.Update(key, val); - } catch (std::runtime_error e) { + } catch (...) { // TODO(JMM/PG) Add failed load list of "fail/needs fix" list if (Globals::my_rank == 0) { std::stringstream ss; - ss << "Failed to load parameter " << fullpath + ss << "Failed to load parameter " << full_path << " from the restart file! Using default value." << std::endl; PARTHENON_WARN(ss); } From 7cc4040428c528924ef75c30b0e122513e87432c Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 19 Aug 2025 14:55:40 +0200 Subject: [PATCH 117/125] Add final logic to opmd --- src/outputs/parthenon_opmd.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 62f56f60c8a9..5e25fdac9458 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -321,8 +321,13 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * auto filename = output_params.file_basename + "." + output_params.file_id; if (signal == SignalHandler::OutputSignal::now) { filename.append(".now"); + } else if (signal == SignalHandler::OutputSignal::final && + output_params.file_label_final) { + filename.append(".final"); + } else { + filename.append(".%05T"); } - filename.append(".%05T.bp"); + filename.append(".bp"); Series series = Series(filename, Access::CREATE, #ifdef MPI_PARALLEL MPI_COMM_WORLD, From e671e67a51309e841b1eb20f4e0d05a9c68a0f3f Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 19 Aug 2025 15:40:54 +0200 Subject: [PATCH 118/125] WIP slices --- src/outputs/parthenon_opmd.cpp | 131 ++++++++++++++++++++++----------- 1 file changed, 89 insertions(+), 42 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 5e25fdac9458..363bed94e231 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -58,6 +58,8 @@ using namespace OutputUtils; namespace OpenPMDUtils { +enum class SubOutputType { Restart, X1Slice, X2Slice, X3Slice }; + template auto GetFlatHostVecFromView(T view) { // Take a view and return a vector containing rank and dims and a flattened (1D) @@ -381,7 +383,32 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * it.setTime(-1.0); it.setDt(-1.0); } - { + + // TODO(reviewers): PG: I didn't want to pollute OutputParams with sth specific to this + // output type. It's not super nice to process `pin` info here but it did the job. Any + // suggestions? + + const auto output_type_str = pin->GetOrAddString( + output_params.block_name, "output_type", "restart", + std::vector{"restart", "x1slice", "x2slice", "x3slice"}, + "Type of output in the file."); + using OpenPMDUtils::SubOutputType; + auto output_type = SubOutputType::Restart; + if (output_type_str == "x1slice") { + output_type = SubOutputType::X1Slice; + } else if (output_type_str == "x2slice") { + output_type = SubOutputType::X2Slice; + } else if (output_type_str == "x3slice") { + output_type = SubOutputType::X3Slice; + } + const auto is_slice = output_type != SubOutputType::Restart; + auto slice_loc = std::numeric_limits::signaling_NaN(); + if (is_slice) { + PARTHENON_REQUIRE_THROWS(pm->ndim == 3, "Slices are only implemented in 3D"); + slice_loc = pin->GetReal(output_params.block_name, "slice_loc"); + } + + if (!is_slice) { PARTHENON_INSTRUMENT_REGION("Dump Params"); for (const auto &[pkg_name, pkg] : pm->packages.AllPackages()) { @@ -390,7 +417,7 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * } } // Then our own - { + if (!is_slice) { PARTHENON_INSTRUMENT_REGION("write input"); // write input key-value pairs std::ostringstream oss; @@ -398,7 +425,7 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * it.setAttribute("InputFile", oss.str()); } - { + if (!is_slice) { // It's not clear we need all these attributes, but they mirror what's done in the // hdf5 output. it.setAttribute("WallTime", Driver::elapsed_main()); @@ -459,7 +486,7 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * Kokkos::Profiling::popRegion(); // write Attributes // Write block metadata - { + if (!is_slice) { // Manually gather all block data first as it allows to use the (simpler) // Attribute interface rather than writing a distributed dataset -- especially as all // data is being read on restart by every rank anyway. @@ -595,16 +622,16 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * auto effective_nx = static_cast(std::pow(2, level)); openPMD::Extent global_extent; if (pm->ndim == 3) { - mesh_record.setGridSpacing(std::vector{dx3, dx2, dx1}); - mesh_record.setAxisLabels({"z", "y", "x"}); - mesh_record.setGridGlobalOffset({ + auto grid_spacing = std::vector{dx3, dx2, dx1}; + auto axis_labels = std::vector{"z", "y", "x"}; + auto global_offset = std::vector{ pm->mesh_size.xmin(X3DIR), pm->mesh_size.xmin(X2DIR), pm->mesh_size.xmin(X1DIR), - }); - mesh_comp.setPosition(std::vector{ - 0.5 - 0.5 * TopologicalOffsetK(te), 0.5 - 0.5 * TopologicalOffsetJ(te), - 0.5 - 0.5 * TopologicalOffsetI(te)}); + }; + auto position = std::vector{0.5 - 0.5 * TopologicalOffsetK(te), + 0.5 - 0.5 * TopologicalOffsetJ(te), + 0.5 - 0.5 * TopologicalOffsetI(te)}; global_extent = { static_cast(pm->mesh_size.nx(X3DIR) / coarsening_factor_) * @@ -619,6 +646,25 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * effective_nx + TopologicalOffsetI(te), }; + int remove_comp = -1; + if (output_type == SubOutputType::X1Slice) { + remove_comp = 0; + } else if (output_type == SubOutputType::X2Slice) { + remove_comp = 1; + } else if (output_type == SubOutputType::X3Slice) { + remove_comp = 2; + } + if (remove_comp >= 0) { + grid_spacing.erase(grid_spacing.begin() + remove_comp); + axis_labels.erase(axis_labels.begin() + remove_comp); + global_offset.erase(global_offset.begin() + remove_comp); + position.erase(position.begin() + remove_comp); + global_extent.erase(global_extent.begin() + remove_comp); + } + mesh_record.setGridSpacing(grid_spacing); + mesh_record.setAxisLabels(axis_labels); + mesh_record.setGridGlobalOffset(global_offset); + mesh_comp.setPosition(position); } else if (pm->ndim == 2) { mesh_record.setGridSpacing(std::vector{dx2, dx1}); mesh_record.setAxisLabels({"y", "x"}); @@ -734,7 +780,7 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * // -------------------------------------------------------------------------------- // // WRITING Sparse metadata // // -------------------------------------------------------------------------------- // - if (num_sparse > 0) { + if (!is_slice && num_sparse > 0) { auto sparse_allocated_global = FlattendedLocalToGlobal(pm, sparse_allocated); it.setAttribute("SparseInfo", sparse_allocated_global); it.setAttribute("SparseFields", sparse_names); @@ -746,39 +792,40 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * // -------------------------------------------------------------------------------- // // WRITING PARTICLE DATA // // -------------------------------------------------------------------------------- // + if (!is_slice) { + + Kokkos::Profiling::pushRegion("write particle data"); + // TODO(pgrete) as above, first wrt differentiating between restart_ (last arg) + AllSwarmInfo all_swarm_info(pm->block_list, output_params.swarms, + DumpOutputMode::RESTART); + for (auto &[swname, swinfo] : all_swarm_info.all_info) { + openPMD::ParticleSpecies swm = it.particles[swname]; + // These indicate particles/meshblock and location in global index + // space where each meshblock starts + auto counts_global = FlattendedLocalToGlobal(pm, swinfo.counts); + swm.setAttribute("counts", counts_global); + auto offsets_global = FlattendedLocalToGlobal(pm, swinfo.offsets); + swm.setAttribute("offsets", offsets_global); + + if (swinfo.global_count == 0) { + continue; + } - Kokkos::Profiling::pushRegion("write particle data"); - // TODO(pgrete) as above, first wrt differentiating between restart_ (last arg) - AllSwarmInfo all_swarm_info(pm->block_list, output_params.swarms, - DumpOutputMode::RESTART); - for (auto &[swname, swinfo] : all_swarm_info.all_info) { - openPMD::ParticleSpecies swm = it.particles[swname]; - // These indicate particles/meshblock and location in global index - // space where each meshblock starts - auto counts_global = FlattendedLocalToGlobal(pm, swinfo.counts); - swm.setAttribute("counts", counts_global); - auto offsets_global = FlattendedLocalToGlobal(pm, swinfo.offsets); - swm.setAttribute("offsets", offsets_global); - - if (swinfo.global_count == 0) { - continue; + OpenPMDUtils::WriteSwarmVar(swinfo, swm, it); + OpenPMDUtils::WriteSwarmVar(swinfo, swm, it); + OpenPMDUtils::WriteSwarmVar(swinfo, swm, it); + + // From the HDF5 output: + // If swarm does not contain an "id" object, generate a sequential + // one for vis. + // BUT PG: this may break things in unpredicable ways + // I'm in favor of enforcing a global id somehow. We shold discuss. + PARTHENON_REQUIRE_THROWS(swinfo.var_info.count(swarm_position::id::name()) != 0 || + swinfo.var_info.count("id") != 0, + "Particles should always carry a unique, persistent id!"); } - - OpenPMDUtils::WriteSwarmVar(swinfo, swm, it); - OpenPMDUtils::WriteSwarmVar(swinfo, swm, it); - OpenPMDUtils::WriteSwarmVar(swinfo, swm, it); - - // From the HDF5 output: - // If swarm does not contain an "id" object, generate a sequential - // one for vis. - // BUT PG: this may break things in unpredicable ways - // I'm in favor of enforcing a global id somehow. We shold discuss. - PARTHENON_REQUIRE_THROWS(swinfo.var_info.count(swarm_position::id::name()) != 0 || - swinfo.var_info.count("id") != 0, - "Particles should always carry a unique, persistent id!"); + Kokkos::Profiling::popRegion(); // write particle data } - Kokkos::Profiling::popRegion(); // write particle data - // The iteration can be closed in order to help free up resources. // The iteration's content will be flushed automatically. // An iteration once closed cannot (yet) be reopened. From 6ab9d02f8e9324ac695223d5d85f2013b6725e1b Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 20 Aug 2025 22:33:29 +0200 Subject: [PATCH 119/125] Logic for slices seems to work. Data not. --- src/outputs/parthenon_opmd.cpp | 46 +++++++++++++++++++++++++++++----- src/outputs/parthenon_opmd.hpp | 5 +++- src/outputs/restart_opmd.cpp | 3 ++- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 363bed94e231..bb75a2a480f5 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -58,8 +58,6 @@ using namespace OutputUtils; namespace OpenPMDUtils { -enum class SubOutputType { Restart, X1Slice, X2Slice, X3Slice }; - template auto GetFlatHostVecFromView(T view) { // Take a view and return a vector containing rank and dims and a flattened (1D) @@ -259,7 +257,8 @@ GetMeshRecordAndComponentNames(const VarInfo &vinfo, const TopologicalElement te std::tuple GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb, - const TopologicalElement te, const int coarsening_factor) { + const TopologicalElement te, const int coarsening_factor, + const SubOutputType output_type) { openPMD::Offset chunk_offset; openPMD::Extent chunk_extent; const auto loc = pm->Forest().GetLegacyTreeLocation(pmb->loc); @@ -277,6 +276,18 @@ GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb, } else { PARTHENON_THROW("1D output for openpmd not yet supported."); } + int remove_comp = -1; + if (output_type == SubOutputType::X1Slice) { + remove_comp = 0; + } else if (output_type == SubOutputType::X2Slice) { + remove_comp = 1; + } else if (output_type == SubOutputType::X3Slice) { + remove_comp = 2; + } + if (remove_comp >= 0) { + chunk_extent.erase(chunk_extent.begin() + remove_comp); + chunk_offset.erase(chunk_offset.begin() + remove_comp); + } return {chunk_offset, chunk_extent}; } } // namespace OpenPMDUtils @@ -392,6 +403,8 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * output_params.block_name, "output_type", "restart", std::vector{"restart", "x1slice", "x2slice", "x3slice"}, "Type of output in the file."); + // C++20 please + // using enum OpenPMDUtils::SubOutputType; using OpenPMDUtils::SubOutputType; auto output_type = SubOutputType::Restart; if (output_type_str == "x1slice") { @@ -401,7 +414,9 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * } else if (output_type_str == "x3slice") { output_type = SubOutputType::X3Slice; } - const auto is_slice = output_type != SubOutputType::Restart; + const auto is_slice = output_type == SubOutputType::X1Slice || + output_type == SubOutputType::X2Slice || + output_type == SubOutputType::X3Slice; auto slice_loc = std::numeric_limits::signaling_NaN(); if (is_slice) { PARTHENON_REQUIRE_THROWS(pm->ndim == 3, "Slices are only implemented in 3D"); @@ -722,6 +737,7 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * component_buffer_view(component_buffer.data(), nk, nj, ni); Kokkos::deep_copy(component_buffer_view, data); #endif + auto &coords = pmb->coords; auto out_var_h = out_var->data.GetHostMirrorAndCopy(); for (const auto &te : vinfo.topological_elements) { auto ib = bounds.GetBoundsI(IndexDomain::interior, te); @@ -749,15 +765,33 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * ((k - kb.s) % coarsening_factor_ != 0)) { continue; } + if (is_slice) { + if (output_type == SubOutputType::X1Slice) { + if (slice_loc < coords.Xf(k, j, i)) continue; + if (slice_loc >= coords.Xf(k, j, i + coarsening_factor_)) + continue; + } else if (output_type == SubOutputType::X2Slice) { + if (slice_loc < coords.Xf(k, j, i)) continue; + if (slice_loc >= coords.Xf(k, j + coarsening_factor_, i)) + continue; + } else if (output_type == SubOutputType::X3Slice) { + if (slice_loc < coords.Xf(k, j, i)) continue; + if (slice_loc >= coords.Xf(k + coarsening_factor_, j, i)) + continue; + } else { + PARTHENON_FAIL("Unclear how I got here."); + } + } tmp_data[tmp_offset] = static_cast( out_var_h(static_cast(te) % 3, t, u, v, k, j, i)); + tmp_offset++; } } } const auto [chunk_offset, chunk_extent] = - OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb, te, - coarsening_factor_); + OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb, te, coarsening_factor_, + output_type); mesh_comp.storeChunkRaw(&tmp_data[comp_offset], chunk_offset, chunk_extent); comp_idx += 1; diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index 0b95b06f93fc..54409e30b1a2 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -23,6 +23,8 @@ namespace parthenon { namespace OpenPMDUtils { +enum class SubOutputType { Restart, X1Slice, X2Slice, X3Slice }; + template void RestoreViewAttribute(const std::string &full_path, T &view, openPMD::Iteration *it); @@ -49,7 +51,8 @@ GetMeshRecordAndComponentNames(const OutputUtils::VarInfo &vinfo, // TODO(pgrete) needs to be updated to properly work with Forests std::tuple GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb, - const TopologicalElement te, const int coarsening_factor); + const TopologicalElement te, const int coarsening_factor, + const SubOutputType outupt_type); } // namespace OpenPMDUtils } // namespace parthenon diff --git a/src/outputs/restart_opmd.cpp b/src/outputs/restart_opmd.cpp index 709272830e40..41cf9350c46e 100644 --- a/src/outputs/restart_opmd.cpp +++ b/src/outputs/restart_opmd.cpp @@ -243,7 +243,8 @@ void RestartReaderOPMD::ReadBlocks(const std::string &var_name, IndexRange block // Restarting from coarsened output not supported at the moment const int coarsening_factor = 1; const auto [chunk_offset, chunk_extent] = - OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb, te, coarsening_factor); + OpenPMDUtils::GetChunkOffsetAndExtent( + pm, pmb, te, coarsening_factor, OpenPMDUtils::SubOutputType::Restart); mesh_comp.loadChunkRaw(&data_vec[comp_offset], chunk_offset, chunk_extent); comp_offset += std::accumulate(chunk_extent.cbegin(), chunk_extent.cend(), 1, std::multiplies{}); From 6cd7c39dc2b313b0b29d854a09ae2ad35e7d2dca Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 21 Aug 2025 12:05:45 +0200 Subject: [PATCH 120/125] Fix final logic --- src/outputs/parthenon_opmd.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index bb75a2a480f5..674dd66cbe5c 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -337,9 +337,9 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * } else if (signal == SignalHandler::OutputSignal::final && output_params.file_label_final) { filename.append(".final"); - } else { - filename.append(".%05T"); } + filename.append(".%05T"); + filename.append(".bp"); Series series = Series(filename, Access::CREATE, #ifdef MPI_PARALLEL From 56081fe0b5a34895e16dc0b372c8a95095e823f5 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 21 Aug 2025 16:12:54 +0200 Subject: [PATCH 121/125] Fix slices --- src/outputs/outputs.cpp | 1 - src/outputs/parthenon_opmd.cpp | 15 +++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/outputs/outputs.cpp b/src/outputs/outputs.cpp index 3036b543cbfe..a0b10e933482 100644 --- a/src/outputs/outputs.cpp +++ b/src/outputs/outputs.cpp @@ -233,7 +233,6 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) { (op.file_type == "corehdf5"); const bool is_openpmd_output = (op.file_type == "openpmd"); - if (is_hdf5_output || is_openpmd_output) { op.single_precision_output = pin->GetOrAddBoolean(op.block_name, "single_precision_output", false); diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index 674dd66cbe5c..f20439ea4005 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -278,11 +278,11 @@ GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb, } int remove_comp = -1; if (output_type == SubOutputType::X1Slice) { - remove_comp = 0; + remove_comp = 2; } else if (output_type == SubOutputType::X2Slice) { remove_comp = 1; } else if (output_type == SubOutputType::X3Slice) { - remove_comp = 2; + remove_comp = 0; } if (remove_comp >= 0) { chunk_extent.erase(chunk_extent.begin() + remove_comp); @@ -663,11 +663,11 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * }; int remove_comp = -1; if (output_type == SubOutputType::X1Slice) { - remove_comp = 0; + remove_comp = 2; } else if (output_type == SubOutputType::X2Slice) { remove_comp = 1; } else if (output_type == SubOutputType::X3Slice) { - remove_comp = 2; + remove_comp = 0; } if (remove_comp >= 0) { grid_spacing.erase(grid_spacing.begin() + remove_comp); @@ -760,11 +760,13 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * for (int k = kb.s; k <= kb.e; ++k) { for (int j = jb.s; j <= jb.e; ++j) { for (int i = ib.s; i <= ib.e; ++i) { + // Skip cells for coarse grained outputs if (((i - ib.s) % coarsening_factor_ != 0) || ((j - jb.s) % coarsening_factor_ != 0) || ((k - kb.s) % coarsening_factor_ != 0)) { continue; } + // Skip cells outside slices if (is_slice) { if (output_type == SubOutputType::X1Slice) { if (slice_loc < coords.Xf(k, j, i)) continue; @@ -789,9 +791,14 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * } } } + // if no data was being selected + if (comp_offset == tmp_offset) { + continue; + } const auto [chunk_offset, chunk_extent] = OpenPMDUtils::GetChunkOffsetAndExtent(pm, pmb, te, coarsening_factor_, output_type); + mesh_comp.storeChunkRaw(&tmp_data[comp_offset], chunk_offset, chunk_extent); comp_idx += 1; From 505cdb47dba9ae82e1a5efba1a936e3e41fc7d89 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 28 Aug 2025 10:27:45 +0200 Subject: [PATCH 122/125] Fix regression test --- tst/regression/test_suites/restart_opmd/parthinput.restart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tst/regression/test_suites/restart_opmd/parthinput.restart b/tst/regression/test_suites/restart_opmd/parthinput.restart index 06f482585edb..5ba796143d19 100644 --- a/tst/regression/test_suites/restart_opmd/parthinput.restart +++ b/tst/regression/test_suites/restart_opmd/parthinput.restart @@ -1,6 +1,6 @@ # ======================================================================================== # Parthenon performance portable AMR framework -# Copyright(C) 2024 The Parthenon collaboration +# Copyright(C) 2024-2025 The Parthenon collaboration # Licensed under the 3-clause BSD License, see LICENSE file for details # ======================================================================================== @@ -57,8 +57,8 @@ fill_derived = false # whether to fill one-copy test vars file_type = openpmd dt = 0.050 +use_final_label = false file_type = rst dt = 0.050 - From 39c2a9ac16b8f33bba0281699e0c43ef2dc4e478 Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Tue, 30 Sep 2025 17:53:32 -0400 Subject: [PATCH 123/125] fix style --- src/outputs/parthenon_opmd.cpp | 3 ++- src/outputs/parthenon_opmd.hpp | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index f20439ea4005..e3360052ca95 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -17,10 +17,12 @@ //! \file parthenon_openpmd.cpp // \brief Output for OpenPMD https://www.openpmd.org/ (supporting various backends) +#include #include #include #include #include +#include #include #include #include @@ -834,7 +836,6 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * // WRITING PARTICLE DATA // // -------------------------------------------------------------------------------- // if (!is_slice) { - Kokkos::Profiling::pushRegion("write particle data"); // TODO(pgrete) as above, first wrt differentiating between restart_ (last arg) AllSwarmInfo all_swarm_info(pm->block_list, output_params.swarms, diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index 54409e30b1a2..46480d949926 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -8,13 +8,14 @@ //! \file restart_opmd.hpp // \brief Provides support for restarting from OpenPMD output -// OpenPMD headers -#include - +// C++ stdlib #include #include #include +// OpenPMD headers +#include + #include "basic_types.hpp" #include "mesh/meshblock.hpp" #include "outputs/output_utils.hpp" From beeacba9ebc2359aa9d607eaa5e56282c60c82d7 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 7 Oct 2025 15:22:34 +0200 Subject: [PATCH 124/125] Impl VariableExists for OpenPMD --- src/outputs/parthenon_opmd.cpp | 81 ++++++++++++++++++---------------- src/outputs/parthenon_opmd.hpp | 11 ++++- src/outputs/restart.hpp | 5 ++- src/outputs/restart_hdf5.hpp | 13 ++++-- src/outputs/restart_opmd.hpp | 59 +++++++++++++------------ src/parthenon_manager.hpp | 2 +- 6 files changed, 97 insertions(+), 74 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index e3360052ca95..c59fbc834ee2 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -156,58 +156,61 @@ void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, auto const dataset = openPMD::Dataset(openPMD::determineDatatype(host_data.data()), {swinfo.global_count}); - // TODO(pgrete) ask OpenPMD group if this is the right approach of if our non-scalar - // partices should be a multi-D `dataset` if is scalar - if (vinfo.tensor_rank == 0) { - // special sauce to align "positions" with standard - std::string particle_record; - std::string particle_record_component; - if (vname == swarm_position::x::name()) { - particle_record = "position"; - particle_record_component = "x"; - } else if (vname == swarm_position::y::name()) { - particle_record = "position"; - particle_record_component = "y"; - } else if (vname == swarm_position::z::name()) { - particle_record = "position"; - particle_record_component = "z"; - } else if (vname == swarm_position::id::name()) { - particle_record = "id"; - particle_record_component = openPMD::MeshRecordComponent::SCALAR; - } else { - particle_record = vname; - particle_record_component = openPMD::MeshRecordComponent::SCALAR; - } + // TODO(pgrete) ask OpenPMD group if this is the right approach (flatten vector and + // tensors with flattended indices as string component names) or if our non-scalar + // particle variables should be a multi-D `dataset` (if possible) + for (auto n = 0; n < vinfo.nvar; n++) { + auto [particle_record, particle_record_component] = + OpenPMDUtils::GetParticleRecordAndComponentNames(vname, vinfo.tensor_rank, n); + openPMD::RecordComponent rc = swm[particle_record][particle_record_component]; rc.resetDataset(dataset); // only write if there's sth to write (otherwise the host_data nullptr is caught) if (swinfo.count_on_rank != 0) { - rc.storeChunk(host_data, {swinfo.global_offset}, {host_data.size()}); + rc.storeChunkRaw(&host_data[n * swinfo.count_on_rank], {swinfo.global_offset}, + {swinfo.count_on_rank}); } // if positional, add offsets - if (particle_record_component != openPMD::MeshRecordComponent::SCALAR) { + if (particle_record == "position") { auto rc_offset = swm["positionOffset"][particle_record_component]; rc_offset.resetDataset(dataset); rc_offset.makeConstant(0.0); } - - // else flatten components - } else { - for (auto n = 0; n < vinfo.nvar; n++) { - openPMD::RecordComponent rc = swm[vname][std::to_string(n)]; - rc.resetDataset(dataset); - // only write if there's sth to write (otherwise the host_data nullptr is caught) - if (swinfo.count_on_rank != 0) { - rc.storeChunkRaw(&host_data[n * swinfo.count_on_rank], {swinfo.global_offset}, - {swinfo.count_on_rank}); - } - } } // Flush because the host buffer is temporary it.seriesFlush(); } } + +std::tuple +GetParticleRecordAndComponentNames(const std::string &vname, const int rank, + const int flat_comp_idx) { + + std::string particle_record; + std::string particle_record_component; + + // special sauce to align "positions" with standard + if (vname == swarm_position::x::name()) { + particle_record = "position"; + particle_record_component = "x"; + } else if (vname == swarm_position::y::name()) { + particle_record = "position"; + particle_record_component = "y"; + } else if (vname == swarm_position::z::name()) { + particle_record = "position"; + particle_record_component = "z"; + } else if (vname == swarm_position::id::name()) { + particle_record = "id"; + particle_record_component = openPMD::MeshRecordComponent::SCALAR; + } else { + particle_record = vname; + particle_record_component = + rank == 0 ? openPMD::MeshRecordComponent::SCALAR : std::to_string(flat_comp_idx); + } + return {particle_record, particle_record_component}; +} + std::tuple GetMeshRecordAndComponentNames(const VarInfo &vinfo, const TopologicalElement te, const int comp_idx, const int level) { @@ -807,8 +810,8 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * } } } // loop over components - } // loop over topological elements - } // out_var->IsAllocated() + } // loop over topological elements + } // out_var->IsAllocated() if (vinfo.is_sparse) { auto sparse_idx = sparse_field_idx.at(vinfo.label); sparse_allocated.at(b_idx * num_sparse + sparse_idx) = @@ -817,7 +820,7 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * } } // loop over blocks it.seriesFlush(); - } // loop over vars + } // loop over vars Kokkos::Profiling::popRegion(); // write all variable data // -------------------------------------------------------------------------------- // diff --git a/src/outputs/parthenon_opmd.hpp b/src/outputs/parthenon_opmd.hpp index 46480d949926..e4c9e3b134a0 100644 --- a/src/outputs/parthenon_opmd.hpp +++ b/src/outputs/parthenon_opmd.hpp @@ -1,6 +1,6 @@ //======================================================================================== // Parthenon performance portable AMR framework -// Copyright(C) 2024 The Parthenon collaboration +// Copyright(C) 2024-2025 The Parthenon collaboration // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== #ifndef OUTPUTS_PARTHENON_OPMD_HPP_ @@ -55,6 +55,15 @@ GetChunkOffsetAndExtent(Mesh *pm, std::shared_ptr pmb, const TopologicalElement te, const int coarsening_factor, const SubOutputType outupt_type); +// Construct OpenPMD Particle "record" name and comonnent identifier. +// - vname is the variable name +// - rank is the variable rank (i.e., 0 is scalar etc) +// - comp_idx is a flattended index over all components of the vectors and tensors, i.e., +// the typical v,u,t indices. +std::tuple +GetParticleRecordAndComponentNames(const std::string &vname, const int rank, + const int flat_comp_idx); + } // namespace OpenPMDUtils } // namespace parthenon diff --git a/src/outputs/restart.hpp b/src/outputs/restart.hpp index 797ed4da803b..e61cfe2dd6d4 100644 --- a/src/outputs/restart.hpp +++ b/src/outputs/restart.hpp @@ -137,8 +137,9 @@ class RestartReader { virtual void ReadParams(const std::string &name, Params &p) = 0; enum class DataType { Field, Swarm, SwarmVar }; - [[nodiscard]] virtual bool VariableExists(const std::string &name, - const DataType data_type) const = 0; + [[nodiscard]] virtual bool + VariableExists(const std::string &name, const DataType data_type, + const std::string swarmvarname = "") const = 0; // closes out the restart file // perhaps belongs in a destructor? diff --git a/src/outputs/restart_hdf5.hpp b/src/outputs/restart_hdf5.hpp index e931d4610397..f73246b47370 100644 --- a/src/outputs/restart_hdf5.hpp +++ b/src/outputs/restart_hdf5.hpp @@ -234,16 +234,23 @@ class RestartReaderHDF5 : public RestartReader { void ReadParams(const std::string &name, Params &p) override; - [[nodiscard]] bool VariableExists(const std::string &name, - const DataType /*data_type*/) const override { + [[nodiscard]] bool VariableExists(const std::string &name, const DataType data_type, + const std::string swarmvarname = "" + + ) const override { #ifdef ENABLE_HDF5 // Make sure dataset exists. Our HDF5 output does not differentiate between // fields and swarms, so we can ignore the data_type. Note, we may eventually // want to fix this as swarms and fields with the same name may cause issues. // disable error handling/printing while probing so missing datasets do not // spam the log, then restore the aborting handler. + std::string full_name = name; + if (data_type == DataType::SwarmVar) { + full_name = name + "/SwarmVars/" + swarmvarname; + } H5Eset_auto(H5E_DEFAULT, NULL, NULL); - auto status = PARTHENON_HDF5_CHECK(H5Oexists_by_name(fh_, name.c_str(), H5P_DEFAULT)); + auto status = + PARTHENON_HDF5_CHECK(H5Oexists_by_name(fh_, full_name.c_str(), H5P_DEFAULT)); H5Eset_auto(H5E_DEFAULT, aborting_error_handler, NULL); return status > 0; #else diff --git a/src/outputs/restart_opmd.hpp b/src/outputs/restart_opmd.hpp index 052c35a8bbdb..6a503aa0cfe7 100644 --- a/src/outputs/restart_opmd.hpp +++ b/src/outputs/restart_opmd.hpp @@ -17,6 +17,7 @@ #include #include "basic_types.hpp" +#include "outputs/parthenon_opmd.hpp" #include "outputs/restart.hpp" #include "pack/swarm_default_names.hpp" @@ -71,36 +72,18 @@ class RestartReaderOPMD : public RestartReader { const auto &shape = m.Shape(); const int rank = shape.size(); - std::size_t nvar = 1; + std::size_t ncomp = 1; for (int i = 0; i < rank; ++i) { - nvar *= shape[rank - 1 - i]; + ncomp *= shape[rank - 1 - i]; } - std::size_t total_count = nvar * count; + std::size_t total_count = ncomp * count; if (data_vec.size() < total_count) { // greedy re-alloc data_vec.resize(total_count); } - std::string particle_record; - std::string particle_record_component; - for (auto n = 0; n < nvar; n++) { - if (varname == swarm_position::x::name()) { - particle_record = "position"; - particle_record_component = "x"; - } else if (varname == swarm_position::y::name()) { - particle_record = "position"; - particle_record_component = "y"; - } else if (varname == swarm_position::z::name()) { - particle_record = "position"; - particle_record_component = "z"; - } else if (varname == swarm_position::id::name()) { - particle_record = "id"; - particle_record_component = openPMD::MeshRecordComponent::SCALAR; - } else { - particle_record = varname; - particle_record_component = - rank == 0 ? openPMD::MeshRecordComponent::SCALAR : std::to_string(n); - } - + for (auto n = 0; n < ncomp; n++) { + auto [particle_record, particle_record_component] = + OpenPMDUtils::GetParticleRecordAndComponentNames(varname, rank, n); openPMD::RecordComponent rc = swm[particle_record][particle_record_component]; rc.loadChunkRaw(&data_vec[n * count], {offset}, {count}); } @@ -155,10 +138,30 @@ class RestartReaderOPMD : public RestartReader { } Kokkos::deep_copy(view, view_h); } - [[nodiscard]] bool VariableExists(const std::string &name, - const DataType data_type) const override { - // TODO(pgrete) needs impl - return true; + [[nodiscard]] bool VariableExists(const std::string &name, const DataType data_type, + const std::string swarmvarname = "") const override { + if (data_type == DataType::Field) { + // Given that MeshRecord labels also carry information about the topological element + // and level, we just check for the prefix (this silently assumes that if one + // matching record is found, then the variable exists on all levels/for all + // components). Might cause issue for edge cases (and or variable combinations that + // contain the `_` separator), but this should not be an issue as the error message + // in the OpenPMD restart reader is clear (about the variable) when reading fails + // later. + for (auto [label, mesh] : it->meshes) { + if (label.compare(0, name.length() + 1, name + "_") == 0) { + return true; + } + } + } else if (data_type == DataType::Swarm) { + return it->particles.contains(name); + } else if (data_type == DataType::SwarmVar) { + // rank = 0, and component index = 0 because we just care about the record name + auto [particle_record, particle_record_component] = + OpenPMDUtils::GetParticleRecordAndComponentNames(swarmvarname, 0, 0); + return it->particles[name].contains(particle_record); + } + return false; } // closes out the restart file // perhaps belongs in a destructor? diff --git a/src/parthenon_manager.hpp b/src/parthenon_manager.hpp index d5851d146fd2..34fe5458e710 100644 --- a/src/parthenon_manager.hpp +++ b/src/parthenon_manager.hpp @@ -73,7 +73,7 @@ class ParthenonManager { auto arrdims = m.GetArrayDims(pswarm->GetBlockPointer(), false); auto var_missing_on_disk = !restartReader->VariableExists( - swarmname + "/SwarmVars/" + varname, RestartReader::DataType::SwarmVar); + swarmname, RestartReader::DataType::SwarmVar, varname); if (Globals::my_rank == 0) { std::cout << "SwarmVar: " << varname << (var_missing_on_disk ? " missing on disk\n" : "\n"); From 00d94763f28ed552c04d3a53669db1db37f3e68e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 7 Oct 2025 17:33:59 +0200 Subject: [PATCH 125/125] Format --- src/outputs/parthenon_opmd.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/outputs/parthenon_opmd.cpp b/src/outputs/parthenon_opmd.cpp index c59fbc834ee2..c393c7025320 100644 --- a/src/outputs/parthenon_opmd.cpp +++ b/src/outputs/parthenon_opmd.cpp @@ -186,7 +186,6 @@ void WriteSwarmVar(const SwarmInfo &swinfo, openPMD::ParticleSpecies swm, std::tuple GetParticleRecordAndComponentNames(const std::string &vname, const int rank, const int flat_comp_idx) { - std::string particle_record; std::string particle_record_component; @@ -810,8 +809,8 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * } } } // loop over components - } // loop over topological elements - } // out_var->IsAllocated() + } // loop over topological elements + } // out_var->IsAllocated() if (vinfo.is_sparse) { auto sparse_idx = sparse_field_idx.at(vinfo.label); sparse_allocated.at(b_idx * num_sparse + sparse_idx) = @@ -820,7 +819,7 @@ void OpenPMDOutput::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime * } } // loop over blocks it.seriesFlush(); - } // loop over vars + } // loop over vars Kokkos::Profiling::popRegion(); // write all variable data // -------------------------------------------------------------------------------- //