diff --git a/CHANGELOG.md b/CHANGELOG.md index ec4f16dd1578..bd24facc4e43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Current develop ### Added (new features/APIs/variables/...) +- [[PR 1314]](https://github.com/parthenon-hpc-lab/parthenon/pull/1314) Add option of user specified BCs in AddBoundaryExchangeTasks - [[PR 1244]](https://github.com/parthenon-hpc-lab/parthenon/pull/1244) Add TaskCollection timeout capability - [[PR 1311]](https://github.com/parthenon-hpc-lab/parthenon/pull/1311) Magnitude refinement criteria, per-package PostInitialize function hooks, mergescv and prettyparams utilities - [[PR 1142]](https://github.com/parthenon-hpc-lab/parthenon/pull/1142) Unify par_dispatch, par_for_outer & par_for_inner overloads diff --git a/doc/sphinx/src/boundary_conditions.rst b/doc/sphinx/src/boundary_conditions.rst index c97cc6b449dd..58077e19e4bf 100644 --- a/doc/sphinx/src/boundary_conditions.rst +++ b/doc/sphinx/src/boundary_conditions.rst @@ -145,3 +145,17 @@ for a more complete example): pkg->UserBoundaryFunctions[BF::inner_x2].push_back(GetMyBC()); ... } + +User override of boundary conditions in `AddBoundaryExchangeTasks`. +------------------------------------------------------------------- + +Sometimes it is desirable to apply different boundary conditions when communicating +on different containers (e.g. when one container represents the value of a variable +and another container represents a Newton-Raphson correction to that variable). To +easily allow this, `AddBoundaryExchangeTasks` can take a final argument with type +`std::function> &, bool)>` or with type +`std::function>, bool)>`. +When this argument is specified, all other boundary conditions will be ignored and +the passed functions will be used as boundary conditions instead. The final boolean +arguments of the functions specify if the boundary condition needs to be applied on +the coarse buffer. \ No newline at end of file diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 78121cd3fac9..db1c70a91017 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -168,6 +168,8 @@ template TaskStatus SendBoundBufs(std::shared_ptr> &); template TaskStatus SendBoundBufs(std::shared_ptr> &); +template TaskStatus +SendBoundBufs(std::shared_ptr> &); template TaskStatus StartReceiveBoundBufs(std::shared_ptr> &md) { @@ -196,6 +198,8 @@ template TaskStatus StartReceiveBoundBufs( std::shared_ptr> &); template TaskStatus StartReceiveBoundBufs(std::shared_ptr> &); +template TaskStatus +StartReceiveBoundBufs(std::shared_ptr> &); template TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { @@ -246,6 +250,8 @@ template TaskStatus ReceiveBoundBufs(std::shared_ptr> &); template TaskStatus ReceiveBoundBufs(std::shared_ptr> &); +template TaskStatus +ReceiveBoundBufs(std::shared_ptr> &); template TaskStatus SetBounds(std::shared_ptr> &md) { @@ -356,6 +362,7 @@ template TaskStatus SetBounds(std::shared_ptr> &); template TaskStatus SetBounds(std::shared_ptr> &); +template TaskStatus SetBounds(std::shared_ptr> &); template TaskStatus ProlongateBounds(std::shared_ptr> &md) { @@ -400,56 +407,55 @@ template TaskStatus ProlongateBounds(std::shared_ptr> &); template TaskStatus ProlongateBounds(std::shared_ptr> &); +template TaskStatus +ProlongateBounds(std::shared_ptr> &); -// Adds all relevant boundary communication to a single task list -template -TaskID AddBoundaryExchangeTasks(TaskID dependency, TaskList &tl, - std::shared_ptr> &md, bool multilevel) { - // TODO(LFR): Splitting up the boundary tasks while doing prolongation can cause some - // possible issues for sparse fields. In particular, the order in which - // fields are allocated and then set could potentially result in different - // results if the default sparse value is non-zero. - // const auto any = BoundaryType::any; - static_assert(bounds == BoundaryType::any || bounds == BoundaryType::gmg_same); - // const auto local = BoundaryType::local; - // const auto nonlocal = BoundaryType::nonlocal; - - // auto send = tl.AddTask(dependency, SendBoundBufs, md); - // auto send_local = tl.AddTask(dependency, SendBoundBufs, md); - - // auto recv_local = tl.AddTask(dependency, ReceiveBoundBufs, md); - // auto set_local = tl.AddTask(recv_local, SetBounds, md); - - // auto recv = tl.AddTask(dependency, ReceiveBoundBufs, md); - // auto set = tl.AddTask(recv, SetBounds, md); +template +TaskStatus ProlongateInternalBounds(std::shared_ptr> &md) { + PARTHENON_INSTRUMENT - // auto cbound = tl.AddTask(set, ApplyCoarseBoundaryConditions, md); + Mesh *pmesh = md->GetMeshPointer(); + auto &cache = md->GetBvarsCache().GetSubCache(bound_type, false); - // auto pro_local = tl.AddTask(cbound | set_local | set, ProlongateBounds, md); - // auto pro = tl.AddTask(cbound | set_local | set, ProlongateBounds, md); + auto [rebuild, nbound] = CheckReceiveBufferCacheForRebuild(md); - // auto out = (pro_local | pro); + if (rebuild) { + if constexpr (bound_type == BoundaryType::gmg_prolongate_recv) { + RebuildBufferCache(md, nbound, BndInfo::GetSetBndInfo, + ProResInfo::GetInteriorProlongate); + } else if constexpr (bound_type == BoundaryType::gmg_restrict_recv) { + RebuildBufferCache(md, nbound, BndInfo::GetSetBndInfo, + ProResInfo::GetNull); + } else { + RebuildBufferCache(md, nbound, BndInfo::GetSetBndInfo, + ProResInfo::GetSet); + } + } - auto send = tl.AddTask(dependency, TF(SendBoundBufs), md); - auto recv = tl.AddTask(dependency, TF(ReceiveBoundBufs), md); - auto set = tl.AddTask(recv, TF(SetBounds), md); + if (nbound > 0 && pmesh->multilevel && md->NumBlocks() > 0) { + auto pmb = md->GetBlockData(0)->GetBlockPointer(); + StateDescriptor *resolved_packages = pmb->resolved_packages.get(); - auto pro = set; - if (md->GetMeshPointer()->multilevel) { - auto cbound = tl.AddTask(set, TF(ApplyBoundaryConditionsOnCoarseOrFineMD), md, true); - pro = tl.AddTask(cbound, TF(ProlongateBounds), md); + // Prolongate from coarse buffer + refinement::ProlongateInternal(resolved_packages, cache.prores_cache, pmb->cellbounds, + pmb->c_cellbounds); } - auto fbound = tl.AddTask(pro, TF(ApplyBoundaryConditionsOnCoarseOrFineMD), md, false); - - return fbound; + return TaskStatus::complete; } -template TaskID -AddBoundaryExchangeTasks(TaskID, TaskList &, - std::shared_ptr> &, bool); +template TaskStatus +ProlongateInternalBounds(std::shared_ptr> &); +template TaskStatus +ProlongateInternalBounds(std::shared_ptr> &); +template TaskStatus +ProlongateInternalBounds(std::shared_ptr> &); +template TaskStatus ProlongateInternalBounds( + std::shared_ptr> &); +template TaskStatus +ProlongateInternalBounds(std::shared_ptr> &); -template TaskID -AddBoundaryExchangeTasks(TaskID, TaskList &, - std::shared_ptr> &, bool); +bool IsMeshMultilevel(std::shared_ptr> &md) { + return md->GetMeshPointer()->multilevel; +} TaskID AddFluxCorrectionTasks(TaskID dependency, TaskList &tl, std::shared_ptr> &md, bool multilevel) { diff --git a/src/bvals/comms/bvals_in_one.hpp b/src/bvals/comms/bvals_in_one.hpp index 0180bd6a8305..091a836cbbb3 100644 --- a/src/bvals/comms/bvals_in_one.hpp +++ b/src/bvals/comms/bvals_in_one.hpp @@ -23,6 +23,7 @@ #include #include "basic_types.hpp" +#include "bvals/boundary_conditions.hpp" #include "bvals/neighbor_block.hpp" #include "coordinates/coordinates.hpp" @@ -62,6 +63,8 @@ inline TaskStatus SetBoundaries(std::shared_ptr> &md) { template TaskStatus ProlongateBounds(std::shared_ptr> &md); +template +TaskStatus ProlongateInternalBounds(std::shared_ptr> &md); inline TaskStatus ProlongateBoundaries(std::shared_ptr> &md) { return ProlongateBounds(md); } @@ -79,14 +82,52 @@ static TaskStatus SetFluxCorrections(std::shared_ptr> &md) { return SetBounds(md); } +// Adds all relevant flux correction tasks to a single task list +TaskID AddFluxCorrectionTasks(TaskID dependency, TaskList &tl, + std::shared_ptr> &md, bool multilevel); + +using BValOnMDFunc_t = std::function> &, bool)>; +using BValOnMDTasks_t = + std::function>, bool)>; +bool IsMeshMultilevel(std::shared_ptr> &md); + // Adds all relevant boundary communication to a single task list template TaskID AddBoundaryExchangeTasks(TaskID dependency, TaskList &tl, - std::shared_ptr> &md, bool multilevel); + std::shared_ptr> &md, bool multilevel, + BValOnMDTasks_t ApplyBCs) { + static_assert(bounds == BoundaryType::any || bounds == BoundaryType::gmg_same); -// Adds all relevant flux correction tasks to a single task list -TaskID AddFluxCorrectionTasks(TaskID dependency, TaskList &tl, - std::shared_ptr> &md, bool multilevel); + auto send = tl.AddTask(dependency, TF(SendBoundBufs), md); + auto recv = tl.AddTask(dependency, TF(ReceiveBoundBufs), md); + auto set = tl.AddTask(recv, TF(SetBounds), md); + + auto pro = set; + if (IsMeshMultilevel(md)) { + auto cbound = ApplyBCs(set, &tl, md, true); + pro = tl.AddTask(cbound, TF(ProlongateBounds), md); + } + auto fbound = ApplyBCs(pro, &tl, md, false); + if (IsMeshMultilevel(md)) { + // Need to prolongate internal bounds after setting physical boundary + // conditions, since the internal prolongation uses the values of the + // normal buffer on shared elements (rather than values in the coarse) + // buffer for prolongation. + fbound = tl.AddTask(fbound, TF(ProlongateInternalBounds), md); + } + return fbound; +} + +template +TaskID AddBoundaryExchangeTasks( + TaskID dependency, TaskList &tl, std::shared_ptr> &md, bool multilevel, + BValOnMDFunc_t ApplyBCs = ApplyBoundaryConditionsOnCoarseOrFineMD) { + return AddBoundaryExchangeTasks( + dependency, tl, md, multilevel, + [&](TaskID id, TaskList *tl, std::shared_ptr> md, bool coarse) { + return tl->AddTask(id, TF(ApplyBCs), md, coarse); + }); +} // These tasks should not be called in down stream code TaskStatus BuildBoundaryBuffers(std::shared_ptr> &md);