From 68b6ce8b12f033e20694144232fb5e563177a2cc Mon Sep 17 00:00:00 2001 From: Tapish Date: Wed, 21 Oct 2026 14:29:51 +0200 Subject: [PATCH] Use alpaka3 iterators for PIC and SPH unary particle traversal --- .../pmacc/particles/algorithm/ForEach.hpp | 158 +++++++----- .../particles/algorithm/ForEachFrameSlot.hpp | 34 +++ .../particles/algorithms/FrameSchedule.hpp | 37 +-- .../pmacc/tests/particles/IterationUT.cpp | 241 ++++++++++++++++++ .../unit/HierarchyForEach/AlpakaIteration.cpp | 205 +++++++++++++++ 5 files changed, 598 insertions(+), 77 deletions(-) create mode 100644 packages/pmacc/include/pmacc/particles/algorithm/ForEachFrameSlot.hpp create mode 100644 packages/pmacc/tests/particles/IterationUT.cpp create mode 100644 tests/spmacc/unit/HierarchyForEach/AlpakaIteration.cpp diff --git a/packages/pmacc/include/pmacc/particles/algorithm/ForEach.hpp b/packages/pmacc/include/pmacc/particles/algorithm/ForEach.hpp index 790bedc8..1500ec0a 100644 --- a/packages/pmacc/include/pmacc/particles/algorithm/ForEach.hpp +++ b/packages/pmacc/include/pmacc/particles/algorithm/ForEach.hpp @@ -27,6 +27,7 @@ #include "pmacc/lockstep/Config.hpp" #include "pmacc/lockstep/ForEach.hpp" #include "pmacc/mappings/kernel/AreaMapping.hpp" +#include "pmacc/particles/algorithm/ForEachFrameSlot.hpp" #include "pmacc/particles/algorithm/detail/ForEach.hpp" #include "pmacc/particles/frame_types.hpp" #include "pmacc/traits/GetNumWorkers.hpp" @@ -83,16 +84,22 @@ namespace pmacc::particles::algorithm } }; - /** Execute a particle or frame functor for each particle/frame. + /** Particle traversal selected explicitly by callback interface. * - * @tparam T_AccessTag Type of the tag to mark if the particle or frame interface will be used. - * valid options: detail::CallParticleFunctor or detail::CallFrameFunctor + * The primary template is intentionally undefined; only the particle and frame interfaces + * below are supported. + * + * @tparam T_AccessTag detail::CallParticleFunctor or detail::CallFrameFunctor * @tparam T_Order Iteration order. valid options: Forward or Reverse * @tparam T_ParBox Type of the particle box to traverse. * @tparam T_Worker lockstep worker */ template - class ForEachParticle + class ForEachParticle; + + /** Legacy collective frame-batch traversal. */ + template + class ForEachParticle { private: static constexpr uint32_t dim = T_ParBox::Dim; @@ -145,22 +152,13 @@ namespace pmacc::particles::algorithm { } - /** Execute unary functor for each particle. + /** Execute a frame callback with the legacy mapping-bound batch context. * - * @attention There is no guarantee in which order particles will be executed. - * If the particle functor is used the algorithm assumes that the frame structure is following - * the rule that all frames but the last are fully filled and in the last frame the low indices - * are contiguously filled. - * - * @tparam T_Functor unary particle functor or frame functor following the interface concept - * detail::FrameFunctorInterface or detail::ParticleFunctorInterface - * @param unaryFunctor Functor executed for each particle/frame. - * The caller must ensure that calling the functor in parallel with - * different workers is data race free. - * - particle functor: 'void operator()(T_Worker const &, ParticleType)'. - * It is not allowed to call a synchronization function within the functor. - * - frame functor: 'void operator()(T_Worker const &, FrameCtx)'. - * Calling synchronization within the functor is allowed. + * @tparam T_Functor Frame functor following detail::FrameFunctorInterface. + * @param unaryFunctor Callable as void(worker, FrameCtx). The context can hold several + * frames and invalid handles; use lockstepForEach() to distribute it. + * The caller must ensure data-race freedom and uniform collective + * entry/termination when synchronizing within a callback. */ template DINLINE void operator()(T_Functor&& unaryFunctor) const @@ -221,48 +219,8 @@ namespace pmacc::particles::algorithm // iterate over all frames in the supercell do { - if constexpr(std::is_same_v) - { - // loop over all particles in the frame - forEachParticleInFrame( - [&](uint32_t const linearIdx, auto const& framePtr, auto const frameConditionData) - { - auto const particleIdx = linearIdx % frameSize; - - auto particle = framePtr[particleIdx]; - if constexpr(std::is_same_v) - { - bool const isParticle = particleIdx < frameConditionData; - if(!isParticle) - particle.setHandleInvalid(); - } - else if constexpr(std::is_same_v) - { - uint32_t const parIdxInSupercell = frameConditionData + linearIdx; - bool const isParticle = parIdxInSupercell < numParticlesInSupercell; - if(!isParticle) - particle.setHandleInvalid(); - } - - - constexpr bool functorReturnsVoid - = std::is_void_v; - PMACC_CASSERT_MSG(__unaryParticleFunctor_must_return_void, functorReturnsVoid); - if(particle.isHandleValid()) - { - auto particleFunctor - = detail::makeParticleFunctorInterface(std::forward(unaryFunctor)); - particleFunctor(worker, particle); - } - }, - framePtrCtx, - frameConditionDataCtx); - } - else if constexpr(std::is_same_v) - { - auto frameFunctor = detail::makeFrameFunctorInterface(std::forward(unaryFunctor)); - frameFunctor(worker, framePtrCtx); - } + auto frameFunctor = detail::makeFrameFunctorInterface(std::forward(unaryFunctor)); + frameFunctor(worker, framePtrCtx); hasMoreWork = false; @@ -299,6 +257,84 @@ namespace pmacc::particles::algorithm } }; + /** Unary traversal: block-uniform linked-frame walking and independent alpaka slot loops. + * + * All frames except the last must be full; the last has a packed live prefix. PIC multiMask + * values (including exchange encodings) are deliberately not interpreted here. Frame order + * is preserved, but no serial order of particle execution is promised. Callbacks must not + * synchronize or mutate the frame topology. Collective frame batches use the frame specialization. + */ + template + class ForEachParticle + { + static_assert(std::is_same_v || std::is_same_v); + static constexpr uint32_t frameSize = T_ParBox::frameSize; + T_Worker m_worker; + T_ParBox m_particlesBox; + DataSpace m_superCellIdx; + + public: + DINLINE ForEachParticle( + T_Worker const& worker, + T_ParBox const& particlesBox, + DataSpace const& superCellIdx) + : m_worker(worker) + , m_particlesBox(particlesBox) + , m_superCellIdx(superCellIdx) + { + } + + DINLINE uint32_t numParticles() const + { + return m_particlesBox.getSuperCell(m_superCellIdx).getNumParticles(); + } + + DINLINE bool hasParticles() const + { + return numParticles() != 0u; + } + + template + DINLINE void operator()(T_Functor&& unaryFunctor) const + { + uint32_t remaining = numParticles(); + if(remaining == 0u) + return; + + // Bind once, including rvalues; every invocation below uses the same lvalue. + auto particleFunctor = detail::makeParticleFunctorInterface(std::forward(unaryFunctor)); + auto frame = T_Order::getFirst(m_particlesBox, m_superCellIdx); + uint32_t activeSlots = frameSize; + if constexpr(std::is_same_v) + activeSlots = m_particlesBox.getSuperCell(m_superCellIdx).getSizeLastFrame(); + + while(frame.isValid() && remaining != 0u) + { + if constexpr(std::is_same_v) + activeSlots = remaining < frameSize ? remaining : frameSize; + + forEachFrameSlot( + m_worker, + [&](uint32_t slot) + { + // Test the packed-prefix bound before constructing/accessing a particle handle. + if(slot < activeSlots) + { + auto particle = frame[slot]; + static_assert( + std::is_void_v, + "Unary particle functors must return void"); + if(particle.isHandleValid()) + particleFunctor(m_worker, particle); + } + }); + remaining -= activeSlots; + frame = T_Order::next(m_particlesBox, frame); + activeSlots = frameSize; + } + } + }; + /** Creates an executor to iterate over all particles in a supercell. * * @return ForEach executor which can be invoked with a particle functor as argument. @see diff --git a/packages/pmacc/include/pmacc/particles/algorithm/ForEachFrameSlot.hpp b/packages/pmacc/include/pmacc/particles/algorithm/ForEachFrameSlot.hpp new file mode 100644 index 00000000..92442403 --- /dev/null +++ b/packages/pmacc/include/pmacc/particles/algorithm/ForEachFrameSlot.hpp @@ -0,0 +1,34 @@ +/* Copyright 2026 Tapish Narwal + * + * This file is part of PMacc. + * SPDX-License-Identifier: GPL-3.0-or-later OR LGPL-3.0-or-later + */ + +#pragma once + +#include "pmacc/attribute/FunctionSpecifier.hpp" + +#include + +#include + +namespace pmacc::particles::algorithm +{ + /** Distribute a frame's slot indices across physical threads, without storage or occupancy semantics. + * + * Calls body(slot) as an lvalue. No callback copies, allocations, or implicit barriers are introduced. + * The callback must not synchronize the block: threads can receive different numbers of slots. + * Slot ownership must not be used to index legacy lockstep::Variable storage. + */ + template + DINLINE void forEachFrameSlot(auto const& worker, auto&& body) + { + for(auto [slot] : alpaka::onAcc::makeIdxMap( + worker.getAcc(), + alpaka::onAcc::worker::linearThreadsInBlock, + alpaka::IdxRange{alpaka::CVec{}}, + alpaka::onAcc::traverse::Flat{}, + alpaka::onAcc::layout::Strided{})) + body(slot); + } +} // namespace pmacc::particles::algorithm diff --git a/packages/pmacc/include/spmacc/particles/algorithms/FrameSchedule.hpp b/packages/pmacc/include/spmacc/particles/algorithms/FrameSchedule.hpp index 70bc6807..b5441f1e 100644 --- a/packages/pmacc/include/spmacc/particles/algorithms/FrameSchedule.hpp +++ b/packages/pmacc/include/spmacc/particles/algorithms/FrameSchedule.hpp @@ -27,7 +27,7 @@ #include "spmacc/particles/views/FrameView.hpp" #include -#include +#include #include #include @@ -93,26 +93,29 @@ namespace pmacc::spearhed { DINLINE void operator()(auto const& worker, int total, auto frameAt, auto frameBody) const { - int const stride = static_cast(worker.gridDomSize()); - for(int f = static_cast(worker.blockDomIdx()); f < total; f += stride) + for(auto [f] : alpaka::onAcc::makeIdxMap( + worker.getAcc(), + alpaka::onAcc::worker::linearBlocksInGrid, + alpaka::IdxRange{alpaka::Vec{total}}, + alpaka::onAcc::traverse::Flat{}, + alpaka::onAcc::layout::Strided{})) frameBody(frameAt(f)); } }; - //! Static contiguous chunks: block i processes frames [i*chunk, (i+1)*chunk). With the index, - //! frameAt is O(1) per call just like GridStride's access, so Contiguous now differs from - //! GridStride only in which frames land in the same block (contiguous slot range vs. strided), - //! not in per-frame access cost -- there is no more cursor to reuse across the chunk. Valid for - //! any grid. + //! Balanced contiguous ranges assigned by alpaka. The first total % numBlocks blocks receive + //! one extra frame; unlike the old ceil-sized chunks, range lengths differ by at most one. + //! Frame access remains O(1). Valid for any nonempty grid. struct Contiguous { DINLINE void operator()(auto const& worker, int total, auto frameAt, auto frameBody) const { - int const numBlocks = static_cast(worker.gridDomSize()); - int const chunk = (total + numBlocks - 1) / numBlocks; - int const first = static_cast(worker.blockDomIdx()) * chunk; - int const last = (first + chunk < total) ? first + chunk : total; - for(int f = first; f < last; ++f) + for(auto [f] : alpaka::onAcc::makeIdxMap( + worker.getAcc(), + alpaka::onAcc::worker::linearBlocksInGrid, + alpaka::IdxRange{alpaka::Vec{total}}, + alpaka::onAcc::traverse::Flat{}, + alpaka::onAcc::layout::Contiguous{})) frameBody(frameAt(f)); } }; @@ -131,12 +134,14 @@ namespace pmacc::spearhed * @tparam T_Frame Any frame handle with a static frameSize and operator[](slot) (e.g. FrameView). * @param worker The alpaka worker of the enclosing lockstep kernel. * @param frame The frame whose live slots are distributed. - * @param body Invoked as body(particle) for each live particle in the frame. + * @param body Invoked as an lvalue, body(particle), for each live particle in the frame. + * Must not synchronize the block; selected slots are not a collective. */ template - DINLINE void lockstepForEachParticle(auto const& worker, T_Frame frame, auto body) + DINLINE void lockstepForEachParticle(auto const& worker, T_Frame frame, auto&& body) { - pmacc::lockstep::makeForEach::frameSize>(worker)( + pmacc::particles::algorithm::forEachFrameSlot::frameSize>( + worker, [&](uint32_t const slot) { auto particle = frame[slot]; diff --git a/packages/pmacc/tests/particles/IterationUT.cpp b/packages/pmacc/tests/particles/IterationUT.cpp new file mode 100644 index 00000000..f6630528 --- /dev/null +++ b/packages/pmacc/tests/particles/IterationUT.cpp @@ -0,0 +1,241 @@ +/* Copyright 2026 Tapish Narwal + * + * This file is part of PMacc. + * SPDX-License-Identifier: GPL-3.0-or-later OR LGPL-3.0-or-later + */ + +#include + +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace +{ + constexpr uint32_t slots = 7u; + constexpr uint32_t capacity = 4u * slots; + + // Real PIC particle handles over minimal storage, without allocating a device heap. + struct Frame + { + using MethodsList = boost::mp11::mp_list<>; + using ValueTypeSeq = boost::mp11::mp_list<>; + using Name = void; + using SuperCellSize = pmacc::math::CT::UInt32; + using ParticleType = pmacc::Particle; + uint32_t number; + uint8_t masks[slots]; + + HDINLINE ParticleType operator[](uint32_t slot) + { + return ParticleType{*this, slot}; + } + }; + + struct SuperCell + { + uint32_t count; + + HDINLINE uint32_t getNumParticles() const + { + return count; + } + + HDINLINE uint32_t getSizeLastFrame() const + { + return count == 0u ? 0u : (count - 1u) % slots + 1u; + } + }; + + struct Box + { + static constexpr uint32_t Dim = TEST_DIM; + static constexpr uint32_t frameSize = slots; + using FramePtr = pmacc::FramePointer; + Frame* frames; + SuperCell cell; + + HDINLINE auto const& getSuperCell(auto const&) const + { + return cell; + } + + HDINLINE FramePtr getFirstFrame(auto const&) const + { + return cell.count == 0u ? nullptr : frames; + } + + HDINLINE FramePtr getLastFrame(auto const&) const + { + return cell.count == 0u ? nullptr : frames + (cell.count - 1u) / slots; + } + + HDINLINE FramePtr getNextFrame(FramePtr frame) const + { + return frame->number + 1u < (cell.count + slots - 1u) / slots ? frame.ptr + 1 : nullptr; + } + + HDINLINE FramePtr getPreviousFrame(FramePtr frame) const + { + return frame->number != 0u ? frame.ptr - 1 : nullptr; + } + }; + + template + struct Visit + { + T_Box visits; + uint32_t calls = 0u; + uint32_t lastFrame = 0u; + bool forward; + bool valid = true; + + DINLINE Visit(T_Box box, bool forwardOrder) : visits(box), forward(forwardOrder) + { + } + + Visit(Visit const&) = delete; + + DINLINE Visit(Visit&& other) + : visits(other.visits) + , calls(other.calls) + , lastFrame(other.lastFrame) + , forward(other.forward) + , valid(other.valid) + { + other.valid = false; + } + + DINLINE void operator()(auto const& worker, auto& particle) & + { + uint32_t const frame = particle.frame->number; + bool const ordered = calls == 0u || (forward ? frame >= lastFrame : frame <= lastFrame); + lastFrame = frame; + ++calls; + uint32_t const index = frame * slots + particle.idx; + // PIC prefix selection must neither filter nor reinterpret exchange mask values. + uint32_t const increment + = valid && ordered && particle.frame->masks[particle.idx] == index % 4u ? 1u : 100u; + alpaka::onAcc::atomicAdd(worker.getAcc(), &visits[index], increment, alpaka::onAcc::scope::Device{}); + } + }; + + template + struct UnaryKernel + { + DINLINE void operator()(auto const& worker, auto visits, uint32_t count) const + { + Frame frames[4]; + for(uint32_t f = 0; f < 4u; ++f) + { + frames[f].number = f; + for(uint32_t s = 0; s < slots; ++s) + frames[f].masks[s] = static_cast((f * slots + s) % 4u); + } + Box box{frames, {count}}; + auto each + = pmacc::particles::algorithm::acc::makeForEach(worker, box, pmacc::DataSpace{}); + constexpr bool forward = std::is_same_v; + if constexpr(T_MoveSensitive) + each(Visit{visits, forward}); + else + { + auto callback = Visit{visits, forward}; + each(callback); + } + } + }; + + struct BatchKernel + { + DINLINE void operator()(auto const& worker, auto visits, uint32_t count) const + { + Frame frames[4]; + for(uint32_t f = 0; f < 4u; ++f) + frames[f].number = f; + Box box{frames, {count}}; + auto each = pmacc::particles::algorithm::acc::makeForEachFrame( + worker, + box, + pmacc::DataSpace{}); + auto slotsInBatch = each.lockstepForEach(); + each( + [&](auto const& w, auto& context) + { + w.sync(); + slotsInBatch( + [&](uint32_t index, auto const& frame) + { + if(frame.isValid()) + { + uint32_t const global = frame->number * slots + index % slots; + alpaka::onAcc::atomicAdd( + w.getAcc(), + &visits[global], + 1u, + alpaka::onAcc::scope::Device{}); + } + }, + context); + w.sync(); + }); + } + }; + + template + void checkVisits(T_Kernel kernel, uint32_t count, bool wholeFrames = false) + { + pmacc::HostDeviceBuffer visits(capacity); + visits.getHostBuffer().setValue(0u); + auto& device = pmacc::Environment<>::get().DeviceContext(); + auto run = [&](auto sender) { caravan::syncWait(caravan::alpaka::withDevice(device, std::move(sender))); }; + run(visits.hostToDevice()); + run(PMACC_LOCKSTEP_KERNEL(kernel).template config( + 1u)(visits.getDeviceBuffer().getDataBox(), count)); + run(visits.deviceToHost()); + uint32_t const expected = wholeFrames ? (count + slots - 1u) / slots * slots : count; + for(uint32_t i = 0; i < capacity; ++i) + { + CAPTURE(T_Threads, count, i); + CHECK(visits.getHostBuffer().data()[i] == (i < expected ? 1u : 0u)); + } + } +} // namespace + +TEST_CASE("PIC unary iteration preserves packed prefixes in both directions", "[particles][iteration]") +{ + namespace acc = pmacc::particles::algorithm::acc; + for(uint32_t count : {0u, 1u, 6u, 7u, 8u, 14u, 17u, capacity}) + { + checkVisits<3>(UnaryKernel{}, count); + checkVisits<7>(UnaryKernel{}, count); + checkVisits<14>(UnaryKernel{}, count); + checkVisits<14>(UnaryKernel{}, count); + } +} + +TEST_CASE("PIC frame batches retain collective entry for incomplete batches", "[particles][iteration][collective]") +{ + for(uint32_t count : {0u, 1u, 7u, 8u, 17u, capacity}) + checkVisits<7>(BatchKernel{}, count, true); + // Exercise batching and an incomplete first batch without triggering the known legacy + // per-thread termination hazard of an incomplete later batch (documented in the progress note). + for(uint32_t count : {0u, 1u, 7u, 8u, capacity}) + checkVisits<14>(BatchKernel{}, count, true); +} + +TEST_CASE("PIC unary iteration binds a move-sensitive callback once", "[particles][iteration][callback]") +{ + namespace acc = pmacc::particles::algorithm::acc; + checkVisits<3>(UnaryKernel{}, capacity); + checkVisits<14>(UnaryKernel{}, capacity); + checkVisits<10>(UnaryKernel{}, 17u); + checkVisits<10>(UnaryKernel{}, 17u); +} diff --git a/tests/spmacc/unit/HierarchyForEach/AlpakaIteration.cpp b/tests/spmacc/unit/HierarchyForEach/AlpakaIteration.cpp new file mode 100644 index 00000000..18115cc9 --- /dev/null +++ b/tests/spmacc/unit/HierarchyForEach/AlpakaIteration.cpp @@ -0,0 +1,205 @@ +/* Copyright 2026 Tapish Narwal + * + * This file is part of SPEARHED. + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +#include "spearhed/test/SpearhedParticleFixture.hpp" + +#include +#include +#include + +#include +#include + +#include + +namespace sp = pmacc::spearhed; + +namespace +{ + constexpr uint32_t slots = 17u; + + struct SparseFrame + { + static constexpr uint32_t frameSize = slots; + uint8_t masks[slots]; + + struct Particle + { + uint32_t slot; + uint8_t mask; + + DINLINE uint8_t operator[](sp::tags::multiMask_t) const + { + return mask; + } + }; + + DINLINE Particle operator[](uint32_t slot) const + { + return {slot, masks[slot]}; + } + }; + + DINLINE bool occupied(uint32_t pattern, uint32_t slot) + { + return pattern == 1u ? slot == 0u : pattern == 2u || (pattern == 3u && slot % 3u == 1u); + } + + template + struct StatefulVisit + { + T_Box output; + T_Worker const& worker; + uint32_t calls = 0u; + + DINLINE StatefulVisit(T_Box box, T_Worker const& w) : output(box), worker(w) + { + } + + StatefulVisit(StatefulVisit const&) = delete; + + StatefulVisit(StatefulVisit&&) = delete; + + DINLINE void operator()(SparseFrame::Particle particle) & + { + ++calls; + alpaka::onAcc::atomicAdd(worker.getAcc(), &output[particle.slot], 1u, alpaka::onAcc::scope::Device{}); + } + }; + + struct SparseKernel + { + DINLINE void operator()(auto const& worker, auto output, uint32_t pattern) const + { + SparseFrame frame; + for(uint32_t slot = 0; slot < slots; ++slot) + frame.masks[slot] = occupied(pattern, slot) ? uint8_t{1} : uint8_t{0}; + auto body = StatefulVisit{output, worker}; + sp::lockstepForEachParticle(worker, frame, body); + output[slots + worker.workerIdx()] = body.calls; + } + }; + + template + void checkSparse() + { + constexpr uint32_t workers = pmacc::traits::GetNumWorkers::value; + pmacc::HostDeviceBuffer output(slots + workers); + for(uint32_t pattern : {0u, 1u, 2u, 3u}) + { + output.getHostBuffer().setValue(0u); + spearhed::test::runDevice(output.hostToDevice()); + spearhed::test::runDevice(PMACC_LOCKSTEP_KERNEL(SparseKernel{}) + .config(1u)(output.getDeviceBuffer().getDataBox(), pattern)); + spearhed::test::runDevice(output.deviceToHost()); + auto data = output.getHostBuffer().data(); + for(uint32_t slot = 0; slot < slots; ++slot) + { + bool const live = pattern == 1u ? slot == 0u : pattern == 2u || (pattern == 3u && slot % 3u == 1u); + CAPTURE(T_Threads, pattern, slot); + CHECK(data[slot] == (live ? 1u : 0u)); + } + for(uint32_t w = 0; w < workers; ++w) + { + uint32_t expected = 0u; + for(uint32_t s = w; s < slots; s += workers) + expected += pattern == 1u ? s == 0u : pattern == 2u || (pattern == 3u && s % 3u == 1u); + CHECK(data[slots + w] == expected); + } + } + } + + struct ScheduleKernel + { + DINLINE void operator()(auto const& worker, auto output, int frames, auto schedule) const + { + schedule( + worker, + frames, + [](int f) { return f; }, + [&](int f) + { + // A frame callback is collective, unlike a selected-particle callback. + worker.sync(); + uint32_t const index = 2u * (static_cast(f) * worker.numWorkers() + worker.workerIdx()); + alpaka::onAcc::atomicAdd(worker.getAcc(), &output[index], 1u, alpaka::onAcc::scope::Device{}); + alpaka::onAcc::atomicAdd( + worker.getAcc(), + &output[index + 1u], + worker.blockDomIdx() + 1u, + alpaka::onAcc::scope::Device{}); + worker.sync(); + }); + } + }; + + template + void checkSchedule(T_Schedule schedule) + { + constexpr uint32_t workers = pmacc::traits::GetNumWorkers::value; + pmacc::HostDeviceBuffer output(2u * slots * workers); + for(int frames : {0, 1, 2, 7, 17}) + for(uint32_t blocks : {1u, 2u, 3u, 7u, 8u, 17u, 24u}) + { + if constexpr(std::is_same_v) + { + if(frames == 0 || blocks != static_cast(frames)) + continue; + } + output.getHostBuffer().setValue(0u); + spearhed::test::runDevice(output.hostToDevice()); + spearhed::test::runDevice( + PMACC_LOCKSTEP_KERNEL(ScheduleKernel{}) + .config(blocks)(output.getDeviceBuffer().getDataBox(), frames, schedule)); + spearhed::test::runDevice(output.deviceToHost()); + for(uint32_t f = 0; f < slots; ++f) + { + uint32_t owner = f % blocks; + if constexpr(std::is_same_v) + { + uint32_t const total = static_cast(frames); + for(uint32_t b = 0; b < blocks; ++b) + { + uint32_t const first = b * (total / blocks) + std::min(b, total % blocks); + uint32_t const last = first + total / blocks + (b < total % blocks ? 1u : 0u); + if(f >= first && f < last) + owner = b; + } + } + for(uint32_t w = 0; w < workers; ++w) + { + CAPTURE(T_Threads, frames, blocks, f, w); + uint32_t const index = 2u * (f * workers + w); + bool const active = f < static_cast(frames); + CHECK(output.getHostBuffer().data()[index] == (active ? 1u : 0u)); + CHECK(output.getHostBuffer().data()[index + 1u] == (active ? owner + 1u : 0u)); + } + } + } + } +} // namespace + +using IterationFixture = spearhed::test::SpearhedParticleFixture; + +TEST_CASE_METHOD( + IterationFixture, + "Alpaka slot iteration filters sparse masks and preserves callback state", + "[particles][iteration]") +{ + checkSparse<3>(); + checkSparse<17>(); + checkSparse<32>(); +} + +TEST_CASE_METHOD( + IterationFixture, + "Alpaka frame schedules assign every frame uniformly across block threads", + "[particles][iteration]") +{ + checkSchedule<3>(sp::GridStride{}); + checkSchedule<17>(sp::Contiguous{}); + checkSchedule<32>(sp::OneToOne{}); +}