From ad7450468c88631efba72320ad2ad6f9718ac36a Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Fri, 28 Jul 2023 14:53:59 -0700 Subject: [PATCH 1/3] Remove use of managed memory from TracerParticleContainer::TimeStamp --- Src/Particle/AMReX_TracerParticles.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Src/Particle/AMReX_TracerParticles.cpp b/Src/Particle/AMReX_TracerParticles.cpp index 2efb30b92d4..26cb4c483e9 100644 --- a/Src/Particle/AMReX_TracerParticles.cpp +++ b/Src/Particle/AMReX_TracerParticles.cpp @@ -242,7 +242,13 @@ TracerParticleContainer::Timestamp (const std::string& basename, const auto& pmap = GetParticles(lev); for (const auto& kv : pmap) { - const auto& pbox = kv.second.GetArrayOfStructs(); + using PinnedTile = amrex::ParticleTile, 0, 0, + amrex::PinnedArenaAllocator>; + PinnedTile pinned_tile; + pinned_tile.define(NumRuntimeRealComps(), NumRuntimeIntComps()); + amrex::copyParticles(pinned_tile, kv.second); + + const auto& pbox = pinned_tile.GetArrayOfStructs(); for (int k = 0; k < pbox.numParticles(); ++k) { const ParticleType& p = pbox[k]; From 55bb60d77e45fd74feb6653b1cfb37a2ccd5137c Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Fri, 28 Jul 2023 15:50:41 -0700 Subject: [PATCH 2/3] fix in different way --- Src/Particle/AMReX_TracerParticles.cpp | 51 ++++++++++++++------------ 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/Src/Particle/AMReX_TracerParticles.cpp b/Src/Particle/AMReX_TracerParticles.cpp index 26cb4c483e9..69382cd4e68 100644 --- a/Src/Particle/AMReX_TracerParticles.cpp +++ b/Src/Particle/AMReX_TracerParticles.cpp @@ -238,27 +238,7 @@ TracerParticleContainer::Timestamp (const std::string& basename, // // Do we have any particles at this level that need writing? // - bool gotwork = false; - - const auto& pmap = GetParticles(lev); - for (const auto& kv : pmap) { - using PinnedTile = amrex::ParticleTile, 0, 0, - amrex::PinnedArenaAllocator>; - PinnedTile pinned_tile; - pinned_tile.define(NumRuntimeRealComps(), NumRuntimeIntComps()); - amrex::copyParticles(pinned_tile, kv.second); - - const auto& pbox = pinned_tile.GetArrayOfStructs(); - for (int k = 0; k < pbox.numParticles(); ++k) - { - const ParticleType& p = pbox[k]; - if (p.id() > 0) { - gotwork = true; - break; - } - } - if (gotwork) break; - } + bool gotwork = NumberOfParticlesAtLevel(lev, true, true) > 0; if (gotwork) { @@ -286,12 +266,35 @@ TracerParticleContainer::Timestamp (const std::string& basename, std::vector vals(M); + const auto& pmap = GetParticles(lev); for (const auto& kv : pmap) { + using PinnedTile = amrex::ParticleTile, 0, 0, + amrex::PinnedArenaAllocator>; + PinnedTile pinned_tile; + pinned_tile.define(NumRuntimeRealComps(), NumRuntimeIntComps()); + pinned_tile.resize(kv.second.numParticles()); + amrex::copyParticles(pinned_tile, kv.second); + int grid = kv.first.first; - const auto& pbox = kv.second.GetArrayOfStructs(); + const auto& pbox = pinned_tile.GetArrayOfStructs(); const Box& bx = ba[grid]; const FArrayBox& fab = mf[grid]; - const auto uccarr = fab.array(); + auto uccarr = fab.array(); + + amrex::Array4* uccarr_ptr = &uccarr; +#ifdef AMREX_USE_GPU + std::unique_ptr hostfab; + { + amrex::Print() << fab.arena()->isManaged() << " " << fab.arena()->isDevice() << "\n"; + hostfab = std::make_unique(fab.box(), fab.nComp(), + The_Pinned_Arena()); + Gpu::dtoh_memcpy_async(hostfab->dataPtr(), fab.dataPtr(), + fab.size()*sizeof(Real)); + Gpu::streamSynchronize(); + auto hostarr = hostfab->const_array(); + uccarr_ptr = &hostarr; + } +#endif for (int k = 0; k < pbox.numParticles(); ++k) { @@ -319,7 +322,7 @@ TracerParticleContainer::Timestamp (const std::string& basename, if (M > 0) { - cic_interpolate(p, plo, dxi, uccarr, &vals[0], M); + cic_interpolate(p, plo, dxi, *uccarr_ptr, &vals[0], M); for (int i = 0; i < M; i++) { From 8847241c6c91735eeafc71907ef95ebe37558793 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Fri, 28 Jul 2023 15:57:53 -0700 Subject: [PATCH 3/3] remove print statement --- Src/Particle/AMReX_TracerParticles.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/Src/Particle/AMReX_TracerParticles.cpp b/Src/Particle/AMReX_TracerParticles.cpp index 69382cd4e68..5d290be43f2 100644 --- a/Src/Particle/AMReX_TracerParticles.cpp +++ b/Src/Particle/AMReX_TracerParticles.cpp @@ -285,7 +285,6 @@ TracerParticleContainer::Timestamp (const std::string& basename, #ifdef AMREX_USE_GPU std::unique_ptr hostfab; { - amrex::Print() << fab.arena()->isManaged() << " " << fab.arena()->isDevice() << "\n"; hostfab = std::make_unique(fab.box(), fab.nComp(), The_Pinned_Arena()); Gpu::dtoh_memcpy_async(hostfab->dataPtr(), fab.dataPtr(),