From a664c25ef69a4856da662857cdca234d359f9a56 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Fri, 1 Mar 2024 13:14:09 -0800 Subject: [PATCH 1/3] Fix GPU restart for pure SoA particles --- Src/Particle/AMReX_ParticleIO.H | 2 +- Src/Particle/AMReX_WriteBinaryParticleData.H | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Src/Particle/AMReX_ParticleIO.H b/Src/Particle/AMReX_ParticleIO.H index 22c0c5a35f9..abab946ab04 100644 --- a/Src/Particle/AMReX_ParticleIO.H +++ b/Src/Particle/AMReX_ParticleIO.H @@ -999,7 +999,7 @@ ParticleContainer_impl& idata, Vector& rdata, const PC& pc, int l idata.resize(np*iChunkSize); int num_output_real = 0; - for (int i = 0; i < pc.NumRealComps() + PC::NStructReal; ++i) { + for (int i = 0; i < (int) write_real_comp.size(); ++i) { if (write_real_comp[i]) { ++num_output_real; } } @@ -249,15 +249,18 @@ packIOData (Vector& idata, Vector& rdata, const PC& pc, int l } } - for (int j = 0; j < PC::SuperParticleType::NReal; j++) { - if (write_real_comp_d_ptr[j]) { + // extra SoA Real components + const int real_start_offset = PC::ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; // pure SoA: skip positions + for (int j = real_start_offset; j < PC::SuperParticleType::NReal; j++) { + const int write_comp_index = PC::NStructReal+j-real_start_offset; + if (write_real_comp_d_ptr[write_comp_index]) { rdata_d_ptr[rout_index] = p.rdata(j); rout_index++; } } for (int j = 0; j < ptd.m_num_runtime_real; j++) { - if (write_real_comp_d_ptr[PC::SuperParticleType::NReal + j]) { + if (write_real_comp_d_ptr[PC::SuperParticleType::NReal+j-real_start_offset]) { rdata_d_ptr[rout_index] = ptd.m_runtime_rdata[j][pindex]; rout_index++; } @@ -361,8 +364,7 @@ packIOData (Vector& idata, Vector& rdata, const PC& pc, int l // extra SoA Real components const int real_start_offset = PC::ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; // pure SoA: skip positions for (int j = real_start_offset; j < pc.NumRealComps(); j++) { - const int write_comp_offset = PC::ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; // pure SoA: skip positions - const int write_comp_index = PC::NStructReal+j-write_comp_offset; + const int write_comp_index = PC::NStructReal+j-real_start_offset; if (write_real_comp[write_comp_index]) { *rptr = (ParticleReal) soa.GetRealData(j)[pindex]; ++rptr; From 5386de9dc0c5e970e1a602fca2c7fb1c38d5f701 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Sat, 2 Mar 2024 18:13:14 -0800 Subject: [PATCH 2/3] fix cpu and async restarts --- Src/Particle/AMReX_ParticleIO.H | 2 +- Src/Particle/AMReX_WriteBinaryParticleData.H | 47 +++++++++++++++----- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/Src/Particle/AMReX_ParticleIO.H b/Src/Particle/AMReX_ParticleIO.H index abab946ab04..27ffb44a904 100644 --- a/Src/Particle/AMReX_ParticleIO.H +++ b/Src/Particle/AMReX_ParticleIO.H @@ -999,7 +999,7 @@ ParticleContainer_impl& idata, Vector& rdata, const PC& pc, int l } } else { - amrex::ignore_unused(is_checkpoint); - // Int: id, cpu uint64_t idcpu = soa.GetIdCPUData()[pindex]; - *iptr = (int) ParticleIDWrapper(idcpu); - iptr += 1; - *iptr = (int) ParticleCPUWrapper(idcpu); - iptr += 1; + if (is_checkpoint) { + std::int32_t xi, yi; + std::uint32_t xu, yu; + xu = (std::uint32_t)((idcpu & 0xFFFFFFFF00000000LL) >> 32); + yu = (std::uint32_t)( idcpu & 0xFFFFFFFFLL); + std::memcpy(&xi, &xu, sizeof(xu)); + std::memcpy(&yi, &yu, sizeof(yu)); + *iptr = xi; + iptr += 1; + *iptr = yi; + iptr += 1; + } else { + // Int: id, cpu + *iptr = (int) ParticleIDWrapper(idcpu); + iptr += 1; + *iptr = (int) ParticleCPUWrapper(idcpu); + iptr += 1; + } // Real: position for (int j = 0; j < AMREX_SPACEDIM; j++) { rptr[j] = soa.GetRealData(j)[pindex]; } @@ -1034,12 +1046,25 @@ void WriteBinaryParticleDataAsync (PC const& pc, } } else { - // Ints: id, cpu uint64_t idcpu = soa.GetIdCPUData()[pindex]; - *iptr = (int) ParticleIDWrapper(idcpu); - iptr += 1; - *iptr = (int) ParticleCPUWrapper(idcpu); - iptr += 1; + if (is_checkpoint) { + std::int32_t xi, yi; + std::uint32_t xu, yu; + xu = (std::uint32_t)((idcpu & 0xFFFFFFFF00000000LL) >> 32); + yu = (std::uint32_t)( idcpu & 0xFFFFFFFFLL); + std::memcpy(&xi, &xu, sizeof(xu)); + std::memcpy(&yi, &yu, sizeof(yu)); + *iptr = xi; + iptr += 1; + *iptr = yi; + iptr += 1; + } else { + // Int: id, cpu + *iptr = (int) ParticleIDWrapper(idcpu); + iptr += 1; + *iptr = (int) ParticleCPUWrapper(idcpu); + iptr += 1; + } } // extra SoA Ints From 3238707a50280a49a85118174b6666c1cb3b9d38 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Wed, 6 Mar 2024 13:25:47 -0600 Subject: [PATCH 3/3] Update Src/Particle/AMReX_WriteBinaryParticleData.H Co-authored-by: Weiqun Zhang --- Src/Particle/AMReX_WriteBinaryParticleData.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/Particle/AMReX_WriteBinaryParticleData.H b/Src/Particle/AMReX_WriteBinaryParticleData.H index 2cd07fc0e25..073384cc12d 100644 --- a/Src/Particle/AMReX_WriteBinaryParticleData.H +++ b/Src/Particle/AMReX_WriteBinaryParticleData.H @@ -252,7 +252,7 @@ packIOData (Vector& idata, Vector& rdata, const PC& pc, int l // extra SoA Real components const int real_start_offset = PC::ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; // pure SoA: skip positions for (int j = real_start_offset; j < PC::SuperParticleType::NReal; j++) { - const int write_comp_index = PC::NStructReal+j-real_start_offset; + const int write_comp_index = j-real_start_offset; if (write_real_comp_d_ptr[write_comp_index]) { rdata_d_ptr[rout_index] = p.rdata(j); rout_index++;