From 9ee80aad41818fb4fd7b84c9cf00e91cbdc7a3e4 Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Fri, 12 Apr 2024 12:19:45 -0700 Subject: [PATCH 01/99] Removal of asserttion which prevented from usung the averaged PSATD algorithms with PML BC --- Source/Parallelization/WarpXComm.cpp | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 9daf37af3f8..fdecc180df2 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -673,11 +673,6 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) { if (patch_type == PatchType::fine) { - if (do_pml && pml[lev]->ok()) - { - WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); - } - const amrex::Periodicity& period = Geom(lev).periodicity(); if ( safe_guard_cells ){ const Vector mf{Efield_avg_fp[lev][0].get(),Efield_avg_fp[lev][1].get(),Efield_avg_fp[lev][2].get()}; @@ -693,11 +688,6 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) } else if (patch_type == PatchType::coarse) { - if (do_pml && pml[lev]->ok()) - { - WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); - } - const amrex::Periodicity& cperiod = Geom(lev-1).periodicity(); if ( safe_guard_cells ) { const Vector mf{Efield_avg_cp[lev][0].get(),Efield_avg_cp[lev][1].get(),Efield_avg_cp[lev][2].get()}; @@ -727,10 +717,6 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) { if (patch_type == PatchType::fine) { - if (do_pml && pml[lev]->ok()) - { - WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); - } const amrex::Periodicity& period = Geom(lev).periodicity(); if ( safe_guard_cells ) { const Vector mf{Bfield_avg_fp[lev][0].get(),Bfield_avg_fp[lev][1].get(),Bfield_avg_fp[lev][2].get()}; @@ -746,11 +732,6 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) } else if (patch_type == PatchType::coarse) { - if (do_pml && pml[lev]->ok()) - { - WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); - } - const amrex::Periodicity& cperiod = Geom(lev-1).periodicity(); if ( safe_guard_cells ){ const Vector mf{Bfield_avg_cp[lev][0].get(),Bfield_avg_cp[lev][1].get(),Bfield_avg_cp[lev][2].get()}; From 3574fa3d4eff06261222786843cb988ee2f428f5 Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Mon, 5 Aug 2024 20:53:39 -0700 Subject: [PATCH 02/99] Added ParticleBoundaryScrapingDiagnostic class. --- Python/pywarpx/picmi.py | 165 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 8be6d9c6212..5fef4c8c055 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -3389,3 +3389,168 @@ def diagnostic_initialize_inputs(self): self.diagnostic.__setattr__(key, expression) else: self.diagnostic.__setattr__(key, value) + +class ParticleBoundaryScrapingDiagnostic(picmistandard.PICMI_ParticleBoundaryScrapingDiagnostic, WarpXDiagnosticBase): + """ + See `Input Parameters `__ for more information. + + Parameters + ---------- + warpx_format: openpmd + Diagnostic file format + + warpx_openpmd_backend: {bp, h5, json}, optional + Openpmd backend file format + + warpx_openpmd_encoding: 'v' (variable based), 'f' (file based) or 'g' (group based), optional + Only read if ``.format = openpmd``. openPMD file output encoding. + File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). + Variable based is an experimental feature with ADIOS2. Default: `'f'`. + + warpx_file_prefix: string, optional + Prefix on the diagnostic file name + + warpx_file_min_digits: integer, optional + Minimum number of digits for the time step number in the file name + + warpx_random_fraction: float or dict, optional + Random fraction of particles to include in the diagnostic. If a float + is given the same fraction will be used for all species, if a dictionary + is given the keys should be species with the value specifying the random + fraction for that species. + + warpx_uniform_stride: integer or dict, optional + Stride to down select to the particles to include in the diagnostic. + If an integer is given the same stride will be used for all species, if + a dictionary is given the keys should be species with the value + specifying the stride for that species. + + warpx_dump_last_timestep: bool, optional + If true, the last timestep is dumped regardless of the diagnostic period/intervals. + + warpx_plot_filter_function: string, optional + Analytic expression to down select the particles to in the diagnostic + """ + def init(self, kw): + + self.format = kw.pop('warpx_format', 'openpmd') + self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) + self.openpmd_encoding = kw.pop('warpx_openpmd_encoding', None) + self.file_prefix = kw.pop('warpx_file_prefix', None) + self.file_min_digits = kw.pop('warpx_file_min_digits', None) + self.random_fraction = kw.pop('warpx_random_fraction', None) + self.uniform_stride = kw.pop('warpx_uniform_stride', None) + self.plot_filter_function = kw.pop('warpx_plot_filter_function', None) + self.dump_last_timestep = kw.pop('warpx_dump_last_timestep', None) + + self.user_defined_kw = {} + if self.plot_filter_function is not None: + # This allows variables to be used in the plot_filter_function, but + # in order not to break other codes, the variables must begin with "warpx_" + for k in list(kw.keys()): + if k.startswith('warpx_') and re.search(r'\b%s\b'%k, self.plot_filter_function): + self.user_defined_kw[k] = kw[k] + del kw[k] + + self.mangle_dict = None + + def diagnostic_initialize_inputs(self): + + self.add_diagnostic() + + self.diagnostic.diag_type = 'BoundaryScraping' + self.diagnostic.format = self.format + self.diagnostic.openpmd_backend = self.openpmd_backend + self.diagnostic.openpmd_encoding = self.openpmd_encoding + self.diagnostic.file_min_digits = self.file_min_digits + self.diagnostic.dump_last_timestep = self.dump_last_timestep + self.diagnostic.intervals = self.period + self.diagnostic.set_or_replace_attr('write_species', True) + if 'fields_to_plot' not in self.diagnostic.argvattrs: + self.diagnostic.fields_to_plot = 'none' + + self.set_write_dir() + + # --- Use a set to ensure that fields don't get repeated. + variables = set() + + if self.data_list is not None: + for dataname in self.data_list: + if dataname == 'position': + if pywarpx.geometry.dims != '1': # because then it's WARPX_DIM_1D_Z + variables.add('x') + if pywarpx.geometry.dims == '3': + variables.add('y') + variables.add('z') + if pywarpx.geometry.dims == 'RZ': + variables.add('theta') + elif dataname == 'momentum': + variables.add('ux') + variables.add('uy') + variables.add('uz') + elif dataname == 'weighting': + variables.add('w') + elif dataname in ['x', 'y', 'z', 'theta', 'ux', 'uy', 'uz']: + if pywarpx.geometry.dims == '1' and (dataname == 'x' or dataname == 'y'): + raise RuntimeError( + f"The attribute {dataname} is not available in mode WARPX_DIM_1D_Z" + f"chosen by dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != '3' and dataname == 'y': + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_3D" + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != 'RZ' and dataname == 'theta': + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_RZ." + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + else: + variables.add(dataname) + else: + # possibly add user defined attributes + variables.add(dataname) + + # --- Convert the set to a sorted list so that the order + # --- is the same on all processors. + variables = list(variables) + variables.sort() + + # species list + if self.species is None: + species_names = pywarpx.particles.species_names + elif np.iterable(self.species): + species_names = [species.name for species in self.species] + else: + species_names = [self.species.name] + + # check if random fraction is specified and whether a value is given per species + random_fraction = {} + random_fraction_default = self.random_fraction + if isinstance(self.random_fraction, dict): + random_fraction_default = 1.0 + for key, val in self.random_fraction.items(): + random_fraction[key.name] = val + + # check if uniform stride is specified and whether a value is given per species + uniform_stride = {} + uniform_stride_default = self.uniform_stride + if isinstance(self.uniform_stride, dict): + uniform_stride_default = 1 + for key, val in self.uniform_stride.items(): + uniform_stride[key.name] = val + + if self.mangle_dict is None: + # Only do this once so that the same variables are used in this distribution + # is used multiple times + self.mangle_dict = pywarpx.my_constants.add_keywords(self.user_defined_kw) + + for name in species_names: + diag = pywarpx.Bucket.Bucket(self.name + '.' + name, + variables = variables, + random_fraction = random_fraction.get(name, random_fraction_default), + uniform_stride = uniform_stride.get(name, uniform_stride_default)) + expression = pywarpx.my_constants.mangle_expression(self.plot_filter_function, self.mangle_dict) + diag.__setattr__('plot_filter_function(t,x,y,z,ux,uy,uz)', expression) + self.diagnostic._species_dict[name] = diag From 0de03c72e4a0617d089cc49df540cc67d4d69644 Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Tue, 6 Aug 2024 10:20:25 -0700 Subject: [PATCH 03/99] Temp commit: testing an unreleased PICMI version. --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8e664ae3096..dd2169c7eb7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,9 +4,9 @@ periodictable~=1.5 # PICMI # note: don't forget to update the version in Docs/requirements.txt, too -picmistandard==0.29.0 +#picmistandard==0.29.0 # for development against an unreleased PICMI version, use: -#picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python +picmistandard @ git+https://github.com/oshapoval/picmi.git@add_boundary_scraping_diags_picmi # optional, some used for testing: # yt From dbd0047b227d2b874150255881cfc6f129687e9f Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Tue, 6 Aug 2024 10:35:25 -0700 Subject: [PATCH 04/99] Temp commit: updated PICMI version. --- Docs/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Docs/requirements.txt b/Docs/requirements.txt index 1ce1ee6c1a0..5cd0625b3ad 100644 --- a/Docs/requirements.txt +++ b/Docs/requirements.txt @@ -13,9 +13,10 @@ openpmd-viewer # for checksumAPI # PICMI API docs # note: keep in sync with version in ../requirements.txt -picmistandard==0.29.0 +#picmistandard==0.29.0 # for development against an unreleased PICMI version, use: # picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python +picmistandard @ git+https://github.com/oshapoval/picmi.git@add_boundary_scraping_diags_picmi pybtex pygments From 4067aecfa492096ac3ba58673abea87ed6cd1459 Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Tue, 6 Aug 2024 11:52:51 -0700 Subject: [PATCH 05/99] Temp commit: updated PICMI version in setup.py --- Python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/setup.py b/Python/setup.py index 31f35eeceac..0d0bcf41872 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -59,7 +59,7 @@ package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", package_data = package_data, - install_requires = ['numpy', 'picmistandard==0.29.0', 'periodictable'], + install_requires = ['numpy', 'picmistandard @ git+https://github.com/oshapoval/picmi.git@add_boundary_scraping_diags_picmi', 'periodictable'], python_requires = '>=3.8', zip_safe=False ) From 66d324e55823b445057fdf62c44f208d525c0ccb Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Tue, 6 Aug 2024 16:03:19 -0700 Subject: [PATCH 06/99] Activated scraping boundary diagnostics in the CI test: spacecraft_charging --- .../spacecraft_charging/PICMI_inputs_rz.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py index 5b57b59fbe3..2c68de5bbe9 100644 --- a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py +++ b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py @@ -194,7 +194,8 @@ def compute_actual_charge_on_spacecraft(): electrons = picmi.Species(particle_type='electron', name='electrons', initial_distribution=[e_dist,e_dist2], - warpx_save_particles_at_eb=1) + warpx_save_particles_at_eb=1, + warpx_save_particles_at_zhi=1) p_dist = picmi.UniformDistribution(density = n, rms_velocity=[v_pth, v_pth, v_pth] ) p_dist2 = picmi.UniformFluxDistribution( @@ -206,7 +207,8 @@ def compute_actual_charge_on_spacecraft(): protons = picmi.Species(particle_type='proton', name='protons', initial_distribution=[p_dist,p_dist2], - warpx_save_particles_at_eb=1) + warpx_save_particles_at_eb=1, + warpx_save_particles_at_zhi=1) ########################## @@ -259,6 +261,14 @@ def compute_actual_charge_on_spacecraft(): warpx_file_prefix = 'spacecraft_charging_plt' ) +part_scraping_boundary_diag = picmi.ParticleBoundaryScrapingDiagnostic(name = 'diag2', + period = diagnostic_interval, + species = [electrons, protons], + warpx_format = 'openpmd', + write_dir = '.', + warpx_file_prefix = 'spacecraft_charging_plt' +) + ########################## # simulation setup ########################## @@ -284,6 +294,7 @@ def compute_actual_charge_on_spacecraft(): sim.add_diagnostic(field_diag) sim.add_diagnostic(part_diag) +sim.add_diagnostic(part_scraping_boundary_diag) ########################## # simulation run From 03c8e7fc4656d326b5f0d9763e3b8078e959e0d9 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 19 Aug 2024 11:37:34 -0700 Subject: [PATCH 07/99] Compute whether particles are inside EB --- .../Particles/Deposition/CurrentDeposition.H | 29 +++++++++++++++++++ Source/Particles/WarpXParticleContainer.cpp | 28 +++++++++++++++--- Source/WarpX.H | 2 +- 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index cb56c559bc0..179daf7d661 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -10,6 +10,7 @@ #include "Particles/Deposition/SharedDepositionUtils.H" #include "ablastr/parallelization/KernelTimer.H" +#include "ablastr/particles/NodalFieldGather.H" #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/ShapeFactors.H" #include "Utils/TextMsg.H" @@ -649,6 +650,9 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, const amrex::Array4& Jx_arr, const amrex::Array4& Jy_arr, const amrex::Array4& Jz_arr, +#ifdef AMREX_USE_EB + const amrex::Array4& distance_to_eb_arr, +#endif long np_to_deposit, amrex::Real dt, amrex::Real relative_time, @@ -735,6 +739,31 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, double const z_new = (zp - xyzmin.z + (relative_time + 0.5_rt*dt)*uzp[ip]*gaminv)*dinv.z; double const z_old = z_new - dt*dinv.z*uzp[ip]*gaminv; +// Only execute for EB (embedded boundary) +#ifdef AMREX_USE_EB + // Check whether the particle is close to the EB at the old and new position + // TODO: abstract in a separate function (doGatherScalarFieldNodal) + // TODO: check calculation for RZ + bool close_to_eb_old = false; + bool close_to_eb_new = false; + { + int i, j, k; + amrex::Real W[AMREX_SPACEDIM][2]; + ablastr::particles::compute_weights( + x_old, y_old, z_old, {AMREX_D_DECL(0,0,0)}, {AMREX_D_DECL(1,1,1)}, i, j, k, W); + amrex::Real distance = ablastr::particles::interp_field_nodal(i, j, k, W, distance_to_eb_arr); + close_to_eb_old = (distance < 1.0_rt); // TODO: tune depth + } + { + int i, j, k; + amrex::Real W[AMREX_SPACEDIM][2]; + ablastr::particles::compute_weights( + x_new, y_new, z_new, {AMREX_D_DECL(0,0,0)}, {AMREX_D_DECL(1,1,1)}, i, j, k, W); + amrex::Real distance = ablastr::particles::interp_field_nodal(i, j, k, W, distance_to_eb_arr); + close_to_eb_new = (distance < 1.0_rt); // TODO: tune depth + } +#endif + #if defined(WARPX_DIM_RZ) Real const vy = (-uxp[ip]*sintheta_mid + uyp[ip]*costheta_mid)*gaminv; #elif defined(WARPX_DIM_XZ) diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index bdce18b7b2b..35a61bb43a7 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -577,29 +577,49 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, else { if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { if (push_type == PushType::Explicit) { +#ifdef AMREX_USE_EB + // signed distance function + auto const distance_to_eb_arr = (*warpx.GetDistanceToEB()[lev])[pti].array(); +#endif if (WarpX::nox == 1){ doEsirkepovDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, +#ifdef AMREX_USE_EB + distance_to_eb_arr, +#endif + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doEsirkepovDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, +#ifdef AMREX_USE_EB + distance_to_eb_arr, +#endif + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doEsirkepovDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, +#ifdef AMREX_USE_EB + distance_to_eb_arr, +#endif + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doEsirkepovDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, +#ifdef AMREX_USE_EB + distance_to_eb_arr, +#endif + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } else if (push_type == PushType::Implicit) { diff --git a/Source/WarpX.H b/Source/WarpX.H index 2caea95e926..ba96d40418d 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -146,7 +146,7 @@ public: [[nodiscard]] HybridPICModel * get_pointer_HybridPICModel () const { return m_hybrid_pic_model.get(); } MultiDiagnostics& GetMultiDiags () {return *multi_diags;} #ifdef AMREX_USE_EB - amrex::Vector >& GetDistanceToEB () {return m_distance_to_eb;} + amrex::Vector > const& GetDistanceToEB () const {return m_distance_to_eb;} #endif ParticleBoundaryBuffer& GetParticleBoundaryBuffer () { return *m_particle_boundary_buffer; } From a1486246059bc32af9885b0d9e9360a58655a25e Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 19 Aug 2024 13:04:15 -0700 Subject: [PATCH 08/99] Implement change of shape factor --- .../Particles/Deposition/CurrentDeposition.H | 5 +- Source/Particles/ShapeFactors.H | 112 ++++++++++++------ 2 files changed, 81 insertions(+), 36 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 179daf7d661..2ce86f8dff2 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -739,13 +739,12 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, double const z_new = (zp - xyzmin.z + (relative_time + 0.5_rt*dt)*uzp[ip]*gaminv)*dinv.z; double const z_old = z_new - dt*dinv.z*uzp[ip]*gaminv; -// Only execute for EB (embedded boundary) + bool close_to_eb_old = false; + bool close_to_eb_new = false; #ifdef AMREX_USE_EB // Check whether the particle is close to the EB at the old and new position // TODO: abstract in a separate function (doGatherScalarFieldNodal) // TODO: check calculation for RZ - bool close_to_eb_old = false; - bool close_to_eb_new = false; { int i, j, k; amrex::Real W[AMREX_SPACEDIM][2]; diff --git a/Source/Particles/ShapeFactors.H b/Source/Particles/ShapeFactors.H index e711cce15ff..b51623f1cc2 100644 --- a/Source/Particles/ShapeFactors.H +++ b/Source/Particles/ShapeFactors.H @@ -31,7 +31,8 @@ struct Compute_shape_factor AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE int operator()( T* const sx, - T xmid) const + T xmid, + bool const close_to_eb=false) const { if constexpr (depos_order == 0){ const auto j = static_cast(xmid + T(0.5)); @@ -47,31 +48,52 @@ struct Compute_shape_factor } else if constexpr (depos_order == 2){ const auto j = static_cast(xmid + T(0.5)); - const T xint = xmid - T(j); - sx[0] = T(0.5)*(T(0.5) - xint)*(T(0.5) - xint); - sx[1] = T(0.75) - xint*xint; - sx[2] = T(0.5)*(T(0.5) + xint)*(T(0.5) + xint); + if (close_to_eb) { + // Use order 1 shape function + const auto j1 = static_cast(xmid); + const T xint = xmid - T(j1); + sx[1+j1-j] = T(1.0) - xint; + sx[2+j1-j] = xint; + } else { + const T xint = xmid - T(j); + sx[0] = T(0.5)*(T(0.5) - xint)*(T(0.5) - xint); + sx[1] = T(0.75) - xint*xint; + sx[2] = T(0.5)*(T(0.5) + xint)*(T(0.5) + xint); + } // index of the leftmost cell where particle deposits return j-1; } else if constexpr (depos_order == 3){ const auto j = static_cast(xmid); const T xint = xmid - T(j); - sx[0] = (T(1.0))/(T(6.0))*(T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - xint); - sx[1] = (T(2.0))/(T(3.0)) - xint*xint*(T(1.0) - xint/(T(2.0))); - sx[2] = (T(2.0))/(T(3.0)) - (T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - T(0.5)*(T(1.0) - xint)); - sx[3] = (T(1.0))/(T(6.0))*xint*xint*xint; + if (close_to_eb) { + sx[1] = T(1.0) - xint; + sx[2] = xint; + } else { + sx[0] = (T(1.0))/(T(6.0))*(T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - xint); + sx[1] = (T(2.0))/(T(3.0)) - xint*xint*(T(1.0) - xint/(T(2.0))); + sx[2] = (T(2.0))/(T(3.0)) - (T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - T(0.5)*(T(1.0) - xint)); + sx[3] = (T(1.0))/(T(6.0))*xint*xint*xint; + } // index of the leftmost cell where particle deposits return j-1; } else if constexpr (depos_order == 4){ const auto j = static_cast(xmid + T(0.5)); - const T xint = xmid - T(j); - sx[0] = (T(1.0))/(T(24.0))*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint); - sx[1] = (T(1.0))/(T(24.0))*(T(4.75) - T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) + xint - xint*xint)); - sx[2] = (T(1.0))/(T(24.0))*(T(14.375) + T(6.0)*xint*xint*(xint*xint - T(2.5))); - sx[3] = (T(1.0))/(T(24.0))*(T(4.75) + T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) - xint - xint*xint)); - sx[4] = (T(1.0))/(T(24.0))*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5)+xint); + if (close_to_eb) { + // Use order 1 shape function + const auto j1 = static_cast(xmid); + const T xint = xmid - T(j1); + sx[2+j1-j] = T(1.0) - xint; + sx[3+j1-j] = xint; + } else { + const T xint = xmid - T(j); + sx[0] = (T(1.0))/(T(24.0))*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint); + sx[1] = (T(1.0))/(T(24.0))*(T(4.75) - T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) + xint - xint*xint)); + sx[2] = (T(1.0))/(T(24.0))*(T(14.375) + T(6.0)*xint*xint*(xint*xint - T(2.5))); + sx[3] = (T(1.0))/(T(24.0))*(T(4.75) + T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) - xint - xint*xint)); + sx[4] = (T(1.0))/(T(24.0))*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5)+xint); + } // index of the leftmost cell where particle deposits return j-2; } @@ -98,7 +120,8 @@ struct Compute_shifted_shape_factor int operator()( T* const sx, const T x_old, - const int i_new) const + const int i_new, + bool const close_to_eb=false) const { if constexpr (depos_order == 0){ const auto i = static_cast(std::floor(x_old + T(0.5))); @@ -116,11 +139,20 @@ struct Compute_shifted_shape_factor } else if constexpr (depos_order == 2){ const auto i = static_cast(x_old + T(0.5)); - const int i_shift = i - (i_new + 1); - const T xint = x_old - T(i); - sx[1+i_shift] = T(0.5)*(T(0.5) - xint)*(T(0.5) - xint); - sx[2+i_shift] = T(0.75) - xint*xint; - sx[3+i_shift] = T(0.5)*(T(0.5) + xint)*(T(0.5) + xint); + if (close_to_eb) { + // Use order 1 shape function + const auto i1 = static_cast(x_old); + const int i_shift = i1 - (i_new + 1); + const T xint = x_old - T(i1); + sx[2+i_shift] = T(1.0) - xint; + sx[3+i_shift] = xint; + } else { + const int i_shift = i - (i_new + 1); + const T xint = x_old - T(i); + sx[1+i_shift] = T(0.5)*(T(0.5) - xint)*(T(0.5) - xint); + sx[2+i_shift] = T(0.75) - xint*xint; + sx[3+i_shift] = T(0.5)*(T(0.5) + xint)*(T(0.5) + xint); + } // index of the leftmost cell where particle deposits return i - 1; } @@ -128,23 +160,37 @@ struct Compute_shifted_shape_factor const auto i = static_cast(x_old); const int i_shift = i - (i_new + 1); const T xint = x_old - T(i); - sx[1+i_shift] = (T(1.0))/(T(6.0))*(T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - xint); - sx[2+i_shift] = (T(2.0))/(T(3.0)) - xint*xint*(T(1.0) - xint/(T(2.0))); - sx[3+i_shift] = (T(2.0))/(T(3.0)) - (T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - T(0.5)*(T(1.0) - xint)); - sx[4+i_shift] = (T(1.0))/(T(6.0))*xint*xint*xint; + if (close_to_eb) { + sx[2+i_shift] = T(1.0) - xint; + sx[3+i_shift] = xint; + } else { + sx[1+i_shift] = (T(1.0))/(T(6.0))*(T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - xint); + sx[2+i_shift] = (T(2.0))/(T(3.0)) - xint*xint*(T(1.0) - xint/(T(2.0))); + sx[3+i_shift] = (T(2.0))/(T(3.0)) - (T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - T(0.5)*(T(1.0) - xint)); + sx[4+i_shift] = (T(1.0))/(T(6.0))*xint*xint*xint; + } // index of the leftmost cell where particle deposits return i - 1; } else if constexpr (depos_order == 4){ const auto i = static_cast(x_old + T(0.5)); - const int i_shift = i - (i_new + 2); - const T xint = x_old - T(i); - sx[1+i_shift] = (T(1.0))/(T(24.0))*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint); - sx[2+i_shift] = (T(1.0))/(T(24.0))*(T(4.75) - T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) + xint - xint*xint)); - sx[3+i_shift] = (T(1.0))/(T(24.0))*(T(14.375) + T(6.0)*xint*xint*(xint*xint - T(2.5))); - sx[4+i_shift] = (T(1.0))/(T(24.0))*(T(4.75) + T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) - xint - xint*xint)); - sx[5+i_shift] = (T(1.0))/(T(24.0))*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5)+xint); - // index of the leftmost cell where particle deposits + if (close_to_eb) { + // Use order 1 shape function + const auto i1 = static_cast(x_old); + const int i_shift = i1 - (i_new + 2); + const T xint = x_old - T(i1); + sx[3+i_shift] = T(1.0) - xint; + sx[4+i_shift] = xint; + } else { + const int i_shift = i - (i_new + 2); + const T xint = x_old - T(i); + sx[1+i_shift] = (T(1.0))/(T(24.0))*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint); + sx[2+i_shift] = (T(1.0))/(T(24.0))*(T(4.75) - T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) + xint - xint*xint)); + sx[3+i_shift] = (T(1.0))/(T(24.0))*(T(14.375) + T(6.0)*xint*xint*(xint*xint - T(2.5))); + sx[4+i_shift] = (T(1.0))/(T(24.0))*(T(4.75) + T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) - xint - xint*xint)); + sx[5+i_shift] = (T(1.0))/(T(24.0))*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5)+xint); + } + // index of the leftmost cell where particle deposits return i - 2; } else{ From c02d68970495082a4e50f241ae2c713126724c84 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 20 Aug 2024 10:27:12 -0700 Subject: [PATCH 09/99] Add automated test --- .../embedded_boundary_removal_depth/inputs | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs b/Examples/Tests/embedded_boundary_removal_depth/inputs new file mode 100644 index 00000000000..08fe316b2f5 --- /dev/null +++ b/Examples/Tests/embedded_boundary_removal_depth/inputs @@ -0,0 +1,47 @@ +max_step = 50 +amr.n_cell = 32 32 32 +amr.max_level = 0 +amr.blocking_factor = 8 +amr.max_grid_size = 256 +geometry.dims = 3 +geometry.prob_lo = -10 -10 -10 +geometry.prob_hi = 10 10 10 +warpx.eb_particle_removal_depth = 1 + +# Boundary condition +boundary.field_lo = pec pec pec +boundary.field_hi = pec pec pec + +algo.charge_deposition = standard +algo.field_gathering = energy-conserving +warpx.cfl = 1.0 +warpx.use_filter = 1 + +# Order of particle shape factors +algo.particle_shape = 2 + +warpx.eb_implicit_function = "(x*x + y*y + z*z - 49)" + +particles.species_names = electron positron + +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "SingleParticle" +electron.single_particle_pos = 0.0 0.0 0.0 +electron.single_particle_u = 1.e20 0.0 0.3e20 # gamma*beta +electron.single_particle_weight = 1.0 + +positron.charge = q_e +positron.mass = m_e +positron.injection_style = "SingleParticle" +positron.single_particle_pos = 0.0 0.0 0.0 +positron.single_particle_u = -1.e20 0.0 -0.3e20 # gamma*beta +positron.single_particle_weight = 1.0 + + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz divE rho +diag1.format = openpmd From 0051a17ea119c481cca057195f2f84584b9570b1 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 20 Aug 2024 10:36:36 -0700 Subject: [PATCH 10/99] Simplify code for computing distance to EB --- Source/Particles/Deposition/CurrentDeposition.H | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 2ce86f8dff2..facb3a7554f 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -746,19 +746,13 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // TODO: abstract in a separate function (doGatherScalarFieldNodal) // TODO: check calculation for RZ { - int i, j, k; - amrex::Real W[AMREX_SPACEDIM][2]; - ablastr::particles::compute_weights( - x_old, y_old, z_old, {AMREX_D_DECL(0,0,0)}, {AMREX_D_DECL(1,1,1)}, i, j, k, W); - amrex::Real distance = ablastr::particles::interp_field_nodal(i, j, k, W, distance_to_eb_arr); + amrex::Real distance = ablastr::particles::doGatherScalarFieldNodal( + x_old, y_old, z_old, distance_to_eb_arr, {AMREX_D_DECL(1,1,1)}, {AMREX_D_DECL(0,0,0)} ); close_to_eb_old = (distance < 1.0_rt); // TODO: tune depth } { - int i, j, k; - amrex::Real W[AMREX_SPACEDIM][2]; - ablastr::particles::compute_weights( - x_new, y_new, z_new, {AMREX_D_DECL(0,0,0)}, {AMREX_D_DECL(1,1,1)}, i, j, k, W); - amrex::Real distance = ablastr::particles::interp_field_nodal(i, j, k, W, distance_to_eb_arr); + amrex::Real distance = ablastr::particles::doGatherScalarFieldNodal( + x_new, y_new, z_new, distance_to_eb_arr, {AMREX_D_DECL(1,1,1)}, {AMREX_D_DECL(0,0,0)} ); close_to_eb_new = (distance < 1.0_rt); // TODO: tune depth } #endif From 765f39658c082e943f44d0f7194f820f47b71042 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 20 Aug 2024 10:45:36 -0700 Subject: [PATCH 11/99] Remove unused parameter --- Examples/Tests/embedded_boundary_removal_depth/inputs | 1 - 1 file changed, 1 deletion(-) diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs b/Examples/Tests/embedded_boundary_removal_depth/inputs index 08fe316b2f5..a35d17d5a42 100644 --- a/Examples/Tests/embedded_boundary_removal_depth/inputs +++ b/Examples/Tests/embedded_boundary_removal_depth/inputs @@ -6,7 +6,6 @@ amr.max_grid_size = 256 geometry.dims = 3 geometry.prob_lo = -10 -10 -10 geometry.prob_hi = 10 10 10 -warpx.eb_particle_removal_depth = 1 # Boundary condition boundary.field_lo = pec pec pec From 08d19e2f552c49df6068033aca8340630933a999 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 20 Aug 2024 11:01:41 -0700 Subject: [PATCH 12/99] Use new shapes --- .../embedded_boundary_removal_depth/inputs | 4 ++-- .../Particles/Deposition/CurrentDeposition.H | 19 +++++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs b/Examples/Tests/embedded_boundary_removal_depth/inputs index a35d17d5a42..cd5ad6863ce 100644 --- a/Examples/Tests/embedded_boundary_removal_depth/inputs +++ b/Examples/Tests/embedded_boundary_removal_depth/inputs @@ -14,10 +14,10 @@ boundary.field_hi = pec pec pec algo.charge_deposition = standard algo.field_gathering = energy-conserving warpx.cfl = 1.0 -warpx.use_filter = 1 +warpx.use_filter = 0 # TODO: use filtering # Order of particle shape factors -algo.particle_shape = 2 +algo.particle_shape = 3 warpx.eb_implicit_function = "(x*x + y*y + z*z - 49)" diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index facb3a7554f..b965b02e7b5 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -748,13 +748,16 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, { amrex::Real distance = ablastr::particles::doGatherScalarFieldNodal( x_old, y_old, z_old, distance_to_eb_arr, {AMREX_D_DECL(1,1,1)}, {AMREX_D_DECL(0,0,0)} ); - close_to_eb_old = (distance < 1.0_rt); // TODO: tune depth + amrex::Print() << "Distance to EB: " << distance << "\n"; + close_to_eb_old = (distance < 1.0_rt) ? true: false; // TODO: tune depth } { amrex::Real distance = ablastr::particles::doGatherScalarFieldNodal( x_new, y_new, z_new, distance_to_eb_arr, {AMREX_D_DECL(1,1,1)}, {AMREX_D_DECL(0,0,0)} ); - close_to_eb_new = (distance < 1.0_rt); // TODO: tune depth + amrex::Print() << "Distance to EB: " << distance << "\n"; + close_to_eb_new = (distance < 1.0_rt) ? true: false; // TODO: tune depth } + amrex::Print() << "Close to EB: " << close_to_eb_old << " " << close_to_eb_new << "\n"; #endif #if defined(WARPX_DIM_RZ) @@ -780,19 +783,19 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #if !defined(WARPX_DIM_1D_Z) double sx_new[depos_order + 3] = {0.}; double sx_old[depos_order + 3] = {0.}; - const int i_new = compute_shape_factor(sx_new+1, x_new); - const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new); + const int i_new = compute_shape_factor(sx_new+1, x_new, close_to_eb_new); + const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new, close_to_eb_old); #endif #if defined(WARPX_DIM_3D) double sy_new[depos_order + 3] = {0.}; double sy_old[depos_order + 3] = {0.}; - const int j_new = compute_shape_factor(sy_new+1, y_new); - const int j_old = compute_shifted_shape_factor(sy_old, y_old, j_new); + const int j_new = compute_shape_factor(sy_new+1, y_new, close_to_eb_new); + const int j_old = compute_shifted_shape_factor(sy_old, y_old, j_new, close_to_eb_old); #endif double sz_new[depos_order + 3] = {0.}; double sz_old[depos_order + 3] = {0.}; - const int k_new = compute_shape_factor(sz_new+1, z_new); - const int k_old = compute_shifted_shape_factor(sz_old, z_old, k_new); + const int k_new = compute_shape_factor(sz_new+1, z_new, close_to_eb_new); + const int k_old = compute_shifted_shape_factor(sz_old, z_old, k_new, close_to_eb_old); // computes min/max positions of current contributions #if !defined(WARPX_DIM_1D_Z) From 0dc0c64161e2450f52dfe04a48fd833f384e0f99 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 20 Aug 2024 11:11:49 -0700 Subject: [PATCH 13/99] Revert "Simplify code for computing distance to EB" This reverts commit 0051a17ea119c481cca057195f2f84584b9570b1. --- .../Particles/Deposition/CurrentDeposition.H | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index b965b02e7b5..ed4b20bfc2e 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -746,16 +746,22 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // TODO: abstract in a separate function (doGatherScalarFieldNodal) // TODO: check calculation for RZ { - amrex::Real distance = ablastr::particles::doGatherScalarFieldNodal( - x_old, y_old, z_old, distance_to_eb_arr, {AMREX_D_DECL(1,1,1)}, {AMREX_D_DECL(0,0,0)} ); + int i, j, k; + amrex::Real W[AMREX_SPACEDIM][2]; + ablastr::particles::compute_weights( + x_old, y_old, z_old, {AMREX_D_DECL(0,0,0)}, {AMREX_D_DECL(1,1,1)}, i, j, k, W); + amrex::Real distance = ablastr::particles::interp_field_nodal(i, j, k, W, distance_to_eb_arr); amrex::Print() << "Distance to EB: " << distance << "\n"; - close_to_eb_old = (distance < 1.0_rt) ? true: false; // TODO: tune depth + close_to_eb_old = (distance < 1.0_rt); // TODO: tune depth } { - amrex::Real distance = ablastr::particles::doGatherScalarFieldNodal( - x_new, y_new, z_new, distance_to_eb_arr, {AMREX_D_DECL(1,1,1)}, {AMREX_D_DECL(0,0,0)} ); + int i, j, k; + amrex::Real W[AMREX_SPACEDIM][2]; + ablastr::particles::compute_weights( + x_new, y_new, z_new, {AMREX_D_DECL(0,0,0)}, {AMREX_D_DECL(1,1,1)}, i, j, k, W); + amrex::Real distance = ablastr::particles::interp_field_nodal(i, j, k, W, distance_to_eb_arr); amrex::Print() << "Distance to EB: " << distance << "\n"; - close_to_eb_new = (distance < 1.0_rt) ? true: false; // TODO: tune depth + close_to_eb_new = (distance < 1.0_rt); // TODO: tune depth } amrex::Print() << "Close to EB: " << close_to_eb_old << " " << close_to_eb_new << "\n"; #endif From 25c1b2626d2044bd36e07fbe37d09f89e38a8565 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 20 Aug 2024 13:08:47 -0700 Subject: [PATCH 14/99] Fix computation of EB distance --- .../Particles/Deposition/CurrentDeposition.H | 34 +++++++++---------- Source/Particles/WarpXParticleContainer.cpp | 19 ++++++----- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index ed4b20bfc2e..ccdda926e0a 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -650,9 +650,6 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, const amrex::Array4& Jx_arr, const amrex::Array4& Jy_arr, const amrex::Array4& Jz_arr, -#ifdef AMREX_USE_EB - const amrex::Array4& distance_to_eb_arr, -#endif long np_to_deposit, amrex::Real dt, amrex::Real relative_time, @@ -660,7 +657,13 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, - [[maybe_unused]]int n_rz_azimuthal_modes) +#ifdef AMREX_USE_EB + const amrex::Array4& distance_to_eb_arr, + const amrex::GpuArray& eb_dxi, + const amrex::GpuArray& eb_plo, +#endif + [[maybe_unused]]int n_rz_azimuthal_modes + ) { using namespace amrex; using namespace amrex::literals; @@ -746,24 +749,21 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // TODO: abstract in a separate function (doGatherScalarFieldNodal) // TODO: check calculation for RZ { - int i, j, k; - amrex::Real W[AMREX_SPACEDIM][2]; - ablastr::particles::compute_weights( - x_old, y_old, z_old, {AMREX_D_DECL(0,0,0)}, {AMREX_D_DECL(1,1,1)}, i, j, k, W); - amrex::Real distance = ablastr::particles::interp_field_nodal(i, j, k, W, distance_to_eb_arr); - amrex::Print() << "Distance to EB: " << distance << "\n"; + amrex::Real distance = ablastr::particles::doGatherScalarFieldNodal( + xp + (relative_time - 0.5_rt*dt)*uxp[ip]*gaminv, + yp + (relative_time - 0.5_rt*dt)*uyp[ip]*gaminv, + zp + (relative_time - 0.5_rt*dt)*uzp[ip]*gaminv, + distance_to_eb_arr, eb_dxi, eb_plo ); close_to_eb_old = (distance < 1.0_rt); // TODO: tune depth } { - int i, j, k; - amrex::Real W[AMREX_SPACEDIM][2]; - ablastr::particles::compute_weights( - x_new, y_new, z_new, {AMREX_D_DECL(0,0,0)}, {AMREX_D_DECL(1,1,1)}, i, j, k, W); - amrex::Real distance = ablastr::particles::interp_field_nodal(i, j, k, W, distance_to_eb_arr); - amrex::Print() << "Distance to EB: " << distance << "\n"; + amrex::Real distance = ablastr::particles::doGatherScalarFieldNodal( + xp + (relative_time + 0.5_rt*dt)*uxp[ip]*gaminv, + yp + (relative_time + 0.5_rt*dt)*uyp[ip]*gaminv, + zp + (relative_time + 0.5_rt*dt)*uzp[ip]*gaminv, + distance_to_eb_arr, eb_dxi, eb_plo ); close_to_eb_new = (distance < 1.0_rt); // TODO: tune depth } - amrex::Print() << "Close to EB: " << close_to_eb_old << " " << close_to_eb_new << "\n"; #endif #if defined(WARPX_DIM_RZ) diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 35a61bb43a7..fadeffc525b 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -580,46 +580,49 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, #ifdef AMREX_USE_EB // signed distance function auto const distance_to_eb_arr = (*warpx.GetDistanceToEB()[lev])[pti].array(); + const Geometry& geom = Geom(lev); + const amrex::GpuArray eb_dxi = geom.InvCellSizeArray(); + const amrex::GpuArray eb_plo = geom.ProbLoArray(); #endif if (WarpX::nox == 1){ doEsirkepovDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, #ifdef AMREX_USE_EB - distance_to_eb_arr, + distance_to_eb_arr, eb_dxi, eb_plo, #endif - np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doEsirkepovDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, #ifdef AMREX_USE_EB - distance_to_eb_arr, + distance_to_eb_arr, eb_dxi, eb_plo, #endif - np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doEsirkepovDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, #ifdef AMREX_USE_EB - distance_to_eb_arr, + distance_to_eb_arr, eb_dxi, eb_plo, #endif - np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doEsirkepovDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, #ifdef AMREX_USE_EB - distance_to_eb_arr, + distance_to_eb_arr, eb_dxi, eb_plo, #endif - np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } else if (push_type == PushType::Implicit) { From 330707adf6a923675175eeae632455c32bc4633e Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 22 Aug 2024 09:10:33 -0700 Subject: [PATCH 15/99] Use latest picmistandard version --- Docs/requirements.txt | 3 +-- Python/setup.py | 2 +- Tools/machines/karolina-it4i/install_dependencies.sh | 2 +- requirements.txt | 4 ++-- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Docs/requirements.txt b/Docs/requirements.txt index 5cd0625b3ad..a8c2af0e474 100644 --- a/Docs/requirements.txt +++ b/Docs/requirements.txt @@ -13,10 +13,9 @@ openpmd-viewer # for checksumAPI # PICMI API docs # note: keep in sync with version in ../requirements.txt -#picmistandard==0.29.0 +picmistandard==0.30.0 # for development against an unreleased PICMI version, use: # picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python -picmistandard @ git+https://github.com/oshapoval/picmi.git@add_boundary_scraping_diags_picmi pybtex pygments diff --git a/Python/setup.py b/Python/setup.py index f82cf428437..f6c17ed3676 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -59,7 +59,7 @@ package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", package_data = package_data, - install_requires = ['numpy', 'picmistandard @ git+https://github.com/oshapoval/picmi.git@add_boundary_scraping_diags_picmi', 'periodictable'], + install_requires = ['numpy', 'picmistandard==0.30.0', 'periodictable'], python_requires = '>=3.8', zip_safe=False ) diff --git a/Tools/machines/karolina-it4i/install_dependencies.sh b/Tools/machines/karolina-it4i/install_dependencies.sh index cba455f3d29..c1b6e93ab00 100755 --- a/Tools/machines/karolina-it4i/install_dependencies.sh +++ b/Tools/machines/karolina-it4i/install_dependencies.sh @@ -53,7 +53,7 @@ python -m pip install --user --upgrade matplotlib #python -m pip install --user --upgrade yt # install or update WarpX dependencies -python -m pip install --user --upgrade picmistandard==0.29.0 +python -m pip install --user --upgrade picmistandard==0.30.0 python -m pip install --user --upgrade lasy # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) diff --git a/requirements.txt b/requirements.txt index dd2169c7eb7..272c4903e94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,9 +4,9 @@ periodictable~=1.5 # PICMI # note: don't forget to update the version in Docs/requirements.txt, too -#picmistandard==0.29.0 +picmistandard==0.30.0 # for development against an unreleased PICMI version, use: -picmistandard @ git+https://github.com/oshapoval/picmi.git@add_boundary_scraping_diags_picmi +#picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python # optional, some used for testing: # yt From 85728ba2fdd19869a0ad1c2f18df28656430c026 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 22 Aug 2024 09:15:48 -0700 Subject: [PATCH 16/99] Revert "Removal of asserttion which prevented from usung the averaged PSATD algorithms with PML BC" This reverts commit 9ee80aad41818fb4fd7b84c9cf00e91cbdc7a3e4. --- Source/Parallelization/WarpXComm.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 5a3849d55c3..6c44df061fd 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -781,6 +781,11 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) { if (patch_type == PatchType::fine) { + if (do_pml && pml[lev]->ok()) + { + WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); + } + const amrex::Periodicity& period = Geom(lev).periodicity(); if ( safe_guard_cells ){ const Vector mf{Efield_avg_fp[lev][0].get(),Efield_avg_fp[lev][1].get(),Efield_avg_fp[lev][2].get()}; @@ -796,6 +801,11 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) } else if (patch_type == PatchType::coarse) { + if (do_pml && pml[lev]->ok()) + { + WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); + } + const amrex::Periodicity& cperiod = Geom(lev-1).periodicity(); if ( safe_guard_cells ) { const Vector mf{Efield_avg_cp[lev][0].get(),Efield_avg_cp[lev][1].get(),Efield_avg_cp[lev][2].get()}; @@ -825,6 +835,10 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) { if (patch_type == PatchType::fine) { + if (do_pml && pml[lev]->ok()) + { + WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); + } const amrex::Periodicity& period = Geom(lev).periodicity(); if ( safe_guard_cells ) { const Vector mf{Bfield_avg_fp[lev][0].get(),Bfield_avg_fp[lev][1].get(),Bfield_avg_fp[lev][2].get()}; @@ -840,6 +854,11 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) } else if (patch_type == PatchType::coarse) { + if (do_pml && pml[lev]->ok()) + { + WARPX_ABORT_WITH_MESSAGE("Averaged Galilean PSATD with PML is not yet implemented"); + } + const amrex::Periodicity& cperiod = Geom(lev-1).periodicity(); if ( safe_guard_cells ){ const Vector mf{Bfield_avg_cp[lev][0].get(),Bfield_avg_cp[lev][1].get(),Bfield_avg_cp[lev][2].get()}; From de29a5564138b03884dd3fc2b95bc3770c35fa39 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 22 Aug 2024 12:07:10 -0700 Subject: [PATCH 17/99] Update Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py --- .../Physics_applications/spacecraft_charging/PICMI_inputs_rz.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py index 3bdef6fee6f..78b7b63db2f 100644 --- a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py +++ b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py @@ -280,8 +280,6 @@ def compute_actual_charge_on_spacecraft(): period=diagnostic_interval, species=[electrons, protons], warpx_format="openpmd", - write_dir=".", - warpx_file_prefix="spacecraft_charging_plt", ) ########################## From 552c8d2488ee0232a365f96ccc790b7d721e65cb Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Tue, 3 Sep 2024 15:13:23 -0700 Subject: [PATCH 18/99] Added a new CI test: embedded_boundary_removal_depth. --- .../{inputs => inputs_3d} | 0 Regression/WarpX-tests.ini | 14 ++++++++++++++ 2 files changed, 14 insertions(+) rename Examples/Tests/embedded_boundary_removal_depth/{inputs => inputs_3d} (100%) diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs b/Examples/Tests/embedded_boundary_removal_depth/inputs_3d similarity index 100% rename from Examples/Tests/embedded_boundary_removal_depth/inputs rename to Examples/Tests/embedded_boundary_removal_depth/inputs_3d diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 3e6a896d1c3..0c4b796ed46 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -637,6 +637,20 @@ useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/embedded_circle/analysis.py +[embedded_boundary_removal_depth] +buildDir = . +inputFile = Examples/Tests/embedded_boundary_removal_depth/inputs_3d +runtime_params = warpx.abort_on_warning_threshold = high +dim = 3 +addToCompileString = USE_EB=TRUE USE_OPENPMD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +analysisRoutine = Examples/Tests/embedded_boundary_removal_depth/analysis.py + [FieldProbe] buildDir = . inputFile = Examples/Tests/field_probe/inputs_2d From a2db7f0ed2479dc8025f17045d9695c17d9622de Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Tue, 3 Sep 2024 15:14:44 -0700 Subject: [PATCH 19/99] Added an analysis.py script. --- .../analysis.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 Examples/Tests/embedded_boundary_removal_depth/analysis.py diff --git a/Examples/Tests/embedded_boundary_removal_depth/analysis.py b/Examples/Tests/embedded_boundary_removal_depth/analysis.py new file mode 100644 index 00000000000..9683a7c53d6 --- /dev/null +++ b/Examples/Tests/embedded_boundary_removal_depth/analysis.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +""" +This analysis script checks for any spurious charge build-up at the embedded boundary, when particles are removed in 3D. +It averages the divergence of the electric field (divE) over the last 20 time steps and compares the results with a specified tolerance. +""" + +import os +import sys + +import matplotlib.pyplot as plt +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries + +yt.funcs.mylog.setLevel(0) +sys.path.insert(1, "../../../../warpx/Regression/Checksum/") +import checksumAPI + +# Open plotfile specified in command line +filename = sys.argv[1] +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, filename, output_format="openpmd") + +ts = OpenPMDTimeSeries("./embedded_boundary_removal_depth_plt/") + +def get_avg_divE(ts, start_avg_iter, end_avg_iter, ar_size ): + avg_divE = np.zeros((ar_size,ar_size)) + for iteration in tqdm.tqdm(ts.iterations[start_avg_iter:end_avg_iter]): + divE = ts_dev.get_field('divE', iteration=iteration, slice_across='y', plot=False, cmap='RdBu') + avg_divE += divE[0] + return avg_divE / (end_avg_iter - start_avg_iter) + +def plot(array, vmax=1e-9): + x = np.linspace(-7, 7, 400) + y = np.sqrt(7**2 - x**2) + + fig, ax = plt.subplots() + ax.plot(x, y, 'k--') + ax.plot(x, -y, 'k--') + cax = ax.imshow(array, cmap='RdBu', extent=[-10, 10, -10, 10], origin='lower') + cbar = fig.colorbar(cax, ax=ax) + ax.set_xlabel('x (m)') + ax.set_ylabel('z (m)') + ax.set_title('Averaged divE') + +ar_size = 32 +start_avg_iter = 20 +end_avg_iter = 50 + +divE_avg = get_avg_divE(ts_new, start_avg_iter, end_avg_iter, ar_size) +plot(divE_avg, vmax=1e-9) +plt.savefig("AverageddivE.png") + +tolerance = 1e-11 +def check_tolerance(array, tolerance): + assert np.all(array <= tolerance), f"Test did not pass: one or more elements exceed the tolerance of {tolerance}." + print("All elements of are within the tolerance.") + +check_tolerance(divE_avg, tolerance) From d7b8333072ca82309af079d6ecf206cc8569ede2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Sep 2024 22:16:17 +0000 Subject: [PATCH 20/99] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../analysis.py | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/Examples/Tests/embedded_boundary_removal_depth/analysis.py b/Examples/Tests/embedded_boundary_removal_depth/analysis.py index 9683a7c53d6..8f746858e02 100644 --- a/Examples/Tests/embedded_boundary_removal_depth/analysis.py +++ b/Examples/Tests/embedded_boundary_removal_depth/analysis.py @@ -23,25 +23,30 @@ ts = OpenPMDTimeSeries("./embedded_boundary_removal_depth_plt/") -def get_avg_divE(ts, start_avg_iter, end_avg_iter, ar_size ): - avg_divE = np.zeros((ar_size,ar_size)) + +def get_avg_divE(ts, start_avg_iter, end_avg_iter, ar_size): + avg_divE = np.zeros((ar_size, ar_size)) for iteration in tqdm.tqdm(ts.iterations[start_avg_iter:end_avg_iter]): - divE = ts_dev.get_field('divE', iteration=iteration, slice_across='y', plot=False, cmap='RdBu') + divE = ts_dev.get_field( + "divE", iteration=iteration, slice_across="y", plot=False, cmap="RdBu" + ) avg_divE += divE[0] - return avg_divE / (end_avg_iter - start_avg_iter) + return avg_divE / (end_avg_iter - start_avg_iter) + def plot(array, vmax=1e-9): x = np.linspace(-7, 7, 400) y = np.sqrt(7**2 - x**2) fig, ax = plt.subplots() - ax.plot(x, y, 'k--') - ax.plot(x, -y, 'k--') - cax = ax.imshow(array, cmap='RdBu', extent=[-10, 10, -10, 10], origin='lower') + ax.plot(x, y, "k--") + ax.plot(x, -y, "k--") + cax = ax.imshow(array, cmap="RdBu", extent=[-10, 10, -10, 10], origin="lower") cbar = fig.colorbar(cax, ax=ax) - ax.set_xlabel('x (m)') - ax.set_ylabel('z (m)') - ax.set_title('Averaged divE') + ax.set_xlabel("x (m)") + ax.set_ylabel("z (m)") + ax.set_title("Averaged divE") + ar_size = 32 start_avg_iter = 20 @@ -52,8 +57,13 @@ def plot(array, vmax=1e-9): plt.savefig("AverageddivE.png") tolerance = 1e-11 + + def check_tolerance(array, tolerance): - assert np.all(array <= tolerance), f"Test did not pass: one or more elements exceed the tolerance of {tolerance}." + assert np.all( + array <= tolerance + ), f"Test did not pass: one or more elements exceed the tolerance of {tolerance}." print("All elements of are within the tolerance.") + check_tolerance(divE_avg, tolerance) From dd684a6fb08a3cf72a9ee96793afd7047c91c594 Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Tue, 17 Sep 2024 12:45:20 -0700 Subject: [PATCH 21/99] Updated embedded_boundary_removal_depth CI test & analysis --- Examples/Tests/CMakeLists.txt | 1 + .../embedded_boundary_removal_depth/analysis.py | 16 +++++++--------- ...puts_test_3d_embedded_boundary_removal_depth} | 0 3 files changed, 8 insertions(+), 9 deletions(-) mode change 100644 => 100755 Examples/Tests/embedded_boundary_removal_depth/analysis.py rename Examples/Tests/embedded_boundary_removal_depth/{inputs_3d => inputs_test_3d_embedded_boundary_removal_depth} (100%) diff --git a/Examples/Tests/CMakeLists.txt b/Examples/Tests/CMakeLists.txt index 6fea9368e78..7625da8b6f2 100644 --- a/Examples/Tests/CMakeLists.txt +++ b/Examples/Tests/CMakeLists.txt @@ -16,6 +16,7 @@ add_subdirectory(electrostatic_sphere_eb) add_subdirectory(embedded_boundary_cube) add_subdirectory(embedded_boundary_diffraction) add_subdirectory(embedded_boundary_python_api) +add_subdirectory(embedded_boundary_removal_depth) add_subdirectory(embedded_boundary_rotated_cube) add_subdirectory(embedded_circle) add_subdirectory(energy_conserving_thermal_plasma) diff --git a/Examples/Tests/embedded_boundary_removal_depth/analysis.py b/Examples/Tests/embedded_boundary_removal_depth/analysis.py old mode 100644 new mode 100755 index 8f746858e02..8a12562fd52 --- a/Examples/Tests/embedded_boundary_removal_depth/analysis.py +++ b/Examples/Tests/embedded_boundary_removal_depth/analysis.py @@ -10,9 +10,10 @@ import matplotlib.pyplot as plt import numpy as np +import tqdm from openpmd_viewer import OpenPMDTimeSeries -yt.funcs.mylog.setLevel(0) +#yt.funcs.mylog.setLevel(0) sys.path.insert(1, "../../../../warpx/Regression/Checksum/") import checksumAPI @@ -20,14 +21,12 @@ filename = sys.argv[1] test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename, output_format="openpmd") - -ts = OpenPMDTimeSeries("./embedded_boundary_removal_depth_plt/") - +print(os.getcwd()) def get_avg_divE(ts, start_avg_iter, end_avg_iter, ar_size): avg_divE = np.zeros((ar_size, ar_size)) for iteration in tqdm.tqdm(ts.iterations[start_avg_iter:end_avg_iter]): - divE = ts_dev.get_field( + divE = ts.get_field( "divE", iteration=iteration, slice_across="y", plot=False, cmap="RdBu" ) avg_divE += divE[0] @@ -47,17 +46,17 @@ def plot(array, vmax=1e-9): ax.set_ylabel("z (m)") ax.set_title("Averaged divE") +ts = OpenPMDTimeSeries("./diags/diag1/") ar_size = 32 start_avg_iter = 20 end_avg_iter = 50 -divE_avg = get_avg_divE(ts_new, start_avg_iter, end_avg_iter, ar_size) +divE_avg = get_avg_divE(ts, start_avg_iter, end_avg_iter, ar_size) plot(divE_avg, vmax=1e-9) plt.savefig("AverageddivE.png") -tolerance = 1e-11 - +tolerance = 1e-9 def check_tolerance(array, tolerance): assert np.all( @@ -65,5 +64,4 @@ def check_tolerance(array, tolerance): ), f"Test did not pass: one or more elements exceed the tolerance of {tolerance}." print("All elements of are within the tolerance.") - check_tolerance(divE_avg, tolerance) diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_3d b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth similarity index 100% rename from Examples/Tests/embedded_boundary_removal_depth/inputs_3d rename to Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth From e27d29e8f82dbd177a81c2016e7ca5b269c105ca Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 19:46:07 +0000 Subject: [PATCH 22/99] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Examples/Tests/embedded_boundary_removal_depth/analysis.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Examples/Tests/embedded_boundary_removal_depth/analysis.py b/Examples/Tests/embedded_boundary_removal_depth/analysis.py index 8a12562fd52..0f970f7dfb2 100755 --- a/Examples/Tests/embedded_boundary_removal_depth/analysis.py +++ b/Examples/Tests/embedded_boundary_removal_depth/analysis.py @@ -13,7 +13,7 @@ import tqdm from openpmd_viewer import OpenPMDTimeSeries -#yt.funcs.mylog.setLevel(0) +# yt.funcs.mylog.setLevel(0) sys.path.insert(1, "../../../../warpx/Regression/Checksum/") import checksumAPI @@ -23,6 +23,7 @@ checksumAPI.evaluate_checksum(test_name, filename, output_format="openpmd") print(os.getcwd()) + def get_avg_divE(ts, start_avg_iter, end_avg_iter, ar_size): avg_divE = np.zeros((ar_size, ar_size)) for iteration in tqdm.tqdm(ts.iterations[start_avg_iter:end_avg_iter]): @@ -46,6 +47,7 @@ def plot(array, vmax=1e-9): ax.set_ylabel("z (m)") ax.set_title("Averaged divE") + ts = OpenPMDTimeSeries("./diags/diag1/") ar_size = 32 @@ -58,10 +60,12 @@ def plot(array, vmax=1e-9): tolerance = 1e-9 + def check_tolerance(array, tolerance): assert np.all( array <= tolerance ), f"Test did not pass: one or more elements exceed the tolerance of {tolerance}." print("All elements of are within the tolerance.") + check_tolerance(divE_avg, tolerance) From 5615c120345f2d3fa2da890ac70fe1858e853045 Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Tue, 17 Sep 2024 12:57:11 -0700 Subject: [PATCH 23/99] Clean-up --- Examples/Tests/embedded_boundary_removal_depth/analysis.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Examples/Tests/embedded_boundary_removal_depth/analysis.py b/Examples/Tests/embedded_boundary_removal_depth/analysis.py index 8a12562fd52..7baac31a74d 100755 --- a/Examples/Tests/embedded_boundary_removal_depth/analysis.py +++ b/Examples/Tests/embedded_boundary_removal_depth/analysis.py @@ -13,7 +13,6 @@ import tqdm from openpmd_viewer import OpenPMDTimeSeries -#yt.funcs.mylog.setLevel(0) sys.path.insert(1, "../../../../warpx/Regression/Checksum/") import checksumAPI @@ -21,7 +20,6 @@ filename = sys.argv[1] test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename, output_format="openpmd") -print(os.getcwd()) def get_avg_divE(ts, start_avg_iter, end_avg_iter, ar_size): avg_divE = np.zeros((ar_size, ar_size)) From 0ceda51e4f1bc7f6ac373c5593e7526d139759af Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Tue, 17 Sep 2024 13:22:42 -0700 Subject: [PATCH 24/99] Added missing CMakeLists.txt for embedded_boundary_removal_depth CI --- .../embedded_boundary_removal_depth/CMakeLists.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt diff --git a/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt b/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt new file mode 100644 index 00000000000..aad011d3c79 --- /dev/null +++ b/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt @@ -0,0 +1,14 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_EB) + add_warpx_test( + test_3d_embedded_boundary_removal_depth # name + 3 # dims + 1 # nprocs + inputs_test_3d_embedded_boundary_removal_depth # inputs + analysis.py # analysis + diags/diag1/ # output + OFF # dependency + ) +endif() From f8b2fc4a2b68c456865058cff3d60a8f4bad1b6f Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Tue, 1 Oct 2024 00:02:37 -0700 Subject: [PATCH 25/99] Changed dimensions of arrays eb_dxi and eb_plo from 3D to AMREX_SPACEDIM --- Source/Particles/Deposition/CurrentDeposition.H | 4 ++-- Source/Particles/WarpXParticleContainer.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index ccdda926e0a..97e63061465 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -659,8 +659,8 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, amrex::Real q, #ifdef AMREX_USE_EB const amrex::Array4& distance_to_eb_arr, - const amrex::GpuArray& eb_dxi, - const amrex::GpuArray& eb_plo, + const amrex::GpuArray& eb_dxi, + const amrex::GpuArray& eb_plo, #endif [[maybe_unused]]int n_rz_azimuthal_modes ) diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 3dcfb20b3f3..d6fa31f87ed 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -584,8 +584,8 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, // signed distance function auto const distance_to_eb_arr = (*warpx.GetDistanceToEB()[lev])[pti].array(); const Geometry& geom = Geom(lev); - const amrex::GpuArray eb_dxi = geom.InvCellSizeArray(); - const amrex::GpuArray eb_plo = geom.ProbLoArray(); + const amrex::GpuArray eb_dxi = geom.InvCellSizeArray(); + const amrex::GpuArray eb_plo = geom.ProbLoArray(); #endif if (WarpX::nox == 1){ doEsirkepovDepositionShapeN<1>( From 736d7cf76d85f21bc32b760d0cefb8556c6f1168 Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Tue, 1 Oct 2024 12:24:39 -0700 Subject: [PATCH 26/99] Clean-up: removed rebudant assignment of the colorbar object to cbar --- Examples/Tests/embedded_boundary_removal_depth/analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Examples/Tests/embedded_boundary_removal_depth/analysis.py b/Examples/Tests/embedded_boundary_removal_depth/analysis.py index db21ca8f2bd..2c40c29badc 100755 --- a/Examples/Tests/embedded_boundary_removal_depth/analysis.py +++ b/Examples/Tests/embedded_boundary_removal_depth/analysis.py @@ -40,7 +40,7 @@ def plot(array, vmax=1e-9): ax.plot(x, y, "k--") ax.plot(x, -y, "k--") cax = ax.imshow(array, cmap="RdBu", extent=[-10, 10, -10, 10], origin="lower") - cbar = fig.colorbar(cax, ax=ax) + fig.colorbar(cax, ax=ax) ax.set_xlabel("x (m)") ax.set_ylabel("z (m)") ax.set_title("Averaged divE") From b64dcf0655aa3c1ba520ea542dee7a3ecffe5888 Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Tue, 29 Oct 2024 12:13:35 -0700 Subject: [PATCH 27/99] Added two more 3D embedded_boundary_removal_depth CIs with shape factor=2,3 --- .../CMakeLists.txt | 28 +++++++++++++++-- ..._boundary_removal_depth => inputs_base_3d} | 3 -- ...mbedded_boundary_removal_depth_sh_factor_1 | 6 ++++ ...mbedded_boundary_removal_depth_sh_factor_2 | 6 ++++ ...mbedded_boundary_removal_depth_sh_factor_3 | 6 ++++ ...ed_boundary_removal_depth_sh_factor_1.json | 31 +++++++++++++++++++ ...ed_boundary_removal_depth_sh_factor_2.json | 31 +++++++++++++++++++ ...ed_boundary_removal_depth_sh_factor_3.json | 31 +++++++++++++++++++ 8 files changed, 137 insertions(+), 5 deletions(-) rename Examples/Tests/embedded_boundary_removal_depth/{inputs_test_3d_embedded_boundary_removal_depth => inputs_base_3d} (95%) create mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_1 create mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_2 create mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_3 create mode 100644 Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_1.json create mode 100644 Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_2.json create mode 100644 Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_3.json diff --git a/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt b/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt index aad011d3c79..5c01c255ad5 100644 --- a/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt +++ b/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt @@ -3,10 +3,34 @@ if(WarpX_EB) add_warpx_test( - test_3d_embedded_boundary_removal_depth # name + test_3d_embedded_boundary_removal_depth_sh_factor_1 # name 3 # dims 1 # nprocs - inputs_test_3d_embedded_boundary_removal_depth # inputs + inputs_test_3d_embedded_boundary_removal_depth_sh_factor_1 # inputs + analysis.py # analysis + diags/diag1/ # output + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_3d_embedded_boundary_removal_depth_sh_factor_2 # name + 3 # dims + 1 # nprocs + inputs_test_3d_embedded_boundary_removal_depth_sh_factor_2 # inputs + analysis.py # analysis + diags/diag1/ # output + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_3d_embedded_boundary_removal_depth_sh_factor_3 # name + 3 # dims + 1 # nprocs + inputs_test_3d_embedded_boundary_removal_depth_sh_factor_3 # inputs analysis.py # analysis diags/diag1/ # output OFF # dependency diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth b/Examples/Tests/embedded_boundary_removal_depth/inputs_base_3d similarity index 95% rename from Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth rename to Examples/Tests/embedded_boundary_removal_depth/inputs_base_3d index cd5ad6863ce..62430a57e3f 100644 --- a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth +++ b/Examples/Tests/embedded_boundary_removal_depth/inputs_base_3d @@ -16,9 +16,6 @@ algo.field_gathering = energy-conserving warpx.cfl = 1.0 warpx.use_filter = 0 # TODO: use filtering -# Order of particle shape factors -algo.particle_shape = 3 - warpx.eb_implicit_function = "(x*x + y*y + z*z - 49)" particles.species_names = electron positron diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_1 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_1 new file mode 100644 index 00000000000..2480459e506 --- /dev/null +++ b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_1 @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +# Order of particle shape factors +algo.particle_shape = 1 diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_2 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_2 new file mode 100644 index 00000000000..268a4bcfb90 --- /dev/null +++ b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_2 @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +# Order of particle shape factors +algo.particle_shape = 2 diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_3 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_3 new file mode 100644 index 00000000000..964d9ab4137 --- /dev/null +++ b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_3 @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_3d + +# test input parameters +# Order of particle shape factors +algo.particle_shape = 3 diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_1.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_1.json new file mode 100644 index 00000000000..0e0cdf5eb0f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_1.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 2.9100687916345874e-15, + "By": 6.121275580503632e-15, + "Bz": 5.9043095451081354e-15, + "Ex": 1.4574231057224582e-06, + "Ey": 1.1648744803916206e-06, + "Ez": 1.16164257835401e-06, + "divE": 6.781029990295743e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } + } + diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_2.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_2.json new file mode 100644 index 00000000000..0e0cdf5eb0f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_2.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 2.9100687916345874e-15, + "By": 6.121275580503632e-15, + "Bz": 5.9043095451081354e-15, + "Ex": 1.4574231057224582e-06, + "Ey": 1.1648744803916206e-06, + "Ez": 1.16164257835401e-06, + "divE": 6.781029990295743e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } + } + diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_3.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_3.json new file mode 100644 index 00000000000..0e0cdf5eb0f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_3.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 2.9100687916345874e-15, + "By": 6.121275580503632e-15, + "Bz": 5.9043095451081354e-15, + "Ex": 1.4574231057224582e-06, + "Ey": 1.1648744803916206e-06, + "Ez": 1.16164257835401e-06, + "divE": 6.781029990295743e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } + } + From a2223c5fe4f4f4bdd35e62f8a54b68ffa7b4ba84 Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Wed, 30 Oct 2024 13:17:16 -0700 Subject: [PATCH 28/99] Added and updated embedded_boundary_removal_depth CIs with shape factor=1,2,3 in 2D,3D and RZ --- .../CMakeLists.txt | 72 +++++++++++++++++++ .../analysis.py | 67 ++++++++++------- .../inputs_base_2d | 45 ++++++++++++ .../inputs_base_rz | 44 ++++++++++++ ...mbedded_boundary_removal_depth_sh_factor_1 | 6 ++ ...mbedded_boundary_removal_depth_sh_factor_2 | 6 ++ ...mbedded_boundary_removal_depth_sh_factor_3 | 6 ++ ...mbedded_boundary_removal_depth_sh_factor_1 | 6 ++ ...mbedded_boundary_removal_depth_sh_factor_2 | 6 ++ ...mbedded_boundary_removal_depth_sh_factor_3 | 6 ++ ...ed_boundary_removal_depth_sh_factor_1.json | 30 ++++++++ ...ed_boundary_removal_depth_sh_factor_2.json | 30 ++++++++ ...ed_boundary_removal_depth_sh_factor_3.json | 31 ++++++++ ...ed_boundary_removal_depth_sh_factor_1.json | 57 ++++++++------- ...ed_boundary_removal_depth_sh_factor_2.json | 57 ++++++++------- ...ed_boundary_removal_depth_sh_factor_1.json | 28 ++++++++ ...ed_boundary_removal_depth_sh_factor_2.json | 28 ++++++++ ...ed_boundary_removal_depth_sh_factor_3.json | 28 ++++++++ 18 files changed, 471 insertions(+), 82 deletions(-) create mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_base_2d create mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_base_rz create mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_1 create mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_2 create mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_3 create mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_1 create mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_2 create mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_3 create mode 100644 Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_1.json create mode 100644 Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_2.json create mode 100644 Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_3.json create mode 100644 Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_1.json create mode 100644 Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_2.json create mode 100644 Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_3.json diff --git a/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt b/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt index 5c01c255ad5..62d75e077a3 100644 --- a/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt +++ b/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt @@ -36,3 +36,75 @@ if(WarpX_EB) OFF # dependency ) endif() + +if(WarpX_EB) + add_warpx_test( + test_2d_embedded_boundary_removal_depth_sh_factor_1 # name + 2 # dims + 1 # nprocs + inputs_test_2d_embedded_boundary_removal_depth_sh_factor_1 # inputs + analysis.py # analysis + diags/diag1/ # output + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_2d_embedded_boundary_removal_depth_sh_factor_2 # name + 2 # dims + 1 # nprocs + inputs_test_2d_embedded_boundary_removal_depth_sh_factor_2 # inputs + analysis.py # analysis + diags/diag1/ # output + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_2d_embedded_boundary_removal_depth_sh_factor_3 # name + 2 # dims + 1 # nprocs + inputs_test_2d_embedded_boundary_removal_depth_sh_factor_3 # inputs + analysis.py # analysis + diags/diag1/ # output + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_rz_embedded_boundary_removal_depth_sh_factor_1 # name + RZ # dims + 1 # nprocs + inputs_test_rz_embedded_boundary_removal_depth_sh_factor_1 # inputs + analysis.py # analysis + diags/diag1/ # output + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_rz_embedded_boundary_removal_depth_sh_factor_2 # name + RZ # dims + 1 # nprocs + inputs_test_rz_embedded_boundary_removal_depth_sh_factor_2 # inputs + analysis.py # analysis + diags/diag1/ # output + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_rz_embedded_boundary_removal_depth_sh_factor_3 # name + RZ # dims + 1 # nprocs + inputs_test_rz_embedded_boundary_removal_depth_sh_factor_3 # inputs + analysis.py # analysis + diags/diag1/ # output + OFF # dependency + ) +endif() \ No newline at end of file diff --git a/Examples/Tests/embedded_boundary_removal_depth/analysis.py b/Examples/Tests/embedded_boundary_removal_depth/analysis.py index 2c40c29badc..dc263bf0305 100755 --- a/Examples/Tests/embedded_boundary_removal_depth/analysis.py +++ b/Examples/Tests/embedded_boundary_removal_depth/analysis.py @@ -21,45 +21,64 @@ test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename, output_format="openpmd") - -def get_avg_divE(ts, start_avg_iter, end_avg_iter, ar_size): - avg_divE = np.zeros((ar_size, ar_size)) +def get_avg_divE (ts, start_avg_iter, end_avg_iter, ncell, test_dim ): for iteration in tqdm.tqdm(ts.iterations[start_avg_iter:end_avg_iter]): - divE = ts.get_field( - "divE", iteration=iteration, slice_across="y", plot=False, cmap="RdBu" - ) - avg_divE += divE[0] - return avg_divE / (end_avg_iter - start_avg_iter) - - -def plot(array, vmax=1e-9): + if test_dim == '3d': + avg_divE = np.zeros((ncell,ncell)) + divE = ts.get_field('divE', iteration=iteration, slice_across='y', plot=False, cmap='RdBu')[0] + elif (test_dim == '2d'): + avg_divE = np.zeros((ncell,ncell)) + divE = ts.get_field('divE', iteration=iteration, plot=False, cmap='RdBu')[0] + elif (test_dim == 'rz'): + avg_divE = np.zeros((ncell,2*ncell)) + divE = ts.get_field('divE', iteration=iteration, plot=False, cmap='RdBu')[0] + avg_divE += divE + return avg_divE / (end_avg_iter - start_avg_iter) + +def parse_dimension_in_test_name (test_name): + test_name = test_name.lower() + if '3d' in test_name: + return '3d' + elif '2d' in test_name: + return '2d' + elif 'rz' in test_name: + return 'rz' + return None + +def plot (array): x = np.linspace(-7, 7, 400) y = np.sqrt(7**2 - x**2) fig, ax = plt.subplots() - ax.plot(x, y, "k--") - ax.plot(x, -y, "k--") - cax = ax.imshow(array, cmap="RdBu", extent=[-10, 10, -10, 10], origin="lower") - fig.colorbar(cax, ax=ax) - ax.set_xlabel("x (m)") - ax.set_ylabel("z (m)") - ax.set_title("Averaged divE") + + # Plot the boundary curves + ax.plot(x, y, 'k--') + ax.plot(x, -y, 'k--') + cax = ax.imshow(array, cmap='RdBu', extent=[-10, 10, -10, 10], origin='lower') + cbar = fig.colorbar(cax, ax=ax) + ax.set_xlabel('x (m)') + ax.set_ylabel('z (m)') + ax.set_title('Averaged divE') ts = OpenPMDTimeSeries("./diags/diag1/") -ar_size = 32 -start_avg_iter = 20 +ncell = 32 +start_avg_iter = 30 end_avg_iter = 50 +test_dim = parse_dimension_in_test_name(test_name) -divE_avg = get_avg_divE(ts, start_avg_iter, end_avg_iter, ar_size) -plot(divE_avg, vmax=1e-9) +divE_avg = get_avg_divE(ts, start_avg_iter, end_avg_iter, ncell, test_dim) +plot(divE_avg) plt.savefig("AverageddivE.png") -tolerance = 1e-9 +if (test_dim =='3d' or test_dim =='rz'): + tolerance = 1e-10 +else: + tolerance = 1e-9 -def check_tolerance(array, tolerance): +def check_tolerance (array, tolerance): assert np.all( array <= tolerance ), f"Test did not pass: one or more elements exceed the tolerance of {tolerance}." diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_base_2d b/Examples/Tests/embedded_boundary_removal_depth/inputs_base_2d new file mode 100644 index 00000000000..bb27233f4cf --- /dev/null +++ b/Examples/Tests/embedded_boundary_removal_depth/inputs_base_2d @@ -0,0 +1,45 @@ +max_step = 50 + +geometry.dims = 2 + +amr.n_cell = 32 32 +amr.max_level = 0 +amr.blocking_factor = 8 +amr.max_grid_size = 256 +geometry.prob_lo = -10 -10 +geometry.prob_hi = 10 10 + + +# Boundary condition +boundary.field_lo = none pec +boundary.field_hi = pec pec + +algo.charge_deposition = standard +algo.field_gathering = energy-conserving +warpx.use_filter = 0 # TODO: use filtering +warpx.const_dt = 1.203645751e-09 +warpx.eb_implicit_function = "(x**2 + y**2 + z**2 - 49)" + +particles.species_names = electron positron + +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "SingleParticle" +electron.single_particle_pos = 0.0 0.0 0.0 +electron.single_particle_u = 1.e20 0.0 0.3e20 # gamma*beta +electron.single_particle_weight = 1.0 + +positron.charge = q_e +positron.mass = m_e +positron.injection_style = "SingleParticle" +positron.single_particle_pos = 0.0 0.0 0.0 +positron.single_particle_u = -1.e20 0.0 -0.3e20 # gamma*beta +positron.single_particle_weight = 1.0 + + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz divE rho +diag1.format = openpmd diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_base_rz b/Examples/Tests/embedded_boundary_removal_depth/inputs_base_rz new file mode 100644 index 00000000000..57c13a4fd4b --- /dev/null +++ b/Examples/Tests/embedded_boundary_removal_depth/inputs_base_rz @@ -0,0 +1,44 @@ +max_step = 70 + +geometry.dims = RZ + +amr.n_cell = 32 32 +amr.max_level = 0 +amr.blocking_factor = 8 +amr.max_grid_size = 256 +geometry.prob_lo = 0 -10 +geometry.prob_hi = 10 10 + + +# Boundary condition +boundary.field_lo = none pec +boundary.field_hi = pec pec + +algo.charge_deposition = standard +algo.field_gathering = energy-conserving +warpx.use_filter = 0 # TODO: use filtering + +warpx.eb_implicit_function = "(x**2 + y**2 + z**2 - 49)" + +particles.species_names = electron positron + +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "SingleParticle" +electron.single_particle_pos = 1.0 0.0 0.0 +electron.single_particle_u = 0.0 0.0 0.3e20 # gamma*beta +electron.single_particle_weight = 1.0 + +positron.charge = q_e +positron.mass = m_e +positron.injection_style = "SingleParticle" +positron.single_particle_pos = 1.0 0.0 0.0 +positron.single_particle_u = 0.0 0.0 -0.3e20 # gamma*beta +positron.single_particle_weight = 1.0 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Er Ez Br Bz divE rho +diag1.format = openpmd diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_1 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_1 new file mode 100644 index 00000000000..221aade2e37 --- /dev/null +++ b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_1 @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +# Order of particle shape factors +algo.particle_shape = 1 diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_2 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_2 new file mode 100644 index 00000000000..41743908169 --- /dev/null +++ b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_2 @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +# Order of particle shape factors +algo.particle_shape = 2 diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_3 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_3 new file mode 100644 index 00000000000..c15166f3796 --- /dev/null +++ b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_3 @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +# Order of particle shape factors +algo.particle_shape = 3 diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_1 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_1 new file mode 100644 index 00000000000..02f7cb4936e --- /dev/null +++ b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_1 @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_rz + +# test input parameters +# Order of particle shape factors +algo.particle_shape = 1 diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_2 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_2 new file mode 100644 index 00000000000..9e0910cf2f8 --- /dev/null +++ b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_2 @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_rz + +# test input parameters +# Order of particle shape factors +algo.particle_shape = 2 diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_3 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_3 new file mode 100644 index 00000000000..28eace99f01 --- /dev/null +++ b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_3 @@ -0,0 +1,6 @@ +# base input parameters +FILE = inputs_base_rz + +# test input parameters +# Order of particle shape factors +algo.particle_shape = 3 diff --git a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_1.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_1.json new file mode 100644 index 00000000000..632c48df963 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_1.json @@ -0,0 +1,30 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 2.3792992316172512e-15, + "Bz": 0.0 , + "Ex": 6.177046470842443e-07, + "Ey": 0.0, + "Ez": 7.259396011803518e-07, + "divE": 2.809306467366024e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_2.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_2.json new file mode 100644 index 00000000000..9112c931b55 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_2.json @@ -0,0 +1,30 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 2.3948084603369097e-15, + "Bz": 0.0, + "Ex": 6.747158562891953e-07, + "Ey": 0.0, + "Ez": 5.541309886315263e-07, + "divE": 2.091715826275267e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_3.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_3.json new file mode 100644 index 00000000000..8d6ddf169af --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_removal_depth_sh_factor_3.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 2.379299231617251e-15, + "Bz": 0.0, + "Ex": 6.177046470842443e-07, + "Ey": 0.0, + "Ez": 7.259396011803522e-07, + "divE": 2.8093064673660275e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } + } + diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_1.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_1.json index 0e0cdf5eb0f..109f5fb4d35 100644 --- a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_1.json +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_1.json @@ -1,31 +1,30 @@ { - "lev=0": { - "Bx": 2.9100687916345874e-15, - "By": 6.121275580503632e-15, - "Bz": 5.9043095451081354e-15, - "Ex": 1.4574231057224582e-06, - "Ey": 1.1648744803916206e-06, - "Ez": 1.16164257835401e-06, - "divE": 6.781029990295743e-07, - "rho": 0.0 - }, - "electron": { - "particle_position_x": 0.0, - "particle_position_y": 0.0, - "particle_position_z": 0.0, - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_weight": 0.0 - }, - "positron": { - "particle_position_x": 0.0, - "particle_position_y": 0.0, - "particle_position_z": 0.0, - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_weight": 0.0 - } + "lev=0": { + "Bx": 3.835377401535272e-15, + "By": 7.634560527952392e-15, + "Bz": 7.670097149513554e-15, + "Ex": 1.837433604148419e-06, + "Ey": 1.4507850267928362e-06, + "Ez": 1.4325637523931794e-06, + "divE": 7.330866223540695e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 } - +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_2.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_2.json index 0e0cdf5eb0f..9ff91af6550 100644 --- a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_2.json +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_removal_depth_sh_factor_2.json @@ -1,31 +1,30 @@ { - "lev=0": { - "Bx": 2.9100687916345874e-15, - "By": 6.121275580503632e-15, - "Bz": 5.9043095451081354e-15, - "Ex": 1.4574231057224582e-06, - "Ey": 1.1648744803916206e-06, - "Ez": 1.16164257835401e-06, - "divE": 6.781029990295743e-07, - "rho": 0.0 - }, - "electron": { - "particle_position_x": 0.0, - "particle_position_y": 0.0, - "particle_position_z": 0.0, - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_weight": 0.0 - }, - "positron": { - "particle_position_x": 0.0, - "particle_position_y": 0.0, - "particle_position_z": 0.0, - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_weight": 0.0 - } + "lev=0": { + "Bx": 2.3948084603369097e-15, + "By": 0.0, + "Bz": 6.747158562891953e-07, + "Ex": 0.0, + "Ey": 5.541309886315263e-07, + "Ez": 0.0, + "divE": 2.091715826275267e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 } - +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_1.json b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_1.json new file mode 100644 index 00000000000..0376427a4f0 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_1.json @@ -0,0 +1,28 @@ +{ + "lev=0": { + "Br": 0.0, + "Bz": 0.0, + "Er": 1.6208621785146114e-07, + "Ez": 2.805848027148827e-07, + "divE": 5.118824286040605e-08, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_2.json b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_2.json new file mode 100644 index 00000000000..0376427a4f0 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_2.json @@ -0,0 +1,28 @@ +{ + "lev=0": { + "Br": 0.0, + "Bz": 0.0, + "Er": 1.6208621785146114e-07, + "Ez": 2.805848027148827e-07, + "divE": 5.118824286040605e-08, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_3.json b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_3.json new file mode 100644 index 00000000000..0376427a4f0 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_removal_depth_sh_factor_3.json @@ -0,0 +1,28 @@ +{ + "lev=0": { + "Br": 0.0, + "Bz": 0.0, + "Er": 1.6208621785146114e-07, + "Ez": 2.805848027148827e-07, + "divE": 5.118824286040605e-08, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} From fc54dd13a2730a77d19256bc1ddfa2088f69bb39 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 20:17:47 +0000 Subject: [PATCH 29/99] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../CMakeLists.txt | 2 +- .../analysis.py | 59 ++++++++++--------- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt b/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt index 62d75e077a3..71426745092 100644 --- a/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt +++ b/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt @@ -107,4 +107,4 @@ if(WarpX_EB) diags/diag1/ # output OFF # dependency ) -endif() \ No newline at end of file +endif() diff --git a/Examples/Tests/embedded_boundary_removal_depth/analysis.py b/Examples/Tests/embedded_boundary_removal_depth/analysis.py index dc263bf0305..3b2c4199082 100755 --- a/Examples/Tests/embedded_boundary_removal_depth/analysis.py +++ b/Examples/Tests/embedded_boundary_removal_depth/analysis.py @@ -21,44 +21,49 @@ test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename, output_format="openpmd") -def get_avg_divE (ts, start_avg_iter, end_avg_iter, ncell, test_dim ): + +def get_avg_divE(ts, start_avg_iter, end_avg_iter, ncell, test_dim): for iteration in tqdm.tqdm(ts.iterations[start_avg_iter:end_avg_iter]): - if test_dim == '3d': - avg_divE = np.zeros((ncell,ncell)) - divE = ts.get_field('divE', iteration=iteration, slice_across='y', plot=False, cmap='RdBu')[0] - elif (test_dim == '2d'): - avg_divE = np.zeros((ncell,ncell)) - divE = ts.get_field('divE', iteration=iteration, plot=False, cmap='RdBu')[0] - elif (test_dim == 'rz'): - avg_divE = np.zeros((ncell,2*ncell)) - divE = ts.get_field('divE', iteration=iteration, plot=False, cmap='RdBu')[0] + if test_dim == "3d": + avg_divE = np.zeros((ncell, ncell)) + divE = ts.get_field( + "divE", iteration=iteration, slice_across="y", plot=False, cmap="RdBu" + )[0] + elif test_dim == "2d": + avg_divE = np.zeros((ncell, ncell)) + divE = ts.get_field("divE", iteration=iteration, plot=False, cmap="RdBu")[0] + elif test_dim == "rz": + avg_divE = np.zeros((ncell, 2 * ncell)) + divE = ts.get_field("divE", iteration=iteration, plot=False, cmap="RdBu")[0] avg_divE += divE - return avg_divE / (end_avg_iter - start_avg_iter) + return avg_divE / (end_avg_iter - start_avg_iter) + -def parse_dimension_in_test_name (test_name): +def parse_dimension_in_test_name(test_name): test_name = test_name.lower() - if '3d' in test_name: - return '3d' - elif '2d' in test_name: - return '2d' - elif 'rz' in test_name: - return 'rz' + if "3d" in test_name: + return "3d" + elif "2d" in test_name: + return "2d" + elif "rz" in test_name: + return "rz" return None -def plot (array): + +def plot(array): x = np.linspace(-7, 7, 400) y = np.sqrt(7**2 - x**2) fig, ax = plt.subplots() # Plot the boundary curves - ax.plot(x, y, 'k--') - ax.plot(x, -y, 'k--') - cax = ax.imshow(array, cmap='RdBu', extent=[-10, 10, -10, 10], origin='lower') + ax.plot(x, y, "k--") + ax.plot(x, -y, "k--") + cax = ax.imshow(array, cmap="RdBu", extent=[-10, 10, -10, 10], origin="lower") cbar = fig.colorbar(cax, ax=ax) - ax.set_xlabel('x (m)') - ax.set_ylabel('z (m)') - ax.set_title('Averaged divE') + ax.set_xlabel("x (m)") + ax.set_ylabel("z (m)") + ax.set_title("Averaged divE") ts = OpenPMDTimeSeries("./diags/diag1/") @@ -72,13 +77,13 @@ def plot (array): plot(divE_avg) plt.savefig("AverageddivE.png") -if (test_dim =='3d' or test_dim =='rz'): +if test_dim == "3d" or test_dim == "rz": tolerance = 1e-10 else: tolerance = 1e-9 -def check_tolerance (array, tolerance): +def check_tolerance(array, tolerance): assert np.all( array <= tolerance ), f"Test did not pass: one or more elements exceed the tolerance of {tolerance}." From 7c07b071f7b259ad8c231aa998afe2169a2590f6 Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Thu, 31 Oct 2024 12:21:48 -0700 Subject: [PATCH 30/99] Clean-up --- Examples/Tests/embedded_boundary_removal_depth/analysis.py | 2 +- Source/Particles/Deposition/CurrentDeposition.H | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Examples/Tests/embedded_boundary_removal_depth/analysis.py b/Examples/Tests/embedded_boundary_removal_depth/analysis.py index dc263bf0305..5a1bfa2dc15 100755 --- a/Examples/Tests/embedded_boundary_removal_depth/analysis.py +++ b/Examples/Tests/embedded_boundary_removal_depth/analysis.py @@ -55,7 +55,7 @@ def plot (array): ax.plot(x, y, 'k--') ax.plot(x, -y, 'k--') cax = ax.imshow(array, cmap='RdBu', extent=[-10, 10, -10, 10], origin='lower') - cbar = fig.colorbar(cax, ax=ax) + fig.colorbar(cax, ax=ax) ax.set_xlabel('x (m)') ax.set_ylabel('z (m)') ax.set_title('Averaged divE') diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 6a01c5dde63..924153dfd21 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -749,7 +749,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // TODO: abstract in a separate function (doGatherScalarFieldNodal) // TODO: check calculation for RZ { - amrex::Real distance = ablastr::particles::doGatherScalarFieldNodal( + const amrex::Real distance = ablastr::particles::doGatherScalarFieldNodal( xp + (relative_time - 0.5_rt*dt)*uxp[ip]*gaminv, yp + (relative_time - 0.5_rt*dt)*uyp[ip]*gaminv, zp + (relative_time - 0.5_rt*dt)*uzp[ip]*gaminv, @@ -757,7 +757,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, close_to_eb_old = (distance < 1.0_rt); // TODO: tune depth } { - amrex::Real distance = ablastr::particles::doGatherScalarFieldNodal( + const amrex::Real distance = ablastr::particles::doGatherScalarFieldNodal( xp + (relative_time + 0.5_rt*dt)*uxp[ip]*gaminv, yp + (relative_time + 0.5_rt*dt)*uyp[ip]*gaminv, zp + (relative_time + 0.5_rt*dt)*uzp[ip]*gaminv, From 1fbad083ab3b680cc36b8a6b0775b64e705bebc3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 31 Oct 2024 19:25:50 +0000 Subject: [PATCH 31/99] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../embedded_boundary_removal_depth/analysis.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Examples/Tests/embedded_boundary_removal_depth/analysis.py b/Examples/Tests/embedded_boundary_removal_depth/analysis.py index bd154879956..6e8ad4d48e3 100755 --- a/Examples/Tests/embedded_boundary_removal_depth/analysis.py +++ b/Examples/Tests/embedded_boundary_removal_depth/analysis.py @@ -57,13 +57,13 @@ def plot(array): fig, ax = plt.subplots() # Plot the boundary curves - ax.plot(x, y, 'k--') - ax.plot(x, -y, 'k--') - cax = ax.imshow(array, cmap='RdBu', extent=[-10, 10, -10, 10], origin='lower') + ax.plot(x, y, "k--") + ax.plot(x, -y, "k--") + cax = ax.imshow(array, cmap="RdBu", extent=[-10, 10, -10, 10], origin="lower") fig.colorbar(cax, ax=ax) - ax.set_xlabel('x (m)') - ax.set_ylabel('z (m)') - ax.set_title('Averaged divE') + ax.set_xlabel("x (m)") + ax.set_ylabel("z (m)") + ax.set_title("Averaged divE") ts = OpenPMDTimeSeries("./diags/diag1/") From d058440b45fa1cbc99cb7eb061d7baed7b3e4aba Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Mon, 4 Nov 2024 16:49:57 -0800 Subject: [PATCH 32/99] Moved distance_to_eb_arr and the correspondinf part of the code inside if (EB::enabled()) condition --- Source/Particles/WarpXParticleContainer.cpp | 99 +++++++++++++-------- 1 file changed, 63 insertions(+), 36 deletions(-) diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index ef6ef09ce68..14c66cc0059 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -586,53 +586,80 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { if (push_type == PushType::Explicit) { #ifdef AMREX_USE_EB - // signed distance function - auto const distance_to_eb_arr = (*warpx.GetDistanceToEB()[lev])[pti].array(); - const Geometry& geom = Geom(lev); - const amrex::GpuArray eb_dxi = geom.InvCellSizeArray(); - const amrex::GpuArray eb_plo = geom.ProbLoArray(); -#endif - if (WarpX::nox == 1){ - doEsirkepovDepositionShapeN<1>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, - np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, -#ifdef AMREX_USE_EB - distance_to_eb_arr, eb_dxi, eb_plo, -#endif - WarpX::n_rz_azimuthal_modes); - } else if (WarpX::nox == 2){ - doEsirkepovDepositionShapeN<2>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, - np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, -#ifdef AMREX_USE_EB - distance_to_eb_arr, eb_dxi, eb_plo, -#endif - WarpX::n_rz_azimuthal_modes); - } else if (WarpX::nox == 3){ - doEsirkepovDepositionShapeN<3>( + if (EB::enabled()) + { + // signed distance function + auto const distance_to_eb_arr = (*warpx.GetDistanceToEB()[lev])[pti].array(); + const Geometry& geom = Geom(lev); + const amrex::GpuArray eb_dxi = geom.InvCellSizeArray(); + const amrex::GpuArray eb_plo = geom.ProbLoArray(); + + if (WarpX::nox == 1){ + doEsirkepovDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, -#ifdef AMREX_USE_EB distance_to_eb_arr, eb_dxi, eb_plo, -#endif WarpX::n_rz_azimuthal_modes); - } else if (WarpX::nox == 4){ - doEsirkepovDepositionShapeN<4>( + } else if (WarpX::nox == 2){ + doEsirkepovDepositionShapeN<2>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, + distance_to_eb_arr, eb_dxi, eb_plo, + WarpX::n_rz_azimuthal_modes); + } else if (WarpX::nox == 3){ + doEsirkepovDepositionShapeN<3>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, + distance_to_eb_arr, eb_dxi, eb_plo, + WarpX::n_rz_azimuthal_modes); + } else if (WarpX::nox == 4){ + doEsirkepovDepositionShapeN<4>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, + distance_to_eb_arr, eb_dxi, eb_plo, + WarpX::n_rz_azimuthal_modes); + } + } +#else + { + if (WarpX::nox == 1){ + doEsirkepovDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, -#ifdef AMREX_USE_EB - distance_to_eb_arr, eb_dxi, eb_plo, -#endif WarpX::n_rz_azimuthal_modes); - } + } else if (WarpX::nox == 2){ + doEsirkepovDepositionShapeN<2>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes); + } else if (WarpX::nox == 3){ + doEsirkepovDepositionShapeN<3>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes); + } else if (WarpX::nox == 4){ + doEsirkepovDepositionShapeN<4>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes); + } +#endif //AMREX_USE_EB } else if (push_type == PushType::Implicit) { #if (AMREX_SPACEDIM >= 2) auto& xp_n = pti.GetAttribs(particle_comps["x_n"]); From 66b029e978455b3d0b5a5e67d9a7f4ff736cbe67 Mon Sep 17 00:00:00 2001 From: Olga Shapoval Date: Mon, 4 Nov 2024 17:15:20 -0800 Subject: [PATCH 33/99] Added missing curly brace --- Source/Particles/WarpXParticleContainer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 14c66cc0059..d93d425798a 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -659,6 +659,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } + } #endif //AMREX_USE_EB } else if (push_type == PushType::Implicit) { #if (AMREX_SPACEDIM >= 2) From 3fa996fc8835fcadcd6c523ca56b0ba5f4f8cd03 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 2 Jan 2025 16:54:39 -0800 Subject: [PATCH 34/99] Modify stair-case approximation --- .../FiniteDifferenceSolver/EvolveE.cpp | 77 +++++++++++++++---- 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index 03a9866fb98..6d30ad8bdca 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -155,11 +155,24 @@ void FiniteDifferenceSolver::EvolveECartesian ( Array4 const& jy = Jfield[1]->array(mfi); Array4 const& jz = Jfield[2]->array(mfi); - amrex::Array4 lx, ly, lz; + // Extract structures for embedded boundaries + bool eb_fully_covered_box = false; + amrex::Array4::value_type> eb_flag_arr; if (EB::enabled()) { - lx = edge_lengths[0]->array(mfi); - ly = edge_lengths[1]->array(mfi); - lz = edge_lengths[2]->array(mfi); +#ifdef AMREX_USE_EB + auto & warpx = WarpX::GetInstance(); + amrex::EBFArrayBoxFactory const& eb_box_factory = warpx.fieldEBFactory(lev); + amrex::FabArray const& eb_flag = eb_box_factory.getMultiEBCellFlagFab(); + amrex::Box const& box = mfi.tilebox( amrex::IntVect::TheCellVector() ); + amrex::FabType const fab_type = eb_flag[mfi].getType(box); + if (fab_type == amrex::FabType::covered) { + eb_fully_covered_box = true; + } else if (!(fab_type == amrex::FabType::regular)) { + // For cells that are not fully covered or regular, + // we need to check each cell is covered or regular + eb_flag_arr = eb_flag.array(mfi); + } +#endif } // Extract stencil coefficients @@ -179,8 +192,21 @@ void FiniteDifferenceSolver::EvolveECartesian ( amrex::ParallelFor(tex, tey, tez, [=] AMREX_GPU_DEVICE (int i, int j, int k){ - // Skip field push if this cell is fully covered by embedded boundaries - if (lx && lx(i, j, k) <= 0) { return; } + + // Stair-case approximation to the embedded boundary: + // Skip field push if this edge touches a partially or fully covered cell + if (eb_fully_covered_box) { return; } + else if (eb_flag_arr) { +#ifdef WARPX_DIM_3D + if ( !eb_flag_arr(i, j-1, k-1).isRegular() || + !eb_flag_arr(i, j-1, k ).isRegular() || + !eb_flag_arr(i, j , k-1).isRegular() || + !eb_flag_arr(i, j , k ).isRegular() ) { return; } +#elif (defined WARPX_DIM_XZ) + if ( !eb_flag_arr(i, j-1, k).isRegular() || + !eb_flag_arr(i, j , k).isRegular() ) { return; } +#endif + } Ex(i, j, k) += c2 * dt * ( - T_Algo::DownwardDz(By, coefs_z, n_coefs_z, i, j, k) @@ -189,14 +215,23 @@ void FiniteDifferenceSolver::EvolveECartesian ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k){ - // Skip field push if this cell is fully covered by embedded boundaries + + // Stair-case approximation to the embedded boundary: + // Skip field push if this edge touches a partially or fully covered cell + if (eb_fully_covered_box) { return; } + else if (eb_flag_arr) { #ifdef WARPX_DIM_3D - if (ly && ly(i,j,k) <= 0) { return; } -#elif defined(WARPX_DIM_XZ) - //In XZ Ey is associated with a mesh node, so we need to check if the mesh node is covered - amrex::ignore_unused(ly); - if (lx && (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0)) { return; } + if ( !eb_flag_arr(i-1, j, k-1).isRegular() || + !eb_flag_arr(i-1, j, k ).isRegular() || + !eb_flag_arr(i , j, k-1).isRegular() || + !eb_flag_arr(i , j, k ).isRegular() ) { return; } +#elif (defined WARPX_DIM_XZ) + if ( !eb_flag_arr(i-1, j-1, k).isRegular() || + !eb_flag_arr(i-1, j , k).isRegular() || + !eb_flag_arr(i , j-1, k).isRegular() || + !eb_flag_arr(i , j , k).isRegular() ) { return; } #endif + } Ey(i, j, k) += c2 * dt * ( - T_Algo::DownwardDx(Bz, coefs_x, n_coefs_x, i, j, k) @@ -205,8 +240,22 @@ void FiniteDifferenceSolver::EvolveECartesian ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k){ - // Skip field push if this cell is fully covered by embedded boundaries - if (lz && lz(i,j,k) <= 0) { return; } + + // Stair-case approximation to the embedded boundary: + // Skip field push if this edge touches a partially or fully covered cell + if (eb_fully_covered_box) { return; } + else if (eb_flag_arr) { +#ifdef WARPX_DIM_3D + if ( !eb_flag_arr(i-1, j-1, k).isRegular() || + !eb_flag_arr(i-1, j , k).isRegular() || + !eb_flag_arr(i , j-1, k).isRegular() || + !eb_flag_arr(i , j , k).isRegular() ) { return; } +#elif (defined WARPX_DIM_XZ) + if ( !eb_flag_arr(i-1, j, k).isRegular() || + !eb_flag_arr(i , j, k).isRegular() ) { return; } +#endif + } + Ez(i, j, k) += c2 * dt * ( - T_Algo::DownwardDy(Bx, coefs_y, n_coefs_y, i, j, k) + T_Algo::DownwardDx(By, coefs_x, n_coefs_x, i, j, k) From 112cce1952f1b1f5753dfcf876351c6a33cfb0af Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Sun, 5 Jan 2025 08:10:14 -0800 Subject: [PATCH 35/99] Fix compilation without EB --- Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index 6d30ad8bdca..bddd69b2393 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -155,11 +155,11 @@ void FiniteDifferenceSolver::EvolveECartesian ( Array4 const& jy = Jfield[1]->array(mfi); Array4 const& jz = Jfield[2]->array(mfi); +#ifdef AMREX_USE_EB // Extract structures for embedded boundaries bool eb_fully_covered_box = false; amrex::Array4::value_type> eb_flag_arr; if (EB::enabled()) { -#ifdef AMREX_USE_EB auto & warpx = WarpX::GetInstance(); amrex::EBFArrayBoxFactory const& eb_box_factory = warpx.fieldEBFactory(lev); amrex::FabArray const& eb_flag = eb_box_factory.getMultiEBCellFlagFab(); @@ -172,8 +172,8 @@ void FiniteDifferenceSolver::EvolveECartesian ( // we need to check each cell is covered or regular eb_flag_arr = eb_flag.array(mfi); } -#endif } +#endif // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); @@ -193,6 +193,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( [=] AMREX_GPU_DEVICE (int i, int j, int k){ +#ifdef AMREX_USE_EB // Stair-case approximation to the embedded boundary: // Skip field push if this edge touches a partially or fully covered cell if (eb_fully_covered_box) { return; } @@ -207,6 +208,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( !eb_flag_arr(i, j , k).isRegular() ) { return; } #endif } +#endif Ex(i, j, k) += c2 * dt * ( - T_Algo::DownwardDz(By, coefs_z, n_coefs_z, i, j, k) @@ -216,6 +218,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( [=] AMREX_GPU_DEVICE (int i, int j, int k){ +#ifdef AMREX_USE_EB // Stair-case approximation to the embedded boundary: // Skip field push if this edge touches a partially or fully covered cell if (eb_fully_covered_box) { return; } @@ -232,6 +235,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( !eb_flag_arr(i , j , k).isRegular() ) { return; } #endif } +#endif Ey(i, j, k) += c2 * dt * ( - T_Algo::DownwardDx(Bz, coefs_x, n_coefs_x, i, j, k) @@ -241,6 +245,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( [=] AMREX_GPU_DEVICE (int i, int j, int k){ +#ifdef AMREX_USE_EB // Stair-case approximation to the embedded boundary: // Skip field push if this edge touches a partially or fully covered cell if (eb_fully_covered_box) { return; } @@ -255,6 +260,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( !eb_flag_arr(i , j, k).isRegular() ) { return; } #endif } +#endif Ez(i, j, k) += c2 * dt * ( - T_Algo::DownwardDy(Bx, coefs_y, n_coefs_y, i, j, k) From b0ccce457a74054099c93454f71f7de04b2b5dd9 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 6 Jan 2025 11:28:46 -0800 Subject: [PATCH 36/99] Slightly modify boundaries --- .../inputs_test_2d_embedded_boundary_cube | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Examples/Tests/embedded_boundary_cube/inputs_test_2d_embedded_boundary_cube b/Examples/Tests/embedded_boundary_cube/inputs_test_2d_embedded_boundary_cube index 684325dc030..4fbf13e7f5c 100644 --- a/Examples/Tests/embedded_boundary_cube/inputs_test_2d_embedded_boundary_cube +++ b/Examples/Tests/embedded_boundary_cube/inputs_test_2d_embedded_boundary_cube @@ -12,10 +12,10 @@ warpx.abort_on_warning_threshold = medium boundary.field_lo = pec pec boundary.field_hi = pec pec -my_constants.xmin = -0.5 -my_constants.zmin = -0.5 -my_constants.xmax = 0.5 -my_constants.zmax = 0.5 +my_constants.xmin = -0.501 +my_constants.zmin = -0.501 +my_constants.xmax = 0.501 +my_constants.zmax = 0.501 # Note that for amrex EB implicit function, >0 is covered, =0 is boundary and <0 is regular. warpx.eb_implicit_function = "max(max(x+xmin,-(x+xmax)), max(z+zmin,-(z+zmax)))" From 76fa82e783c2420ca9b589db2b48da838532d1ea Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 8 Jan 2025 06:46:16 -0800 Subject: [PATCH 37/99] Added new arrays with flags --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 55 ++++++++++++++++++++++++- Source/Initialization/WarpXInitData.cpp | 5 ++- Source/WarpX.H | 19 +++++++-- Source/WarpX.cpp | 10 +++++ 4 files changed, 83 insertions(+), 6 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index edbc97a8efe..d87b891dd99 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -291,9 +291,60 @@ WarpX::ScaleAreas (ablastr::fields::VectorField& face_areas, } } +void +WarpX::MarkUpdateECells ( + amrex::EBFArrayBoxFactory const & eb_fact, + int const lev ) +{ + + using ablastr::fields::Direction; + using warpx::fields::FieldType; + + // Extract structures for embedded boundaries + amrex::FabArray const& eb_flag = eb_fact.getMultiEBCellFlagFab(); + + for (int idim = 0; idim < 3; ++idim) { + + // TODO: Handle guard cells + + for (amrex::MFIter mfi(*m_fields.get(FieldType::Efield_fp, Direction{idim}, lev)); mfi.isValid(); ++mfi) { + + const amrex::Box& box = mfi.tilebox(); + auto const & update_E_arr = m_eb_update_E[lev][idim]->array(mfi); + + amrex::FabType const fab_type = eb_flag[mfi].getType(mfi.tilebox(amrex::IntVect::TheCellVector())); + + if (fab_type == amrex::FabType::regular) { + + // every cell in box is all regular + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + update_E_arr(i, j, k) = 1; + }); + + } else if (fab_type == amrex::FabType::covered) { + + // every cell in box is all covered + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + update_E_arr(i, j, k) = 0; + }); + + } else { + + auto const & flag = eb_flag[mfi].array(); + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + update_E_arr(i, j, k) = flag(i, j, k).isRegular(); + }); + + } + + } + + } + +} void -WarpX::MarkCells () +WarpX::MarkExtensionCells () { using ablastr::fields::Direction; using warpx::fields::FieldType; @@ -302,7 +353,7 @@ WarpX::MarkCells () auto const &cell_size = CellSize(maxLevel()); #if !defined(WARPX_DIM_3D) && !defined(WARPX_DIM_XZ) - WARPX_ABORT_WITH_MESSAGE("MarkCells only implemented in 2D and 3D"); + WARPX_ABORT_WITH_MESSAGE("MarkExtensionCells only implemented in 2D and 3D"); #endif for (int idim = 0; idim < 3; ++idim) { diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index daecfac8bed..b0f429b3e74 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1300,6 +1300,9 @@ void WarpX::InitializeEBGridData (int lev) auto const eb_fact = fieldEBFactory(lev); + MarkUpdateECells( eb_fact, lev ); + + // TODO: move inside if condition for ECT auto edge_lengths_lev = m_fields.get_alldirs(FieldType::edge_lengths, lev); ComputeEdgeLengths(edge_lengths_lev, eb_fact); ScaleEdges(edge_lengths_lev, CellSize(lev)); @@ -1309,7 +1312,7 @@ void WarpX::InitializeEBGridData (int lev) ScaleAreas(face_areas_lev, CellSize(lev)); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - MarkCells(); + MarkExtensionCells(); ComputeFaceExtensions(); } } diff --git a/Source/WarpX.H b/Source/WarpX.H index 73998d6faf2..bb771c18ece 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1013,6 +1013,14 @@ public: void InitEB (); #ifdef AMREX_USE_EB + /** Set a flag to indicate which cells are in the valid domain and should be updated + * by the field solve, and which cells are in the EB and should not be updated. + * TODO: Add detail about criterion for Yee and ECT solver + */ + void MarkUpdateECells ( + amrex::EBFArrayBoxFactory const & eb_fact, + int const lev ); + /** * \brief Compute the length of the mesh edges. Here the length is a value in [0, 1]. * An edge of length 0 is fully covered. @@ -1044,7 +1052,7 @@ public: * - 2 for stable cells which have been intruded * Here we cannot know if a cell is intruded or not so we initialize all stable cells with 1 */ - void MarkCells(); + void MarkExtensionCells(); #endif /** @@ -1393,17 +1401,22 @@ private: mutable amrex::Vector,3 > > Afield_dotMask; mutable amrex::Vector< std::unique_ptr > phi_dotMask; + /** EB: Flag to indicate whether the E location is inside the EB and therefore E should not be updated. + * (One array per level and per direction, due to staggering) + */ + amrex::Vector,3 > > m_eb_update_E; + /** EB: for every mesh face flag_info_face contains a: * * 0 if the face needs to be extended * * 1 if the face is large enough to lend area to other faces * * 2 if the face is actually intruded by other face - * It is initialized in WarpX::MarkCells + * It is initialized in WarpX::MarkExtensionCells * This is only used for the ECT solver.*/ amrex::Vector, 3 > > m_flag_info_face; /** EB: for every mesh face face flag_ext_face contains a: * * 1 if the face needs to be extended * * 0 otherwise - * It is initialized in WarpX::MarkCells and then modified in WarpX::ComputeOneWayExtensions + * It is initialized in WarpX::MarkExtensionCells and then modified in WarpX::ComputeOneWayExtensions * and in WarpX::ComputeEightWaysExtensions * This is only used for the ECT solver.*/ amrex::Vector, 3 > > m_flag_ext_face; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 965235e1078..6b2c025c89f 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -323,6 +323,7 @@ WarpX::WarpX () Afield_dotMask.resize(nlevs_max); phi_dotMask.resize(nlevs_max); + m_eb_update_E.resize(nlevs_max); m_flag_info_face.resize(nlevs_max); m_flag_ext_face.resize(nlevs_max); m_borrowing.resize(nlevs_max); @@ -2301,6 +2302,15 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm // EB info are needed only at the finest level if (lev == maxLevel()) { if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { + + AllocInitMultiFab(m_eb_update_E[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_E[x]"); + AllocInitMultiFab(m_eb_update_E[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_E[y]"); + AllocInitMultiFab(m_eb_update_E[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_E[z]"); + + // TODO: do not allocate these arrays anymore with the Yee solver //! EB: Lengths of the mesh edges m_fields.alloc_init(FieldType::edge_lengths, Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); From 3624a7232e5cb9551f0d9b55dc65888f1ac6e4ca Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 8 Jan 2025 07:01:48 -0800 Subject: [PATCH 38/99] Finalize initialization of the array --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 39 +++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index d87b891dd99..676c28a1e9a 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -316,14 +316,15 @@ WarpX::MarkUpdateECells ( if (fab_type == amrex::FabType::regular) { - // every cell in box is all regular + // Every cell in box is all regular: update E in every cell + // TODO: We actually need to check guard cells amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { update_E_arr(i, j, k) = 1; }); } else if (fab_type == amrex::FabType::covered) { - // every cell in box is all covered + // Every cell in box is all covered: do not update E amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { update_E_arr(i, j, k) = 0; }); @@ -331,10 +332,42 @@ WarpX::MarkUpdateECells ( } else { auto const & flag = eb_flag[mfi].array(); + auto index_type = m_fields.get(FieldType::Efield_fp, Direction{idim}, lev)->ixType(); + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - update_E_arr(i, j, k) = flag(i, j, k).isRegular(); + + // If neighboring cells are partially covered: do not update E + int update_E = 1; + + // Check neighbors in the first spatial direction + if ( index_type.nodeCentered(0) ) { + if ( !flag(i, j, k).isRegular() || !flag(i-1, j, k).isRegular() ) { update_E = 0; } + } else { + if ( !flag(i, j, k).isRegular() ) { update_E = 0; } + } + +#if AMREX_SPACEDIM > 1 + // Check neighbors in the second spatial direction + if ( index_type.nodeCentered(1) ) { + if ( !flag(i, j, k).isRegular() || !flag(i, j-1, k).isRegular() ) { update_E = 0; } + } else { + if ( !flag(i, j, k).isRegular() ) { update_E = 0; } + } +#endif + +#if AMREX_SPACEDIM > 2 + // Check neighbors in the third spatial direction + if ( index_type.nodeCentered(2) ) { + if ( !flag(i, j, k).isRegular() || !flag(i, j, k-1).isRegular() ) { update_E = 0; } + } else { + if ( !flag(i, j, k).isRegular() ) { update_E = 0; } + } +#endif + update_E_arr(i, j, k) = update_E; }); + // TODO: Handle the case of the ECT solver + } } From 3181d45e840eccc8222ccd90e5de5b3d4cabfbe3 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 8 Jan 2025 11:38:32 -0800 Subject: [PATCH 39/99] Use guard cell information --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 676c28a1e9a..1259c9f8835 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -305,38 +305,38 @@ WarpX::MarkUpdateECells ( for (int idim = 0; idim < 3; ++idim) { - // TODO: Handle guard cells - for (amrex::MFIter mfi(*m_fields.get(FieldType::Efield_fp, Direction{idim}, lev)); mfi.isValid(); ++mfi) { const amrex::Box& box = mfi.tilebox(); auto const & update_E_arr = m_eb_update_E[lev][idim]->array(mfi); - amrex::FabType const fab_type = eb_flag[mfi].getType(mfi.tilebox(amrex::IntVect::TheCellVector())); + // Check if the box (including one layer of guard cells) contains a mix of covered and regular cells + const amrex::Box& eb_info_box = mfi.tilebox(amrex::IntVect::TheCellVector()).grow(1); + amrex::FabType const fab_type = eb_flag[mfi].getType( eb_info_box ); - if (fab_type == amrex::FabType::regular) { + if (fab_type == amrex::FabType::regular) { // All cells in the box are regular // Every cell in box is all regular: update E in every cell - // TODO: We actually need to check guard cells amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { update_E_arr(i, j, k) = 1; }); - } else if (fab_type == amrex::FabType::covered) { + } else if (fab_type == amrex::FabType::covered) { // All cells in the box are covered // Every cell in box is all covered: do not update E amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { update_E_arr(i, j, k) = 0; }); - } else { + } else { // The box contains a mix of covered and regular cells auto const & flag = eb_flag[mfi].array(); auto index_type = m_fields.get(FieldType::Efield_fp, Direction{idim}, lev)->ixType(); amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - // If neighboring cells are partially covered: do not update E + // Stair-case approximation: If neighboring cells are either partially + // or fully covered (i.e. if they are not regular cells): do not update E int update_E = 1; // Check neighbors in the first spatial direction From 38d967d83b3f1dbc99114d1a329f6a8658334078 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 8 Jan 2025 16:28:05 -0800 Subject: [PATCH 40/99] Remove edge_length in field update --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 4 +- .../FiniteDifferenceSolver/EvolveE.cpp | 136 +++++------------- .../FiniteDifferenceSolver.H | 5 +- .../ImplicitSolvers/WarpXImplicitOps.cpp | 2 + Source/FieldSolver/WarpXPushFieldsEM.cpp | 2 + 5 files changed, 40 insertions(+), 109 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 1259c9f8835..29dd1da9279 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -337,15 +337,14 @@ WarpX::MarkUpdateECells ( // Stair-case approximation: If neighboring cells are either partially // or fully covered (i.e. if they are not regular cells): do not update E - int update_E = 1; + int update_E = 1; // Check neighbors in the first spatial direction if ( index_type.nodeCentered(0) ) { if ( !flag(i, j, k).isRegular() || !flag(i-1, j, k).isRegular() ) { update_E = 0; } } else { if ( !flag(i, j, k).isRegular() ) { update_E = 0; } } - #if AMREX_SPACEDIM > 1 // Check neighbors in the second spatial direction if ( index_type.nodeCentered(1) ) { @@ -354,7 +353,6 @@ WarpX::MarkUpdateECells ( if ( !flag(i, j, k).isRegular() ) { update_E = 0; } } #endif - #if AMREX_SPACEDIM > 2 // Check neighbors in the third spatial direction if ( index_type.nodeCentered(2) ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index bddd69b2393..8f9d13dcb45 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -55,6 +55,7 @@ void FiniteDifferenceSolver::EvolveE ( int lev, PatchType patch_type, ablastr::fields::VectorField const& Efield, + std::array< std::unique_ptr,3 >& eb_update_E, amrex::Real const dt ) { @@ -72,40 +73,23 @@ void FiniteDifferenceSolver::EvolveE ( fields.get(FieldType::F_fp, lev) : fields.get(FieldType::F_cp, lev); } - ablastr::fields::VectorField edge_lengths; - if (fields.has_vector(FieldType::edge_lengths, lev)) { - edge_lengths = fields.get_alldirs(FieldType::edge_lengths, lev); - } - ablastr::fields::VectorField face_areas; - if (fields.has_vector(FieldType::face_areas, lev)) { - face_areas = fields.get_alldirs(FieldType::face_areas, lev); - } - ablastr::fields::VectorField area_mod; - if (fields.has_vector(FieldType::area_mod, lev)) { - area_mod = fields.get_alldirs(FieldType::area_mod, lev); - } - ablastr::fields::VectorField ECTRhofield; - if (fields.has_vector(FieldType::ECTRhofield, lev)) { - ECTRhofield = fields.get_alldirs(FieldType::ECTRhofield, lev); - } - // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ if (m_fdtd_algo == ElectromagneticSolverAlgo::Yee){ - EvolveECylindrical ( Efield, Bfield, Jfield, edge_lengths, Ffield, lev, dt ); + EvolveECylindrical ( Efield, Bfield, Jfield, eb_update_E, Ffield, lev, dt ); #else if (m_grid_type == GridType::Collocated) { - EvolveECartesian ( Efield, Bfield, Jfield, edge_lengths, Ffield, lev, dt ); + EvolveECartesian ( Efield, Bfield, Jfield, eb_update_E, Ffield, lev, dt ); } else if (m_fdtd_algo == ElectromagneticSolverAlgo::Yee || m_fdtd_algo == ElectromagneticSolverAlgo::ECT) { - EvolveECartesian ( Efield, Bfield, Jfield, edge_lengths, Ffield, lev, dt ); + EvolveECartesian ( Efield, Bfield, Jfield, eb_update_E, Ffield, lev, dt ); } else if (m_fdtd_algo == ElectromagneticSolverAlgo::CKC) { - EvolveECartesian ( Efield, Bfield, Jfield, edge_lengths, Ffield, lev, dt ); + EvolveECartesian ( Efield, Bfield, Jfield, eb_update_E, Ffield, lev, dt ); #endif } else { @@ -122,7 +106,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, - VectorField const& edge_lengths, + std::array< std::unique_ptr,3 >& eb_update_E, amrex::MultiFab const* Ffield, int lev, amrex::Real const dt ) { @@ -155,25 +139,13 @@ void FiniteDifferenceSolver::EvolveECartesian ( Array4 const& jy = Jfield[1]->array(mfi); Array4 const& jz = Jfield[2]->array(mfi); -#ifdef AMREX_USE_EB - // Extract structures for embedded boundaries - bool eb_fully_covered_box = false; - amrex::Array4::value_type> eb_flag_arr; + // Extract structures indicating whether the E field should be updated + amrex::Array4 update_Ex_arr, update_Ey_arr, update_Ez_arr; if (EB::enabled()) { - auto & warpx = WarpX::GetInstance(); - amrex::EBFArrayBoxFactory const& eb_box_factory = warpx.fieldEBFactory(lev); - amrex::FabArray const& eb_flag = eb_box_factory.getMultiEBCellFlagFab(); - amrex::Box const& box = mfi.tilebox( amrex::IntVect::TheCellVector() ); - amrex::FabType const fab_type = eb_flag[mfi].getType(box); - if (fab_type == amrex::FabType::covered) { - eb_fully_covered_box = true; - } else if (!(fab_type == amrex::FabType::regular)) { - // For cells that are not fully covered or regular, - // we need to check each cell is covered or regular - eb_flag_arr = eb_flag.array(mfi); - } + update_Ex_arr = eb_update_E[0]->array(mfi); + update_Ey_arr = eb_update_E[1]->array(mfi); + update_Ez_arr = eb_update_E[2]->array(mfi); } -#endif // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); @@ -193,22 +165,8 @@ void FiniteDifferenceSolver::EvolveECartesian ( [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - // Stair-case approximation to the embedded boundary: - // Skip field push if this edge touches a partially or fully covered cell - if (eb_fully_covered_box) { return; } - else if (eb_flag_arr) { -#ifdef WARPX_DIM_3D - if ( !eb_flag_arr(i, j-1, k-1).isRegular() || - !eb_flag_arr(i, j-1, k ).isRegular() || - !eb_flag_arr(i, j , k-1).isRegular() || - !eb_flag_arr(i, j , k ).isRegular() ) { return; } -#elif (defined WARPX_DIM_XZ) - if ( !eb_flag_arr(i, j-1, k).isRegular() || - !eb_flag_arr(i, j , k).isRegular() ) { return; } -#endif - } -#endif + // Skip field push in the embedded boundaries + if (update_Ex_arr && update_Ex_arr(i, j, k) == 0) { return; } Ex(i, j, k) += c2 * dt * ( - T_Algo::DownwardDz(By, coefs_z, n_coefs_z, i, j, k) @@ -218,24 +176,8 @@ void FiniteDifferenceSolver::EvolveECartesian ( [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - // Stair-case approximation to the embedded boundary: - // Skip field push if this edge touches a partially or fully covered cell - if (eb_fully_covered_box) { return; } - else if (eb_flag_arr) { -#ifdef WARPX_DIM_3D - if ( !eb_flag_arr(i-1, j, k-1).isRegular() || - !eb_flag_arr(i-1, j, k ).isRegular() || - !eb_flag_arr(i , j, k-1).isRegular() || - !eb_flag_arr(i , j, k ).isRegular() ) { return; } -#elif (defined WARPX_DIM_XZ) - if ( !eb_flag_arr(i-1, j-1, k).isRegular() || - !eb_flag_arr(i-1, j , k).isRegular() || - !eb_flag_arr(i , j-1, k).isRegular() || - !eb_flag_arr(i , j , k).isRegular() ) { return; } -#endif - } -#endif + // Skip field push in the embedded boundaries + if (update_Ey_arr && update_Ey_arr(i, j, k) == 0) { return; } Ey(i, j, k) += c2 * dt * ( - T_Algo::DownwardDx(Bz, coefs_x, n_coefs_x, i, j, k) @@ -245,22 +187,8 @@ void FiniteDifferenceSolver::EvolveECartesian ( [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - // Stair-case approximation to the embedded boundary: - // Skip field push if this edge touches a partially or fully covered cell - if (eb_fully_covered_box) { return; } - else if (eb_flag_arr) { -#ifdef WARPX_DIM_3D - if ( !eb_flag_arr(i-1, j-1, k).isRegular() || - !eb_flag_arr(i-1, j , k).isRegular() || - !eb_flag_arr(i , j-1, k).isRegular() || - !eb_flag_arr(i , j , k).isRegular() ) { return; } -#elif (defined WARPX_DIM_XZ) - if ( !eb_flag_arr(i-1, j, k).isRegular() || - !eb_flag_arr(i , j, k).isRegular() ) { return; } -#endif - } -#endif + // Skip field push in the embedded boundaries + if (update_Ez_arr && update_Ez_arr(i, j, k) == 0) { return; } Ez(i, j, k) += c2 * dt * ( - T_Algo::DownwardDy(Bx, coefs_y, n_coefs_y, i, j, k) @@ -311,14 +239,10 @@ void FiniteDifferenceSolver::EvolveECylindrical ( ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, - ablastr::fields::VectorField const& edge_lengths, + std::array< std::unique_ptr,3 >& eb_update_E, amrex::MultiFab const* Ffield, int lev, amrex::Real const dt ) { -#ifndef AMREX_USE_EB - amrex::ignore_unused(edge_lengths); -#endif - amrex::LayoutData* cost = WarpX::getCosts(lev); // Loop through the grids, and over the tiles within each grid @@ -343,10 +267,12 @@ void FiniteDifferenceSolver::EvolveECylindrical ( Array4 const& jt = Jfield[1]->array(mfi); Array4 const& jz = Jfield[2]->array(mfi); - amrex::Array4 lr, lz; + // Extract structures indicating whether the E field should be updated + amrex::Array4 update_Er_arr, update_Et_arr, update_Ez_arr; if (EB::enabled()) { - lr = edge_lengths[0]->array(mfi); - lz = edge_lengths[2]->array(mfi); + update_Er_arr = eb_update_E[0]->array(mfi); + update_Et_arr = eb_update_E[1]->array(mfi); + update_Ez_arr = eb_update_E[2]->array(mfi); } // Extract stencil coefficients @@ -371,8 +297,9 @@ void FiniteDifferenceSolver::EvolveECylindrical ( amrex::ParallelFor(ter, tet, tez, [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ - // Skip field push if this cell is fully covered by embedded boundaries - if (lr && lr(i, j, 0) <= 0) { return; } + + // Skip field push in the embedded boundaries + if (update_Er_arr && update_Er_arr(i, j, k) == 0) { return; } Real const r = rmin + (i + 0.5_rt)*dr; // r on cell-centered point (Er is cell-centered in r) Er(i, j, 0, 0) += c2 * dt*( @@ -391,9 +318,9 @@ void FiniteDifferenceSolver::EvolveECylindrical ( }, [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ - // Skip field push if this cell is fully covered by embedded boundaries - // The Et field is at a node, so we need to check if the node is covered - if (lr && (lr(i, j, 0)<=0 || lr(i-1, j, 0)<=0 || lz(i, j-1, 0)<=0 || lz(i, j, 0)<=0)) { return; } + + // Skip field push in the embedded boundaries + if (update_Et_arr && update_Et_arr(i, j, k) == 0) { return; } Real const r = rmin + i*dr; // r on a nodal grid (Et is nodal in r) if (r != 0) { // Off-axis, regular Maxwell equations @@ -436,8 +363,9 @@ void FiniteDifferenceSolver::EvolveECylindrical ( }, [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ - // Skip field push if this cell is fully covered by embedded boundaries - if (lz && lz(i, j, 0) <= 0) { return; } + + // Skip field push in the embedded boundaries + if (update_Ez_arr && update_Ez_arr(i, j, k) == 0) { return; } Real const r = rmin + i*dr; // r on a nodal grid (Ez is nodal in r) if (r != 0) { // Off-axis, regular Maxwell equations diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 45c06584fda..64d93f33935 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -63,6 +63,7 @@ class FiniteDifferenceSolver int lev, PatchType patch_type, ablastr::fields::VectorField const& Efield, + std::array< std::unique_ptr,3 >& eb_update_E, amrex::Real dt ); void EvolveF ( amrex::MultiFab* Ffield, @@ -215,7 +216,7 @@ class FiniteDifferenceSolver ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, - ablastr::fields::VectorField const& edge_lengths, + std::array< std::unique_ptr,3 >& eb_update_E, amrex::MultiFab const* Ffield, int lev, amrex::Real dt ); @@ -267,7 +268,7 @@ class FiniteDifferenceSolver ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, - ablastr::fields::VectorField const& edge_lengths, + std::array< std::unique_ptr,3 >& eb_update_E, amrex::MultiFab const* Ffield, int lev, amrex::Real dt ); diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index eaf96cf77ec..9b62bd91b0c 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -385,12 +385,14 @@ WarpX::ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real a_dt, War lev, patch_type, a_Erhs_vec.getArrayVec()[lev], + m_eb_update_E[lev], a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveE( m_fields, lev, patch_type, a_Erhs_vec.getArrayVec()[lev], + m_eb_update_E[lev], a_dt ); } diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 24640fc63c7..a77fb390fff 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -963,12 +963,14 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt, amrex::Real sta lev, patch_type, m_fields.get_alldirs(FieldType::Efield_fp, lev), + m_eb_update_E[lev], a_dt ); } else { m_fdtd_solver_cp[lev]->EvolveE( m_fields, lev, patch_type, m_fields.get_alldirs(FieldType::Efield_cp, lev), + m_eb_update_E[lev], a_dt ); } From 79ed11409f3f1fb521dd4dade7e333ef731c5bff Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 8 Jan 2025 16:38:13 -0800 Subject: [PATCH 41/99] Fix const-ness --- Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp | 6 +++--- .../FiniteDifferenceSolver/FiniteDifferenceSolver.H | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index 8f9d13dcb45..f8bc4ade0b3 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -55,7 +55,7 @@ void FiniteDifferenceSolver::EvolveE ( int lev, PatchType patch_type, ablastr::fields::VectorField const& Efield, - std::array< std::unique_ptr,3 >& eb_update_E, + std::array< std::unique_ptr,3 > const& eb_update_E, amrex::Real const dt ) { @@ -106,7 +106,7 @@ void FiniteDifferenceSolver::EvolveECartesian ( ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, - std::array< std::unique_ptr,3 >& eb_update_E, + std::array< std::unique_ptr,3> const& eb_update_E, amrex::MultiFab const* Ffield, int lev, amrex::Real const dt ) { @@ -239,7 +239,7 @@ void FiniteDifferenceSolver::EvolveECylindrical ( ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, - std::array< std::unique_ptr,3 >& eb_update_E, + std::array< std::unique_ptr,3 > const& eb_update_E, amrex::MultiFab const* Ffield, int lev, amrex::Real const dt ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 64d93f33935..07596dc8562 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -63,7 +63,7 @@ class FiniteDifferenceSolver int lev, PatchType patch_type, ablastr::fields::VectorField const& Efield, - std::array< std::unique_ptr,3 >& eb_update_E, + std::array< std::unique_ptr,3 > const& eb_update_E, amrex::Real dt ); void EvolveF ( amrex::MultiFab* Ffield, @@ -216,7 +216,7 @@ class FiniteDifferenceSolver ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, - std::array< std::unique_ptr,3 >& eb_update_E, + std::array< std::unique_ptr,3 > const& eb_update_E, amrex::MultiFab const* Ffield, int lev, amrex::Real dt ); @@ -268,7 +268,7 @@ class FiniteDifferenceSolver ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, - std::array< std::unique_ptr,3 >& eb_update_E, + std::array< std::unique_ptr,3 > const& eb_update_E, amrex::MultiFab const* Ffield, int lev, amrex::Real dt ); From 8516d1ad8defd31f2dc958386d2de49ba6f21174 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 8 Jan 2025 16:55:51 -0800 Subject: [PATCH 42/99] Fix compilation warning --- Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index f8bc4ade0b3..509027c5569 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -110,10 +110,6 @@ void FiniteDifferenceSolver::EvolveECartesian ( amrex::MultiFab const* Ffield, int lev, amrex::Real const dt ) { -#ifndef AMREX_USE_EB - amrex::ignore_unused(edge_lengths); -#endif - amrex::LayoutData* cost = WarpX::getCosts(lev); Real constexpr c2 = PhysConst::c * PhysConst::c; From 3c0867eff6e622dc5aa0a96e60f28abafb264f2d Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 10 Jan 2025 06:12:49 -0800 Subject: [PATCH 43/99] Updated init functions --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 39 +++++++++++++------------ Source/Initialization/WarpXInitData.cpp | 6 +++- Source/WarpX.H | 24 +++++++++++---- 3 files changed, 43 insertions(+), 26 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 29dd1da9279..059667d605c 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -292,9 +292,10 @@ WarpX::ScaleAreas (ablastr::fields::VectorField& face_areas, } void -WarpX::MarkUpdateECells ( - amrex::EBFArrayBoxFactory const & eb_fact, - int const lev ) +WarpX::MarkUpdateCells ( + std::array< std::unique_ptr,3> & eb_update, + ablastr::fields::VectorField const& field, + amrex::EBFArrayBoxFactory const & eb_fact ) { using ablastr::fields::Direction; @@ -305,10 +306,10 @@ WarpX::MarkUpdateECells ( for (int idim = 0; idim < 3; ++idim) { - for (amrex::MFIter mfi(*m_fields.get(FieldType::Efield_fp, Direction{idim}, lev)); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(*field[idim]); mfi.isValid(); ++mfi) { const amrex::Box& box = mfi.tilebox(); - auto const & update_E_arr = m_eb_update_E[lev][idim]->array(mfi); + auto const & eb_update_arr = eb_update[idim]->array(mfi); // Check if the box (including one layer of guard cells) contains a mix of covered and regular cells const amrex::Box& eb_info_box = mfi.tilebox(amrex::IntVect::TheCellVector()).grow(1); @@ -316,52 +317,52 @@ WarpX::MarkUpdateECells ( if (fab_type == amrex::FabType::regular) { // All cells in the box are regular - // Every cell in box is all regular: update E in every cell + // Every cell in box is all regular: update field in every cell amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - update_E_arr(i, j, k) = 1; + eb_update_arr(i, j, k) = 1; }); } else if (fab_type == amrex::FabType::covered) { // All cells in the box are covered - // Every cell in box is all covered: do not update E + // Every cell in box is all covered: do not update field amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - update_E_arr(i, j, k) = 0; + eb_update_arr(i, j, k) = 0; }); } else { // The box contains a mix of covered and regular cells auto const & flag = eb_flag[mfi].array(); - auto index_type = m_fields.get(FieldType::Efield_fp, Direction{idim}, lev)->ixType(); + auto index_type = field[idim]->ixType(); amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { // Stair-case approximation: If neighboring cells are either partially - // or fully covered (i.e. if they are not regular cells): do not update E + // or fully covered (i.e. if they are not regular cells): do not update field - int update_E = 1; + int eb_update = 1; // Check neighbors in the first spatial direction if ( index_type.nodeCentered(0) ) { - if ( !flag(i, j, k).isRegular() || !flag(i-1, j, k).isRegular() ) { update_E = 0; } + if ( !flag(i, j, k).isRegular() || !flag(i-1, j, k).isRegular() ) { eb_update = 0; } } else { - if ( !flag(i, j, k).isRegular() ) { update_E = 0; } + if ( !flag(i, j, k).isRegular() ) { eb_update = 0; } } #if AMREX_SPACEDIM > 1 // Check neighbors in the second spatial direction if ( index_type.nodeCentered(1) ) { - if ( !flag(i, j, k).isRegular() || !flag(i, j-1, k).isRegular() ) { update_E = 0; } + if ( !flag(i, j, k).isRegular() || !flag(i, j-1, k).isRegular() ) { eb_update = 0; } } else { - if ( !flag(i, j, k).isRegular() ) { update_E = 0; } + if ( !flag(i, j, k).isRegular() ) { eb_update = 0; } } #endif #if AMREX_SPACEDIM > 2 // Check neighbors in the third spatial direction if ( index_type.nodeCentered(2) ) { - if ( !flag(i, j, k).isRegular() || !flag(i, j, k-1).isRegular() ) { update_E = 0; } + if ( !flag(i, j, k).isRegular() || !flag(i, j, k-1).isRegular() ) { eb_update = 0; } } else { - if ( !flag(i, j, k).isRegular() ) { update_E = 0; } + if ( !flag(i, j, k).isRegular() ) { eb_update = 0; } } #endif - update_E_arr(i, j, k) = update_E; + eb_update_arr(i, j, k) = eb_update; }); // TODO: Handle the case of the ECT solver diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index b0f429b3e74..945376a8eae 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1300,7 +1300,11 @@ void WarpX::InitializeEBGridData (int lev) auto const eb_fact = fieldEBFactory(lev); - MarkUpdateECells( eb_fact, lev ); + // Mark on which grid points E should be updated + MarkUpdateCells( + m_eb_update_E[lev], + m_fields.get_alldirs(FieldType::Efield_fp, lev), + eb_fact ); // TODO: move inside if condition for ECT auto edge_lengths_lev = m_fields.get_alldirs(FieldType::edge_lengths, lev); diff --git a/Source/WarpX.H b/Source/WarpX.H index bb771c18ece..063555ea7a7 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1013,13 +1013,25 @@ public: void InitEB (); #ifdef AMREX_USE_EB - /** Set a flag to indicate which cells are in the valid domain and should be updated - * by the field solve, and which cells are in the EB and should not be updated. - * TODO: Add detail about criterion for Yee and ECT solver + /** Set a flag to indicate on which grid points the field `field` + * should be updated. + * + * More specifically, this function fill the iMultiFabs in `eb_update` + * (which have the same indexType as the MultiFabs in `field`) with 1 + * or 0, depending on whether the grid point is in the valid region + * or in the embedded boundary. + * + * This uses a stair-case approximation of the embedded boundary + * (unless the ECT solver is used): If a grid point touches cells + * that are either partially or fully covered by the embedded + * boundary: do not update field. + * + * TODO Discuss ECT solver */ - void MarkUpdateECells ( - amrex::EBFArrayBoxFactory const & eb_fact, - int const lev ); + void MarkUpdateCells ( + std::array< std::unique_ptr,3> & eb_update, + ablastr::fields::VectorField const & field, + amrex::EBFArrayBoxFactory const & eb_fact ); /** * \brief Compute the length of the mesh edges. Here the length is a value in [0, 1]. From 28d43a13825aea0642596b14226339e4fc164213 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 10 Jan 2025 06:21:16 -0800 Subject: [PATCH 44/99] Fix initialization criterion --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 44 +++++++++++++------------ 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 059667d605c..d2a2c991165 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -337,31 +337,33 @@ WarpX::MarkUpdateCells ( amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { // Stair-case approximation: If neighboring cells are either partially - // or fully covered (i.e. if they are not regular cells): do not update field - - int eb_update = 1; - // Check neighbors in the first spatial direction - if ( index_type.nodeCentered(0) ) { - if ( !flag(i, j, k).isRegular() || !flag(i-1, j, k).isRegular() ) { eb_update = 0; } - } else { - if ( !flag(i, j, k).isRegular() ) { eb_update = 0; } - } + // or fully covered: do not update field + + // Check neighbors in the each spatial direction + int i_start = 0; + int j_start = 0; + int k_start = 0; + // If nodal in a given direction, we need to start from -1 (left-neighboring cell) + if ( index_type.nodeCentered(0) ) { i_start = -1; }; #if AMREX_SPACEDIM > 1 - // Check neighbors in the second spatial direction - if ( index_type.nodeCentered(1) ) { - if ( !flag(i, j, k).isRegular() || !flag(i, j-1, k).isRegular() ) { eb_update = 0; } - } else { - if ( !flag(i, j, k).isRegular() ) { eb_update = 0; } - } + if ( index_type.nodeCentered(1) ) { j_start = -1; }; #endif #if AMREX_SPACEDIM > 2 - // Check neighbors in the third spatial direction - if ( index_type.nodeCentered(2) ) { - if ( !flag(i, j, k).isRegular() || !flag(i, j, k-1).isRegular() ) { eb_update = 0; } - } else { - if ( !flag(i, j, k).isRegular() ) { eb_update = 0; } - } + if ( index_type.nodeCentered(2) ) { k_start = -1; }; #endif + // Loop over neighboring cells + int eb_update = 1; + for (int ii = i_start; ii <= 1; ++ii) { + for (int jj = j_start; jj <= 1; ++jj) { + for (int kk = k_start; kk <= 1; ++kk) { + // If one of the neighboring is either partially or fully covered + // (i.e. if they are not regular cells), do not update field + if ( !flag(i+ii, j+jj, k+kk).isRegular() ) { + eb_update = 0; + } + } + } + } eb_update_arr(i, j, k) = eb_update; }); From 02147690ebfda487ba0f8709d8b0f5727f1d0061 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 10 Jan 2025 06:29:04 -0800 Subject: [PATCH 45/99] Add flag for B --- Source/Initialization/WarpXInitData.cpp | 5 +++++ Source/WarpX.H | 5 +++-- Source/WarpX.cpp | 8 ++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 945376a8eae..1b541f85d8e 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1305,6 +1305,11 @@ void WarpX::InitializeEBGridData (int lev) m_eb_update_E[lev], m_fields.get_alldirs(FieldType::Efield_fp, lev), eb_fact ); + // Mark on which grid points B should be updated + MarkUpdateCells( + m_eb_update_B[lev], + m_fields.get_alldirs(FieldType::Bfield_fp, lev), + eb_fact ); // TODO: move inside if condition for ECT auto edge_lengths_lev = m_fields.get_alldirs(FieldType::edge_lengths, lev); diff --git a/Source/WarpX.H b/Source/WarpX.H index 063555ea7a7..5bd823ad8ba 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1413,10 +1413,11 @@ private: mutable amrex::Vector,3 > > Afield_dotMask; mutable amrex::Vector< std::unique_ptr > phi_dotMask; - /** EB: Flag to indicate whether the E location is inside the EB and therefore E should not be updated. - * (One array per level and per direction, due to staggering) + /** EB: Flag to indicate whether a gridpoint is inside the embedded boundary and therefore + * whether the E or B should not be updated. (One array per level and per direction, due to staggering) */ amrex::Vector,3 > > m_eb_update_E; + amrex::Vector,3 > > m_eb_update_B; /** EB: for every mesh face flag_info_face contains a: * * 0 if the face needs to be extended diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 6b2c025c89f..cf709ca304c 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -324,6 +324,7 @@ WarpX::WarpX () phi_dotMask.resize(nlevs_max); m_eb_update_E.resize(nlevs_max); + m_eb_update_B.resize(nlevs_max); m_flag_info_face.resize(nlevs_max); m_flag_ext_face.resize(nlevs_max); m_borrowing.resize(nlevs_max); @@ -2310,6 +2311,13 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(m_eb_update_E[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_eb_update_E[z]"); + AllocInitMultiFab(m_eb_update_B[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_B[x]"); + AllocInitMultiFab(m_eb_update_B[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_B[y]"); + AllocInitMultiFab(m_eb_update_B[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_B[z]"); + // TODO: do not allocate these arrays anymore with the Yee solver //! EB: Lengths of the mesh edges m_fields.alloc_init(FieldType::edge_lengths, Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), From ab87809509adca02439740e7f8c7d72a03406b73 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 10 Jan 2025 09:12:40 -0800 Subject: [PATCH 46/99] Fix RZ compilation error --- Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp index 509027c5569..926f52aa8ee 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveE.cpp @@ -295,7 +295,7 @@ void FiniteDifferenceSolver::EvolveECylindrical ( [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ // Skip field push in the embedded boundaries - if (update_Er_arr && update_Er_arr(i, j, k) == 0) { return; } + if (update_Er_arr && update_Er_arr(i, j, 0) == 0) { return; } Real const r = rmin + (i + 0.5_rt)*dr; // r on cell-centered point (Er is cell-centered in r) Er(i, j, 0, 0) += c2 * dt*( @@ -316,7 +316,7 @@ void FiniteDifferenceSolver::EvolveECylindrical ( [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ // Skip field push in the embedded boundaries - if (update_Et_arr && update_Et_arr(i, j, k) == 0) { return; } + if (update_Et_arr && update_Et_arr(i, j, 0) == 0) { return; } Real const r = rmin + i*dr; // r on a nodal grid (Et is nodal in r) if (r != 0) { // Off-axis, regular Maxwell equations @@ -361,7 +361,7 @@ void FiniteDifferenceSolver::EvolveECylindrical ( [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ // Skip field push in the embedded boundaries - if (update_Ez_arr && update_Ez_arr(i, j, k) == 0) { return; } + if (update_Ez_arr && update_Ez_arr(i, j, 0) == 0) { return; } Real const r = rmin + i*dr; // r on a nodal grid (Ez is nodal in r) if (r != 0) { // Off-axis, regular Maxwell equations From 78bb8c1bb20ad12c0e5dd8316af4448a9edf1f8e Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Sat, 11 Jan 2025 09:52:12 -0800 Subject: [PATCH 47/99] Update initialization of external fields --- .../HybridPICModel/HybridPICModel.cpp | 10 +- Source/Initialization/WarpXInitData.cpp | 105 +++++------------- Source/WarpX.H | 12 +- 3 files changed, 38 insertions(+), 89 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 20989cbeca9..c4d48fc0c26 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -226,9 +226,8 @@ void HybridPICModel::InitData () m_J_external[0], m_J_external[1], m_J_external[2], - lev, PatchType::fine, 'e', - warpx.m_fields.get_alldirs(FieldType::edge_lengths, lev), - warpx.m_fields.get_alldirs(FieldType::face_areas, lev)); + lev, PatchType::fine, + warpx.m_eb_update_E); } } @@ -244,9 +243,8 @@ void HybridPICModel::GetCurrentExternal () m_J_external[0], m_J_external[1], m_J_external[2], - lev, PatchType::fine, 'e', - warpx.m_fields.get_alldirs(FieldType::edge_lengths, lev), - warpx.m_fields.get_alldirs(FieldType::face_areas, lev)); + lev, PatchType::fine, + warpx.m_eb_update_E); } } diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 1b541f85d8e..57f1802e939 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -976,23 +976,21 @@ WarpX::InitLevelData (int lev, Real /*time*/) // The default maxlevel_extEMfield_init value is the total number of levels in the simulation if ((m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) && (lev > 0) && (lev <= maxlevel_extEMfield_init)) { + + // TODO: raise error when EB is activated ComputeExternalFieldOnGridUsingParser( FieldType::Bfield_aux, m_p_ext_field_params->Bxfield_parser->compile<4>(), m_p_ext_field_params->Byfield_parser->compile<4>(), m_p_ext_field_params->Bzfield_parser->compile<4>(), - lev, PatchType::fine, 'f', - m_fields.get_alldirs(FieldType::edge_lengths, lev), - m_fields.get_alldirs(FieldType::face_areas, lev)); + lev, PatchType::fine, m_eb_update_B); ComputeExternalFieldOnGridUsingParser( FieldType::Bfield_cp, m_p_ext_field_params->Bxfield_parser->compile<4>(), m_p_ext_field_params->Byfield_parser->compile<4>(), m_p_ext_field_params->Bzfield_parser->compile<4>(), - lev, PatchType::coarse, 'f', - m_fields.get_alldirs(FieldType::edge_lengths, lev), - m_fields.get_mr_levels_alldirs(FieldType::face_areas, max_level)[lev]); + lev, PatchType::coarse, m_eb_update_E); } // if the input string for the E-field is "parse_e_ext_grid_function", @@ -1023,18 +1021,14 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_p_ext_field_params->Exfield_parser->compile<4>(), m_p_ext_field_params->Eyfield_parser->compile<4>(), m_p_ext_field_params->Ezfield_parser->compile<4>(), - lev, PatchType::fine, 'e', - m_fields.get_alldirs(FieldType::edge_lengths, lev), - m_fields.get_alldirs(FieldType::face_areas, lev)); + lev, PatchType::fine, m_eb_update_E); ComputeExternalFieldOnGridUsingParser( FieldType::Efield_cp, m_p_ext_field_params->Exfield_parser->compile<4>(), m_p_ext_field_params->Eyfield_parser->compile<4>(), m_p_ext_field_params->Ezfield_parser->compile<4>(), - lev, PatchType::coarse, 'e', - m_fields.get_alldirs(FieldType::edge_lengths, lev), - m_fields.get_alldirs(FieldType::face_areas, lev)); + lev, PatchType::coarse, m_eb_update_E); #ifdef AMREX_USE_EB if (eb_enabled) { if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { @@ -1068,9 +1062,8 @@ void WarpX::ComputeExternalFieldOnGridUsingParser ( amrex::ParserExecutor<4> const& fx_parser, amrex::ParserExecutor<4> const& fy_parser, amrex::ParserExecutor<4> const& fz_parser, - int lev, PatchType patch_type, [[maybe_unused]] const char topology, - std::optional const& edge_lengths, - std::optional const& face_areas) + int lev, PatchType patch_type, + amrex::Vector,3 > > const& eb_update_field) { auto t = gett_new(lev); @@ -1093,8 +1086,6 @@ void WarpX::ComputeExternalFieldOnGridUsingParser ( const amrex::IntVect y_nodal_flag = mfy->ixType().toIntVect(); const amrex::IntVect z_nodal_flag = mfz->ixType().toIntVect(); - const bool eb_enabled = EB::enabled(); - for ( MFIter mfi(*mfx, TilingIfNotGPU()); mfi.isValid(); ++mfi) { const amrex::Box& tbx = mfi.tilebox( x_nodal_flag, mfx->nGrowVect() ); const amrex::Box& tby = mfi.tilebox( y_nodal_flag, mfy->nGrowVect() ); @@ -1104,44 +1095,19 @@ void WarpX::ComputeExternalFieldOnGridUsingParser ( auto const& mfyfab = mfy->array(mfi); auto const& mfzfab = mfz->array(mfi); - amrex::Array4 lx, ly, lz, Sx, Sy, Sz; - if (eb_enabled) { - if (edge_lengths.has_value()) { - const auto& edge_lengths_array = edge_lengths.value(); - lx = edge_lengths_array[0]->array(mfi); - ly = edge_lengths_array[1]->array(mfi); - lz = edge_lengths_array[2]->array(mfi); - } - if (face_areas.has_value()) { - const auto& face_areas_array = face_areas.value(); - Sx = face_areas_array[0]->array(mfi); - Sy = face_areas_array[1]->array(mfi); - Sz = face_areas_array[2]->array(mfi); - } - } - -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::Dim3 lx_lo, lx_hi, lz_lo, lz_hi; -#endif - if (eb_enabled) { -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - lx_lo = amrex::lbound(lx); - lx_hi = amrex::ubound(lx); - lz_lo = amrex::lbound(lz); - lz_hi = amrex::ubound(lz); -#endif + amrex::Array4 update_fx_arr, update_fy_arr, update_fz_arr; + if (EB::enabled()) { + update_fx_arr = eb_update_field[lev][0]->array(mfi); + update_fy_arr = eb_update_field[lev][1]->array(mfi); + update_fz_arr = eb_update_field[lev][2]->array(mfi); } amrex::ParallelFor (tbx, tby, tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) { -#ifdef AMREX_USE_EB -#ifdef WARPX_DIM_3D - if(lx && ((topology=='e' and lx(i, j, k)<=0) or (topology=='f' and Sx(i, j, k)<=0))) { return; } -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - //In XZ and RZ Ex is associated with a x-edge, while Bx is associated with a z-edge - if(lx && ((topology=='e' and lx(i, j, k)<=0) or (topology=='f' and lz(i, j, k)<=0))) { return; } -#endif -#endif + + // Do not set fields inside the embedded boundary + if (update_fx_arr && update_fx_arr(i,j,k) == 0) { return; } + // Shift required in the x-, y-, or z- position // depending on the index type of the multifab #if defined(WARPX_DIM_1D_Z) @@ -1167,19 +1133,10 @@ void WarpX::ComputeExternalFieldOnGridUsingParser ( mfxfab(i,j,k) = fx_parser(x,y,z,t); }, [=] AMREX_GPU_DEVICE (int i, int j, int k) { -#ifdef AMREX_USE_EB -#ifdef WARPX_DIM_3D - if(ly && ((topology=='e' and ly(i, j, k)<=0) or (topology=='f' and Sy(i, j, k)<=0))) { return; } -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - //In XZ and RZ Ey is associated with a mesh node, so we need to check if the mesh node is covered - if(lx && - ((topology=='e' and (lx(std::min(i , lx_hi.x), std::min(j , lx_hi.y), k)<=0 - || lx(std::max(i-1, lx_lo.x), std::min(j , lx_hi.y), k)<=0 - || lz(std::min(i , lz_hi.x), std::min(j , lz_hi.y), k)<=0 - || lz(std::min(i , lz_hi.x), std::max(j-1, lz_lo.y), k)<=0)) or - (topology=='f' and Sy(i,j,k)<=0))) { return; } -#endif -#endif + + // Do not set fields inside the embedded boundary + if (update_fy_arr && update_fy_arr(i,j,k) == 0) { return; } + #if defined(WARPX_DIM_1D_Z) const amrex::Real x = 0._rt; const amrex::Real y = 0._rt; @@ -1203,14 +1160,10 @@ void WarpX::ComputeExternalFieldOnGridUsingParser ( mfyfab(i,j,k) = fy_parser(x,y,z,t); }, [=] AMREX_GPU_DEVICE (int i, int j, int k) { -#ifdef AMREX_USE_EB -#ifdef WARPX_DIM_3D - if(lz && ((topology=='e' and lz(i, j, k)<=0) or (topology=='f' and Sz(i, j, k)<=0))) { return; } -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - //In XZ and RZ Ez is associated with a z-edge, while Bz is associated with a x-edge - if(lz && ((topology=='e' and lz(i, j, k)<=0) or (topology=='f' and lx(i, j, k)<=0))) { return; } -#endif -#endif + + // Do not set fields inside the embedded boundary + if (update_fz_arr && update_fz_arr(i,j,k) == 0) { return; } + #if defined(WARPX_DIM_1D_Z) const amrex::Real x = 0._rt; const amrex::Real y = 0._rt; @@ -1407,9 +1360,7 @@ WarpX::LoadExternalFields (int const lev) m_p_ext_field_params->Bxfield_parser->compile<4>(), m_p_ext_field_params->Byfield_parser->compile<4>(), m_p_ext_field_params->Bzfield_parser->compile<4>(), - lev, PatchType::fine, 'f', - m_fields.get_alldirs(FieldType::edge_lengths, lev), - m_fields.get_alldirs(FieldType::face_areas, lev)); + lev, PatchType::fine, m_eb_update_B); } else if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { #if defined(WARPX_DIM_RZ) @@ -1432,9 +1383,7 @@ WarpX::LoadExternalFields (int const lev) m_p_ext_field_params->Exfield_parser->compile<4>(), m_p_ext_field_params->Eyfield_parser->compile<4>(), m_p_ext_field_params->Ezfield_parser->compile<4>(), - lev, PatchType::fine, 'e', - m_fields.get_alldirs(FieldType::edge_lengths, lev), - m_fields.get_alldirs(FieldType::face_areas, lev)); + lev, PatchType::fine, m_eb_update_E ); } else if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { #if defined(WARPX_DIM_RZ) diff --git a/Source/WarpX.H b/Source/WarpX.H index 5bd823ad8ba..8fb1eaf4408 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -937,9 +937,8 @@ public: amrex::ParserExecutor<4> const& fx_parser, amrex::ParserExecutor<4> const& fy_parser, amrex::ParserExecutor<4> const& fz_parser, - int lev, PatchType patch_type, [[maybe_unused]] char topology, - std::optional const& edge_lengths = std::nullopt, - std::optional const& face_areas = std::nullopt); + int lev, PatchType patch_type, + amrex::Vector,3 > > const& eb_update_field); /** * \brief Load field values from a user-specified openPMD file, @@ -1416,8 +1415,11 @@ private: /** EB: Flag to indicate whether a gridpoint is inside the embedded boundary and therefore * whether the E or B should not be updated. (One array per level and per direction, due to staggering) */ - amrex::Vector,3 > > m_eb_update_E; - amrex::Vector,3 > > m_eb_update_B; + // TODO: leave as private, add setters, getters + public: + amrex::Vector,3 > > m_eb_update_E; + amrex::Vector,3 > > m_eb_update_B; + private: /** EB: for every mesh face flag_info_face contains a: * * 0 if the face needs to be extended From df9ae82e0e0ce90360ec3c2e0aa6246e64eab3a9 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Sat, 11 Jan 2025 11:12:01 -0800 Subject: [PATCH 48/99] Fix setting of eb_update --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index d2a2c991165..c6cb4f7bceb 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -353,12 +353,12 @@ WarpX::MarkUpdateCells ( #endif // Loop over neighboring cells int eb_update = 1; - for (int ii = i_start; ii <= 1; ++ii) { - for (int jj = j_start; jj <= 1; ++jj) { - for (int kk = k_start; kk <= 1; ++kk) { + for (int i_shift = i_start; i_shift <= 0; ++i_shift) { + for (int j_shift = j_start; j_shift <= 0; ++j_shift) { + for (int k_shift = k_start; k_shift <= 0; ++k_shift) { // If one of the neighboring is either partially or fully covered // (i.e. if they are not regular cells), do not update field - if ( !flag(i+ii, j+jj, k+kk).isRegular() ) { + if ( !flag(i+i_shift, j+j_shift, k+k_shift).isRegular() ) { eb_update = 0; } } From 59579b4da85f58e7196fe61eb28135c628665722 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Sat, 11 Jan 2025 12:26:28 -0800 Subject: [PATCH 49/99] Implement ECT condition for B --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 71 +++++++++++++++++++++++-- Source/Initialization/WarpXInitData.cpp | 27 ++++++---- Source/WarpX.H | 11 ++++ 3 files changed, 95 insertions(+), 14 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index c6cb4f7bceb..8fe0a9d2fa8 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -306,10 +306,13 @@ WarpX::MarkUpdateCells ( for (int idim = 0; idim < 3; ++idim) { +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif for (amrex::MFIter mfi(*field[idim]); mfi.isValid(); ++mfi) { const amrex::Box& box = mfi.tilebox(); - auto const & eb_update_arr = eb_update[idim]->array(mfi); + amrex::Array4 const & eb_update_arr = eb_update[idim]->array(mfi); // Check if the box (including one layer of guard cells) contains a mix of covered and regular cells const amrex::Box& eb_info_box = mfi.tilebox(amrex::IntVect::TheCellVector()).grow(1); @@ -367,8 +370,6 @@ WarpX::MarkUpdateCells ( eb_update_arr(i, j, k) = eb_update; }); - // TODO: Handle the case of the ECT solver - } } @@ -377,6 +378,70 @@ WarpX::MarkUpdateCells ( } +void +WarpX::MarkECTUpdateECells ( + std::array< std::unique_ptr,3> & eb_update_E, + ablastr::fields::VectorField const& edge_lengths ) +{ +} + +void +WarpX::MarkECTUpdateBCells ( + std::array< std::unique_ptr,3> & eb_update_B, + ablastr::fields::VectorField const& face_areas, + ablastr::fields::VectorField const& edge_lengths ) +{ + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( amrex::MFIter mfi(*eb_update_B[0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { + + const amrex::Box& tbx = mfi.tilebox( eb_update_B[0]->ixType().toIntVect(), eb_update_B[0]->nGrowVect() ); + const amrex::Box& tby = mfi.tilebox( eb_update_B[1]->ixType().toIntVect(), eb_update_B[1]->nGrowVect() ); + const amrex::Box& tbz = mfi.tilebox( eb_update_B[2]->ixType().toIntVect(), eb_update_B[2]->nGrowVect() ); + + amrex::Array4 const & eb_update_Bx_arr = eb_update_B[0]->array(mfi); + amrex::Array4 const & eb_update_By_arr = eb_update_B[1]->array(mfi); + amrex::Array4 const & eb_update_Bz_arr = eb_update_B[2]->array(mfi); + +#ifdef WARPX_DIM_3D + amrex::Array4 const & Sx_arr = face_areas[0]->array(mfi); + amrex::Array4 const & Sy_arr = face_areas[1]->array(mfi); + amrex::Array4 const & Sz_arr = face_areas[2]->array(mfi); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Array4 const & Sy_arr = face_areas[1]->array(mfi); + amrex::Array4 const & lx_arr = edge_lengths[0]->array(mfi); + amrex::Array4 const & lz_arr = edge_lengths[2]->array(mfi); +#endif + amrex::ParallelFor (tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { +#ifdef WARPX_DIM_3D + // In 3D: do not update Bx if the face on which it lives is fully covered + eb_update_Bx_arr(i, j, k) = (Sx_arr(i, j, k) == 0)? 0 : 1; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + //In XZ and RZ, Bx lives on a z-edge ; do not update if fully covered + eb_update_Bx_arr(i, j, k) = (lz_arr(i, j, k) == 0)? 0 : 1; +#endif + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // Do not update By if the face on which it lives is fully covered + eb_update_By_arr(i, j, k) = (Sy_arr(i, j, k) == 0)? 0 : 1; + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { +#ifdef WARPX_DIM_3D + // In 3D: do not update Bz if the face on which it lives is fully covered + eb_update_Bz_arr(i, j, k) = (Sz_arr(i, j, k) == 0)? 0 : 1; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + //In XZ and RZ, Bz lives on a x-edge ; do not update if fully covered + eb_update_Bz_arr(i, j, k) = (lx_arr(i, j, k) == 0)? 0 : 1; +#endif + } + ); + + } +} + void WarpX::MarkExtensionCells () { diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 57f1802e939..64d675878c7 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1253,17 +1253,6 @@ void WarpX::InitializeEBGridData (int lev) auto const eb_fact = fieldEBFactory(lev); - // Mark on which grid points E should be updated - MarkUpdateCells( - m_eb_update_E[lev], - m_fields.get_alldirs(FieldType::Efield_fp, lev), - eb_fact ); - // Mark on which grid points B should be updated - MarkUpdateCells( - m_eb_update_B[lev], - m_fields.get_alldirs(FieldType::Bfield_fp, lev), - eb_fact ); - // TODO: move inside if condition for ECT auto edge_lengths_lev = m_fields.get_alldirs(FieldType::edge_lengths, lev); ComputeEdgeLengths(edge_lengths_lev, eb_fact); @@ -1274,8 +1263,24 @@ void WarpX::InitializeEBGridData (int lev) ScaleAreas(face_areas_lev, CellSize(lev)); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { + // Mark on which grid points E should be updated + MarkECTUpdateECells( m_eb_update_E[lev], edge_lengths_lev ); + // Mark on which grid points B should be updated + MarkECTUpdateBCells( m_eb_update_B[lev], face_areas_lev, edge_lengths_lev); + // Compute additional quantities required for the ECT solver MarkExtensionCells(); ComputeFaceExtensions(); + } else { + // Mark on which grid points E should be updated (stair-case approximation) + MarkUpdateCells( + m_eb_update_E[lev], + m_fields.get_alldirs(FieldType::Efield_fp, lev), + eb_fact ); + // Mark on which grid points B should be updated (stair-case approximation) + MarkUpdateCells( + m_eb_update_B[lev], + m_fields.get_alldirs(FieldType::Bfield_fp, lev), + eb_fact ); } } diff --git a/Source/WarpX.H b/Source/WarpX.H index 8fb1eaf4408..0dce1b75905 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1032,6 +1032,17 @@ public: ablastr::fields::VectorField const & field, amrex::EBFArrayBoxFactory const & eb_fact ); + // TODO documentation + void MarkECTUpdateECells ( + std::array< std::unique_ptr,3> & eb_update_B, + ablastr::fields::VectorField const& edge_lengths ); + + // TODO documentation + void MarkECTUpdateBCells ( + std::array< std::unique_ptr,3> & eb_update_B, + ablastr::fields::VectorField const& face_areas, + ablastr::fields::VectorField const& edge_lengths ); + /** * \brief Compute the length of the mesh edges. Here the length is a value in [0, 1]. * An edge of length 0 is fully covered. From 4fd4dc7a0eddcd7f5811d539c2ad9475e93c2f81 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Sat, 11 Jan 2025 12:36:32 -0800 Subject: [PATCH 50/99] Do not use any guard cells --- Source/WarpX.cpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index cf709ca304c..1df4bcde2e2 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2304,19 +2304,19 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (lev == maxLevel()) { if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - AllocInitMultiFab(m_eb_update_E[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_eb_update_E[x]"); - AllocInitMultiFab(m_eb_update_E[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_eb_update_E[y]"); - AllocInitMultiFab(m_eb_update_E[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_eb_update_E[z]"); - - AllocInitMultiFab(m_eb_update_B[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_eb_update_B[x]"); - AllocInitMultiFab(m_eb_update_B[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_eb_update_B[y]"); - AllocInitMultiFab(m_eb_update_B[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_eb_update_B[z]"); + AllocInitMultiFab(m_eb_update_E[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, 0, + amrex::IntVect::TheZeroVector(), lev, "m_eb_update_E[x]"); + AllocInitMultiFab(m_eb_update_E[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, 0, + amrex::IntVect::TheZeroVector(), lev, "m_eb_update_E[y]"); + AllocInitMultiFab(m_eb_update_E[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, 0, + amrex::IntVect::TheZeroVector(), lev, "m_eb_update_E[z]"); + + AllocInitMultiFab(m_eb_update_B[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, 0, + amrex::IntVect::TheZeroVector(), lev, "m_eb_update_B[x]"); + AllocInitMultiFab(m_eb_update_B[lev][1], amrex::convert(ba, By_nodal_flag), dm, 0, + amrex::IntVect::TheZeroVector(), lev, "m_eb_update_B[y]"); + AllocInitMultiFab(m_eb_update_B[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, 0, + amrex::IntVect::TheZeroVector(), lev, "m_eb_update_B[z]"); // TODO: do not allocate these arrays anymore with the Yee solver //! EB: Lengths of the mesh edges From 4232db7040633d3a3bd3d671b2845c1bfc48fa82 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Sat, 11 Jan 2025 15:53:11 -0800 Subject: [PATCH 51/99] Fixed ECT solver --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 8fe0a9d2fa8..559da0472a2 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -383,6 +383,60 @@ WarpX::MarkECTUpdateECells ( std::array< std::unique_ptr,3> & eb_update_E, ablastr::fields::VectorField const& edge_lengths ) { + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( amrex::MFIter mfi(*eb_update_E[0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { + + const amrex::Box& tbx = mfi.tilebox( eb_update_E[0]->ixType().toIntVect(), eb_update_E[0]->nGrowVect() ); + const amrex::Box& tby = mfi.tilebox( eb_update_E[1]->ixType().toIntVect(), eb_update_E[1]->nGrowVect() ); + const amrex::Box& tbz = mfi.tilebox( eb_update_E[2]->ixType().toIntVect(), eb_update_E[2]->nGrowVect() ); + + amrex::Array4 const & eb_update_Ex_arr = eb_update_E[0]->array(mfi); + amrex::Array4 const & eb_update_Ey_arr = eb_update_E[1]->array(mfi); + amrex::Array4 const & eb_update_Ez_arr = eb_update_E[2]->array(mfi); + + amrex::Array4 const & lx_arr = edge_lengths[0]->array(mfi); + amrex::Array4 const & lz_arr = edge_lengths[2]->array(mfi); +#if defined(WARPX_DIM_3D) + amrex::Array4 const & ly_arr = edge_lengths[1]->array(mfi); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::Dim3 lx_lo = amrex::lbound(lx_arr); + amrex::Dim3 lx_hi = amrex::ubound(lx_arr); + amrex::Dim3 lz_lo = amrex::lbound(lz_arr); + amrex::Dim3 lz_hi = amrex::ubound(lz_arr); +#endif + + amrex::ParallelFor (tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // Do not update Ex if the edge on which it lives is fully covered + eb_update_Ex_arr(i, j, k) = (lx_arr(i, j, k) == 0)? 0 : 1; + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { +#ifdef WARPX_DIM_3D + // In 3D: Do not update Ey if the edge on which it lives is fully covered + eb_update_Ey_arr(i, j, k) = (ly_arr(i, j, k) == 0)? 0 : 1; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + // In XZ and RZ: Ey is associated with a mesh node, + // so we need to check if the mesh node is covered + if((lx_arr(std::min(i , lx_hi.x), std::min(j , lx_hi.y), k)==0) + ||(lx_arr(std::max(i-1, lx_lo.x), std::min(j , lx_hi.y), k)==0) + ||(lz_arr(std::min(i , lz_hi.x), std::min(j , lz_hi.y), k)==0) + ||(lz_arr(std::min(i , lz_hi.x), std::max(j-1, lz_lo.y), k)==0)) { + eb_update_Ey_arr(i, j, k) = 0; + } else { + eb_update_Ey_arr(i, j, k) = 1; + } +#endif + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // Do not update Ez if the edge on which it lives is fully covered + eb_update_Ez_arr(i, j, k) = (lz_arr(i, j, k) == 0)? 0 : 1; + } + ); + + } } void From f45b257f2c6f5bf8d6ca9c15c1bcd971cceb4563 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Sat, 11 Jan 2025 16:00:04 -0800 Subject: [PATCH 52/99] Revert "Do not use any guard cells" This reverts commit 4fd4dc7a0eddcd7f5811d539c2ad9475e93c2f81. --- Source/WarpX.cpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 1df4bcde2e2..cf709ca304c 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2304,19 +2304,19 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (lev == maxLevel()) { if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - AllocInitMultiFab(m_eb_update_E[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, 0, - amrex::IntVect::TheZeroVector(), lev, "m_eb_update_E[x]"); - AllocInitMultiFab(m_eb_update_E[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, 0, - amrex::IntVect::TheZeroVector(), lev, "m_eb_update_E[y]"); - AllocInitMultiFab(m_eb_update_E[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, 0, - amrex::IntVect::TheZeroVector(), lev, "m_eb_update_E[z]"); - - AllocInitMultiFab(m_eb_update_B[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, 0, - amrex::IntVect::TheZeroVector(), lev, "m_eb_update_B[x]"); - AllocInitMultiFab(m_eb_update_B[lev][1], amrex::convert(ba, By_nodal_flag), dm, 0, - amrex::IntVect::TheZeroVector(), lev, "m_eb_update_B[y]"); - AllocInitMultiFab(m_eb_update_B[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, 0, - amrex::IntVect::TheZeroVector(), lev, "m_eb_update_B[z]"); + AllocInitMultiFab(m_eb_update_E[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_E[x]"); + AllocInitMultiFab(m_eb_update_E[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_E[y]"); + AllocInitMultiFab(m_eb_update_E[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_E[z]"); + + AllocInitMultiFab(m_eb_update_B[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_B[x]"); + AllocInitMultiFab(m_eb_update_B[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_B[y]"); + AllocInitMultiFab(m_eb_update_B[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_update_B[z]"); // TODO: do not allocate these arrays anymore with the Yee solver //! EB: Lengths of the mesh edges From 2003d8593385b504f76ac5c6e619ccef9b4b5edf Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Sat, 11 Jan 2025 18:49:37 -0800 Subject: [PATCH 53/99] Fix compilation errors --- .../embedded_boundary_cube/inputs_base_3d | 4 +- .../inputs_test_2d_embedded_boundary_cube | 8 ++-- Source/EmbeddedBoundary/WarpXInitEB.cpp | 1 + .../FiniteDifferenceSolver.H | 4 +- .../MacroscopicEvolveE.cpp | 44 +++++++++---------- Source/FieldSolver/WarpXPushFieldsEM.cpp | 2 +- 6 files changed, 30 insertions(+), 33 deletions(-) diff --git a/Examples/Tests/embedded_boundary_cube/inputs_base_3d b/Examples/Tests/embedded_boundary_cube/inputs_base_3d index 9710701d871..70ddd8f8f64 100644 --- a/Examples/Tests/embedded_boundary_cube/inputs_base_3d +++ b/Examples/Tests/embedded_boundary_cube/inputs_base_3d @@ -13,8 +13,8 @@ boundary.field_lo = pec pec pec boundary.field_hi = pec pec pec eb2.geom_type = box -eb2.box_lo = -0.5 -0.5 -0.5 -eb2.box_hi = 0.5 0.5 0.5 +eb2.box_lo = -0.501 -0.501 -0.501 # Ensures that the stair-case EB is exactly at -0.5 +eb2.box_hi = 0.501 0.501 0.501 # Ensures that the stair-case EB is exactly at 0.5 eb2.box_has_fluid_inside = true # Alternatively one could use parser to build EB # Note that for amrex EB implicit function, >0 is covered, =0 is boundary and <0 is regular. diff --git a/Examples/Tests/embedded_boundary_cube/inputs_test_2d_embedded_boundary_cube b/Examples/Tests/embedded_boundary_cube/inputs_test_2d_embedded_boundary_cube index 4fbf13e7f5c..46272052c2c 100644 --- a/Examples/Tests/embedded_boundary_cube/inputs_test_2d_embedded_boundary_cube +++ b/Examples/Tests/embedded_boundary_cube/inputs_test_2d_embedded_boundary_cube @@ -12,10 +12,10 @@ warpx.abort_on_warning_threshold = medium boundary.field_lo = pec pec boundary.field_hi = pec pec -my_constants.xmin = -0.501 -my_constants.zmin = -0.501 -my_constants.xmax = 0.501 -my_constants.zmax = 0.501 +my_constants.xmin = -0.501 # Ensures that the stair-case EB is exactly at -0.5 +my_constants.zmin = -0.501 # Ensures that the stair-case EB is exactly at -0.5 +my_constants.xmax = 0.501 # Ensures that the stair-case EB is exactly at 0.5 +my_constants.zmax = 0.501 # Ensures that the stair-case EB is exactly at 0.5 # Note that for amrex EB implicit function, >0 is covered, =0 is boundary and <0 is regular. warpx.eb_implicit_function = "max(max(x+xmin,-(x+xmax)), max(z+zmin,-(z+zmax)))" diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 559da0472a2..fca6cc3114b 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -463,6 +463,7 @@ WarpX::MarkECTUpdateBCells ( amrex::Array4 const & Sx_arr = face_areas[0]->array(mfi); amrex::Array4 const & Sy_arr = face_areas[1]->array(mfi); amrex::Array4 const & Sz_arr = face_areas[2]->array(mfi); + amrex::ignore_unused(edge_lengths); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::Array4 const & Sy_arr = face_areas[1]->array(mfi); amrex::Array4 const & lx_arr = edge_lengths[0]->array(mfi); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 07596dc8562..7726a2ed5bd 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -111,7 +111,7 @@ class FiniteDifferenceSolver ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, - ablastr::fields::VectorField const& edge_lengths, + std::array< std::unique_ptr,3 > const& eb_update_E, amrex::Real dt, std::unique_ptr const& macroscopic_properties); @@ -313,7 +313,7 @@ class FiniteDifferenceSolver ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, - ablastr::fields::VectorField const& edge_lengths, + std::array< std::unique_ptr,3 > const& eb_update_E, amrex::Real dt, std::unique_ptr const& macroscopic_properties); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp index 708728c4e5b..20dab1b2d12 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp @@ -40,7 +40,7 @@ void FiniteDifferenceSolver::MacroscopicEvolveE ( ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, - VectorField const& edge_lengths, + std::array< std::unique_ptr,3 > const& eb_update_E, amrex::Real const dt, std::unique_ptr const& macroscopic_properties) { @@ -48,7 +48,7 @@ void FiniteDifferenceSolver::MacroscopicEvolveE ( // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) #ifdef WARPX_DIM_RZ - amrex::ignore_unused(Efield, Bfield, Jfield, edge_lengths, dt, macroscopic_properties); + amrex::ignore_unused(Efield, Bfield, Jfield, eb_update_E, dt, macroscopic_properties); WARPX_ABORT_WITH_MESSAGE("currently macro E-push does not work for RZ"); #else @@ -61,13 +61,13 @@ void FiniteDifferenceSolver::MacroscopicEvolveE ( if (WarpX::macroscopic_solver_algo == MacroscopicSolverAlgo::LaxWendroff) { MacroscopicEvolveECartesian - ( Efield, Bfield, Jfield, edge_lengths, dt, macroscopic_properties); + ( Efield, Bfield, Jfield, eb_update_E, dt, macroscopic_properties); } if (WarpX::macroscopic_solver_algo == MacroscopicSolverAlgo::BackwardEuler) { MacroscopicEvolveECartesian - ( Efield, Bfield, Jfield, edge_lengths, dt, macroscopic_properties); + ( Efield, Bfield, Jfield, eb_update_E, dt, macroscopic_properties); } @@ -78,12 +78,12 @@ void FiniteDifferenceSolver::MacroscopicEvolveE ( if (WarpX::macroscopic_solver_algo == MacroscopicSolverAlgo::LaxWendroff) { MacroscopicEvolveECartesian - ( Efield, Bfield, Jfield, edge_lengths, dt, macroscopic_properties); + ( Efield, Bfield, Jfield, eb_update_E, dt, macroscopic_properties); } else if (WarpX::macroscopic_solver_algo == MacroscopicSolverAlgo::BackwardEuler) { MacroscopicEvolveECartesian - ( Efield, Bfield, Jfield, edge_lengths, dt, macroscopic_properties); + ( Efield, Bfield, Jfield, eb_update_E, dt, macroscopic_properties); } @@ -103,7 +103,7 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( ablastr::fields::VectorField const& Efield, ablastr::fields::VectorField const& Bfield, ablastr::fields::VectorField const& Jfield, - ablastr::fields::VectorField const& edge_lengths, + std::array< std::unique_ptr,3 > const& eb_update_E, amrex::Real const dt, std::unique_ptr const& macroscopic_properties) { @@ -141,15 +141,12 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( Array4 const& jy = Jfield[1]->array(mfi); Array4 const& jz = Jfield[2]->array(mfi); - amrex::Array4 eb_lx, eb_ly, eb_lz; + amrex::Array4 update_Ex_arr, update_Ey_arr, update_Ez_arr; if (EB::enabled()) { - eb_lx = edge_lengths[0]->array(mfi); - eb_ly = edge_lengths[1]->array(mfi); - eb_lz = edge_lengths[2]->array(mfi); + update_Ex_arr = eb_update_E[0]->array(mfi); + update_Ey_arr = eb_update_E[1]->array(mfi); + update_Ez_arr = eb_update_E[2]->array(mfi); } -#ifdef WARPX_DIM_XZ - amrex::ignore_unused(eb_ly); -#endif // material prop // amrex::Array4 const& sigma_arr = sigma_mf.array(mfi); @@ -180,8 +177,9 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( // Loop over the cells and update the fields amrex::ParallelFor(tex, tey, tez, [=] AMREX_GPU_DEVICE (int i, int j, int k){ - // Skip field push if this cell is fully covered by embedded boundaries - if (eb_lx && eb_lx(i, j, k) <= 0) { return; } + + // Skip field push in the embedded boundaries + if (update_Ex_arr && update_Ex_arr(i, j, k) == 0) { return; } // Interpolate conductivity, sigma, to Ex position on the grid amrex::Real const sigma_interp = ablastr::coarsen::sample::Interp(sigma_arr, sigma_stag, @@ -198,12 +196,9 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef WARPX_DIM_3D - if (eb_ly && eb_ly(i,j,k) <= 0) { return; } -#elif defined(WARPX_DIM_XZ) - //In XZ Ey is associated with a mesh node, so we need to check if the mesh node is covered - if (eb_lx && (eb_lx(i, j, k)<=0 || eb_lx(i-1, j, k)<=0 || eb_lz(i, j, k)<=0 || eb_lz(i, j-1, k)<=0)) { return; } -#endif + + // Skip field push in the embedded boundaries + if (update_Ey_arr && update_Ey_arr(i, j, k) == 0) { return; } // Interpolate conductivity, sigma, to Ey position on the grid amrex::Real const sigma_interp = ablastr::coarsen::sample::Interp(sigma_arr, sigma_stag, @@ -221,8 +216,9 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k){ - // Skip field push if this cell is fully covered by embedded boundaries - if (eb_lz && eb_lz(i, j, k) <= 0) { return; } + + // Skip field push in the embedded boundaries + if (update_Ez_arr && update_Ez_arr(i, j, k) == 0) { return; } // Interpolate conductivity, sigma, to Ez position on the grid amrex::Real const sigma_interp = ablastr::coarsen::sample::Interp(sigma_arr, sigma_stag, diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index a77fb390fff..139988b69a2 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -1157,7 +1157,7 @@ WarpX::MacroscopicEvolveE (int lev, PatchType patch_type, amrex::Real a_dt, amre m_fields.get_alldirs(FieldType::Efield_fp, lev), m_fields.get_alldirs(FieldType::Bfield_fp, lev), m_fields.get_alldirs(FieldType::current_fp, lev), - m_fields.get_alldirs(FieldType::edge_lengths, lev), + m_eb_update_E[lev], a_dt, m_macroscopic_properties); if (do_pml && pml[lev]->ok()) { From d6e64af21c125a31c8b65e850a95c6ea71cbc742 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Sat, 11 Jan 2025 20:34:59 -0800 Subject: [PATCH 54/99] Fix compilation error --- .../FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp index 20dab1b2d12..33d368925f7 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp @@ -107,10 +107,6 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( amrex::Real const dt, std::unique_ptr const& macroscopic_properties) { -#ifndef AMREX_USE_EB - amrex::ignore_unused(edge_lengths); -#endif - amrex::MultiFab& sigma_mf = macroscopic_properties->getsigma_mf(); amrex::MultiFab& epsilon_mf = macroscopic_properties->getepsilon_mf(); amrex::MultiFab& mu_mf = macroscopic_properties->getmu_mf(); From e6d93f9f9be813e2a743819145a1f0c195a1aebd Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Sun, 12 Jan 2025 07:10:05 -0800 Subject: [PATCH 55/99] Fix const-ness --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index fca6cc3114b..3cc29d17b39 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -343,16 +343,17 @@ WarpX::MarkUpdateCells ( // or fully covered: do not update field // Check neighbors in the each spatial direction - int i_start = 0; - int j_start = 0; - int k_start = 0; // If nodal in a given direction, we need to start from -1 (left-neighboring cell) - if ( index_type.nodeCentered(0) ) { i_start = -1; }; + int const i_start = ( index_type.nodeCentered(0) )? -1 : 0; #if AMREX_SPACEDIM > 1 - if ( index_type.nodeCentered(1) ) { j_start = -1; }; + int const j_start = ( index_type.nodeCentered(1) )? -1 : 0; +#else + int const j_start = 0; #endif #if AMREX_SPACEDIM > 2 - if ( index_type.nodeCentered(2) ) { k_start = -1; }; + int const k_start = ( index_type.nodeCentered(2) )? -1 : 0; +#else + int const k_start = 0; #endif // Loop over neighboring cells int eb_update = 1; @@ -402,10 +403,10 @@ WarpX::MarkECTUpdateECells ( #if defined(WARPX_DIM_3D) amrex::Array4 const & ly_arr = edge_lengths[1]->array(mfi); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::Dim3 lx_lo = amrex::lbound(lx_arr); - amrex::Dim3 lx_hi = amrex::ubound(lx_arr); - amrex::Dim3 lz_lo = amrex::lbound(lz_arr); - amrex::Dim3 lz_hi = amrex::ubound(lz_arr); + amrex::Dim3 const lx_lo = amrex::lbound(lx_arr); + amrex::Dim3 const lx_hi = amrex::ubound(lx_arr); + amrex::Dim3 const lz_lo = amrex::lbound(lz_arr); + amrex::Dim3 const lz_hi = amrex::ubound(lz_arr); #endif amrex::ParallelFor (tbx, tby, tbz, From eb4308775f1e8dd66ea5d9e2db993dcd21aa0ef8 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Sun, 12 Jan 2025 08:28:15 -0800 Subject: [PATCH 56/99] Activate load-balancing --- Source/Parallelization/WarpXRegrid.cpp | 10 ++++++++++ Source/WarpX.H | 4 ++-- Source/WarpX.cpp | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index a0a2d4929df..c12b9f7e0f7 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -174,6 +174,14 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi using ablastr::fields::Direction; using warpx::fields::FieldType; + const auto RemakeMultiFab = [&](auto& mf){ + if (mf == nullptr) { return; } + const IntVect& ng = mf->nGrowVect(); + auto pmf = std::remove_reference_t{}; + AllocInitMultiFab(pmf, mf->boxArray(), dm, mf->nComp(), ng, lev, mf->tags()[0]); + mf = std::move(pmf); + }; + bool const eb_enabled = EB::enabled(); if (ba == boxArray(lev)) { @@ -187,6 +195,8 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi { if (eb_enabled) { if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { + RemakeMultiFab( m_eb_update_E[lev][idim] ); + RemakeMultiFab( m_eb_update_B[lev][idim] ); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { m_borrowing[lev][idim] = std::make_unique>(amrex::convert(ba, Bfield_fp[lev][idim]->ixType().toIntVect()), dm); } diff --git a/Source/WarpX.H b/Source/WarpX.H index 0dce1b75905..5e4650e3745 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1015,7 +1015,7 @@ public: /** Set a flag to indicate on which grid points the field `field` * should be updated. * - * More specifically, this function fill the iMultiFabs in `eb_update` + * More specifically, this function fills the iMultiFabs in `eb_update` * (which have the same indexType as the MultiFabs in `field`) with 1 * or 0, depending on whether the grid point is in the valid region * or in the embedded boundary. @@ -1034,7 +1034,7 @@ public: // TODO documentation void MarkECTUpdateECells ( - std::array< std::unique_ptr,3> & eb_update_B, + std::array< std::unique_ptr,3> & eb_update_E, ablastr::fields::VectorField const& edge_lengths ); // TODO documentation diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index cf709ca304c..265603eae24 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2304,14 +2304,14 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (lev == maxLevel()) { if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - AllocInitMultiFab(m_eb_update_E[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, + AllocInitMultiFab(m_eb_update_E[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_eb_update_E[x]"); AllocInitMultiFab(m_eb_update_E[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_eb_update_E[y]"); AllocInitMultiFab(m_eb_update_E[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_eb_update_E[z]"); - AllocInitMultiFab(m_eb_update_B[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, + AllocInitMultiFab(m_eb_update_B[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_eb_update_B[x]"); AllocInitMultiFab(m_eb_update_B[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_eb_update_B[y]"); From 2eb3383a1d26dbb6dd7a2a756e4227dd3e5248fa Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Sun, 12 Jan 2025 11:10:17 -0800 Subject: [PATCH 57/99] Update checksums --- .../test_2d_embedded_boundary_cube.json | 4 ++-- .../benchmarks_json/test_2d_field_probe.json | 8 ++++---- .../benchmarks_json/test_3d_eb_picmi.json | 14 +++++++------- .../test_3d_embedded_boundary_cube.json | 10 +++++----- ...est_3d_embedded_boundary_cube_macroscopic.json | 10 +++++----- .../test_3d_embedded_boundary_rotated_cube.json | 15 +++++++-------- .../test_3d_particle_absorption.json | 14 +++++++------- .../benchmarks_json/test_3d_particle_scrape.json | 14 +++++++------- .../test_3d_particle_scrape_picmi.json | 14 +++++++------- .../test_rz_embedded_boundary_diffraction.json | 12 ++++++------ 10 files changed, 57 insertions(+), 58 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_cube.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_cube.json index a3e609bd9a9..dbb5ffa39ae 100644 --- a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_cube.json +++ b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_cube.json @@ -3,8 +3,8 @@ "Bx": 9.263694545408503e-05, "By": 0.00031905198933489145, "Bz": 7.328424783762594e-05, - "Ex": 8553.906698053046, + "Ex": 8553.90669811286, "Ey": 60867.04830538045, - "Ez": 8.439422682267567e-07 + "Ez": 4.223902107031194e-06 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_2d_field_probe.json b/Regression/Checksum/benchmarks_json/test_2d_field_probe.json index cb82acfc067..8aabe6c8301 100644 --- a/Regression/Checksum/benchmarks_json/test_2d_field_probe.json +++ b/Regression/Checksum/benchmarks_json/test_2d_field_probe.json @@ -1,10 +1,10 @@ { "lev=0": { "Bx": 0.0, - "By": 126826.78487921853, + "By": 123510.69657444415, "Bz": 0.0, - "Ex": 32517064310550.266, + "Ex": 31206368949280.34, "Ey": 0.0, - "Ez": 17321323003697.61 + "Ez": 16921005306450.537 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_eb_picmi.json b/Regression/Checksum/benchmarks_json/test_3d_eb_picmi.json index ad0d2cee5a3..1f9f0a77b5a 100644 --- a/Regression/Checksum/benchmarks_json/test_3d_eb_picmi.json +++ b/Regression/Checksum/benchmarks_json/test_3d_eb_picmi.json @@ -1,10 +1,10 @@ { "lev=0": { - "Bx": 148673.005859208, - "By": 148673.00585920806, - "Bz": 3371.758117878558, - "Ex": 55378581103426.71, - "Ey": 55378581103426.72, - "Ez": 68412803445328.25 + "Bx": 144495.08082507108, + "By": 144495.08082507114, + "Bz": 8481.958724628861, + "Ex": 54500496182517.92, + "Ey": 54500496182517.91, + "Ez": 70231240245509.39 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_cube.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_cube.json index 58ee8806540..9563c52adbe 100644 --- a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_cube.json +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_cube.json @@ -1,10 +1,10 @@ { "lev=0": { - "Bx": 4.060477854092961e-18, + "Bx": 4.166971025838921e-18, "By": 0.006628374119786834, "Bz": 0.006628374119786834, - "Ex": 5102618.4711524295, - "Ey": 6.323754160591239e-05, - "Ez": 6.323754160591239e-05 + "Ex": 5102618.471153786, + "Ey": 1.4283859321773714e-05, + "Ez": 1.4283859321773714e-05 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_cube_macroscopic.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_cube_macroscopic.json index 8cc6af7cb93..67bdbea18ca 100644 --- a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_cube_macroscopic.json +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_cube_macroscopic.json @@ -1,10 +1,10 @@ { "lev=0": { - "Bx": 4.20930075273562e-18, + "Bx": 4.228863291892693e-18, "By": 0.005101824310293573, "Bz": 0.005101824310293573, - "Ex": 4414725.184731115, - "Ey": 6.32375413967707e-05, - "Ez": 6.32375413967707e-05 + "Ex": 4414725.184732471, + "Ey": 1.4283895626502055e-05, + "Ez": 1.4283895626502055e-05 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_rotated_cube.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_rotated_cube.json index b2b4aa569c1..d45887ca932 100644 --- a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_rotated_cube.json +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_rotated_cube.json @@ -1,11 +1,10 @@ { "lev=0": { - "Bx": 1.280747509243305e-05, - "By": 2.473900144296397e-02, - "Bz": 2.473890786894079e-02, - "Ex": 1.025322901921306e+07, - "Ey": 1.042254197269831e+04, - "Ez": 1.040011664019071e+04 + "Bx": 1.252616939910365e-05, + "By": 0.02473895628331097, + "Bz": 0.024738956316621142, + "Ex": 10253221.850298548, + "Ey": 10387.334582977643, + "Ez": 10387.532806510022 } -} - +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_particle_absorption.json b/Regression/Checksum/benchmarks_json/test_3d_particle_absorption.json index ce6e2fcf79b..3dc9d956b79 100644 --- a/Regression/Checksum/benchmarks_json/test_3d_particle_absorption.json +++ b/Regression/Checksum/benchmarks_json/test_3d_particle_absorption.json @@ -1,10 +1,10 @@ { "lev=0": { - "Bx": 202106.71291347666, - "By": 202106.71291347663, - "Bz": 3371.897999274175, - "Ex": 38304043178806.11, - "Ey": 38304043178806.11, - "Ez": 83057027925874.84 + "Bx": 198610.0530604908, + "By": 198610.0530604909, + "Bz": 8482.656173586969, + "Ex": 37232105734622.53, + "Ey": 37232105734622.54, + "Ez": 85094015810307.19 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_particle_scrape.json b/Regression/Checksum/benchmarks_json/test_3d_particle_scrape.json index b03a954397a..9437ebed275 100644 --- a/Regression/Checksum/benchmarks_json/test_3d_particle_scrape.json +++ b/Regression/Checksum/benchmarks_json/test_3d_particle_scrape.json @@ -1,10 +1,10 @@ { "lev=0": { - "Bx": 148673.005859208, - "By": 148673.00585920803, - "Bz": 3371.758117878557, - "Ex": 55378581103426.695, - "Ey": 55378581103426.7, - "Ez": 68412803445328.25 + "Bx": 144495.08082507108, + "By": 144495.0808250711, + "Bz": 8481.95872462886, + "Ex": 54500496182517.914, + "Ey": 54500496182517.914, + "Ez": 70231240245509.4 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_particle_scrape_picmi.json b/Regression/Checksum/benchmarks_json/test_3d_particle_scrape_picmi.json index b03a954397a..1f9f0a77b5a 100644 --- a/Regression/Checksum/benchmarks_json/test_3d_particle_scrape_picmi.json +++ b/Regression/Checksum/benchmarks_json/test_3d_particle_scrape_picmi.json @@ -1,10 +1,10 @@ { "lev=0": { - "Bx": 148673.005859208, - "By": 148673.00585920803, - "Bz": 3371.758117878557, - "Ex": 55378581103426.695, - "Ey": 55378581103426.7, - "Ez": 68412803445328.25 + "Bx": 144495.08082507108, + "By": 144495.08082507114, + "Bz": 8481.958724628861, + "Ex": 54500496182517.92, + "Ey": 54500496182517.91, + "Ez": 70231240245509.39 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_diffraction.json b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_diffraction.json index 0e5fad8db8a..e4b9d9c07ff 100644 --- a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_diffraction.json +++ b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_diffraction.json @@ -1,10 +1,10 @@ { "lev=0": { - "Br": 6.821267675779345e-19, - "Bt": 5.564905732478707e-05, - "Bz": 2.368259586613272e-19, - "Er": 16503.98082446463, - "Et": 1.5299584682447838e-10, - "Ez": 1466.854467399728 + "Br": 6.7914286131989935e-19, + "Bt": 5.4557350206853276e-05, + "Bz": 2.357229221622199e-19, + "Er": 16481.39008058988, + "Et": 1.5258937379236053e-10, + "Ez": 1508.1064116028576 } } \ No newline at end of file From 8981179a16ccd0c80eb61bbd7ba107cd9f7364e2 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Sun, 12 Jan 2025 14:26:49 -0800 Subject: [PATCH 58/99] Add documentation --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 6 +- .../HybridPICModel/HybridPICModel.cpp | 4 +- Source/Initialization/WarpXInitData.cpp | 8 +-- Source/WarpX.H | 58 ++++++++++++------- 4 files changed, 45 insertions(+), 31 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 3cc29d17b39..d7eb98489dc 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -292,7 +292,7 @@ WarpX::ScaleAreas (ablastr::fields::VectorField& face_areas, } void -WarpX::MarkUpdateCells ( +WarpX::MarkUpdateCellsStairCase ( std::array< std::unique_ptr,3> & eb_update, ablastr::fields::VectorField const& field, amrex::EBFArrayBoxFactory const & eb_fact ) @@ -380,7 +380,7 @@ WarpX::MarkUpdateCells ( } void -WarpX::MarkECTUpdateECells ( +WarpX::MarkUpdateECellsECT ( std::array< std::unique_ptr,3> & eb_update_E, ablastr::fields::VectorField const& edge_lengths ) { @@ -441,7 +441,7 @@ WarpX::MarkECTUpdateECells ( } void -WarpX::MarkECTUpdateBCells ( +WarpX::MarkUpdateBCellsECT ( std::array< std::unique_ptr,3> & eb_update_B, ablastr::fields::VectorField const& face_areas, ablastr::fields::VectorField const& edge_lengths ) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index c4d48fc0c26..abda59e40ba 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -227,7 +227,7 @@ void HybridPICModel::InitData () m_J_external[1], m_J_external[2], lev, PatchType::fine, - warpx.m_eb_update_E); + warpx.GetEBUpdateEFlag()); } } @@ -244,7 +244,7 @@ void HybridPICModel::GetCurrentExternal () m_J_external[1], m_J_external[2], lev, PatchType::fine, - warpx.m_eb_update_E); + warpx.GetEBUpdateEFlag()); } } diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 64d675878c7..c8c7d1f0337 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1264,20 +1264,20 @@ void WarpX::InitializeEBGridData (int lev) if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { // Mark on which grid points E should be updated - MarkECTUpdateECells( m_eb_update_E[lev], edge_lengths_lev ); + MarkUpdateECellsECT( m_eb_update_E[lev], edge_lengths_lev ); // Mark on which grid points B should be updated - MarkECTUpdateBCells( m_eb_update_B[lev], face_areas_lev, edge_lengths_lev); + MarkUpdateBCellsECT( m_eb_update_B[lev], face_areas_lev, edge_lengths_lev); // Compute additional quantities required for the ECT solver MarkExtensionCells(); ComputeFaceExtensions(); } else { // Mark on which grid points E should be updated (stair-case approximation) - MarkUpdateCells( + MarkUpdateCellsStairCase( m_eb_update_E[lev], m_fields.get_alldirs(FieldType::Efield_fp, lev), eb_fact ); // Mark on which grid points B should be updated (stair-case approximation) - MarkUpdateCells( + MarkUpdateCellsStairCase( m_eb_update_B[lev], m_fields.get_alldirs(FieldType::Bfield_fp, lev), eb_fact ); diff --git a/Source/WarpX.H b/Source/WarpX.H index 5e4650e3745..ad28c672ad5 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -169,6 +169,7 @@ public: } #endif ParticleBoundaryBuffer& GetParticleBoundaryBuffer () { return *m_particle_boundary_buffer; } + amrex::Vector,3 > >& GetEBUpdateEFlag() { return m_eb_update_E; } static void shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, int num_shift, int dir, int lev, bool update_cost_flag, @@ -1012,33 +1013,49 @@ public: void InitEB (); #ifdef AMREX_USE_EB - /** Set a flag to indicate on which grid points the field `field` - * should be updated. + /** \brief Set a flag to indicate on which grid points the field `field` + * should be updated, depending on their position relative to the embedded boundary. + * + * This function is used by all finite-difference solvers, except the + * ECT solver, which instead uses `MarkUpdateECellsECT` and `MarkUpdateBCellsECT`. + * It uses a stair-case approximation of the embedded boundary: + * If a grid point touches cells that are either partially or fully covered + * by the embedded boundary: the corresponding field is not updated. * * More specifically, this function fills the iMultiFabs in `eb_update` * (which have the same indexType as the MultiFabs in `field`) with 1 - * or 0, depending on whether the grid point is in the valid region - * or in the embedded boundary. - * - * This uses a stair-case approximation of the embedded boundary - * (unless the ECT solver is used): If a grid point touches cells - * that are either partially or fully covered by the embedded - * boundary: do not update field. - * - * TODO Discuss ECT solver + * or 0, depending on whether the grid point should be updated or not. */ - void MarkUpdateCells ( + void MarkUpdateCellsStairCase ( std::array< std::unique_ptr,3> & eb_update, ablastr::fields::VectorField const & field, amrex::EBFArrayBoxFactory const & eb_fact ); - // TODO documentation - void MarkECTUpdateECells ( + /** \brief Set a flag to indicate on which grid points the E field + * should be updated, depending on their position relative to the embedded boundary. + * + * This function is used by ECT solver. The E field is not updated if + * the edge on which it is defined is fully covered by the embedded boundary. + * + * More specifically, this function fills the iMultiFabs in `eb_update_E` + * (which have the same indexType as the E field) with 1 or 0, depending + * on whether the grid point should be updated or not. + */ + void MarkUpdateECellsECT ( std::array< std::unique_ptr,3> & eb_update_E, ablastr::fields::VectorField const& edge_lengths ); - // TODO documentation - void MarkECTUpdateBCells ( + /** \brief Set a flag to indicate on which grid points the B field + * should be updated, depending on their position relative to the embedded boundary. + * + * This function is used by ECT solver. The B field is not updated if + * the face on which it is defined is fully covered by the embedded boundary. + * + * More specifically, this function fills the iMultiFabs in `eb_update_B` + * (which have the same indexType as the B field) with 1 or 0, depending + * on whether the grid point should be updated or not. + */ + void MarkUpdateBCellsECT ( std::array< std::unique_ptr,3> & eb_update_B, ablastr::fields::VectorField const& face_areas, ablastr::fields::VectorField const& edge_lengths ); @@ -1425,12 +1442,9 @@ private: /** EB: Flag to indicate whether a gridpoint is inside the embedded boundary and therefore * whether the E or B should not be updated. (One array per level and per direction, due to staggering) - */ - // TODO: leave as private, add setters, getters - public: - amrex::Vector,3 > > m_eb_update_E; - amrex::Vector,3 > > m_eb_update_B; - private: + */ + amrex::Vector,3 > > m_eb_update_E; + amrex::Vector,3 > > m_eb_update_B; /** EB: for every mesh face flag_info_face contains a: * * 0 if the face needs to be extended From 8182df74a86d528604ed3c2b461bacedbfdb25a5 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 14 Jan 2025 12:52:35 -0800 Subject: [PATCH 59/99] Apply suggestions from code review --- Source/Initialization/WarpXInitData.cpp | 1 - Source/WarpX.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index c8c7d1f0337..5575e6211f9 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1253,7 +1253,6 @@ void WarpX::InitializeEBGridData (int lev) auto const eb_fact = fieldEBFactory(lev); - // TODO: move inside if condition for ECT auto edge_lengths_lev = m_fields.get_alldirs(FieldType::edge_lengths, lev); ComputeEdgeLengths(edge_lengths_lev, eb_fact); ScaleEdges(edge_lengths_lev, CellSize(lev)); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 265603eae24..db9abb851c1 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2318,7 +2318,6 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(m_eb_update_B[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_eb_update_B[z]"); - // TODO: do not allocate these arrays anymore with the Yee solver //! EB: Lengths of the mesh edges m_fields.alloc_init(FieldType::edge_lengths, Direction{0}, lev, amrex::convert(ba, Ex_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, 0.0_rt); From fb20d08b5584849aa8db8b6f72a50f7cbb9b82eb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 Jan 2025 21:53:04 +0000 Subject: [PATCH 60/99] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Examples/Tests/embedded_boundary_removal_depth/analysis.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Examples/Tests/embedded_boundary_removal_depth/analysis.py b/Examples/Tests/embedded_boundary_removal_depth/analysis.py index 6e8ad4d48e3..9b59cac56d4 100755 --- a/Examples/Tests/embedded_boundary_removal_depth/analysis.py +++ b/Examples/Tests/embedded_boundary_removal_depth/analysis.py @@ -84,9 +84,9 @@ def plot(array): def check_tolerance(array, tolerance): - assert np.all( - array <= tolerance - ), f"Test did not pass: one or more elements exceed the tolerance of {tolerance}." + assert np.all(array <= tolerance), ( + f"Test did not pass: one or more elements exceed the tolerance of {tolerance}." + ) print("All elements of are within the tolerance.") From f2a84a694f73e6a801e321d071db8fc426f560fd Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 15 Jan 2025 12:01:13 -0800 Subject: [PATCH 61/99] Add new tests for particle absorption on EB with EM solver Add file for checksum tests Update comment --- Examples/Tests/CMakeLists.txt | 1 + .../CMakeLists.txt | 39 ++++++++++++++ .../analysis.py | 53 +++++++++++++++++++ .../analysis_default_regression.py | 1 + .../inputs_base | 35 ++++++++++++ ...oundary_em_particle_absorption_sh_factor_1 | 11 ++++ ...oundary_em_particle_absorption_sh_factor_1 | 11 ++++ ...oundary_em_particle_absorption_sh_factor_1 | 11 ++++ ...ry_em_particle_absorption_sh_factor_1.json | 24 +++++++++ ...ry_em_particle_absorption_sh_factor_1.json | 24 +++++++++ ...ry_em_particle_absorption_sh_factor_1.json | 24 +++++++++ 11 files changed, 234 insertions(+) create mode 100644 Examples/Tests/embedded_boundary_em_particle_absorption/CMakeLists.txt create mode 100755 Examples/Tests/embedded_boundary_em_particle_absorption/analysis.py create mode 120000 Examples/Tests/embedded_boundary_em_particle_absorption/analysis_default_regression.py create mode 100644 Examples/Tests/embedded_boundary_em_particle_absorption/inputs_base create mode 100644 Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_1 create mode 100644 Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_1 create mode 100644 Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_1 create mode 100644 Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_1.json create mode 100644 Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_1.json create mode 100644 Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_1.json diff --git a/Examples/Tests/CMakeLists.txt b/Examples/Tests/CMakeLists.txt index 9d11a7b6115..21268bbe7a0 100644 --- a/Examples/Tests/CMakeLists.txt +++ b/Examples/Tests/CMakeLists.txt @@ -15,6 +15,7 @@ add_subdirectory(electrostatic_sphere) add_subdirectory(electrostatic_sphere_eb) add_subdirectory(embedded_boundary_cube) add_subdirectory(embedded_boundary_diffraction) +add_subdirectory(embedded_boundary_em_particle_absorption) add_subdirectory(embedded_boundary_python_api) add_subdirectory(embedded_boundary_removal_depth) add_subdirectory(embedded_boundary_rotated_cube) diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/CMakeLists.txt b/Examples/Tests/embedded_boundary_em_particle_absorption/CMakeLists.txt new file mode 100644 index 00000000000..fb5d54c0dbe --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/CMakeLists.txt @@ -0,0 +1,39 @@ +# Add tests (alphabetical order) ############################################## +# + +if(WarpX_EB) + add_warpx_test( + test_3d_embedded_boundary_em_particle_absorption_sh_factor_1 # name + 3 # dims + 1 # nprocs + inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_1 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() + + +if(WarpX_EB) + add_warpx_test( + test_2d_embedded_boundary_em_particle_absorption_sh_factor_1 # name + 2 # dims + 1 # nprocs + inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_1 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_rz_embedded_boundary_em_particle_absorption_sh_factor_1 # name + RZ # dims + 1 # nprocs + inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_1 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/analysis.py b/Examples/Tests/embedded_boundary_em_particle_absorption/analysis.py new file mode 100755 index 00000000000..28017f20ba5 --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/analysis.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +""" +This analysis script checks that there is no spurious charge build-up when a particle is absorbed by an embedded boundary. + +More specifically, this test simulates two particles of oppposite charge that are initialized at +the same position and then move in opposite directions. The particles are surrounded by a cylindrical +embedded boundary, and are absorbed when their trajectory intersects this boundary. With an +electromagnetic solver, this can lead to spurious charge build-up (i.e., div(E)!= rho/epsion_0) +that remains at the position where particle was absorbed. + +Note that, in this test, there will also be a (non-spurious) component of div(E) that propagates +along the embedded boundary, due to electromagnetic waves reflecting on this boundary. +When checking for static, spurious charge build-up, we average div(E) in time to remove this component. + +The test is performed in 2D, 3D and RZ. +(In 2D, the cylindrical embeded boundary becomes two parallel plates) +""" + +from openpmd_viewer import OpenPMDTimeSeries + +ts = OpenPMDTimeSeries("./diags/diag1/") + +divE_stacked = ts.iterate( + lambda iteration: ts.get_field("divE", iteration=iteration)[0] +) +start_avg_iter = 25 +end_avg_iter = 100 +divE_avg = divE_stacked[start_avg_iter:end_avg_iter].mean(axis=0) + +# Adjust the tolerance so that the remaining error due to the propagating +# div(E) (after averaging) is below this tolerance, but so that any typical +# spurious charge build-up is above this tolerance. This is dimension-dependent. +dim = ts.fields_metadata["divE"]["geometry"] +if dim == "3dcartesian": + tolerance = 5e-11 +elif dim == "2dcartesian": + tolerance = 3.5e-10 +elif dim == "thetaMode": + # In RZ: there are issues with divE on axis + # Set the few cells around the axis to 0 for this test + divE_avg[13:19] = 0 + tolerance = 4e-12 + + +def check_tolerance(array, tolerance): + assert abs(array).max() <= tolerance, ( + f"Test did not pass: the max error {abs(array).max()} exceeded the tolerance of {tolerance}." + ) + print("All elements of are within the tolerance.") + + +check_tolerance(divE_avg, tolerance) diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/analysis_default_regression.py b/Examples/Tests/embedded_boundary_em_particle_absorption/analysis_default_regression.py new file mode 120000 index 00000000000..d8ce3fca419 --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/analysis_default_regression.py @@ -0,0 +1 @@ +../../analysis_default_regression.py \ No newline at end of file diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_base b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_base new file mode 100644 index 00000000000..6c940d2298e --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_base @@ -0,0 +1,35 @@ +max_step = 100 +amr.max_level = 0 +amr.blocking_factor = 8 +amr.max_grid_size = 256 + +algo.charge_deposition = standard +algo.field_gathering = energy-conserving +warpx.const_dt = 1.17957283598e-09 +warpx.use_filter = 0 + +my_constants.R = 6.35 +warpx.eb_implicit_function = "(x**2 + y**2 - R**2)" + +particles.species_names = electron positron + +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "SingleParticle" +electron.single_particle_pos = 0.0 0.0 0.0 +electron.single_particle_u = 1.e20 0.0 0.4843221e20 # gamma*beta +electron.single_particle_weight = 1.0 + +positron.charge = q_e +positron.mass = m_e +positron.injection_style = "SingleParticle" +positron.single_particle_pos = 0.0 0.0 0.0 +positron.single_particle_u = -1.e20 0.0 -0.4843221e20 # gamma*beta +positron.single_particle_weight = 1.0 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = divE rho +diag1.format = openpmd diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_1 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_1 new file mode 100644 index 00000000000..99110df1634 --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_1 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = 2 +amr.n_cell = 32 32 +geometry.prob_lo = -10 -10 +geometry.prob_hi = 10 10 +boundary.field_lo = pec absorbing_silver_mueller +boundary.field_hi = pec absorbing_silver_mueller + +algo.particle_shape = 1 diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_1 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_1 new file mode 100644 index 00000000000..ea977877a2d --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_1 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = 3 +amr.n_cell = 32 32 32 +geometry.prob_lo = -10 -10 -10 +geometry.prob_hi = 10 10 10 +boundary.field_lo = pec pec absorbing_silver_mueller +boundary.field_hi = pec pec absorbing_silver_mueller + +algo.particle_shape = 1 diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_1 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_1 new file mode 100644 index 00000000000..7faf7fd8934 --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_1 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = RZ +amr.n_cell = 16 32 +geometry.prob_lo = 0 -10 +geometry.prob_hi = 10 10 +boundary.field_lo = none absorbing_silver_mueller +boundary.field_hi = pec absorbing_silver_mueller + +algo.particle_shape = 1 diff --git a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_1.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_1.json new file mode 100644 index 00000000000..a9bd697f08b --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_1.json @@ -0,0 +1,24 @@ +{ + "electron": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_weight": 0.0 + }, + "lev=0": { + "divE": 1.6305300553737757e-07, + "rho": 0.0 + }, + "positron": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_1.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_1.json new file mode 100644 index 00000000000..e230b921942 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_1.json @@ -0,0 +1,24 @@ +{ + "electron": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_weight": 0.0 + }, + "lev=0": { + "divE": 8.149809003960999e-07, + "rho": 0.0 + }, + "positron": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_1.json b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_1.json new file mode 100644 index 00000000000..47869c8abe5 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_1.json @@ -0,0 +1,24 @@ +{ + "electron": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_weight": 0.0 + }, + "lev=0": { + "divE": 5.999732410984964e-08, + "rho": 0.0 + }, + "positron": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file From 4c859167e521029fcfd26b32f94e0a6bbcfba30a Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 16 Jan 2025 16:31:30 -0800 Subject: [PATCH 62/99] Remove previous tests --- .../CMakeLists.txt | 110 ------------------ .../analysis.py | 93 --------------- .../inputs_base_2d | 45 ------- .../inputs_base_3d | 43 ------- .../inputs_base_rz | 44 ------- ...mbedded_boundary_removal_depth_sh_factor_1 | 6 - ...mbedded_boundary_removal_depth_sh_factor_2 | 6 - ...mbedded_boundary_removal_depth_sh_factor_3 | 6 - ...mbedded_boundary_removal_depth_sh_factor_1 | 6 - ...mbedded_boundary_removal_depth_sh_factor_2 | 6 - ...mbedded_boundary_removal_depth_sh_factor_3 | 6 - ...mbedded_boundary_removal_depth_sh_factor_1 | 6 - ...mbedded_boundary_removal_depth_sh_factor_2 | 6 - ...mbedded_boundary_removal_depth_sh_factor_3 | 6 - 14 files changed, 389 deletions(-) delete mode 100644 Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt delete mode 100755 Examples/Tests/embedded_boundary_removal_depth/analysis.py delete mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_base_2d delete mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_base_3d delete mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_base_rz delete mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_1 delete mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_2 delete mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_3 delete mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_1 delete mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_2 delete mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_3 delete mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_1 delete mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_2 delete mode 100644 Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_3 diff --git a/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt b/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt deleted file mode 100644 index 71426745092..00000000000 --- a/Examples/Tests/embedded_boundary_removal_depth/CMakeLists.txt +++ /dev/null @@ -1,110 +0,0 @@ -# Add tests (alphabetical order) ############################################## -# - -if(WarpX_EB) - add_warpx_test( - test_3d_embedded_boundary_removal_depth_sh_factor_1 # name - 3 # dims - 1 # nprocs - inputs_test_3d_embedded_boundary_removal_depth_sh_factor_1 # inputs - analysis.py # analysis - diags/diag1/ # output - OFF # dependency - ) -endif() - -if(WarpX_EB) - add_warpx_test( - test_3d_embedded_boundary_removal_depth_sh_factor_2 # name - 3 # dims - 1 # nprocs - inputs_test_3d_embedded_boundary_removal_depth_sh_factor_2 # inputs - analysis.py # analysis - diags/diag1/ # output - OFF # dependency - ) -endif() - -if(WarpX_EB) - add_warpx_test( - test_3d_embedded_boundary_removal_depth_sh_factor_3 # name - 3 # dims - 1 # nprocs - inputs_test_3d_embedded_boundary_removal_depth_sh_factor_3 # inputs - analysis.py # analysis - diags/diag1/ # output - OFF # dependency - ) -endif() - -if(WarpX_EB) - add_warpx_test( - test_2d_embedded_boundary_removal_depth_sh_factor_1 # name - 2 # dims - 1 # nprocs - inputs_test_2d_embedded_boundary_removal_depth_sh_factor_1 # inputs - analysis.py # analysis - diags/diag1/ # output - OFF # dependency - ) -endif() - -if(WarpX_EB) - add_warpx_test( - test_2d_embedded_boundary_removal_depth_sh_factor_2 # name - 2 # dims - 1 # nprocs - inputs_test_2d_embedded_boundary_removal_depth_sh_factor_2 # inputs - analysis.py # analysis - diags/diag1/ # output - OFF # dependency - ) -endif() - -if(WarpX_EB) - add_warpx_test( - test_2d_embedded_boundary_removal_depth_sh_factor_3 # name - 2 # dims - 1 # nprocs - inputs_test_2d_embedded_boundary_removal_depth_sh_factor_3 # inputs - analysis.py # analysis - diags/diag1/ # output - OFF # dependency - ) -endif() - -if(WarpX_EB) - add_warpx_test( - test_rz_embedded_boundary_removal_depth_sh_factor_1 # name - RZ # dims - 1 # nprocs - inputs_test_rz_embedded_boundary_removal_depth_sh_factor_1 # inputs - analysis.py # analysis - diags/diag1/ # output - OFF # dependency - ) -endif() - -if(WarpX_EB) - add_warpx_test( - test_rz_embedded_boundary_removal_depth_sh_factor_2 # name - RZ # dims - 1 # nprocs - inputs_test_rz_embedded_boundary_removal_depth_sh_factor_2 # inputs - analysis.py # analysis - diags/diag1/ # output - OFF # dependency - ) -endif() - -if(WarpX_EB) - add_warpx_test( - test_rz_embedded_boundary_removal_depth_sh_factor_3 # name - RZ # dims - 1 # nprocs - inputs_test_rz_embedded_boundary_removal_depth_sh_factor_3 # inputs - analysis.py # analysis - diags/diag1/ # output - OFF # dependency - ) -endif() diff --git a/Examples/Tests/embedded_boundary_removal_depth/analysis.py b/Examples/Tests/embedded_boundary_removal_depth/analysis.py deleted file mode 100755 index 6e8ad4d48e3..00000000000 --- a/Examples/Tests/embedded_boundary_removal_depth/analysis.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python - -""" -This analysis script checks for any spurious charge build-up at the embedded boundary, when particles are removed in 3D. -It averages the divergence of the electric field (divE) over the last 20 time steps and compares the results with a specified tolerance. -""" - -import os -import sys - -import matplotlib.pyplot as plt -import numpy as np -import tqdm -from openpmd_viewer import OpenPMDTimeSeries - -sys.path.insert(1, "../../../../warpx/Regression/Checksum/") -import checksumAPI - -# Open plotfile specified in command line -filename = sys.argv[1] -test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, filename, output_format="openpmd") - - -def get_avg_divE(ts, start_avg_iter, end_avg_iter, ncell, test_dim): - for iteration in tqdm.tqdm(ts.iterations[start_avg_iter:end_avg_iter]): - if test_dim == "3d": - avg_divE = np.zeros((ncell, ncell)) - divE = ts.get_field( - "divE", iteration=iteration, slice_across="y", plot=False, cmap="RdBu" - )[0] - elif test_dim == "2d": - avg_divE = np.zeros((ncell, ncell)) - divE = ts.get_field("divE", iteration=iteration, plot=False, cmap="RdBu")[0] - elif test_dim == "rz": - avg_divE = np.zeros((ncell, 2 * ncell)) - divE = ts.get_field("divE", iteration=iteration, plot=False, cmap="RdBu")[0] - avg_divE += divE - return avg_divE / (end_avg_iter - start_avg_iter) - - -def parse_dimension_in_test_name(test_name): - test_name = test_name.lower() - if "3d" in test_name: - return "3d" - elif "2d" in test_name: - return "2d" - elif "rz" in test_name: - return "rz" - return None - - -def plot(array): - x = np.linspace(-7, 7, 400) - y = np.sqrt(7**2 - x**2) - - fig, ax = plt.subplots() - - # Plot the boundary curves - ax.plot(x, y, "k--") - ax.plot(x, -y, "k--") - cax = ax.imshow(array, cmap="RdBu", extent=[-10, 10, -10, 10], origin="lower") - fig.colorbar(cax, ax=ax) - ax.set_xlabel("x (m)") - ax.set_ylabel("z (m)") - ax.set_title("Averaged divE") - - -ts = OpenPMDTimeSeries("./diags/diag1/") - -ncell = 32 -start_avg_iter = 30 -end_avg_iter = 50 -test_dim = parse_dimension_in_test_name(test_name) - -divE_avg = get_avg_divE(ts, start_avg_iter, end_avg_iter, ncell, test_dim) -plot(divE_avg) -plt.savefig("AverageddivE.png") - -if test_dim == "3d" or test_dim == "rz": - tolerance = 1e-10 -else: - tolerance = 1e-9 - - -def check_tolerance(array, tolerance): - assert np.all( - array <= tolerance - ), f"Test did not pass: one or more elements exceed the tolerance of {tolerance}." - print("All elements of are within the tolerance.") - - -check_tolerance(divE_avg, tolerance) diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_base_2d b/Examples/Tests/embedded_boundary_removal_depth/inputs_base_2d deleted file mode 100644 index bb27233f4cf..00000000000 --- a/Examples/Tests/embedded_boundary_removal_depth/inputs_base_2d +++ /dev/null @@ -1,45 +0,0 @@ -max_step = 50 - -geometry.dims = 2 - -amr.n_cell = 32 32 -amr.max_level = 0 -amr.blocking_factor = 8 -amr.max_grid_size = 256 -geometry.prob_lo = -10 -10 -geometry.prob_hi = 10 10 - - -# Boundary condition -boundary.field_lo = none pec -boundary.field_hi = pec pec - -algo.charge_deposition = standard -algo.field_gathering = energy-conserving -warpx.use_filter = 0 # TODO: use filtering -warpx.const_dt = 1.203645751e-09 -warpx.eb_implicit_function = "(x**2 + y**2 + z**2 - 49)" - -particles.species_names = electron positron - -electron.charge = -q_e -electron.mass = m_e -electron.injection_style = "SingleParticle" -electron.single_particle_pos = 0.0 0.0 0.0 -electron.single_particle_u = 1.e20 0.0 0.3e20 # gamma*beta -electron.single_particle_weight = 1.0 - -positron.charge = q_e -positron.mass = m_e -positron.injection_style = "SingleParticle" -positron.single_particle_pos = 0.0 0.0 0.0 -positron.single_particle_u = -1.e20 0.0 -0.3e20 # gamma*beta -positron.single_particle_weight = 1.0 - - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 1 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz divE rho -diag1.format = openpmd diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_base_3d b/Examples/Tests/embedded_boundary_removal_depth/inputs_base_3d deleted file mode 100644 index 62430a57e3f..00000000000 --- a/Examples/Tests/embedded_boundary_removal_depth/inputs_base_3d +++ /dev/null @@ -1,43 +0,0 @@ -max_step = 50 -amr.n_cell = 32 32 32 -amr.max_level = 0 -amr.blocking_factor = 8 -amr.max_grid_size = 256 -geometry.dims = 3 -geometry.prob_lo = -10 -10 -10 -geometry.prob_hi = 10 10 10 - -# Boundary condition -boundary.field_lo = pec pec pec -boundary.field_hi = pec pec pec - -algo.charge_deposition = standard -algo.field_gathering = energy-conserving -warpx.cfl = 1.0 -warpx.use_filter = 0 # TODO: use filtering - -warpx.eb_implicit_function = "(x*x + y*y + z*z - 49)" - -particles.species_names = electron positron - -electron.charge = -q_e -electron.mass = m_e -electron.injection_style = "SingleParticle" -electron.single_particle_pos = 0.0 0.0 0.0 -electron.single_particle_u = 1.e20 0.0 0.3e20 # gamma*beta -electron.single_particle_weight = 1.0 - -positron.charge = q_e -positron.mass = m_e -positron.injection_style = "SingleParticle" -positron.single_particle_pos = 0.0 0.0 0.0 -positron.single_particle_u = -1.e20 0.0 -0.3e20 # gamma*beta -positron.single_particle_weight = 1.0 - - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 1 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz divE rho -diag1.format = openpmd diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_base_rz b/Examples/Tests/embedded_boundary_removal_depth/inputs_base_rz deleted file mode 100644 index 57c13a4fd4b..00000000000 --- a/Examples/Tests/embedded_boundary_removal_depth/inputs_base_rz +++ /dev/null @@ -1,44 +0,0 @@ -max_step = 70 - -geometry.dims = RZ - -amr.n_cell = 32 32 -amr.max_level = 0 -amr.blocking_factor = 8 -amr.max_grid_size = 256 -geometry.prob_lo = 0 -10 -geometry.prob_hi = 10 10 - - -# Boundary condition -boundary.field_lo = none pec -boundary.field_hi = pec pec - -algo.charge_deposition = standard -algo.field_gathering = energy-conserving -warpx.use_filter = 0 # TODO: use filtering - -warpx.eb_implicit_function = "(x**2 + y**2 + z**2 - 49)" - -particles.species_names = electron positron - -electron.charge = -q_e -electron.mass = m_e -electron.injection_style = "SingleParticle" -electron.single_particle_pos = 1.0 0.0 0.0 -electron.single_particle_u = 0.0 0.0 0.3e20 # gamma*beta -electron.single_particle_weight = 1.0 - -positron.charge = q_e -positron.mass = m_e -positron.injection_style = "SingleParticle" -positron.single_particle_pos = 1.0 0.0 0.0 -positron.single_particle_u = 0.0 0.0 -0.3e20 # gamma*beta -positron.single_particle_weight = 1.0 - -# Diagnostics -diagnostics.diags_names = diag1 -diag1.intervals = 1 -diag1.diag_type = Full -diag1.fields_to_plot = Er Ez Br Bz divE rho -diag1.format = openpmd diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_1 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_1 deleted file mode 100644 index 221aade2e37..00000000000 --- a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_1 +++ /dev/null @@ -1,6 +0,0 @@ -# base input parameters -FILE = inputs_base_2d - -# test input parameters -# Order of particle shape factors -algo.particle_shape = 1 diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_2 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_2 deleted file mode 100644 index 41743908169..00000000000 --- a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_2 +++ /dev/null @@ -1,6 +0,0 @@ -# base input parameters -FILE = inputs_base_2d - -# test input parameters -# Order of particle shape factors -algo.particle_shape = 2 diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_3 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_3 deleted file mode 100644 index c15166f3796..00000000000 --- a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_2d_embedded_boundary_removal_depth_sh_factor_3 +++ /dev/null @@ -1,6 +0,0 @@ -# base input parameters -FILE = inputs_base_2d - -# test input parameters -# Order of particle shape factors -algo.particle_shape = 3 diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_1 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_1 deleted file mode 100644 index 2480459e506..00000000000 --- a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_1 +++ /dev/null @@ -1,6 +0,0 @@ -# base input parameters -FILE = inputs_base_3d - -# test input parameters -# Order of particle shape factors -algo.particle_shape = 1 diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_2 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_2 deleted file mode 100644 index 268a4bcfb90..00000000000 --- a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_2 +++ /dev/null @@ -1,6 +0,0 @@ -# base input parameters -FILE = inputs_base_3d - -# test input parameters -# Order of particle shape factors -algo.particle_shape = 2 diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_3 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_3 deleted file mode 100644 index 964d9ab4137..00000000000 --- a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_3d_embedded_boundary_removal_depth_sh_factor_3 +++ /dev/null @@ -1,6 +0,0 @@ -# base input parameters -FILE = inputs_base_3d - -# test input parameters -# Order of particle shape factors -algo.particle_shape = 3 diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_1 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_1 deleted file mode 100644 index 02f7cb4936e..00000000000 --- a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_1 +++ /dev/null @@ -1,6 +0,0 @@ -# base input parameters -FILE = inputs_base_rz - -# test input parameters -# Order of particle shape factors -algo.particle_shape = 1 diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_2 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_2 deleted file mode 100644 index 9e0910cf2f8..00000000000 --- a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_2 +++ /dev/null @@ -1,6 +0,0 @@ -# base input parameters -FILE = inputs_base_rz - -# test input parameters -# Order of particle shape factors -algo.particle_shape = 2 diff --git a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_3 b/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_3 deleted file mode 100644 index 28eace99f01..00000000000 --- a/Examples/Tests/embedded_boundary_removal_depth/inputs_test_rz_embedded_boundary_removal_depth_sh_factor_3 +++ /dev/null @@ -1,6 +0,0 @@ -# base input parameters -FILE = inputs_base_rz - -# test input parameters -# Order of particle shape factors -algo.particle_shape = 3 From 46980a52b0358b6ffc8003058688b8839b745cf3 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 16 Jan 2025 12:21:54 -0800 Subject: [PATCH 63/99] Update checksums --- ...ry_em_particle_absorption_sh_factor_1.json | 20 +++++++++---------- ...ry_em_particle_absorption_sh_factor_1.json | 20 +++++++++---------- ...ry_em_particle_absorption_sh_factor_1.json | 20 +++++++++---------- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_1.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_1.json index a9bd697f08b..de3d125c744 100644 --- a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_1.json +++ b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_1.json @@ -1,24 +1,24 @@ { + "lev=0": { + "divE": 3.059581906777539e-08, + "rho": 0.0 + }, "electron": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, "particle_position_x": 0.0, "particle_position_y": 0.0, "particle_position_z": 0.0, - "particle_weight": 0.0 - }, - "lev=0": { - "divE": 1.6305300553737757e-07, - "rho": 0.0 - }, - "positron": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { "particle_position_x": 0.0, "particle_position_y": 0.0, "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, "particle_weight": 0.0 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_1.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_1.json index e230b921942..d3e08d9723e 100644 --- a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_1.json +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_1.json @@ -1,24 +1,24 @@ { + "lev=0": { + "divE": 4.928354322096152e-07, + "rho": 0.0 + }, "electron": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, "particle_position_x": 0.0, "particle_position_y": 0.0, "particle_position_z": 0.0, - "particle_weight": 0.0 - }, - "lev=0": { - "divE": 8.149809003960999e-07, - "rho": 0.0 - }, - "positron": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { "particle_position_x": 0.0, "particle_position_y": 0.0, "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, "particle_weight": 0.0 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_1.json b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_1.json index 47869c8abe5..30d7d0ba081 100644 --- a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_1.json +++ b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_1.json @@ -1,24 +1,24 @@ { + "lev=0": { + "divE": 1.4599714697029335e-08, + "rho": 0.0 + }, "electron": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, "particle_position_x": 0.0, "particle_position_y": 0.0, "particle_position_z": 0.0, - "particle_weight": 0.0 - }, - "lev=0": { - "divE": 5.999732410984964e-08, - "rho": 0.0 - }, - "positron": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { "particle_position_x": 0.0, "particle_position_y": 0.0, "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, "particle_weight": 0.0 } } \ No newline at end of file From 5c36e23f9e5c4c35f6311d50d65bf39b061d0f42 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 16 Jan 2025 14:10:44 -0800 Subject: [PATCH 64/99] Update tolerance --- .../Tests/embedded_boundary_em_particle_absorption/analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/analysis.py b/Examples/Tests/embedded_boundary_em_particle_absorption/analysis.py index 28017f20ba5..b5723a3422a 100755 --- a/Examples/Tests/embedded_boundary_em_particle_absorption/analysis.py +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/analysis.py @@ -33,7 +33,7 @@ # spurious charge build-up is above this tolerance. This is dimension-dependent. dim = ts.fields_metadata["divE"]["geometry"] if dim == "3dcartesian": - tolerance = 5e-11 + tolerance = 7e-11 elif dim == "2dcartesian": tolerance = 3.5e-10 elif dim == "thetaMode": From e2493a4360b41c4db7e01dc0164e6dd737462eb4 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 16 Jan 2025 16:35:40 -0800 Subject: [PATCH 65/99] Remove test --- Examples/Tests/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/Examples/Tests/CMakeLists.txt b/Examples/Tests/CMakeLists.txt index 21268bbe7a0..d9e9404ae3e 100644 --- a/Examples/Tests/CMakeLists.txt +++ b/Examples/Tests/CMakeLists.txt @@ -17,7 +17,6 @@ add_subdirectory(embedded_boundary_cube) add_subdirectory(embedded_boundary_diffraction) add_subdirectory(embedded_boundary_em_particle_absorption) add_subdirectory(embedded_boundary_python_api) -add_subdirectory(embedded_boundary_removal_depth) add_subdirectory(embedded_boundary_rotated_cube) add_subdirectory(embedded_circle) add_subdirectory(energy_conserving_thermal_plasma) From 6cefa699f3a1526eafe4143191daf90b96e11a50 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 17 Jan 2025 05:44:29 -0800 Subject: [PATCH 66/99] Add tests for higher-order shape factors --- .../CMakeLists.txt | 71 +++++++++++++++++++ ...oundary_em_particle_absorption_sh_factor_2 | 11 +++ ...oundary_em_particle_absorption_sh_factor_3 | 11 +++ ...oundary_em_particle_absorption_sh_factor_2 | 11 +++ ...oundary_em_particle_absorption_sh_factor_3 | 11 +++ ...oundary_em_particle_absorption_sh_factor_2 | 11 +++ ...oundary_em_particle_absorption_sh_factor_3 | 11 +++ 7 files changed, 137 insertions(+) create mode 100644 Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_2 create mode 100644 Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_3 create mode 100644 Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_2 create mode 100644 Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_3 create mode 100644 Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_2 create mode 100644 Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_3 diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/CMakeLists.txt b/Examples/Tests/embedded_boundary_em_particle_absorption/CMakeLists.txt index fb5d54c0dbe..0aa5b48b2b7 100644 --- a/Examples/Tests/embedded_boundary_em_particle_absorption/CMakeLists.txt +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/CMakeLists.txt @@ -13,6 +13,29 @@ if(WarpX_EB) ) endif() +if(WarpX_EB) + add_warpx_test( + test_3d_embedded_boundary_em_particle_absorption_sh_factor_2 # name + 3 # dims + 1 # nprocs + inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_2 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_3d_embedded_boundary_em_particle_absorption_sh_factor_3 # name + 3 # dims + 1 # nprocs + inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_3 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() if(WarpX_EB) add_warpx_test( @@ -26,6 +49,30 @@ if(WarpX_EB) ) endif() +if(WarpX_EB) + add_warpx_test( + test_2d_embedded_boundary_em_particle_absorption_sh_factor_2 # name + 2 # dims + 1 # nprocs + inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_2 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_2d_embedded_boundary_em_particle_absorption_sh_factor_3 # name + 2 # dims + 1 # nprocs + inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_3 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() + if(WarpX_EB) add_warpx_test( test_rz_embedded_boundary_em_particle_absorption_sh_factor_1 # name @@ -37,3 +84,27 @@ if(WarpX_EB) OFF # dependency ) endif() + +if(WarpX_EB) + add_warpx_test( + test_rz_embedded_boundary_em_particle_absorption_sh_factor_2 # name + RZ # dims + 1 # nprocs + inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_2 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() + +if(WarpX_EB) + add_warpx_test( + test_rz_embedded_boundary_em_particle_absorption_sh_factor_3 # name + RZ # dims + 1 # nprocs + inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_3 # inputs + "analysis.py" # analysis + "analysis_default_regression.py --path diags/diag1" # checksum + OFF # dependency + ) +endif() diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_2 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_2 new file mode 100644 index 00000000000..4cdbfc5dd5e --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_2 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = 2 +amr.n_cell = 32 32 +geometry.prob_lo = -10 -10 +geometry.prob_hi = 10 10 +boundary.field_lo = pec absorbing_silver_mueller +boundary.field_hi = pec absorbing_silver_mueller + +algo.particle_shape = 2 diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_3 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_3 new file mode 100644 index 00000000000..6113f0668fe --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_2d_embedded_boundary_em_particle_absorption_sh_factor_3 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = 2 +amr.n_cell = 32 32 +geometry.prob_lo = -10 -10 +geometry.prob_hi = 10 10 +boundary.field_lo = pec absorbing_silver_mueller +boundary.field_hi = pec absorbing_silver_mueller + +algo.particle_shape = 3 diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_2 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_2 new file mode 100644 index 00000000000..ea977877a2d --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_2 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = 3 +amr.n_cell = 32 32 32 +geometry.prob_lo = -10 -10 -10 +geometry.prob_hi = 10 10 10 +boundary.field_lo = pec pec absorbing_silver_mueller +boundary.field_hi = pec pec absorbing_silver_mueller + +algo.particle_shape = 1 diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_3 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_3 new file mode 100644 index 00000000000..553e5e058e7 --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_3d_embedded_boundary_em_particle_absorption_sh_factor_3 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = 3 +amr.n_cell = 32 32 32 +geometry.prob_lo = -10 -10 -10 +geometry.prob_hi = 10 10 10 +boundary.field_lo = pec pec absorbing_silver_mueller +boundary.field_hi = pec pec absorbing_silver_mueller + +algo.particle_shape = 3 diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_2 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_2 new file mode 100644 index 00000000000..7faf7fd8934 --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_2 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = RZ +amr.n_cell = 16 32 +geometry.prob_lo = 0 -10 +geometry.prob_hi = 10 10 +boundary.field_lo = none absorbing_silver_mueller +boundary.field_hi = pec absorbing_silver_mueller + +algo.particle_shape = 1 diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_3 b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_3 new file mode 100644 index 00000000000..aad65594c96 --- /dev/null +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/inputs_test_rz_embedded_boundary_em_particle_absorption_sh_factor_3 @@ -0,0 +1,11 @@ +# base input parameters +FILE = inputs_base + +geometry.dims = RZ +amr.n_cell = 16 32 +geometry.prob_lo = 0 -10 +geometry.prob_hi = 10 10 +boundary.field_lo = none absorbing_silver_mueller +boundary.field_hi = pec absorbing_silver_mueller + +algo.particle_shape = 3 From 0980516a26ab1ccdcf89fd3a539ce8eb0452ca8a Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 17 Jan 2025 08:19:50 -0800 Subject: [PATCH 67/99] Minor cleanup --- Source/Initialization/WarpXInitData.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 5575e6211f9..4db7096eb05 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -977,7 +977,6 @@ WarpX::InitLevelData (int lev, Real /*time*/) if ((m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) && (lev > 0) && (lev <= maxlevel_extEMfield_init)) { - // TODO: raise error when EB is activated ComputeExternalFieldOnGridUsingParser( FieldType::Bfield_aux, m_p_ext_field_params->Bxfield_parser->compile<4>(), @@ -990,7 +989,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_p_ext_field_params->Bxfield_parser->compile<4>(), m_p_ext_field_params->Byfield_parser->compile<4>(), m_p_ext_field_params->Bzfield_parser->compile<4>(), - lev, PatchType::coarse, m_eb_update_E); + lev, PatchType::coarse, m_eb_update_B); } // if the input string for the E-field is "parse_e_ext_grid_function", From ace399b6a3d044436563165a27bc93cb25b190b3 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Sat, 18 Jan 2025 08:34:54 -0800 Subject: [PATCH 68/99] Modify initialization in ECT solver --- .../test_3d_embedded_boundary_rotated_cube.json | 12 ++++++------ Source/Initialization/WarpXInitData.cpp | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_rotated_cube.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_rotated_cube.json index d45887ca932..118214948a5 100644 --- a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_rotated_cube.json +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_rotated_cube.json @@ -1,10 +1,10 @@ { "lev=0": { - "Bx": 1.252616939910365e-05, - "By": 0.02473895628331097, - "Bz": 0.024738956316621142, - "Ex": 10253221.850298548, - "Ey": 10387.334582977643, - "Ez": 10387.532806510022 + "Bx": 1.280747509243305e-05, + "By": 2.473900144296397e-02, + "Bz": 2.473890786894079e-02, + "Ex": 1.025322901921306e+07, + "Ey": 1.042254197269831e+04, + "Ez": 1.040011664019071e+04 } } \ No newline at end of file diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 4db7096eb05..248448e817e 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1261,13 +1261,13 @@ void WarpX::InitializeEBGridData (int lev) ScaleAreas(face_areas_lev, CellSize(lev)); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { + // Compute additional quantities required for the ECT solver + MarkExtensionCells(); + ComputeFaceExtensions(); // Mark on which grid points E should be updated MarkUpdateECellsECT( m_eb_update_E[lev], edge_lengths_lev ); // Mark on which grid points B should be updated MarkUpdateBCellsECT( m_eb_update_B[lev], face_areas_lev, edge_lengths_lev); - // Compute additional quantities required for the ECT solver - MarkExtensionCells(); - ComputeFaceExtensions(); } else { // Mark on which grid points E should be updated (stair-case approximation) MarkUpdateCellsStairCase( From 16ec9b2867d754e292ba5cf3e8e36d7da2782496 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Sat, 18 Jan 2025 08:53:19 -0800 Subject: [PATCH 69/99] Update comments --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 37 +++++++++++++++---------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index d7eb98489dc..0ddd8c22d1f 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -339,30 +339,39 @@ WarpX::MarkUpdateCellsStairCase ( amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - // Stair-case approximation: If neighboring cells are either partially - // or fully covered: do not update field - - // Check neighbors in the each spatial direction - // If nodal in a given direction, we need to start from -1 (left-neighboring cell) - int const i_start = ( index_type.nodeCentered(0) )? -1 : 0; + // Stair-case approximation: If neighboring cells of this gridpoint + // are either partially or fully covered: do not update field + + // The number of cells that we need to check depend on the index type + // of the `eb_update_arr` in each direction. + // If `eb_update_arr` is nodal in a given direction, we need to check the cells + // to the left and right of this nodal gridpoint. + // For instance, if `eb_update_arr` is nodal in the first dimension, we + // to check the cells at index i-1 and at index i, since, with AMReX indexing conventions, + // these are the neighboring cells for the nodal gripoint at index i. + // If `eb_update_arr` is cell-centerd in a given direction, we only need to check + // the cell at the same position (e.g., in the first dimension: the cell at index i). + int const i_start = ( index_type.nodeCentered(0) )? i-1 : i; #if AMREX_SPACEDIM > 1 - int const j_start = ( index_type.nodeCentered(1) )? -1 : 0; + int const j_start = ( index_type.nodeCentered(1) )? j-1 : j; #else - int const j_start = 0; + int const j_start = j; #endif #if AMREX_SPACEDIM > 2 - int const k_start = ( index_type.nodeCentered(2) )? -1 : 0; + int const k_start = ( index_type.nodeCentered(2) )? k-1 : k; #else - int const k_start = 0; + int const k_start = k; #endif // Loop over neighboring cells int eb_update = 1; - for (int i_shift = i_start; i_shift <= 0; ++i_shift) { - for (int j_shift = j_start; j_shift <= 0; ++j_shift) { - for (int k_shift = k_start; k_shift <= 0; ++k_shift) { + for (int i_cell = i_start; i_cell <= i; ++i_cell) { + for (int j_cell = j_start; j_cell <= j; ++j_cell) { + for (int k_cell = k_start; k_cell <= k; ++k_cell) { // If one of the neighboring is either partially or fully covered // (i.e. if they are not regular cells), do not update field - if ( !flag(i+i_shift, j+j_shift, k+k_shift).isRegular() ) { + // (Note that `flag` is a cell-centered object, and `isRegular` + // returns `false` if the cell is either partially or fully covered.) + if ( !flag(i_cell, j_cell, k_cell).isRegular() ) { eb_update = 0; } } From 45c4d30f9860c61547ced95330345b954001a5c8 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 21 Jan 2025 05:57:02 -0800 Subject: [PATCH 70/99] Allocate new flag --- Source/Parallelization/WarpXRegrid.cpp | 1 + Source/WarpX.H | 6 ++++++ Source/WarpX.cpp | 6 ++++++ 3 files changed, 13 insertions(+) diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index c12b9f7e0f7..2e99946a599 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -194,6 +194,7 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi for (int idim=0; idim < 3; ++idim) { if (eb_enabled) { + RemakeMultiFab( m_eb_reduce_particle_shape[lev] ); if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { RemakeMultiFab( m_eb_update_E[lev][idim] ); RemakeMultiFab( m_eb_update_B[lev][idim] ); diff --git a/Source/WarpX.H b/Source/WarpX.H index 50001e7e882..5ea036fcac7 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1446,6 +1446,12 @@ private: amrex::Vector,3 > > m_eb_update_E; amrex::Vector,3 > > m_eb_update_B; + /** EB: Flag to indicate whether particles should use order-1 shape factor in a given cell, + * when depositing charge/current. This is needed with the EM solver, in order to avoid + * error in charge conservation when a particle is too close to the embedded boundary. + */ + amrex::Vector< std::unique_ptr > m_eb_reduce_particle_shape; + /** EB: for every mesh face flag_info_face contains a: * * 0 if the face needs to be extended * * 1 if the face is large enough to lend area to other faces diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 265603eae24..54223601910 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -325,6 +325,8 @@ WarpX::WarpX () m_eb_update_E.resize(nlevs_max); m_eb_update_B.resize(nlevs_max); + m_eb_reduce_particle_shape.resize(nlevs_max); + m_flag_info_face.resize(nlevs_max); m_flag_ext_face.resize(nlevs_max); m_borrowing.resize(nlevs_max); @@ -2299,9 +2301,13 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm amrex::IntVect const ng_ls(2); //EB level set m_fields.alloc_init(FieldType::distance_to_eb, lev, amrex::convert(ba, IntVect::TheNodeVector()), dm, nc_ls, ng_ls, 0.0_rt); + // Whether to reduce the particle shape to order 1 when close to the EB + AllocInitMultiFab(m_eb_reduce_particle_shape[lev], amrex::convert(ba, IntVect::TheCellVector()), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_eb_reduce_particle_shape"); // EB info are needed only at the finest level if (lev == maxLevel()) { + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { AllocInitMultiFab(m_eb_update_E[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, From 7587b4deb5fabda3203637193ae496e3a8cb751c Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 21 Jan 2025 06:39:47 -0800 Subject: [PATCH 71/99] Initialize new data structures --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 86 +++++++++++++++++++++++++ Source/Initialization/WarpXInitData.cpp | 11 +--- Source/WarpX.H | 6 ++ 3 files changed, 94 insertions(+), 9 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 0ddd8c22d1f..a1dca9d6316 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -291,6 +291,92 @@ WarpX::ScaleAreas (ablastr::fields::VectorField& face_areas, } } +void +WarpX::MarkReducedShapeCells ( + std::unique_ptr & eb_reduce_particle_shape, + amrex::EBFArrayBoxFactory const & eb_fact ) +{ + + // TODO: handle guard cells: + // - Allocate as many cells as rho/J + // - Initialize to 1 + // - Call Redistribute + + // Extract structures for embedded boundaries + amrex::FabArray const& eb_flag = eb_fact.getMultiEBCellFlagFab(); + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (amrex::MFIter mfi(*eb_reduce_particle_shape); mfi.isValid(); ++mfi) { + + const amrex::Box& box = mfi.tilebox(); + amrex::Array4 const & eb_reduce_particle_shape_arr = eb_reduce_particle_shape->array(mfi); + + // Check if the box (including one layer of guard cells) contains a mix of covered and regular cells + const amrex::Box& eb_info_box = mfi.tilebox(amrex::IntVect::TheCellVector()).grow(1); + amrex::FabType const fab_type = eb_flag[mfi].getType( eb_info_box ); + + if (fab_type == amrex::FabType::regular) { // All cells in the box are regular + + // Every cell in box is regular: do not reduce particle shape in any cell + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + eb_reduce_particle_shape_arr(i, j, k) = 0; + }); + + } else if (fab_type == amrex::FabType::covered) { // All cells in the box are covered + + // Every cell in box is fully covered: reduce particle shape + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + eb_reduce_particle_shape_arr(i, j, k) = 1; + }); + + } else { // The box contains a mix of covered and regular cells + + auto const & flag = eb_flag[mfi].array(); + + amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { + + // Check if any of the neighboring cells are either partially or fully covered + // In this case, reduce particle shape to order 1 + // (This ensures that the particle never deposits any charge in a partially or + // fully covered cell, even with higher-order shapes) + int const i_start = i-1; + int const i_end = i+1; +#if AMREX_SPACEDIM > 1 + int const j_start = j-1; + int const j_end = j+1; +#else + int const j_start = j; + int const j_end = j; +#endif +#if AMREX_SPACEDIM > 2 + int const k_start = k-1; + int const k_end = k+1; +#else + int const k_start = k; + int const k_end = k; +#endif + int reduce_shape = 0; + for (int i_cell = i_start; i_cell <= i_end; ++i_cell) { + for (int j_cell = j_start; j_cell <= j_end; ++j_cell) { + for (int k_cell = k_start; k_cell <= k_end; ++k_cell) { + // `isRegular` returns `false` if the cell is either partially or fully covered. + if ( !flag(i_cell, j_cell, k_cell).isRegular() ) { + reduce_shape = 1; + } + } + } + } + eb_reduce_particle_shape_arr(i, j, k) = reduce_shape; + }); + + } + + } + +} + void WarpX::MarkUpdateCellsStairCase ( std::array< std::unique_ptr,3> & eb_update, diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 84b31f9396d..bba5d05bcf5 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1227,20 +1227,12 @@ void WarpX::InitializeEBGridData (int lev) #ifdef AMREX_USE_EB if (lev == maxLevel()) { - // Throw a warning if EB is on and particle_shape > 1 - if ((nox > 1 or noy > 1 or noz > 1) and EB::enabled()) - { - ablastr::warn_manager::WMRecordWarning("Particles", - "when algo.particle_shape > 1, numerical artifacts will be present when\n" - "particles are close to embedded boundaries"); - } + auto const eb_fact = fieldEBFactory(lev); if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD ) { using warpx::fields::FieldType; - auto const eb_fact = fieldEBFactory(lev); - auto edge_lengths_lev = m_fields.get_alldirs(FieldType::edge_lengths, lev); ComputeEdgeLengths(edge_lengths_lev, eb_fact); ScaleEdges(edge_lengths_lev, CellSize(lev)); @@ -1272,6 +1264,7 @@ void WarpX::InitializeEBGridData (int lev) } ComputeDistanceToEB(); + MarkReducedShapeCells( m_eb_reduce_particle_shape[lev], eb_fact ); } #else diff --git a/Source/WarpX.H b/Source/WarpX.H index 8a6bb8f2868..b7bafecaa0d 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1013,6 +1013,12 @@ public: void InitEB (); #ifdef AMREX_USE_EB + + /** TODO */ + void MarkReducedShapeCells ( + std::unique_ptr & eb_reduce_particle_shape, + amrex::EBFArrayBoxFactory const & eb_fact ); + /** \brief Set a flag to indicate on which grid points the field `field` * should be updated, depending on their position relative to the embedded boundary. * From 16552ef8d4194d14c9e82f5f592af3025493bb00 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 22 Jan 2025 12:45:33 -0800 Subject: [PATCH 72/99] Revert "WarpX header: remove unused GetDistanceToEB function (#5589)" This reverts commit e0c17e755f318fdac59aa37a8e51e1bbef5e396a. --- Source/Initialization/WarpXInitData.cpp | 2 -- Source/Particles/WarpXParticleContainer.cpp | 2 +- Source/WarpX.H | 6 ++++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 0d5800c15a3..0a6934cfd10 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1233,8 +1233,6 @@ void WarpX::InitializeEBGridData (int lev) { using warpx::fields::FieldType; - auto const eb_fact = fieldEBFactory(lev); - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { auto edge_lengths_lev = m_fields.get_alldirs(FieldType::edge_lengths, lev); diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index d93d425798a..98ccc30c259 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -370,7 +370,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, if (do_not_deposit) { return; } // Number of guard cells for local deposition of J - const WarpX& warpx = WarpX::GetInstance(); + WarpX& warpx = WarpX::GetInstance(); const amrex::IntVect& ng_J = warpx.get_ng_depos_J(); diff --git a/Source/WarpX.H b/Source/WarpX.H index d6015fd6c2b..7d6168e6785 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -162,6 +162,12 @@ public: HybridPICModel& GetHybridPICModel () { return *m_hybrid_pic_model; } [[nodiscard]] HybridPICModel * get_pointer_HybridPICModel () const { return m_hybrid_pic_model.get(); } MultiDiagnostics& GetMultiDiags () {return *multi_diags;} +#ifdef AMREX_USE_EB + ablastr::fields::MultiLevelScalarField GetDistanceToEB () { + using warpx::fields::FieldType; + return m_fields.get_mr_levels(FieldType::distance_to_eb, finestLevel()); + } +#endif ParticleBoundaryBuffer& GetParticleBoundaryBuffer () { return *m_particle_boundary_buffer; } amrex::Vector,3 > >& GetEBUpdateEFlag() { return m_eb_update_E; } From 2de126d36754809c6d2e241387cfdf96614652e2 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 22 Jan 2025 13:38:07 -0800 Subject: [PATCH 73/99] Do not use distance to EB anymore --- .../Particles/Deposition/CurrentDeposition.H | 20 ++++++++++--------- Source/Particles/Pusher/GetAndSetPosition.H | 18 +++++++++++++++++ Source/Particles/WarpXParticleContainer.cpp | 10 +++++----- Source/WarpX.H | 1 + 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 924153dfd21..dc9cc176671 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -658,7 +658,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, amrex::Dim3 lo, amrex::Real q, #ifdef AMREX_USE_EB - const amrex::Array4& distance_to_eb_arr, + const amrex::Array4& eb_reduce_particle_shape, const amrex::GpuArray& eb_dxi, const amrex::GpuArray& eb_plo, #endif @@ -749,20 +749,22 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // TODO: abstract in a separate function (doGatherScalarFieldNodal) // TODO: check calculation for RZ { - const amrex::Real distance = ablastr::particles::doGatherScalarFieldNodal( + WarpXParticleContainer::SuperParticleType p; + set_particle_position( p, xp + (relative_time - 0.5_rt*dt)*uxp[ip]*gaminv, yp + (relative_time - 0.5_rt*dt)*uyp[ip]*gaminv, - zp + (relative_time - 0.5_rt*dt)*uzp[ip]*gaminv, - distance_to_eb_arr, eb_dxi, eb_plo ); - close_to_eb_old = (distance < 1.0_rt); // TODO: tune depth + zp + (relative_time - 0.5_rt*dt)*uzp[ip]*gaminv ); + const auto [ii, jj, kk] = amrex::getParticleCell(p, eb_plo, eb_dxi).dim3(); + close_to_eb_old = (eb_reduce_particle_shape(ii,jj,kk) == 1); } { - const amrex::Real distance = ablastr::particles::doGatherScalarFieldNodal( + WarpXParticleContainer::SuperParticleType p; + set_particle_position( p, xp + (relative_time + 0.5_rt*dt)*uxp[ip]*gaminv, yp + (relative_time + 0.5_rt*dt)*uyp[ip]*gaminv, - zp + (relative_time + 0.5_rt*dt)*uzp[ip]*gaminv, - distance_to_eb_arr, eb_dxi, eb_plo ); - close_to_eb_new = (distance < 1.0_rt); // TODO: tune depth + zp + (relative_time + 0.5_rt*dt)*uzp[ip]*gaminv ); + const auto [ii, jj, kk] = amrex::getParticleCell(p, eb_plo, eb_dxi).dim3(); + close_to_eb_new = (eb_reduce_particle_shape(ii,jj,kk) == 1); } #endif diff --git a/Source/Particles/Pusher/GetAndSetPosition.H b/Source/Particles/Pusher/GetAndSetPosition.H index ab06fe3d6cd..22749d73731 100644 --- a/Source/Particles/Pusher/GetAndSetPosition.H +++ b/Source/Particles/Pusher/GetAndSetPosition.H @@ -53,6 +53,24 @@ void get_particle_position (const WarpXParticleContainer::SuperParticleType& p, #endif } +template +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void set_particle_position (WarpXParticleContainer::SuperParticleType& p, + amrex::ParticleReal const& x, + amrex::ParticleReal const& y, + amrex::ParticleReal const& z) noexcept +{ +#if defined(WARPX_DIM_RZ) + p.pos(PIdx::x) = std::sqrt(x*x + y*y); +#elif defined(WARPX_DIM_3D) + p.pos(PIdx::x) = x; + p.pos(PIdx::y) = y; +#elif defined(WARPX_DIM_XZ) + p.pos(PIdx::x) = x; +#endif + p.pos(PIdx::z) = z; +} + /** \brief Functor that can be used to extract the positions of the macroparticles * inside a ParallelFor kernel * diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 98ccc30c259..53d51e6b7c0 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -589,7 +589,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, if (EB::enabled()) { // signed distance function - auto const distance_to_eb_arr = (*warpx.GetDistanceToEB()[lev])[pti].array(); + auto const eb_reduce_particle_shape = (*warpx.GetEBReduceParticleShapeFlag()[lev])[pti].array(); const Geometry& geom = Geom(lev); const amrex::GpuArray eb_dxi = geom.InvCellSizeArray(); const amrex::GpuArray eb_plo = geom.ProbLoArray(); @@ -600,7 +600,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - distance_to_eb_arr, eb_dxi, eb_plo, + eb_reduce_particle_shape, eb_dxi, eb_plo, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doEsirkepovDepositionShapeN<2>( @@ -608,7 +608,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - distance_to_eb_arr, eb_dxi, eb_plo, + eb_reduce_particle_shape, eb_dxi, eb_plo, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doEsirkepovDepositionShapeN<3>( @@ -616,7 +616,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - distance_to_eb_arr, eb_dxi, eb_plo, + eb_reduce_particle_shape, eb_dxi, eb_plo, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doEsirkepovDepositionShapeN<4>( @@ -624,7 +624,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - distance_to_eb_arr, eb_dxi, eb_plo, + eb_reduce_particle_shape, eb_dxi, eb_plo, WarpX::n_rz_azimuthal_modes); } } diff --git a/Source/WarpX.H b/Source/WarpX.H index 7d6168e6785..8844f9edca5 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -170,6 +170,7 @@ public: #endif ParticleBoundaryBuffer& GetParticleBoundaryBuffer () { return *m_particle_boundary_buffer; } amrex::Vector,3 > >& GetEBUpdateEFlag() { return m_eb_update_E; } + amrex::Vector< std::unique_ptr >& GetEBReduceParticleShapeFlag() { return m_eb_reduce_particle_shape; } static void shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, int num_shift, int dir, int lev, bool update_cost_flag, From 41bdbf9ef0e43d1734168722f17b660d2cd9839e Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 22 Jan 2025 13:41:30 -0800 Subject: [PATCH 74/99] Code cleanup --- Source/WarpX.H | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index 8844f9edca5..2781deb4b72 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -162,12 +162,6 @@ public: HybridPICModel& GetHybridPICModel () { return *m_hybrid_pic_model; } [[nodiscard]] HybridPICModel * get_pointer_HybridPICModel () const { return m_hybrid_pic_model.get(); } MultiDiagnostics& GetMultiDiags () {return *multi_diags;} -#ifdef AMREX_USE_EB - ablastr::fields::MultiLevelScalarField GetDistanceToEB () { - using warpx::fields::FieldType; - return m_fields.get_mr_levels(FieldType::distance_to_eb, finestLevel()); - } -#endif ParticleBoundaryBuffer& GetParticleBoundaryBuffer () { return *m_particle_boundary_buffer; } amrex::Vector,3 > >& GetEBUpdateEFlag() { return m_eb_update_E; } amrex::Vector< std::unique_ptr >& GetEBReduceParticleShapeFlag() { return m_eb_reduce_particle_shape; } From 5ae42edb33e3bd41456d6d4b2852b54d09332610 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 22 Jan 2025 14:56:37 -0800 Subject: [PATCH 75/99] Simplify code --- .../Particles/Deposition/CurrentDeposition.H | 24 ++++++++----------- Source/Particles/WarpXParticleContainer.cpp | 11 ++++----- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index dc9cc176671..339fb908680 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -659,8 +659,6 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, amrex::Real q, #ifdef AMREX_USE_EB const amrex::Array4& eb_reduce_particle_shape, - const amrex::GpuArray& eb_dxi, - const amrex::GpuArray& eb_plo, #endif [[maybe_unused]]int n_rz_azimuthal_modes ) @@ -750,21 +748,19 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // TODO: check calculation for RZ { WarpXParticleContainer::SuperParticleType p; - set_particle_position( p, - xp + (relative_time - 0.5_rt*dt)*uxp[ip]*gaminv, - yp + (relative_time - 0.5_rt*dt)*uyp[ip]*gaminv, - zp + (relative_time - 0.5_rt*dt)*uzp[ip]*gaminv ); - const auto [ii, jj, kk] = amrex::getParticleCell(p, eb_plo, eb_dxi).dim3(); - close_to_eb_old = (eb_reduce_particle_shape(ii,jj,kk) == 1); + set_particle_position( p, x_old, y_old, z_old); + close_to_eb_old = eb_reduce_particle_shape( + lo.x + int(p.pos(0)), + lo.y + int(p.pos(1)), + lo.z + int(p.pos(2))); } { WarpXParticleContainer::SuperParticleType p; - set_particle_position( p, - xp + (relative_time + 0.5_rt*dt)*uxp[ip]*gaminv, - yp + (relative_time + 0.5_rt*dt)*uyp[ip]*gaminv, - zp + (relative_time + 0.5_rt*dt)*uzp[ip]*gaminv ); - const auto [ii, jj, kk] = amrex::getParticleCell(p, eb_plo, eb_dxi).dim3(); - close_to_eb_new = (eb_reduce_particle_shape(ii,jj,kk) == 1); + set_particle_position( p, x_new, y_new, z_new); + close_to_eb_new = eb_reduce_particle_shape( + lo.x + int(p.pos(0)), + lo.y + int(p.pos(1)), + lo.z + int(p.pos(2))); } #endif diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 53d51e6b7c0..d0beee15a47 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -590,9 +590,6 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, { // signed distance function auto const eb_reduce_particle_shape = (*warpx.GetEBReduceParticleShapeFlag()[lev])[pti].array(); - const Geometry& geom = Geom(lev); - const amrex::GpuArray eb_dxi = geom.InvCellSizeArray(); - const amrex::GpuArray eb_plo = geom.ProbLoArray(); if (WarpX::nox == 1){ doEsirkepovDepositionShapeN<1>( @@ -600,7 +597,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - eb_reduce_particle_shape, eb_dxi, eb_plo, + eb_reduce_particle_shape, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doEsirkepovDepositionShapeN<2>( @@ -608,7 +605,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - eb_reduce_particle_shape, eb_dxi, eb_plo, + eb_reduce_particle_shape, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doEsirkepovDepositionShapeN<3>( @@ -616,7 +613,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - eb_reduce_particle_shape, eb_dxi, eb_plo, + eb_reduce_particle_shape, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doEsirkepovDepositionShapeN<4>( @@ -624,7 +621,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - eb_reduce_particle_shape, eb_dxi, eb_plo, + eb_reduce_particle_shape, WarpX::n_rz_azimuthal_modes); } } From a126db2efad7c4b34df530f1b019a102fa1b0ed5 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 22 Jan 2025 15:06:55 -0800 Subject: [PATCH 76/99] Simplify code further --- .../Particles/Deposition/CurrentDeposition.H | 37 ++++++++++--------- Source/Particles/Pusher/GetAndSetPosition.H | 18 --------- 2 files changed, 19 insertions(+), 36 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 339fb908680..bc6835d0ce7 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -744,24 +744,25 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, bool close_to_eb_new = false; #ifdef AMREX_USE_EB // Check whether the particle is close to the EB at the old and new position - // TODO: abstract in a separate function (doGatherScalarFieldNodal) - // TODO: check calculation for RZ - { - WarpXParticleContainer::SuperParticleType p; - set_particle_position( p, x_old, y_old, z_old); - close_to_eb_old = eb_reduce_particle_shape( - lo.x + int(p.pos(0)), - lo.y + int(p.pos(1)), - lo.z + int(p.pos(2))); - } - { - WarpXParticleContainer::SuperParticleType p; - set_particle_position( p, x_new, y_new, z_new); - close_to_eb_new = eb_reduce_particle_shape( - lo.x + int(p.pos(0)), - lo.y + int(p.pos(1)), - lo.z + int(p.pos(2))); - } +#if defined(WARPX_DIM_3D) + close_to_eb_old = eb_reduce_particle_shape( + lo.x + int(x_old), + lo.y + int(y_old), + lo.z + int(z_old)); + close_to_eb_new = eb_reduce_particle_shape( + lo.x + int(x_new), + lo.y + int(y_new), + lo.z + int(z_new)); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + close_to_eb_old = eb_reduce_particle_shape( + lo.x + int(x_old), + lo.y + int(z_old), + 0); + close_to_eb_new = eb_reduce_particle_shape( + lo.x + int(x_new), + lo.y + int(z_new), + 0); +#endif #endif #if defined(WARPX_DIM_RZ) diff --git a/Source/Particles/Pusher/GetAndSetPosition.H b/Source/Particles/Pusher/GetAndSetPosition.H index 22749d73731..ab06fe3d6cd 100644 --- a/Source/Particles/Pusher/GetAndSetPosition.H +++ b/Source/Particles/Pusher/GetAndSetPosition.H @@ -53,24 +53,6 @@ void get_particle_position (const WarpXParticleContainer::SuperParticleType& p, #endif } -template -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE -void set_particle_position (WarpXParticleContainer::SuperParticleType& p, - amrex::ParticleReal const& x, - amrex::ParticleReal const& y, - amrex::ParticleReal const& z) noexcept -{ -#if defined(WARPX_DIM_RZ) - p.pos(PIdx::x) = std::sqrt(x*x + y*y); -#elif defined(WARPX_DIM_3D) - p.pos(PIdx::x) = x; - p.pos(PIdx::y) = y; -#elif defined(WARPX_DIM_XZ) - p.pos(PIdx::x) = x; -#endif - p.pos(PIdx::z) = z; -} - /** \brief Functor that can be used to extract the positions of the macroparticles * inside a ParallelFor kernel * From ebd0a9b0d9791aad3d646d5bfa4666c976b41b3f Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 22 Jan 2025 18:56:34 -0800 Subject: [PATCH 77/99] Update Source/Particles/Deposition/CurrentDeposition.H --- Source/Particles/Deposition/CurrentDeposition.H | 1 - 1 file changed, 1 deletion(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index bc6835d0ce7..08432e02d3d 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -10,7 +10,6 @@ #include "Particles/Deposition/SharedDepositionUtils.H" #include "ablastr/parallelization/KernelTimer.H" -#include "ablastr/particles/NodalFieldGather.H" #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/ShapeFactors.H" #include "Utils/TextMsg.H" From 3f8a954ff1f0a37db177e0eabb8bd57fdb424ca7 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 23 Jan 2025 05:31:17 -0800 Subject: [PATCH 78/99] Add 1D implementation --- Source/Particles/Deposition/CurrentDeposition.H | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 08432e02d3d..6bc2ebb3f58 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -761,6 +761,13 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, lo.x + int(x_new), lo.y + int(z_new), 0); +#elif defined(WARPX_DIM_1D_Z) + close_to_eb_old = eb_reduce_particle_shape( + lo.x + int(z_old), + 0, 0); + close_to_eb_new = eb_reduce_particle_shape( + lo.x + int(z_new), + 0, 0); #endif #endif From a444821f53a1787036cdcefeb70143b5748edda4 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 23 Jan 2025 05:50:53 -0800 Subject: [PATCH 79/99] Update guard cell allocation --- Source/WarpX.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 6f8e1f59a25..50a7fb4ea8f 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2333,7 +2333,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm m_fields.alloc_init(FieldType::distance_to_eb, lev, amrex::convert(ba, IntVect::TheNodeVector()), dm, nc_ls, ng_ls, 0.0_rt); // Whether to reduce the particle shape to order 1 when close to the EB AllocInitMultiFab(m_eb_reduce_particle_shape[lev], amrex::convert(ba, IntVect::TheCellVector()), dm, ncomps, - guard_cells.ng_FieldSolver, lev, "m_eb_reduce_particle_shape"); + ngRho, lev, "m_eb_reduce_particle_shape"); // EB info are needed only at the finest level if (lev == maxLevel()) { From 9cf22328da6077651a9fb1d5a16475f050146eac Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 23 Jan 2025 05:57:01 -0800 Subject: [PATCH 80/99] Update initialization --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index cc3a48556dd..28121d42d3e 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -296,11 +296,9 @@ WarpX::MarkReducedShapeCells ( std::unique_ptr & eb_reduce_particle_shape, amrex::EBFArrayBoxFactory const & eb_fact ) { - - // TODO: handle guard cells: - // - Allocate as many cells as rho/J - // - Initialize to 1 - // - Call Redistribute + // Pre-fill array with 1, including in the ghost cells outside of the domain. + // (The guard cells in the domain will be updated by `FillBoundary` at the end of this function.) + eb_reduce_particle_shape->setVal(1, eb_reduce_particle_shape->nGrow()); // Extract structures for embedded boundaries amrex::FabArray const& eb_flag = eb_fact.getMultiEBCellFlagFab(); @@ -375,6 +373,9 @@ WarpX::MarkReducedShapeCells ( } + // FillBoundary to set the values in the guard cells + eb_reduce_particle_shape->FillBoundary(Geom(0).periodicity()); + } void From 3291a7fbce9a59e05c87cf6892015beaf58abeec Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 23 Jan 2025 06:34:33 -0800 Subject: [PATCH 81/99] Only call new kernel when compiling with EB --- .../Particles/Deposition/CurrentDeposition.H | 68 +++++++------- Source/Particles/WarpXParticleContainer.cpp | 90 ++++++------------- 2 files changed, 64 insertions(+), 94 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 6bc2ebb3f58..77630e810f3 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -638,6 +638,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. + * TODO: add documentation for eb_reduce_particle_shape */ template void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, @@ -656,10 +657,9 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, -#ifdef AMREX_USE_EB - const amrex::Array4& eb_reduce_particle_shape, -#endif - [[maybe_unused]]int n_rz_azimuthal_modes + [[maybe_unused]]int n_rz_azimuthal_modes, + const amrex::Array4 eb_reduce_particle_shape, + bool embedded_boundary_enabled ) { using namespace amrex; @@ -684,9 +684,14 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #endif // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr - amrex::ParallelFor( - np_to_deposit, - [=] AMREX_GPU_DEVICE (long const ip) { + + // (Compile 2 versions of the kernel: with and without embedded boundaries) + enum eb_flags : int { has_eb, no_eb }; + const int eb_runtime_flag = embedded_boundary_enabled ? has_eb : no_eb; + + amrex::ParallelFor( TypeList>{}, + {eb_runtime_flag}, + np_to_deposit, [=] AMREX_GPU_DEVICE (long ip, auto eb_control) { // --- Get particle quantities Real const gaminv = 1.0_rt/std::sqrt(1.0_rt + uxp[ip]*uxp[ip]*clightsq + uyp[ip]*uyp[ip]*clightsq @@ -741,35 +746,36 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, bool close_to_eb_old = false; bool close_to_eb_new = false; -#ifdef AMREX_USE_EB + + if constexpr (eb_control == has_eb) { // Check whether the particle is close to the EB at the old and new position #if defined(WARPX_DIM_3D) - close_to_eb_old = eb_reduce_particle_shape( - lo.x + int(x_old), - lo.y + int(y_old), - lo.z + int(z_old)); - close_to_eb_new = eb_reduce_particle_shape( - lo.x + int(x_new), - lo.y + int(y_new), - lo.z + int(z_new)); + close_to_eb_old = eb_reduce_particle_shape( + lo.x + int(x_old), + lo.y + int(y_old), + lo.z + int(z_old)); + close_to_eb_new = eb_reduce_particle_shape( + lo.x + int(x_new), + lo.y + int(y_new), + lo.z + int(z_new)); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - close_to_eb_old = eb_reduce_particle_shape( - lo.x + int(x_old), - lo.y + int(z_old), - 0); - close_to_eb_new = eb_reduce_particle_shape( - lo.x + int(x_new), - lo.y + int(z_new), - 0); + close_to_eb_old = eb_reduce_particle_shape( + lo.x + int(x_old), + lo.y + int(z_old), + 0); + close_to_eb_new = eb_reduce_particle_shape( + lo.x + int(x_new), + lo.y + int(z_new), + 0); #elif defined(WARPX_DIM_1D_Z) - close_to_eb_old = eb_reduce_particle_shape( - lo.x + int(z_old), - 0, 0); - close_to_eb_new = eb_reduce_particle_shape( - lo.x + int(z_new), - 0, 0); -#endif + close_to_eb_old = eb_reduce_particle_shape( + lo.x + int(z_old), + 0, 0); + close_to_eb_new = eb_reduce_particle_shape( + lo.x + int(z_new), + 0, 0); #endif + } #if defined(WARPX_DIM_RZ) Real const vy = (-uxp[ip]*sintheta_mid + uyp[ip]*costheta_mid)*gaminv; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index d0beee15a47..e7713c6d069 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -585,79 +585,43 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, else { if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { if (push_type == PushType::Explicit) { -#ifdef AMREX_USE_EB - if (EB::enabled()) - { - // signed distance function - auto const eb_reduce_particle_shape = (*warpx.GetEBReduceParticleShapeFlag()[lev])[pti].array(); - if (WarpX::nox == 1){ - doEsirkepovDepositionShapeN<1>( + auto const eb_reduce_particle_shape = (*warpx.GetEBReduceParticleShapeFlag()[lev])[pti].array(); + + if (WarpX::nox == 1){ + doEsirkepovDepositionShapeN<1>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, + eb_reduce_particle_shape, EB::enabled() ); + } else if (WarpX::nox == 2){ + doEsirkepovDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - eb_reduce_particle_shape, - WarpX::n_rz_azimuthal_modes); - } else if (WarpX::nox == 2){ - doEsirkepovDepositionShapeN<2>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, - np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - eb_reduce_particle_shape, - WarpX::n_rz_azimuthal_modes); - } else if (WarpX::nox == 3){ - doEsirkepovDepositionShapeN<3>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, - np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - eb_reduce_particle_shape, - WarpX::n_rz_azimuthal_modes); - } else if (WarpX::nox == 4){ - doEsirkepovDepositionShapeN<4>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, - np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - eb_reduce_particle_shape, - WarpX::n_rz_azimuthal_modes); - } - } -#else - { - if (WarpX::nox == 1){ - doEsirkepovDepositionShapeN<1>( + WarpX::n_rz_azimuthal_modes, + eb_reduce_particle_shape, EB::enabled() ); + } else if (WarpX::nox == 3){ + doEsirkepovDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes); - } else if (WarpX::nox == 2){ - doEsirkepovDepositionShapeN<2>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, - np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes); - } else if (WarpX::nox == 3){ - doEsirkepovDepositionShapeN<3>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, - np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes); - } else if (WarpX::nox == 4){ - doEsirkepovDepositionShapeN<4>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, - np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes); - } + WarpX::n_rz_azimuthal_modes, + eb_reduce_particle_shape, EB::enabled() ); + } else if (WarpX::nox == 4){ + doEsirkepovDepositionShapeN<4>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, + eb_reduce_particle_shape, EB::enabled() ); } -#endif //AMREX_USE_EB + } else if (push_type == PushType::Implicit) { #if (AMREX_SPACEDIM >= 2) auto& xp_n = pti.GetAttribs(particle_comps["x_n"]); From 152955814b07c3a579f2db53f0aad4c6795dbb6d Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 23 Jan 2025 09:00:14 -0800 Subject: [PATCH 82/99] Fix errors when EB are not present --- Source/Particles/Deposition/CurrentDeposition.H | 2 +- Source/Particles/WarpXParticleContainer.cpp | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 77630e810f3..a24221ca24d 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -658,7 +658,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, amrex::Dim3 lo, amrex::Real q, [[maybe_unused]]int n_rz_azimuthal_modes, - const amrex::Array4 eb_reduce_particle_shape, + const amrex::Array4& eb_reduce_particle_shape, bool embedded_boundary_enabled ) { diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index e7713c6d069..c60f54ba02d 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -586,7 +586,10 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { if (push_type == PushType::Explicit) { - auto const eb_reduce_particle_shape = (*warpx.GetEBReduceParticleShapeFlag()[lev])[pti].array(); + amrex::Array4 eb_reduce_particle_shape; + if (EB::enabled()) { + eb_reduce_particle_shape = (*warpx.GetEBReduceParticleShapeFlag()[lev])[pti].array(); + } if (WarpX::nox == 1){ doEsirkepovDepositionShapeN<1>( From b79d9d8c4d1dd7376d531504f5b93c384699d0f9 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 23 Jan 2025 09:03:35 -0800 Subject: [PATCH 83/99] Do not call more expensive kernel for shape > 1 --- Source/Particles/Deposition/CurrentDeposition.H | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index a24221ca24d..ee50d4daa75 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -687,7 +687,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // (Compile 2 versions of the kernel: with and without embedded boundaries) enum eb_flags : int { has_eb, no_eb }; - const int eb_runtime_flag = embedded_boundary_enabled ? has_eb : no_eb; + const int eb_runtime_flag = (embedded_boundary_enabled && (depos_order>1))? has_eb : no_eb; amrex::ParallelFor( TypeList>{}, {eb_runtime_flag}, @@ -744,11 +744,10 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, double const z_new = (zp - xyzmin.z + (relative_time + 0.5_rt*dt)*uzp[ip]*gaminv)*dinv.z; double const z_old = z_new - dt*dinv.z*uzp[ip]*gaminv; + // Check whether the particle is close to the EB at the old and new position bool close_to_eb_old = false; bool close_to_eb_new = false; - if constexpr (eb_control == has_eb) { - // Check whether the particle is close to the EB at the old and new position #if defined(WARPX_DIM_3D) close_to_eb_old = eb_reduce_particle_shape( lo.x + int(x_old), From 2592afe067207620422196e86e7fdca1a1d8671a Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 23 Jan 2025 10:17:17 -0800 Subject: [PATCH 84/99] Update const-ness --- Source/Particles/WarpXParticleContainer.cpp | 2 +- Source/WarpX.H | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index c60f54ba02d..21b76485907 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -370,7 +370,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, if (do_not_deposit) { return; } // Number of guard cells for local deposition of J - WarpX& warpx = WarpX::GetInstance(); + const WarpX& warpx = WarpX::GetInstance(); const amrex::IntVect& ng_J = warpx.get_ng_depos_J(); diff --git a/Source/WarpX.H b/Source/WarpX.H index 2781deb4b72..375bbebb3df 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -164,7 +164,7 @@ public: MultiDiagnostics& GetMultiDiags () {return *multi_diags;} ParticleBoundaryBuffer& GetParticleBoundaryBuffer () { return *m_particle_boundary_buffer; } amrex::Vector,3 > >& GetEBUpdateEFlag() { return m_eb_update_E; } - amrex::Vector< std::unique_ptr >& GetEBReduceParticleShapeFlag() { return m_eb_reduce_particle_shape; } + amrex::Vector< std::unique_ptr > const & GetEBReduceParticleShapeFlag() const { return m_eb_reduce_particle_shape; } static void shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, int num_shift, int dir, int lev, bool update_cost_flag, From 7c216659ba9c96654001f4a3caec3a893a717c76 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 23 Jan 2025 10:46:17 -0800 Subject: [PATCH 85/99] Add checksum files --- ...ry_em_particle_absorption_sh_factor_2.json | 24 +++++++++++++++++++ ...ry_em_particle_absorption_sh_factor_3.json | 24 +++++++++++++++++++ ...ry_em_particle_absorption_sh_factor_2.json | 24 +++++++++++++++++++ ...ry_em_particle_absorption_sh_factor_3.json | 24 +++++++++++++++++++ ...ry_em_particle_absorption_sh_factor_2.json | 24 +++++++++++++++++++ ...ry_em_particle_absorption_sh_factor_3.json | 24 +++++++++++++++++++ 6 files changed, 144 insertions(+) create mode 100644 Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_2.json create mode 100644 Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_3.json create mode 100644 Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_2.json create mode 100644 Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_3.json create mode 100644 Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_2.json create mode 100644 Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_3.json diff --git a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_2.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_2.json new file mode 100644 index 00000000000..de3d125c744 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_2.json @@ -0,0 +1,24 @@ +{ + "lev=0": { + "divE": 3.059581906777539e-08, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_3.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_3.json new file mode 100644 index 00000000000..de3d125c744 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_3.json @@ -0,0 +1,24 @@ +{ + "lev=0": { + "divE": 3.059581906777539e-08, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_2.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_2.json new file mode 100644 index 00000000000..d3e08d9723e --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_2.json @@ -0,0 +1,24 @@ +{ + "lev=0": { + "divE": 4.928354322096152e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_3.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_3.json new file mode 100644 index 00000000000..d3e08d9723e --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_3.json @@ -0,0 +1,24 @@ +{ + "lev=0": { + "divE": 4.928354322096152e-07, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_2.json b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_2.json new file mode 100644 index 00000000000..30d7d0ba081 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_2.json @@ -0,0 +1,24 @@ +{ + "lev=0": { + "divE": 1.4599714697029335e-08, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_3.json b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_3.json new file mode 100644 index 00000000000..30d7d0ba081 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_3.json @@ -0,0 +1,24 @@ +{ + "lev=0": { + "divE": 1.4599714697029335e-08, + "rho": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "positron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} \ No newline at end of file From a0e7c25c65612064d7ba84936c0e68cd8a477114 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 23 Jan 2025 11:17:27 -0800 Subject: [PATCH 86/99] Update syntax for shape factors --- Source/Particles/Deposition/CurrentDeposition.H | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index ee50d4daa75..60bc1b91a18 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -790,6 +790,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // [ijk]_new: leftmost grid point that the particle touches const Compute_shape_factor< depos_order > compute_shape_factor; const Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; + const Compute_shifted_shape_factor< 1 > compute_shifted_shape_factor_order1; // Shape factor arrays // Note that there are extra values above and below @@ -799,8 +800,17 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #if !defined(WARPX_DIM_1D_Z) double sx_new[depos_order + 3] = {0.}; double sx_old[depos_order + 3] = {0.}; - const int i_new = compute_shape_factor(sx_new+1, x_new, close_to_eb_new); - const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new, close_to_eb_old); + const int i_new = compute_shape_factor(sx_new+1, x_new ); + const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new); + // If particle is close to the embedded boundary, recompute deposition with order 1 shape + if (close_to_eb_new) { + for (int i=0; i Date: Thu, 23 Jan 2025 11:21:54 -0800 Subject: [PATCH 87/99] Update deposition kernel --- .../Particles/Deposition/CurrentDeposition.H | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 60bc1b91a18..349be1c296c 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -804,24 +804,42 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new); // If particle is close to the embedded boundary, recompute deposition with order 1 shape if (close_to_eb_new) { - for (int i=0; i Date: Thu, 23 Jan 2025 11:25:04 -0800 Subject: [PATCH 88/99] Use original shape factor functions --- Source/Particles/ShapeFactors.H | 112 ++++++++++---------------------- 1 file changed, 33 insertions(+), 79 deletions(-) diff --git a/Source/Particles/ShapeFactors.H b/Source/Particles/ShapeFactors.H index b51623f1cc2..e711cce15ff 100644 --- a/Source/Particles/ShapeFactors.H +++ b/Source/Particles/ShapeFactors.H @@ -31,8 +31,7 @@ struct Compute_shape_factor AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE int operator()( T* const sx, - T xmid, - bool const close_to_eb=false) const + T xmid) const { if constexpr (depos_order == 0){ const auto j = static_cast(xmid + T(0.5)); @@ -48,52 +47,31 @@ struct Compute_shape_factor } else if constexpr (depos_order == 2){ const auto j = static_cast(xmid + T(0.5)); - if (close_to_eb) { - // Use order 1 shape function - const auto j1 = static_cast(xmid); - const T xint = xmid - T(j1); - sx[1+j1-j] = T(1.0) - xint; - sx[2+j1-j] = xint; - } else { - const T xint = xmid - T(j); - sx[0] = T(0.5)*(T(0.5) - xint)*(T(0.5) - xint); - sx[1] = T(0.75) - xint*xint; - sx[2] = T(0.5)*(T(0.5) + xint)*(T(0.5) + xint); - } + const T xint = xmid - T(j); + sx[0] = T(0.5)*(T(0.5) - xint)*(T(0.5) - xint); + sx[1] = T(0.75) - xint*xint; + sx[2] = T(0.5)*(T(0.5) + xint)*(T(0.5) + xint); // index of the leftmost cell where particle deposits return j-1; } else if constexpr (depos_order == 3){ const auto j = static_cast(xmid); const T xint = xmid - T(j); - if (close_to_eb) { - sx[1] = T(1.0) - xint; - sx[2] = xint; - } else { - sx[0] = (T(1.0))/(T(6.0))*(T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - xint); - sx[1] = (T(2.0))/(T(3.0)) - xint*xint*(T(1.0) - xint/(T(2.0))); - sx[2] = (T(2.0))/(T(3.0)) - (T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - T(0.5)*(T(1.0) - xint)); - sx[3] = (T(1.0))/(T(6.0))*xint*xint*xint; - } + sx[0] = (T(1.0))/(T(6.0))*(T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - xint); + sx[1] = (T(2.0))/(T(3.0)) - xint*xint*(T(1.0) - xint/(T(2.0))); + sx[2] = (T(2.0))/(T(3.0)) - (T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - T(0.5)*(T(1.0) - xint)); + sx[3] = (T(1.0))/(T(6.0))*xint*xint*xint; // index of the leftmost cell where particle deposits return j-1; } else if constexpr (depos_order == 4){ const auto j = static_cast(xmid + T(0.5)); - if (close_to_eb) { - // Use order 1 shape function - const auto j1 = static_cast(xmid); - const T xint = xmid - T(j1); - sx[2+j1-j] = T(1.0) - xint; - sx[3+j1-j] = xint; - } else { - const T xint = xmid - T(j); - sx[0] = (T(1.0))/(T(24.0))*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint); - sx[1] = (T(1.0))/(T(24.0))*(T(4.75) - T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) + xint - xint*xint)); - sx[2] = (T(1.0))/(T(24.0))*(T(14.375) + T(6.0)*xint*xint*(xint*xint - T(2.5))); - sx[3] = (T(1.0))/(T(24.0))*(T(4.75) + T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) - xint - xint*xint)); - sx[4] = (T(1.0))/(T(24.0))*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5)+xint); - } + const T xint = xmid - T(j); + sx[0] = (T(1.0))/(T(24.0))*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint); + sx[1] = (T(1.0))/(T(24.0))*(T(4.75) - T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) + xint - xint*xint)); + sx[2] = (T(1.0))/(T(24.0))*(T(14.375) + T(6.0)*xint*xint*(xint*xint - T(2.5))); + sx[3] = (T(1.0))/(T(24.0))*(T(4.75) + T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) - xint - xint*xint)); + sx[4] = (T(1.0))/(T(24.0))*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5)+xint); // index of the leftmost cell where particle deposits return j-2; } @@ -120,8 +98,7 @@ struct Compute_shifted_shape_factor int operator()( T* const sx, const T x_old, - const int i_new, - bool const close_to_eb=false) const + const int i_new) const { if constexpr (depos_order == 0){ const auto i = static_cast(std::floor(x_old + T(0.5))); @@ -139,20 +116,11 @@ struct Compute_shifted_shape_factor } else if constexpr (depos_order == 2){ const auto i = static_cast(x_old + T(0.5)); - if (close_to_eb) { - // Use order 1 shape function - const auto i1 = static_cast(x_old); - const int i_shift = i1 - (i_new + 1); - const T xint = x_old - T(i1); - sx[2+i_shift] = T(1.0) - xint; - sx[3+i_shift] = xint; - } else { - const int i_shift = i - (i_new + 1); - const T xint = x_old - T(i); - sx[1+i_shift] = T(0.5)*(T(0.5) - xint)*(T(0.5) - xint); - sx[2+i_shift] = T(0.75) - xint*xint; - sx[3+i_shift] = T(0.5)*(T(0.5) + xint)*(T(0.5) + xint); - } + const int i_shift = i - (i_new + 1); + const T xint = x_old - T(i); + sx[1+i_shift] = T(0.5)*(T(0.5) - xint)*(T(0.5) - xint); + sx[2+i_shift] = T(0.75) - xint*xint; + sx[3+i_shift] = T(0.5)*(T(0.5) + xint)*(T(0.5) + xint); // index of the leftmost cell where particle deposits return i - 1; } @@ -160,37 +128,23 @@ struct Compute_shifted_shape_factor const auto i = static_cast(x_old); const int i_shift = i - (i_new + 1); const T xint = x_old - T(i); - if (close_to_eb) { - sx[2+i_shift] = T(1.0) - xint; - sx[3+i_shift] = xint; - } else { - sx[1+i_shift] = (T(1.0))/(T(6.0))*(T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - xint); - sx[2+i_shift] = (T(2.0))/(T(3.0)) - xint*xint*(T(1.0) - xint/(T(2.0))); - sx[3+i_shift] = (T(2.0))/(T(3.0)) - (T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - T(0.5)*(T(1.0) - xint)); - sx[4+i_shift] = (T(1.0))/(T(6.0))*xint*xint*xint; - } + sx[1+i_shift] = (T(1.0))/(T(6.0))*(T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - xint); + sx[2+i_shift] = (T(2.0))/(T(3.0)) - xint*xint*(T(1.0) - xint/(T(2.0))); + sx[3+i_shift] = (T(2.0))/(T(3.0)) - (T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - T(0.5)*(T(1.0) - xint)); + sx[4+i_shift] = (T(1.0))/(T(6.0))*xint*xint*xint; // index of the leftmost cell where particle deposits return i - 1; } else if constexpr (depos_order == 4){ const auto i = static_cast(x_old + T(0.5)); - if (close_to_eb) { - // Use order 1 shape function - const auto i1 = static_cast(x_old); - const int i_shift = i1 - (i_new + 2); - const T xint = x_old - T(i1); - sx[3+i_shift] = T(1.0) - xint; - sx[4+i_shift] = xint; - } else { - const int i_shift = i - (i_new + 2); - const T xint = x_old - T(i); - sx[1+i_shift] = (T(1.0))/(T(24.0))*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint); - sx[2+i_shift] = (T(1.0))/(T(24.0))*(T(4.75) - T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) + xint - xint*xint)); - sx[3+i_shift] = (T(1.0))/(T(24.0))*(T(14.375) + T(6.0)*xint*xint*(xint*xint - T(2.5))); - sx[4+i_shift] = (T(1.0))/(T(24.0))*(T(4.75) + T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) - xint - xint*xint)); - sx[5+i_shift] = (T(1.0))/(T(24.0))*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5)+xint); - } - // index of the leftmost cell where particle deposits + const int i_shift = i - (i_new + 2); + const T xint = x_old - T(i); + sx[1+i_shift] = (T(1.0))/(T(24.0))*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint); + sx[2+i_shift] = (T(1.0))/(T(24.0))*(T(4.75) - T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) + xint - xint*xint)); + sx[3+i_shift] = (T(1.0))/(T(24.0))*(T(14.375) + T(6.0)*xint*xint*(xint*xint - T(2.5))); + sx[4+i_shift] = (T(1.0))/(T(24.0))*(T(4.75) + T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) - xint - xint*xint)); + sx[5+i_shift] = (T(1.0))/(T(24.0))*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5)+xint); + // index of the leftmost cell where particle deposits return i - 2; } else{ From 2903e4a37c9b8abb719ffdc88915c05d3ce967d5 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 23 Jan 2025 11:29:01 -0800 Subject: [PATCH 89/99] Do not compile modified shape factor if unused --- .../Particles/Deposition/CurrentDeposition.H | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 349be1c296c..0e8e3c835b4 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -790,6 +790,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // [ijk]_new: leftmost grid point that the particle touches const Compute_shape_factor< depos_order > compute_shape_factor; const Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; + // When using special treatment close to the embedded boundary, we need order 1 deposition const Compute_shifted_shape_factor< 1 > compute_shifted_shape_factor_order1; // Shape factor arrays @@ -803,13 +804,15 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, const int i_new = compute_shape_factor(sx_new+1, x_new ); const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new); // If particle is close to the embedded boundary, recompute deposition with order 1 shape - if (close_to_eb_new) { - for (int i=0; i& GetPosition, const int j_new = compute_shape_factor(sy_new+1, y_new); const int j_old = compute_shifted_shape_factor(sy_old, y_old, j_new); // If particle is close to the embedded boundary, recompute deposition with order 1 shape - if (close_to_eb_new) { - for (int j=0; j& GetPosition, const int k_new = compute_shape_factor(sz_new+1, z_new ); const int k_old = compute_shifted_shape_factor(sz_old, z_old, k_new ); // If particle is close to the embedded boundary, recompute deposition with order 1 shape - if (close_to_eb_new) { - for (int k=0; k Date: Thu, 23 Jan 2025 11:40:34 -0800 Subject: [PATCH 90/99] Apply suggestions from code review --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 28121d42d3e..0640ef8c537 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -296,9 +296,9 @@ WarpX::MarkReducedShapeCells ( std::unique_ptr & eb_reduce_particle_shape, amrex::EBFArrayBoxFactory const & eb_fact ) { - // Pre-fill array with 1, including in the ghost cells outside of the domain. + // Pre-fill array with 0, including in the ghost cells outside of the domain. // (The guard cells in the domain will be updated by `FillBoundary` at the end of this function.) - eb_reduce_particle_shape->setVal(1, eb_reduce_particle_shape->nGrow()); + eb_reduce_particle_shape->setVal(0, eb_reduce_particle_shape->nGrow()); // Extract structures for embedded boundaries amrex::FabArray const& eb_flag = eb_fact.getMultiEBCellFlagFab(); From 378efb9b472b69f6d02d16ee744a4ea3c3877275 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 24 Jan 2025 06:08:06 -0800 Subject: [PATCH 91/99] Fix GPU compilation --- Source/Particles/Deposition/CurrentDeposition.H | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 0e8e3c835b4..a05195a313c 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -747,6 +747,8 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // Check whether the particle is close to the EB at the old and new position bool close_to_eb_old = false; bool close_to_eb_new = false; + amrex::ignore_unused(eb_reduce_particle_shape); // Needed to avoid compilation error with nvcc + amrex::ignore_unused(lo); // Needed to avoid compilation error with nvcc if constexpr (eb_control == has_eb) { #if defined(WARPX_DIM_3D) close_to_eb_old = eb_reduce_particle_shape( From 9aa5272360c7b5ddcf1776ad4f2da750920a6f8b Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 24 Jan 2025 06:34:13 -0800 Subject: [PATCH 92/99] Correct error in analysis script --- .../Tests/embedded_boundary_em_particle_absorption/analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Examples/Tests/embedded_boundary_em_particle_absorption/analysis.py b/Examples/Tests/embedded_boundary_em_particle_absorption/analysis.py index 7647c23d846..3ba44a8ac1b 100755 --- a/Examples/Tests/embedded_boundary_em_particle_absorption/analysis.py +++ b/Examples/Tests/embedded_boundary_em_particle_absorption/analysis.py @@ -39,7 +39,7 @@ elif dim == "thetaMode": # In RZ: there are issues with divE on axis # Set the few cells around the axis to 0 for this test - divE_avg[13:19] = 0 + divE_avg[:, 13:19] = 0 tolerance = 4e-12 From e4205334dbf5ed0c8837eeac25fbf5e73d64e3a5 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 24 Jan 2025 08:43:33 -0800 Subject: [PATCH 93/99] Update checksum --- ...2d_embedded_boundary_em_particle_absorption_sh_factor_2.json | 2 +- ...2d_embedded_boundary_em_particle_absorption_sh_factor_3.json | 2 +- ...3d_embedded_boundary_em_particle_absorption_sh_factor_3.json | 2 +- ...rz_embedded_boundary_em_particle_absorption_sh_factor_3.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_2.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_2.json index de3d125c744..481aba1f36b 100644 --- a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_2.json +++ b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_2.json @@ -1,6 +1,6 @@ { "lev=0": { - "divE": 3.059581906777539e-08, + "divE": 2.4521053721245334e-08, "rho": 0.0 }, "electron": { diff --git a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_3.json b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_3.json index de3d125c744..82b6c6849ac 100644 --- a/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_3.json +++ b/Regression/Checksum/benchmarks_json/test_2d_embedded_boundary_em_particle_absorption_sh_factor_3.json @@ -1,6 +1,6 @@ { "lev=0": { - "divE": 3.059581906777539e-08, + "divE": 2.2059346534892452e-08, "rho": 0.0 }, "electron": { diff --git a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_3.json b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_3.json index d3e08d9723e..23c03c7e7bc 100644 --- a/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_3.json +++ b/Regression/Checksum/benchmarks_json/test_3d_embedded_boundary_em_particle_absorption_sh_factor_3.json @@ -1,6 +1,6 @@ { "lev=0": { - "divE": 4.928354322096152e-07, + "divE": 4.3355127342920327e-07, "rho": 0.0 }, "electron": { diff --git a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_3.json b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_3.json index 30d7d0ba081..76baf73cc3a 100644 --- a/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_3.json +++ b/Regression/Checksum/benchmarks_json/test_rz_embedded_boundary_em_particle_absorption_sh_factor_3.json @@ -1,6 +1,6 @@ { "lev=0": { - "divE": 1.4599714697029335e-08, + "divE": 1.3292471881599093e-08, "rho": 0.0 }, "electron": { From fd180f52b0cdeb0ba27d84334672a486df5a523d Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 24 Jan 2025 09:15:01 -0800 Subject: [PATCH 94/99] Clarify code --- .../Particles/Deposition/CurrentDeposition.H | 57 ++++++++++--------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index a05195a313c..7809ded50ca 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -638,7 +638,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * TODO: add documentation for eb_reduce_particle_shape + * TODO: add documentation for reduced_particle_shape_mask */ template void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, @@ -658,8 +658,8 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, amrex::Dim3 lo, amrex::Real q, [[maybe_unused]]int n_rz_azimuthal_modes, - const amrex::Array4& eb_reduce_particle_shape, - bool embedded_boundary_enabled + const amrex::Array4& reduced_particle_shape_mask, + bool enable_reduced_shape ) { using namespace amrex; @@ -686,12 +686,12 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr // (Compile 2 versions of the kernel: with and without embedded boundaries) - enum eb_flags : int { has_eb, no_eb }; - const int eb_runtime_flag = (embedded_boundary_enabled && (depos_order>1))? has_eb : no_eb; + enum eb_flags : int { has_reduced_shape, no_reduced_shape }; + const int reduce_shape_runtime_flag = (enable_reduced_shape && (depos_order>1))? has_reduced_shape : no_reduced_shape; - amrex::ParallelFor( TypeList>{}, - {eb_runtime_flag}, - np_to_deposit, [=] AMREX_GPU_DEVICE (long ip, auto eb_control) { + amrex::ParallelFor( TypeList>{}, + {reduce_shape_runtime_flag}, + np_to_deposit, [=] AMREX_GPU_DEVICE (long ip, auto reduce_shape_control) { // --- Get particle quantities Real const gaminv = 1.0_rt/std::sqrt(1.0_rt + uxp[ip]*uxp[ip]*clightsq + uyp[ip]*uyp[ip]*clightsq @@ -745,34 +745,34 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, double const z_old = z_new - dt*dinv.z*uzp[ip]*gaminv; // Check whether the particle is close to the EB at the old and new position - bool close_to_eb_old = false; - bool close_to_eb_new = false; - amrex::ignore_unused(eb_reduce_particle_shape); // Needed to avoid compilation error with nvcc + bool reduce_shape_old = false; + bool reduce_shape_new = false; + amrex::ignore_unused(reduced_particle_shape_mask); // Needed to avoid compilation error with nvcc amrex::ignore_unused(lo); // Needed to avoid compilation error with nvcc - if constexpr (eb_control == has_eb) { + if constexpr (reduce_shape_control == has_reduced_shape) { #if defined(WARPX_DIM_3D) - close_to_eb_old = eb_reduce_particle_shape( + reduce_shape_old = reduced_particle_shape_mask( lo.x + int(x_old), lo.y + int(y_old), lo.z + int(z_old)); - close_to_eb_new = eb_reduce_particle_shape( + reduce_shape_new = reduced_particle_shape_mask( lo.x + int(x_new), lo.y + int(y_new), lo.z + int(z_new)); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - close_to_eb_old = eb_reduce_particle_shape( + reduce_shape_old = reduced_particle_shape_mask( lo.x + int(x_old), lo.y + int(z_old), 0); - close_to_eb_new = eb_reduce_particle_shape( + reduce_shape_new = reduced_particle_shape_mask( lo.x + int(x_new), lo.y + int(z_new), 0); #elif defined(WARPX_DIM_1D_Z) - close_to_eb_old = eb_reduce_particle_shape( + reduce_shape_old = reduced_particle_shape_mask( lo.x + int(z_old), 0, 0); - close_to_eb_new = eb_reduce_particle_shape( + reduce_shape_new = reduced_particle_shape_mask( lo.x + int(z_new), 0, 0); #endif @@ -792,8 +792,9 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // [ijk]_new: leftmost grid point that the particle touches const Compute_shape_factor< depos_order > compute_shape_factor; const Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; - // When using special treatment close to the embedded boundary, we need order 1 deposition + // In cells marked by reduced_particle_shape_mask, we need order 1 deposition const Compute_shifted_shape_factor< 1 > compute_shifted_shape_factor_order1; + amrex::ignore_unused(compute_shifted_shape_factor_order1); // unused for `no_reduced_shape` // Shape factor arrays // Note that there are extra values above and below @@ -806,12 +807,12 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, const int i_new = compute_shape_factor(sx_new+1, x_new ); const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new); // If particle is close to the embedded boundary, recompute deposition with order 1 shape - if constexpr (eb_control == has_eb) { - if (close_to_eb_new) { + if constexpr (reduce_shape_control == has_reduced_shape) { + if (reduce_shape_new) { for (int i=0; i& GetPosition, const int j_new = compute_shape_factor(sy_new+1, y_new); const int j_old = compute_shifted_shape_factor(sy_old, y_old, j_new); // If particle is close to the embedded boundary, recompute deposition with order 1 shape - if constexpr (eb_control == has_eb) { - if (close_to_eb_new) { + if constexpr (reduce_shape_control == has_reduced_shape) { + if (reduce_shape_new) { for (int j=0; j& GetPosition, const int k_new = compute_shape_factor(sz_new+1, z_new ); const int k_old = compute_shifted_shape_factor(sz_old, z_old, k_new ); // If particle is close to the embedded boundary, recompute deposition with order 1 shape - if constexpr (eb_control == has_eb) { - if (close_to_eb_new) { + if constexpr (reduce_shape_control == has_reduced_shape) { + if (reduce_shape_new) { for (int k=0; k Date: Fri, 24 Jan 2025 10:53:11 -0800 Subject: [PATCH 95/99] Add more documentation --- .../Particles/Deposition/CurrentDeposition.H | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 7809ded50ca..d5ce971716f 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -810,12 +810,14 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, if constexpr (reduce_shape_control == has_reduced_shape) { if (reduce_shape_new) { for (int i=0; i& GetPosition, if constexpr (reduce_shape_control == has_reduced_shape) { if (reduce_shape_new) { for (int j=0; j& GetPosition, if constexpr (reduce_shape_control == has_reduced_shape) { if (reduce_shape_new) { for (int k=0; k Date: Fri, 24 Jan 2025 11:23:11 -0800 Subject: [PATCH 96/99] Code refactoring --- Source/EmbeddedBoundary/WarpXInitEB.cpp | 22 ++++++++------ Source/Initialization/WarpXInitData.cpp | 2 +- .../Particles/Deposition/CurrentDeposition.H | 29 ++++++++++--------- Source/WarpX.H | 3 +- 4 files changed, 31 insertions(+), 25 deletions(-) diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 0640ef8c537..271f12231b0 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -294,7 +294,8 @@ WarpX::ScaleAreas (ablastr::fields::VectorField& face_areas, void WarpX::MarkReducedShapeCells ( std::unique_ptr & eb_reduce_particle_shape, - amrex::EBFArrayBoxFactory const & eb_fact ) + amrex::EBFArrayBoxFactory const & eb_fact, + int const particle_shape_order ) { // Pre-fill array with 0, including in the ghost cells outside of the domain. // (The guard cells in the domain will be updated by `FillBoundary` at the end of this function.) @@ -335,22 +336,25 @@ WarpX::MarkReducedShapeCells ( amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - // Check if any of the neighboring cells are either partially or fully covered - // In this case, reduce particle shape to order 1 + // Check if any of the neighboring cells over which the particle shape might extend + // are either partially or fully covered. In this case, set eb_reduce_particle_shape_arr + // to one for this cell, to indicate that the particle should use an order 1 shape // (This ensures that the particle never deposits any charge in a partially or // fully covered cell, even with higher-order shapes) - int const i_start = i-1; - int const i_end = i+1; + // Note: in the code below `particle_shape_order/2` corresponds to the number of neighboring cells + // over which the shape factor could extend, in each direction. + int const i_start = i-particle_shape_order/2; + int const i_end = i+particle_shape_order/2; #if AMREX_SPACEDIM > 1 - int const j_start = j-1; - int const j_end = j+1; + int const j_start = j-particle_shape_order/2; + int const j_end = j+particle_shape_order/2; #else int const j_start = j; int const j_end = j; #endif #if AMREX_SPACEDIM > 2 - int const k_start = k-1; - int const k_end = k+1; + int const k_start = k-particle_shape_order/2; + int const k_end = k+particle_shape_order/2; #else int const k_start = k; int const k_end = k; diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 0a6934cfd10..f9f70e5a9e0 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1268,7 +1268,7 @@ void WarpX::InitializeEBGridData (int lev) } ComputeDistanceToEB(); - MarkReducedShapeCells( m_eb_reduce_particle_shape[lev], eb_fact ); + MarkReducedShapeCells( m_eb_reduce_particle_shape[lev], eb_fact, WarpX::nox ); } #else diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index d5ce971716f..980edea4649 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -747,33 +747,34 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // Check whether the particle is close to the EB at the old and new position bool reduce_shape_old = false; bool reduce_shape_new = false; - amrex::ignore_unused(reduced_particle_shape_mask); // Needed to avoid compilation error with nvcc - amrex::ignore_unused(lo); // Needed to avoid compilation error with nvcc +#ifdef AMREX_USE_CUDA + amrex::ignore_unused(reduced_particle_shape_mask, lo); // Needed to avoid compilation error with nvcc +#endif if constexpr (reduce_shape_control == has_reduced_shape) { #if defined(WARPX_DIM_3D) reduce_shape_old = reduced_particle_shape_mask( - lo.x + int(x_old), - lo.y + int(y_old), - lo.z + int(z_old)); + lo.x + int(amrex::Math::floor(x_old)), + lo.y + int(amrex::Math::floor(y_old)), + lo.z + int(amrex::Math::floor(z_old))); reduce_shape_new = reduced_particle_shape_mask( - lo.x + int(x_new), - lo.y + int(y_new), - lo.z + int(z_new)); + lo.x + int(amrex::Math::floor(x_new)), + lo.y + int(amrex::Math::floor(y_new)), + lo.z + int(amrex::Math::floor(z_new))); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) reduce_shape_old = reduced_particle_shape_mask( - lo.x + int(x_old), - lo.y + int(z_old), + lo.x + int(amrex::Math::floor(x_old)), + lo.y + int(amrex::Math::floor(z_old)), 0); reduce_shape_new = reduced_particle_shape_mask( - lo.x + int(x_new), - lo.y + int(z_new), + lo.x + int(amrex::Math::floor(x_new)), + lo.y + int(amrex::Math::floor(z_new)), 0); #elif defined(WARPX_DIM_1D_Z) reduce_shape_old = reduced_particle_shape_mask( - lo.x + int(z_old), + lo.x + int(amrex::Math::floor(z_old)), 0, 0); reduce_shape_new = reduced_particle_shape_mask( - lo.x + int(z_new), + lo.x + int(amrex::Math::floor(z_new)), 0, 0); #endif } diff --git a/Source/WarpX.H b/Source/WarpX.H index 375bbebb3df..1bd87f30aab 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1011,7 +1011,8 @@ public: /** TODO */ void MarkReducedShapeCells ( std::unique_ptr & eb_reduce_particle_shape, - amrex::EBFArrayBoxFactory const & eb_fact ); + amrex::EBFArrayBoxFactory const & eb_fact, + int const particle_shape_order ); /** \brief Set a flag to indicate on which grid points the field `field` * should be updated, depending on their position relative to the embedded boundary. From 8665c30cac42a40466fbd05d5f22ee296882f426 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 24 Jan 2025 13:26:10 -0800 Subject: [PATCH 97/99] Update documentation --- .../Particles/Deposition/CurrentDeposition.H | 6 +++-- Source/WarpX.H | 22 +++++++++++++++---- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 980edea4649..3900f28d417 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -638,7 +638,9 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * TODO: add documentation for reduced_particle_shape_mask + * \param reduced_particle_shape_mask Array4 of int, Mask that indicates whether a particle + * should use its regular shape factor or a reduced, order-1 shape factor instead in a given cell. + * \param enable_reduced_shape Flag to indicate whether to use the reduced shape factor */ template void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, @@ -685,7 +687,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr - // (Compile 2 versions of the kernel: with and without embedded boundaries) + // (Compile 2 versions of the kernel: with and without reduced shape) enum eb_flags : int { has_reduced_shape, no_reduced_shape }; const int reduce_shape_runtime_flag = (enable_reduced_shape && (depos_order>1))? has_reduced_shape : no_reduced_shape; diff --git a/Source/WarpX.H b/Source/WarpX.H index 1bd87f30aab..d2a9b75f2ce 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1008,7 +1008,20 @@ public: #ifdef AMREX_USE_EB - /** TODO */ + /** \brief Set a flag to indicate in which cells a particle should deposit charge/current + * with a reduced, order 1 shape. + * + * More specifically, the flag is set to 1 if any of the neighboring cells over which the + * particle shape might extend are either partially or fully covered by an embedded boundary. + * This ensures that a particle in this cell deposits with an order 1 shape, which in turn + * makes sure that the particle never deposits any charge in a partially or fully covered cell. + * + * \param[in] eb_reduce_particle_shape multifab to be filled with 1s and 0s + * \param[in] eb_fact EB factory + * \param[in] particle_shape_order order of the particle shape function + */ + + void MarkReducedShapeCells ( std::unique_ptr & eb_reduce_particle_shape, amrex::EBFArrayBoxFactory const & eb_fact, @@ -1435,9 +1448,10 @@ private: amrex::Vector,3 > > m_eb_update_E; amrex::Vector,3 > > m_eb_update_B; - /** EB: Flag to indicate whether particles should use order-1 shape factor in a given cell, - * when depositing charge/current. This is needed with the EM solver, in order to avoid - * error in charge conservation when a particle is too close to the embedded boundary. + /** EB: Mask that indicates whether a particle should use its regular shape factor (mask set to 0) + * or a reduced, order-1 shape factor instead (mask set to 1) in a given cell, when depositing charge/current. + * The flag is typically set to 1 in cells that are close to the embedded boundary, in order to avoid + * errors in charge conservation when a particle is too close to the embedded boundary. */ amrex::Vector< std::unique_ptr > m_eb_reduce_particle_shape; From 39214e3556c3e8e61fc5d1474a87dcc24355f681 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 27 Jan 2025 14:37:05 -0800 Subject: [PATCH 98/99] Minor refactor --- .../Particles/Deposition/CurrentDeposition.H | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 3900f28d417..bc870257d8f 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -659,7 +659,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, - [[maybe_unused]]int n_rz_azimuthal_modes, + [[maybe_unused]] int n_rz_azimuthal_modes, const amrex::Array4& reduced_particle_shape_mask, bool enable_reduced_shape ) @@ -747,8 +747,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, double const z_old = z_new - dt*dinv.z*uzp[ip]*gaminv; // Check whether the particle is close to the EB at the old and new position - bool reduce_shape_old = false; - bool reduce_shape_new = false; + bool reduce_shape_old, reduce_shape_new; #ifdef AMREX_USE_CUDA amrex::ignore_unused(reduced_particle_shape_mask, lo); // Needed to avoid compilation error with nvcc #endif @@ -779,6 +778,9 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, lo.x + int(amrex::Math::floor(z_new)), 0, 0); #endif + } else { + reduce_shape_old = false; + reduce_shape_new = false; } #if defined(WARPX_DIM_RZ) @@ -812,11 +814,11 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // If particle is close to the embedded boundary, recompute deposition with order 1 shape if constexpr (reduce_shape_control == has_reduced_shape) { if (reduce_shape_new) { - for (int i=0; i& GetPosition, // If particle is close to the embedded boundary, recompute deposition with order 1 shape if constexpr (reduce_shape_control == has_reduced_shape) { if (reduce_shape_new) { - for (int j=0; j& GetPosition, // If particle is close to the embedded boundary, recompute deposition with order 1 shape if constexpr (reduce_shape_control == has_reduced_shape) { if (reduce_shape_new) { - for (int k=0; k Date: Tue, 28 Jan 2025 07:10:51 -0800 Subject: [PATCH 99/99] Correct const error --- Source/WarpX.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index d2a9b75f2ce..a5a8ea81438 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1025,7 +1025,7 @@ public: void MarkReducedShapeCells ( std::unique_ptr & eb_reduce_particle_shape, amrex::EBFArrayBoxFactory const & eb_fact, - int const particle_shape_order ); + int particle_shape_order ); /** \brief Set a flag to indicate on which grid points the field `field` * should be updated, depending on their position relative to the embedded boundary.