Skip to content

Commit

Permalink
paper: more references, expand related work,
Browse files Browse the repository at this point in the history
define I, software -> references
cf. #137
  • Loading branch information
Theresa Pollinger committed Dec 10, 2024
1 parent ca294c5 commit 4642d4d
Show file tree
Hide file tree
Showing 2 changed files with 150 additions and 45 deletions.
116 changes: 104 additions & 12 deletions paper.bib
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,16 @@ @inproceedings{griebelCombinationTechniqueSolution1992
date = {1992},
pages = {263--281},
publisher = {IMACS, Elsevier, North Holland},
url = {https://ins.uni-bonn.de/media/public/publication-media/griesiam.ps.gz},
}

@article{bungartzSparseGrids2004,
title = {Sparse {{Grids}}},
author = {Bungartz, Hans-Joachim and Griebel, Michael},
date = {2004},
journaltitle = {Acta Numerica},
volume = {13},
pages = {147--269},
}

@phdthesis{heeneMassivelyParallelCombination2018,
Expand All @@ -27,18 +37,19 @@ @phdthesis{obersteiner_spatially_2021
date = {2021}
}

@inproceedings{pollingerLeveragingComputePower2023,
title = {Leveraging the {{Compute Power}} of {{Two HPC Systems}} for {{Higher-Dimensional Grid-Based Simulations}} with the {{Widely-Distributed Sparse Grid Combination Technique}}},
booktitle = {Proceedings of the {{International Conference}} for {{High Performance Computing}}, {{Networking}}, {{Storage}} and {{Analysis}}},
author = {Pollinger, Theresa and Van Craen, Alexander and Niethammer, Christoph and Breyer, Marcel and Pflüger, Dirk},
date = {2023-11-11},
series = {{{SC}} '23},
publisher = {{Association for Computing Machinery}},
location = {{New York, NY, USA}},
doi = {10.1145/3581784.3607036},
url = {https://dl.acm.org/doi/10.1145/3581784.3607036},
abstract = {Grid-based simulations of hot fusion plasmas are often severely limited by computational and memory resources; the grids live in four- to six-dimensional space and thus suffer the curse of dimensionality. However, high resolutions are required to fully capture the physics of interest. The sparse grid combination technique is a multi-scale method in which many anisotropically coarse resolved grids are used to approximate a fine-scale solution---and it alleviates the curse of dimensionality. This paper presents the core concepts of the widely-distributed combination technique, which allows us to use the compute power and memory of more than one HPC system for the same simulation. We apply the sparse grid combination technique to a six-dimensional advection problem serving as a proxy for plasma simulations. The full-grid solution approximated by the combination technique would contain ≈ 5 ZB if computed with conventional grid-based methods. Even the combination technique simulation operates on ≈ 1 × 1011 double-precision degrees of freedom, or 988 GB, plus the supporting sparse grid data structures. We propose a new approach to divide the compute load between the two HPC systems, requiring only 76 GB to be exchanged. Based on this, we have realized the first synchronous combination technique simulation using two HPC systems, in our case the two German Tier-0 supercomputers HAWK and SuperMUC-NG. On two systems, the simulation can be computed at an average overhead of ≈ 35 \% (108 s per combination step) for file I/O and transfer. The presented concepts apply to any pair of HPC systems if high-speed data transfer is possible.},
keywords = {combination technique,coupling HPC systems,higher-dimensional simulation,multi-level methods,plasma turbulence,UFTP},
@inproceedings{pollingerRealizingJointExtremeScale2024,
title = {Realizing {{Joint Extreme-Scale Simulations}} on {{Multiple Supercomputers}}—{{Two Superfacility Case Studies}}},
author = {Pollinger, Theresa and Craen, Alexander Van and Offenhäuser, Philipp and Pflüger, Dirk},
date = {2024-11-09},
pages = {1568--1584},
publisher = {IEEE Computer Society},
doi = {10.1109/SC41406.2024.00104},
url = {https://www.computer.org/csdl/proceedings-article/sc/2024/529100b568/21HUWDIjYlO},
urldate = {2024-11-17},
abstract = {High-dimensional grid-based simulations serve as both a tool and a challenge in researching various domains. The main challenge of these approaches is the well-known curse of dimensionality, amplified by the need for fine resolutions in highfidelity applications. The combination technique (CT) provides a straightforward way of performing such simulations while alleviating the curse of dimensionality. Recent work demonstrated the potential of the CT to join multiple systems simultaneously to perform a single high-dimensional simulation. This paper shows how to extend this to three or more systems and addresses some remaining challenges: load balancing on heterogeneous hardware; utilizing compression to maximize the communication bandwidth; efficient I/O management through hardware mapping; and improving memory utilization through algorithmic optimizations. Combining these contributions, we demonstrate the feasibility of the CT for extreme-scale Superfacility scenarios of 46 trillion DOF on two systems and 35 trillion DOF on three systems. Scenarios at these resolutions would be intractable with full-grid solvers ({$>$} 1,000 nonillion DOF each).},
eventtitle = {2024 {{SC24}}: {{International Conference}} for {{High Performance Computing}}, {{Networking}}, {{Storage}} and {{Analysis SC}}},
isbn = {9798350352917},
langid = {english},
}

@article{pollingerStableMassconservingSparse2023,
Expand Down Expand Up @@ -98,3 +109,84 @@ @online{piazzolaSparseGridsMatlab2022
abstract = {The Sparse Grids Matlab kit is a collection of Matlab functions for high-dimensional interpolation and quadrature, based on the combination technique form of sparse grids. It is lightweight, high-level and easy to use, good for quick prototyping and teaching. It is somehow geared towards Uncertainty Quantification (UQ), but it is flexible enough for other purposes. The goal of this paper is to give an overview of the implementation structure of the Sparse Grids Matlab kit and to showcase its potentialities, guided by some illustrative tests and a final comprehensive example on forward and inverse UQ analysis.},
keywords = {Computer Science - Mathematical Software,Mathematics - Numerical Analysis},
}

@article{brizardFoundationsNonlinearGyrokinetic2007,
title = {Foundations of Nonlinear Gyrokinetic Theory},
author = {Brizard, Alain J. and Hahm, Taik Soo},
date = {2007},
journaltitle = {Reviews of modern physics},
volume = {79},
number = {2},
pages = {421},
publisher = {APS},
}

@article{einkemmerMassMomentumEnergy2021,
title = {A Mass, Momentum, and Energy Conservative Dynamical Low-Rank Scheme for the {{Vlasov}} Equation},
author = {Einkemmer, Lukas and Joseph, Ilon},
date = {2021-10-15},
journaltitle = {Journal of Computational Physics},
shortjournal = {Journal of Computational Physics},
volume = {443},
pages = {110495},
issn = {0021-9991},
doi = {10.1016/j.jcp.2021.110495},
url = {https://www.sciencedirect.com/science/article/pii/S0021999121003909},
urldate = {2023-09-25},
abstract = {The primary challenge in solving kinetic equations, such as the Vlasov equation, is the high-dimensional phase space. In this context, dynamical low-rank approximations have emerged as a promising way to reduce the high computational cost imposed by such problems. However, a major disadvantage of this approach is that the physical structure of the underlying problem is not preserved. In this paper, we propose a dynamical low-rank algorithm that conserves mass, momentum, and energy as well as the corresponding continuity equations. We also show how this approach can be combined with a conservative time and space discretization.},
keywords = {Complexity reduction,Conservative numerical methods,Dynamical low-rank approximation,Kinetic equation,Vlasov equation},
}

@software{tamelliniLorenzotamelliniSparsegridsmatlabkit2024,
title = {Lorenzo-Tamellini/Sparse-Grids-Matlab-Kit},
author = {family=tamellini, prefix=lorenzo-, useprefix=true},
date = {2024-12-05T13:17:27Z},
origdate = {2023-05-06T12:13:51Z},
url = {https://github.com/lorenzo-tamellini/sparse-grids-matlab-kit},
urldate = {2024-12-10},
abstract = {repository for the Sparse Grids Matlab Kit source code. Full info on https://sites.google.com/view/sparse-grids-kit}
}

@software{DisCoTeccombischemeutilities2022,
title = {{{DisCoTec-combischeme-utilities}}},
date = {2022-10-18T05:44:12Z},
origdate = {2022-10-07T08:29:19Z},
url = {https://github.com/SGpp/DisCoTec-combischeme-utilities},
urldate = {2023-03-21},
abstract = {Python utilities to create (combination technique schemes) input files for the DisCoTec code},
organization = {SG++ development team}
}

@article{verboncoeurParticleSimulationPlasmas2005,
title = {Particle Simulation of Plasmas: Review and Advances},
shorttitle = {Particle Simulation of Plasmas},
author = {Verboncoeur, John P.},
date = {2005},
journaltitle = {Plasma Physics and Controlled Fusion},
volume = {47},
pages = {A231},
publisher = {IOP Publishing},
url = {https://iopscience.iop.org/article/10.1088/0741-3335/47/5A/017/meta?casa_token=aElJiSgwIgcAAAAA:xrmEQjmliSWfz_8HcmH7scqGnsFqepsQEqxMBxI-q8uEAvsFo6MSKB46NsoEgCI-oC1k3A3K9z8tyHKEtLsYUptMhe8gWQ},
urldate = {2024-12-04},
issue = {5A},
}
@software{SGppSGpp2024,
title = {{{SGpp}}/{{SGpp}}},
date = {2024-12-05T10:19:33Z},
origdate = {2018-03-05T12:33:52Z},
url = {https://github.com/SGpp/SGpp},
urldate = {2024-12-10},
abstract = {SG⁺⁺ – the numerical library for Sparse Grids in all their variants.},
organization = {SG++ development team},
keywords = {adaptive-sparse-grids,b-splines,cpp,interpolation,java,machine-learning,matlab,numerics,optimization,pde,python,quadrature,regression,sparse-grids}
}

@software{obersteinerSparseSpACESparseGrid2023,
title = {{{sparseSpACE}} - {{The Sparse Grid Spatially Adaptive Combination Environment}}},
author = {Obersteiner, Michael},
date = {2023-05-09T06:55:23Z},
origdate = {2019-09-02T09:22:49Z},
url = {https://github.com/obersteiner/sparseSpACE},
urldate = {2023-08-07},
abstract = {sparseSpACE - the Sparse Grid Spatially Adaptive Combination Environment implements different variants of the spatially adaptive combination technique.}
}
79 changes: 46 additions & 33 deletions paper.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,17 +37,18 @@ bibliography: paper.bib

# Summary

`DisCoTec` is a C++ framework for the sparse grid combination technique,
`DisCoTec` is a C++ framework for the sparse grid combination technique [@griebelCombinationTechniqueSolution1992],
designed for massively parallel settings.
It is implemented with shared-memory parallelism via OpenMP and
distributed-memory parallelism via MPI, and is intended to be used in
conjunction with existing simulation codes.
For simulation codes that can handle nested structured grids, little to no
adaptation work is needed for use with the `DisCoTec` framework.
`DisCoTec` demonstrates its superiority in higher-dimensional time-dependent
simulations, such as high-fidelity plasma simulations in 4- to 6-dimensions
[@pollingerStableMassconservingSparse2023] and even for simulations in two
dimensions, improvements may be observed.
The combination technique with `DisCoTec` demonstrates its superiority in
memory-per-precision for higher-dimensional time-dependent simulations, such
as high-fidelity plasma turbulence simulations
in four to six dimensions
and even for simulations in two dimensions, improvements can be observed [@pollingerStableMassconservingSparse2023].

A central part of the combination technique at scale is the transformation of
grid coefficients into a multi-scale basis.
Expand All @@ -58,7 +59,8 @@ from the model order reduction provided by the underlying sparse grid approach
used by `DisCoTec`, without requiring any multi-scale operations.
An additional feature of `DisCoTec` is the possibility of performing
widely-distributed simulations of higher-dimensional problems, where multiple
HPC systems collaborate to solve a joint simulation, as demonstrated in [@pollingerLeveragingComputePower2023].
High-Performance Computing (HPC) systems collaborate to solve a joint simulation,
as demonstrated in [@pollingerRealizingJointExtremeScale2024].
Thus, `DisCoTec` can leverage the compute power and main memory of multiple HPC
systems, with comparatively low and manageable transfer costs due to the
combination technique.
Expand All @@ -68,21 +70,28 @@ combination technique.
Higher-dimensional problems (by which we mean more than three space
dimensions and one time dimension) quickly require infeasible amounts of
computational resources such as memory and core-hours as the problem size
increases---they are haunted by the
so-called 'curse of dimensionality'.
An example of this are high-fidelity plasma simulations in the field of confined
fusion research.
Current approaches to this problem include dimensionally-reduced models
(which may not always be applicable), and restricting computations to a very limited resolution.
increases---they are haunted by the so-called 'curse of dimensionality'.
An example of this are high-fidelity plasma turbulence simulations in the field
of confined fusion research.
Currently employed approaches to this problem include dimensionally-reduced models,
such as gyrokinetics [@brizardFoundationsNonlinearGyrokinetic2007]
(which may not always be applicable),
particle-in-cell methods (which suffer from inherent noise [@verboncoeurParticleSimulationPlasmas2005]),
and restricting computations to a very limited resolution.
A further---still developing but very promising---approach to the problem are
low-rank methods [@einkemmerMassMomentumEnergy2021].
Multi-scale (hierarchical) methods, such as the sparse grid combination
technique that `DisCoTec` employs, provide an alternative approach to addressing the curse of dimensionality.
While some implementations of the sparse grid combination technique are
available in the context of UQ, there is currently no other implementation for
technique (CT) that `DisCoTec` employs,
provide an alternative approach to addressing the curse of dimensionality by
considering only those resolutions where the highest amount of information is expected
[@bungartzSparseGrids2004].
While some implementations of the CT are
available, there is currently no other implementation for
parallel simulations that require distributed computing.

`DisCoTec` is a C++ framework for the sparse grid combination technique.
Targeted at HPC systems, it is used for parallel simulations [@heeneMassivelyParallelCombination2018],
drawing on distributed-memory parallelism via MPI and shared-memory parallelism
drawing on distributed-memory parallelism via MPI and shared-memory parallelism
via OpenMP.
It is designed to be used in combination with existing simulation codes,
which can be used with `DisCoTec` in a black-box fashion.
Expand All @@ -104,12 +113,13 @@ By updating each other's information throughout the simulation, the component gr
still obtain an accurate solution of the overall problem [@griebelCombinationTechniqueSolution1992].
This is enabled by an intermedate transformation into a multi-scale (hierarchical)
basis, and application of the combination formula
$$ f^{(\text{s})} = \sum_{\vec{l} \in \mathcal{I} } c_{\vec{l}} f_{\vec{l}} $$
where $f^{(\text{s})}$ is the sparse grid approximation, and $f_{\vec{l}}$ are
$$ f^{(\text{s})} = \sum_{\vec{\ell} \in \mathcal{I} } c_{\vec{\ell}} f_{\vec{\ell}} $$
where $f^{(\text{s})}$ is the sparse grid approximation, and $f_{\vec{\ell}}$ are
the component grid functions.
In \autoref{fig:combischeme-2d}, the coefficients $c_{\vec{l}}$ are $-1$ for the coarser
component grids (red background) and $1$ for the finer component grids (orange
background).
The set of all used levels $\vec{\ell}$ is often called a combination scheme $\mathcal{I}$.
In \autoref{fig:combischeme-2d}, the coefficients $c_{\vec{\ell}}$ are $-1$ for
the coarser component grids (red background) and $1$ for the finer component grids
(orange background).
In summary, each of the grids will run (one or more) time steps of the simulation,
then exchange information with the other grids, and repeat this process until
the simulation is finished.
Expand All @@ -133,19 +143,22 @@ Using `DisCoTec`, kinetic simulations were demonstrated to scale up to hundreds
of thousands of CPU cores [@pollingerStableMassconservingHighdimensional2024].
By putting a special focus on saving memory, most of the memory is available for
use by the black-box solver, even at high core counts.
In addition, OpenMP parallelism can be used to further increase parallelism and
to decrease main memory usage.
In addition, OpenMP parallelism can be used to further increase parallelism while
being more lightweight than MPI in terms of memory.

Through highly parallel I/O operations, `DisCoTec` can be used to perform
simulations on multiple HPC systems simultaneously, if there exists a tool for
sufficiently fast file transfer between the systems [@pollingerLeveragingComputePower2023].
simulations on multiple High Performance Computing (HPC) systems simultaneously,
if there exists a tool for
sufficiently fast file transfer between the systems [@pollingerStableMassconservingHighdimensional2024].
The `DisCoTec` repository contains example scripts and documentation for
utilizing UFTP as an example of a transfer tool, but the approach is not limited
to UFTP.

`DisCoTec` provides a conveniently automated way of installing using a
`DisCoTec` provides a conveniently automated way of installation using a
[`spack` package](https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/discotec/package.py)
[@gamblinSpackPackageManager2015].
[@gamblinSpackPackageManager2015],
which can be used to install `DisCoTec` and its whole dependency tree
in an automated manner optimized for HPC hardware.

# State of the field

Expand All @@ -154,17 +167,17 @@ sparse grids and the combination technique.
We will give a brief overview and outline the differences and
application areas of the codes.

The C++ code [`SG++`](https://github.com/SGpp/SGpp) allows to directly utilize
sparse grids and apply them to a variety of different tasks such as interpolation,
The C++ code `SG++`[@SGppSGpp2024] provides a direct interface to
sparse grids and applying them to a variety of different tasks such as interpolation,
quadrature, optimization, PDEs, regression, and classification.
With the help of wrappers, the framework can be used from various other programming
languages such as Python and Matlab.
The code targets direct implementations within sparse grids and provides a basic
implementation of the combination technique.
Although offering parallelization for some of the tasks, the code mainly targets
single-node computations.
Although offering parallelization for some of the tasks, the code
mainly targets single-node computations.

The [`Sparse Grids Matlab Kit`](https://github.com/lorenzo-tamellini/sparse-grids-matlab-kit)
The `Sparse Grids Matlab Kit`[@tamelliniLorenzotamelliniSparsegridsmatlabkit2024]
by Piazzola and Tamellini was originally designed for teaching purposes and
uncertainty quantification with the combination technique [@piazzolaSparseGridsMatlab2022].
It offers a user friendly MATLAB interface for the combination technique.
Expand All @@ -173,7 +186,7 @@ of component grid collocation points.
The code is designed for usage on a single node which limits the parallelism
to shared memory.

The [`sparseSpACE`](https://github.com/obersteiner/sparseSpACE) project offers
The `sparseSpACE` [@obersteinerSparseSpACESparseGrid2023] project offers
different variants of the combination technique including a spatially adaptive
combination technique.
It provides implementations for various applications such as numerical integration,
Expand Down

0 comments on commit 4642d4d

Please sign in to comment.