From 3cf5291e6755f02fd497eddf7bca0714e4c28317 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 8 Mar 2022 09:38:48 -0800 Subject: [PATCH 01/70] Python: Do Not Strip Symbols In Debug (#1219) Avoid stripping symbols for Python debug builds, so we can see lines in coredumps and debugger runs. --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ce713eced..c776fbc47a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -692,7 +692,9 @@ if(openPMD_HAVE_PYTHON) else() pybind11_extension(openPMD.py) endif() - pybind11_strip(openPMD.py) + if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo) + pybind11_strip(openPMD.py) + endif() set_target_properties(openPMD.py PROPERTIES CXX_VISIBILITY_PRESET "hidden" CUDA_VISIBILITY_PRESET "hidden") From eec3f0617335828a84d793c4525a18577f7c66b6 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 8 Mar 2022 10:17:57 -0800 Subject: [PATCH 02/70] Clang-Format 13: JSON (#1222) * Clang-Format 13: JSON Add support for JSON formatting with `clang-format-13`. X-ref: - https://burnicki.pl/en/2021/10/19/git-clang-format-json-support-issue.html - https://clang.llvm.org/docs/ClangFormatStyleOptions.html - https://reviews.llvm.org/D93528 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .clang-format | 6 + .../workflows/clang-format/clang-format.sh | 4 +- .pre-commit-config.yaml | 3 +- .rodare.json | 310 +++++++++--------- docs/source/details/adios2.json | 4 +- docs/source/details/config_layout.json | 8 +- 6 files changed, 171 insertions(+), 164 deletions(-) diff --git a/.clang-format b/.clang-format index c47a8364e6..afc8312d97 100644 --- a/.clang-format +++ b/.clang-format @@ -1,3 +1,4 @@ +--- Language: Cpp BasedOnStyle: LLVM @@ -64,3 +65,8 @@ SpacesInParentheses: false SpacesInSquareBrackets: false Standard: Auto StatementMacros: ["OPENPMD_private", "OPENPMD_protected"] + +--- +Language: Json +BasedOnStyle: llvm +... diff --git a/.github/workflows/clang-format/clang-format.sh b/.github/workflows/clang-format/clang-format.sh index 863b68ba67..0b99c629ab 100755 --- a/.github/workflows/clang-format/clang-format.sh +++ b/.github/workflows/clang-format/clang-format.sh @@ -2,11 +2,11 @@ if (( $# > 0 )); then # received arguments, format those files - clang-format-12 -i "$@" + clang-format-13 -i "$@" else # received no arguments, find files on our own find include/ src/ test/ examples/ \ -regextype egrep \ -type f -regex '.*\.(hpp|cpp|hpp\.in)$' \ - | xargs clang-format-12 -i + | xargs clang-format-13 -i fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e15ddbc1d..c36ffc186a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,7 @@ repos: - id: check-json - id: check-toml - id: check-yaml + args: [--allow-multiple-documents] - id: check-added-large-files - id: requirements-txt-fixer # - id: fix-encoding-pragma @@ -61,7 +62,7 @@ repos: # files: (\.cmake|CMakeLists.txt)(.in)?$ # C++ formatting -# clang-format v12 +# clang-format v13 # to run manually, use .github/workflows/clang-format/clang-format.sh - repo: https://github.com/pre-commit/mirrors-clang-format rev: v13.0.1 diff --git a/.rodare.json b/.rodare.json index 63c816bb3d..f2324fc670 100644 --- a/.rodare.json +++ b/.rodare.json @@ -1,158 +1,158 @@ { - "creators": [ - { - "name": "Huebl, Axel", - "affiliation": "Lawrence Berkeley National Laboratory", - "orcid": "0000-0003-1943-7141" - }, - { - "name": "Poeschel, Franz", - "affiliation": "Center for Advanced Systems Understanding (CASUS)", - "orcid": "0000-0001-7042-5088" - }, - { - "name": "Koller, Fabian", - "affiliation": "Helmholtz-Zentrum Dresden-Rossendorf", - "orcid": "0000-0001-8704-1769" - }, - { - "name": "Gu, Junmin", - "affiliation": "Lawrence Berkeley National Laboratory", - "orcid": "0000-0002-1521-8534" - } + "creators": [ + { + "name": "Huebl, Axel", + "affiliation": "Lawrence Berkeley National Laboratory", + "orcid": "0000-0003-1943-7141" + }, + { + "name": "Poeschel, Franz", + "affiliation": "Center for Advanced Systems Understanding (CASUS)", + "orcid": "0000-0001-7042-5088" + }, + { + "name": "Koller, Fabian", + "affiliation": "Helmholtz-Zentrum Dresden-Rossendorf", + "orcid": "0000-0001-8704-1769" + }, + { + "name": "Gu, Junmin", + "affiliation": "Lawrence Berkeley National Laboratory", + "orcid": "0000-0002-1521-8534" + } - ], - "contributors": [ - { - "affiliation": "EU XFEL GmbH", - "name": "Fortmann-Grote, Carsten", - "orcid": "0000-0002-2579-5546", - "type": "Other" - }, - { - "affiliation": "Warsaw University of Technology", - "name": "Stańczak, Dominik", - "orcid": "0000-0001-6291-8843", - "type": "Other" - }, - { - "affiliation": "Fermi National Accelerator Laboratory", - "name": "Amundson, James", - "orcid": "0000-0001-6918-2728", - "type": "Other" - }, - { - "affiliation": "Anaconda, Inc.", - "name": "Donnelly, Ray", - "type": "Other" - }, - { - "affiliation": "Helmholtz-Zentrum Dresden-Rossendorf", - "name": "Widera, René", - "orcid": "0000-0003-1642-0459", - "type": "Other" - }, - { - "affiliation": "Helmholtz-Zentrum Dresden-Rossendorf", - "name": "Zenker, Erik", - "orcid": "0000-0001-9417-8712", - "type": "Other" - }, - { - "affiliation": "Helmholtz-Zentrum Dresden-Rossendorf", - "name": "Bastrakov, Sergei", - "orcid": "0000-0003-3396-6154", - "type": "Other" - }, - { - "affiliation": "Lawrence Berkeley National Laboratory", - "name": "Lehe, Rémi", - "orcid": "0000-0002-3656-9659", - "type": "Other" - }, - { - "affiliation": "Lawrence Berkeley National Laboratory", - "name": "Amorim, Lígia Diana", - "orcid": "0000-0002-1445-0032", - "type": "Other" - }, - { - "affiliation": "Helmholtz-Zentrum Dresden-Rossendorf", - "name": "Bastrakova, Kseniia", - "type": "Other" - }, - { - "affiliation": "Helmholtz-Zentrum Dresden-Rossendorf", - "name": "Pausch, Richard", - "orcid": "0000-0001-7990-9564", - "type": "Other" - }, - { - "affiliation": "Helmholtz-Zentrum Dresden-Rossendorf", - "name": "Ordyna, Paweł", - "type": "Other" - }, - { - "affiliation": "Oak Ridge National Laboratory", - "name": "Ganyushin, Dmitry", - "orcid": "0000-0001-7337-2161", - "type": "Other" - }, - { - "affiliation": "NVIDIA", - "name": "Kirkham, John", - "type": "Other" - }, - { - "affiliation": "Perimeter Institute for Theoretical Physics", - "name": "Schnetter, Erik", - "orcid": "0000-0002-4518-9017", - "type": "Other" - }, - { - "affiliation": "Lawrence Berkeley National Laboratory", - "name": "Bez, Jean Luca", - "orcid": "0000-0002-3915-1135", - "type": "Other" - } - ], - "title": "C++ & Python API for Scientific I/O with openPMD", - "access_right": "open", - "upload_type": "software", - "license": "LGPL-3.0", - "pub_id": "27579", - "description": "openPMD is an open metadata format for open data workflows in open science. This library provides a common high-level API for openPMD writing and reading. It provides a common interface to I/O libraries and file formats such as HDF5 and ADIOS. Where supported, openPMD-api implements both serial and MPI parallel I/O capabilities.", - "keywords": [ - "openPMD", - "Open Science", - "Open Data", - "HDF5", - "ADIOS", - "data", - "MPI", - "HPC", - "research", - "file-format", - "file-handling" - ], - "notes": "Supported by the Exascale Computing Project (17-SC-20-SC), a collaborative effort of two U.S. Department of Energy organizations (Office of Science and the National Nuclear Security Administration). Supported by the Consortium for Advanced Modeling of Particles Accelerators (CAMPA), funded by the U.S. DOE Office of Science under Contract No. DE-AC02-05CH11231. This work was partially funded by the Center of Advanced Systems Understanding (CASUS), which is financed by Germany's Federal Ministry of Education and Research (BMBF) and by the Saxon Ministry for Science, Culture and Tourism (SMWK) with tax funds on the basis of the budget approved by the Saxon State Parliament.", - "grants": [ - { - "id": "654220" - } - ], - "related_identifiers": [ - { - "identifier": "DOI:10.5281/zenodo.1167843", - "relation": "isCitedBy" - }, - { - "identifier": "DOI:10.5281/zenodo.1069534", - "relation": "isCitedBy" - }, - { - "identifier": "DOI:10.5281/zenodo.33624", - "relation": "isCitedBy" - } - ] + ], + "contributors": [ + { + "affiliation": "EU XFEL GmbH", + "name": "Fortmann-Grote, Carsten", + "orcid": "0000-0002-2579-5546", + "type": "Other" + }, + { + "affiliation": "Warsaw University of Technology", + "name": "Stańczak, Dominik", + "orcid": "0000-0001-6291-8843", + "type": "Other" + }, + { + "affiliation": "Fermi National Accelerator Laboratory", + "name": "Amundson, James", + "orcid": "0000-0001-6918-2728", + "type": "Other" + }, + { + "affiliation": "Anaconda, Inc.", + "name": "Donnelly, Ray", + "type": "Other" + }, + { + "affiliation": "Helmholtz-Zentrum Dresden-Rossendorf", + "name": "Widera, René", + "orcid": "0000-0003-1642-0459", + "type": "Other" + }, + { + "affiliation": "Helmholtz-Zentrum Dresden-Rossendorf", + "name": "Zenker, Erik", + "orcid": "0000-0001-9417-8712", + "type": "Other" + }, + { + "affiliation": "Helmholtz-Zentrum Dresden-Rossendorf", + "name": "Bastrakov, Sergei", + "orcid": "0000-0003-3396-6154", + "type": "Other" + }, + { + "affiliation": "Lawrence Berkeley National Laboratory", + "name": "Lehe, Rémi", + "orcid": "0000-0002-3656-9659", + "type": "Other" + }, + { + "affiliation": "Lawrence Berkeley National Laboratory", + "name": "Amorim, Lígia Diana", + "orcid": "0000-0002-1445-0032", + "type": "Other" + }, + { + "affiliation": "Helmholtz-Zentrum Dresden-Rossendorf", + "name": "Bastrakova, Kseniia", + "type": "Other" + }, + { + "affiliation": "Helmholtz-Zentrum Dresden-Rossendorf", + "name": "Pausch, Richard", + "orcid": "0000-0001-7990-9564", + "type": "Other" + }, + { + "affiliation": "Helmholtz-Zentrum Dresden-Rossendorf", + "name": "Ordyna, Paweł", + "type": "Other" + }, + { + "affiliation": "Oak Ridge National Laboratory", + "name": "Ganyushin, Dmitry", + "orcid": "0000-0001-7337-2161", + "type": "Other" + }, + { + "affiliation": "NVIDIA", + "name": "Kirkham, John", + "type": "Other" + }, + { + "affiliation": "Perimeter Institute for Theoretical Physics", + "name": "Schnetter, Erik", + "orcid": "0000-0002-4518-9017", + "type": "Other" + }, + { + "affiliation": "Lawrence Berkeley National Laboratory", + "name": "Bez, Jean Luca", + "orcid": "0000-0002-3915-1135", + "type": "Other" + } + ], + "title": "C++ & Python API for Scientific I/O with openPMD", + "access_right": "open", + "upload_type": "software", + "license": "LGPL-3.0", + "pub_id": "27579", + "description": "openPMD is an open metadata format for open data workflows in open science. This library provides a common high-level API for openPMD writing and reading. It provides a common interface to I/O libraries and file formats such as HDF5 and ADIOS. Where supported, openPMD-api implements both serial and MPI parallel I/O capabilities.", + "keywords": [ + "openPMD", + "Open Science", + "Open Data", + "HDF5", + "ADIOS", + "data", + "MPI", + "HPC", + "research", + "file-format", + "file-handling" + ], + "notes": "Supported by the Exascale Computing Project (17-SC-20-SC), a collaborative effort of two U.S. Department of Energy organizations (Office of Science and the National Nuclear Security Administration). Supported by the Consortium for Advanced Modeling of Particles Accelerators (CAMPA), funded by the U.S. DOE Office of Science under Contract No. DE-AC02-05CH11231. This work was partially funded by the Center of Advanced Systems Understanding (CASUS), which is financed by Germany's Federal Ministry of Education and Research (BMBF) and by the Saxon Ministry for Science, Culture and Tourism (SMWK) with tax funds on the basis of the budget approved by the Saxon State Parliament.", + "grants": [ + { + "id": "654220" + } + ], + "related_identifiers": [ + { + "identifier": "DOI:10.5281/zenodo.1167843", + "relation": "isCitedBy" + }, + { + "identifier": "DOI:10.5281/zenodo.1069534", + "relation": "isCitedBy" + }, + { + "identifier": "DOI:10.5281/zenodo.33624", + "relation": "isCitedBy" + } + ] } diff --git a/docs/source/details/adios2.json b/docs/source/details/adios2.json index 456c83707b..d71061c0bc 100644 --- a/docs/source/details/adios2.json +++ b/docs/source/details/adios2.json @@ -12,8 +12,8 @@ { "type": "blosc", "parameters": { - "clevel": "1", - "doshuffle": "BLOSC_BITSHUFFLE" + "clevel": "1", + "doshuffle": "BLOSC_BITSHUFFLE" } } ] diff --git a/docs/source/details/config_layout.json b/docs/source/details/config_layout.json index 8960916860..382060f364 100644 --- a/docs/source/details/config_layout.json +++ b/docs/source/details/config_layout.json @@ -1,6 +1,6 @@ { - "adios1": "put ADIOS config here", - "adios2": "put ADIOS2 config here", - "hdf5": "put HDF5 config here", - "json": "put JSON config here" + "adios1": "put ADIOS config here", + "adios2": "put ADIOS2 config here", + "hdf5": "put HDF5 config here", + "json": "put JSON config here" } From 272b137f64aff09d9edf8d521073f4f7880b17ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Sun, 13 Mar 2022 21:00:18 +0100 Subject: [PATCH 03/70] Fix use after free in ADIOS1IOHandler (#1224) --- include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp | 7 ++++++- src/IO/ADIOS/ADIOS1IOHandler.cpp | 2 +- src/IO/ADIOS/CommonADIOS1IOHandler.cpp | 4 ++-- src/IO/ADIOS/ParallelADIOS1IOHandler.cpp | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp index 74733781e9..ea89f033d6 100644 --- a/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp @@ -103,7 +103,12 @@ class CommonADIOS1IOHandlerImpl : public AbstractIOHandlerImpl m_openWriteFileHandles; std::unordered_map, ADIOS_FILE *> m_openReadFileHandles; - std::unordered_map > + struct ScheduledRead + { + ADIOS_SELECTION *selection; + std::shared_ptr data; // needed to avoid early freeing + }; + std::unordered_map > m_scheduledReads; std::unordered_map > m_attributeWrites; diff --git a/src/IO/ADIOS/ADIOS1IOHandler.cpp b/src/IO/ADIOS/ADIOS1IOHandler.cpp index 3c38b1adc6..3a6dc803d2 100644 --- a/src/IO/ADIOS/ADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS1IOHandler.cpp @@ -296,7 +296,7 @@ std::future ADIOS1IOHandlerImpl::flush() "dataset reading"); for (auto &sel : file.second) - adios_selection_delete(sel); + adios_selection_delete(sel.selection); } m_scheduledReads.clear(); diff --git a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp index 345052ddfc..e50207a74d 100644 --- a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp @@ -728,7 +728,7 @@ void CommonADIOS1IOHandlerImpl::closeFile( "dataset reading"); for (auto &sel : scheduled->second) - adios_selection_delete(sel); + adios_selection_delete(sel.selection); m_scheduledReads.erase(scheduled); } close(handle_read->second); @@ -1183,7 +1183,7 @@ void CommonADIOS1IOHandlerImpl::readDataset( "[ADIOS1] Internal error: Failed to schedule ADIOS read during dataset " "reading"); - m_scheduledReads[f].push_back(sel); + m_scheduledReads[f].push_back({sel, parameters.data}); } template diff --git a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp index 20a2d3936e..df6f817098 100644 --- a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp @@ -316,7 +316,7 @@ std::future ParallelADIOS1IOHandlerImpl::flush() "dataset reading"); for (auto &sel : file.second) - adios_selection_delete(sel); + adios_selection_delete(sel.selection); } m_scheduledReads.clear(); From ff8b413d3d3aa77e1ee5cdacf00fadd84637d27f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 14 Mar 2022 02:57:18 +0100 Subject: [PATCH 04/70] Pass-through flushing parameters (#1226) * Pass-through flushing parameters * CI fixes --- include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp | 4 +- .../openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp | 2 +- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 6 +- .../IO/ADIOS/ParallelADIOS1IOHandler.hpp | 2 +- .../IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp | 2 +- include/openPMD/IO/AbstractIOHandler.hpp | 21 +++- include/openPMD/IO/AbstractIOHandlerImpl.hpp | 2 +- include/openPMD/IO/DummyIOHandler.hpp | 2 +- include/openPMD/IO/HDF5/HDF5IOHandler.hpp | 2 +- .../openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp | 2 +- include/openPMD/IO/JSON/JSONIOHandler.hpp | 2 +- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 2 +- include/openPMD/Iteration.hpp | 9 +- include/openPMD/Mesh.hpp | 3 +- include/openPMD/ParticleSpecies.hpp | 2 +- include/openPMD/Record.hpp | 3 +- include/openPMD/RecordComponent.hpp | 2 +- include/openPMD/RecordComponent.tpp | 16 ++-- include/openPMD/Series.hpp | 14 ++- include/openPMD/Span.hpp | 2 +- include/openPMD/backend/Attributable.hpp | 4 +- include/openPMD/backend/BaseRecord.hpp | 18 ++-- include/openPMD/backend/Container.hpp | 9 +- include/openPMD/backend/PatchRecord.hpp | 3 +- .../openPMD/backend/PatchRecordComponent.hpp | 2 +- include/openPMD/backend/Writable.hpp | 2 +- src/IO/ADIOS/ADIOS1IOHandler.cpp | 4 +- src/IO/ADIOS/ADIOS2IOHandler.cpp | 12 ++- src/IO/ADIOS/ParallelADIOS1IOHandler.cpp | 4 +- src/IO/DummyIOHandler.cpp | 2 +- src/IO/HDF5/HDF5IOHandler.cpp | 4 +- src/IO/HDF5/ParallelHDF5IOHandler.cpp | 4 +- src/IO/JSON/JSONIOHandler.cpp | 2 +- src/Iteration.cpp | 64 +++++++------ src/Mesh.cpp | 33 +++---- src/ParticlePatches.cpp | 6 +- src/ParticleSpecies.cpp | 27 +++--- src/Record.cpp | 21 ++-- src/RecordComponent.cpp | 13 +-- src/Series.cpp | 96 +++++++++---------- src/backend/Attributable.cpp | 14 +-- src/backend/BaseRecordComponent.cpp | 2 +- src/backend/MeshRecordComponent.cpp | 2 +- src/backend/PatchRecord.cpp | 18 ++-- src/backend/PatchRecordComponent.cpp | 7 +- src/backend/Writable.cpp | 6 +- 46 files changed, 259 insertions(+), 220 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp index ead72e2a5e..a967433cf4 100644 --- a/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp @@ -50,7 +50,7 @@ class OPENPMDAPI_EXPORT ADIOS1IOHandler : public AbstractIOHandler return "ADIOS1"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; void enqueue(IOTask const &) override; @@ -72,7 +72,7 @@ class OPENPMDAPI_EXPORT ADIOS1IOHandler : public AbstractIOHandler return "DUMMY_ADIOS1"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; private: std::unique_ptr m_impl; diff --git a/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp b/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp index 27bbdbcf59..42305bc78c 100644 --- a/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp @@ -51,7 +51,7 @@ class OPENPMDAPI_EXPORT ADIOS1IOHandlerImpl virtual void init(); - std::future flush() override; + std::future flush(); virtual int64_t open_write(Writable *); virtual ADIOS_FILE *open_read(std::string const &name); diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index c3a91070e2..8b94d3b51b 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -142,7 +142,7 @@ class ADIOS2IOHandlerImpl ~ADIOS2IOHandlerImpl() override; - std::future flush() override; + std::future flush(internal::FlushParams const &); void createFile(Writable *, Parameter const &) override; @@ -1262,7 +1262,7 @@ class ADIOS2IOHandler : public AbstractIOHandler // we must not throw in a destructor try { - this->flush(); + this->flush(internal::defaultFlushParams); } catch (std::exception const &ex) { @@ -1301,6 +1301,6 @@ class ADIOS2IOHandler : public AbstractIOHandler return "ADIOS2"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; }; // ADIOS2IOHandler } // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp index c56440c4fe..eefac95854 100644 --- a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp @@ -54,7 +54,7 @@ class OPENPMDAPI_EXPORT ParallelADIOS1IOHandler : public AbstractIOHandler return "MPI_ADIOS1"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; #if openPMD_HAVE_ADIOS1 void enqueue(IOTask const &) override; #endif diff --git a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp index e0c7504c90..0b1bb2ca34 100644 --- a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp +++ b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp @@ -53,7 +53,7 @@ class OPENPMDAPI_EXPORT ParallelADIOS1IOHandlerImpl virtual void init(); - std::future flush() override; + std::future flush(); virtual int64_t open_write(Writable *); virtual ADIOS_FILE *open_read(std::string const &name); diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index dafa896a76..ff8b2fcccb 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -87,6 +87,24 @@ enum class FlushLevel : unsigned char SkeletonOnly }; +namespace internal +{ + /** + * Parameters recursively passed through the openPMD hierarchy when + * flushing. + * + */ + struct FlushParams + { + FlushLevel flushLevel = FlushLevel::InternalFlush; + }; + + /* + * To be used for reading + */ + constexpr FlushParams defaultFlushParams{}; +} // namespace internal + /** Interface for communicating between logical and physically persistent data. * * Input and output operations are channeled through a task queue that is @@ -123,7 +141,7 @@ class AbstractIOHandler * @return Future indicating the completion state of the operation for * backends that decide to implement this operation asynchronously. */ - virtual std::future flush() = 0; + virtual std::future flush(internal::FlushParams const &) = 0; /** The currently used backend */ virtual std::string backendName() const = 0; @@ -132,7 +150,6 @@ class AbstractIOHandler Access const m_backendAccess; Access const m_frontendAccess; std::queue m_work; - FlushLevel m_flushLevel = FlushLevel::InternalFlush; }; // AbstractIOHandler } // namespace openPMD diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index 01aeb4e9fc..170cf4b81a 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -40,7 +40,7 @@ class AbstractIOHandlerImpl virtual ~AbstractIOHandlerImpl() = default; - virtual std::future flush() + std::future flush() { using namespace auxiliary; diff --git a/include/openPMD/IO/DummyIOHandler.hpp b/include/openPMD/IO/DummyIOHandler.hpp index 8a84bb0919..9a4f3c3852 100644 --- a/include/openPMD/IO/DummyIOHandler.hpp +++ b/include/openPMD/IO/DummyIOHandler.hpp @@ -44,6 +44,6 @@ class DummyIOHandler : public AbstractIOHandler /** No-op consistent with the IOHandler interface to enable library use * without IO. */ - std::future flush() override; + std::future flush(internal::FlushParams const &) override; }; // DummyIOHandler } // namespace openPMD diff --git a/include/openPMD/IO/HDF5/HDF5IOHandler.hpp b/include/openPMD/IO/HDF5/HDF5IOHandler.hpp index 9c2433a7da..85d6ab9d40 100644 --- a/include/openPMD/IO/HDF5/HDF5IOHandler.hpp +++ b/include/openPMD/IO/HDF5/HDF5IOHandler.hpp @@ -42,7 +42,7 @@ class HDF5IOHandler : public AbstractIOHandler return "HDF5"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; private: std::unique_ptr m_impl; diff --git a/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp b/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp index 512e3edbb2..e1c8d52257 100644 --- a/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp +++ b/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp @@ -48,7 +48,7 @@ class ParallelHDF5IOHandler : public AbstractIOHandler return "MPI_HDF5"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; private: std::unique_ptr m_impl; diff --git a/include/openPMD/IO/JSON/JSONIOHandler.hpp b/include/openPMD/IO/JSON/JSONIOHandler.hpp index 1c1302bb55..37b00fa165 100644 --- a/include/openPMD/IO/JSON/JSONIOHandler.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandler.hpp @@ -38,7 +38,7 @@ class JSONIOHandler : public AbstractIOHandler return "JSON"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; private: JSONIOHandlerImpl m_impl; diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 20dfd441da..d090d0b687 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -210,7 +210,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl void listAttributes(Writable *, Parameter &) override; - std::future flush() override; + std::future flush(); private: using FILEHANDLE = std::fstream; diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 246ae195e5..3d8ccb4567 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -242,10 +242,11 @@ class Iteration : public Attributable return *m_iterationData; } - void flushFileBased(std::string const &, uint64_t); - void flushGroupBased(uint64_t); - void flushVariableBased(uint64_t); - void flush(); + void flushFileBased( + std::string const &, uint64_t, internal::FlushParams const &); + void flushGroupBased(uint64_t, internal::FlushParams const &); + void flushVariableBased(uint64_t, internal::FlushParams const &); + void flush(internal::FlushParams const &); void deferParseAccess(internal::DeferredParseAccess); /* * Control flow for read(), readFileBased(), readGroupBased() and diff --git a/include/openPMD/Mesh.hpp b/include/openPMD/Mesh.hpp index 1dac4760ee..17ce9373de 100644 --- a/include/openPMD/Mesh.hpp +++ b/include/openPMD/Mesh.hpp @@ -228,7 +228,8 @@ class Mesh : public BaseRecord private: Mesh(); - void flush_impl(std::string const &) override; + void + flush_impl(std::string const &, internal::FlushParams const &) override; void read() override; }; // Mesh diff --git a/include/openPMD/ParticleSpecies.hpp b/include/openPMD/ParticleSpecies.hpp index fc80960ca5..0257cd474f 100644 --- a/include/openPMD/ParticleSpecies.hpp +++ b/include/openPMD/ParticleSpecies.hpp @@ -43,7 +43,7 @@ class ParticleSpecies : public Container ParticleSpecies(); void read(); - void flush(std::string const &) override; + void flush(std::string const &, internal::FlushParams const &) override; /** * @brief Check recursively whether this ParticleSpecies is dirty. diff --git a/include/openPMD/Record.hpp b/include/openPMD/Record.hpp index 10bf0a5666..4f7ee51c28 100644 --- a/include/openPMD/Record.hpp +++ b/include/openPMD/Record.hpp @@ -50,7 +50,8 @@ class Record : public BaseRecord private: Record(); - void flush_impl(std::string const &) override; + void + flush_impl(std::string const &, internal::FlushParams const &) override; void read() override; }; // Record diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index 70ea8281f3..d30f7684f2 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -289,7 +289,7 @@ class RecordComponent : public BaseRecordComponent static constexpr char const *const SCALAR = "\vScalar"; private: - void flush(std::string const &); + void flush(std::string const &, internal::FlushParams const &); virtual void read(); /** diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index e73b59a5ae..7e757fd7fd 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -280,7 +280,7 @@ RecordComponent::storeChunk( Offset o, Extent e, F && createBuffer ) * Flush the openPMD hierarchy to the backend without flushing any actual * data yet. */ - seriesFlush( FlushLevel::SkeletonOnly ); + seriesFlush({FlushLevel::SkeletonOnly}); size_t size = 1; for( auto ext : e ) @@ -305,16 +305,16 @@ RecordComponent::storeChunk( Offset o, Extent e, F && createBuffer ) getBufferView.offset = o; getBufferView.extent = e; getBufferView.dtype = getDatatype(); - IOHandler()->enqueue( IOTask( this, getBufferView ) ); - IOHandler()->flush(); + IOHandler()->enqueue(IOTask(this, getBufferView)); + IOHandler()->flush(internal::defaultFlushParams); auto &out = *getBufferView.out; - if( !out.backendManagedBuffer ) + if (!out.backendManagedBuffer) { - auto data = std::forward< F >( createBuffer )( size ); - out.ptr = static_cast< void * >( data.get() ); - storeChunk( std::move( data ), std::move( o ), std::move( e ) ); + auto data = std::forward(createBuffer)(size); + out.ptr = static_cast(data.get()); + storeChunk(std::move(data), std::move(o), std::move(e)); } - return DynamicMemoryView< T >{ std::move( getBufferView ), size, *this }; + return DynamicMemoryView{std::move(getBufferView), size, *this}; } template< typename T > diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 527ef4538f..0649a9919b 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -532,16 +532,19 @@ OPENPMD_private * * @param begin Start of the range of iterations to flush. * @param end End of the range of iterations to flush. - * @param level Flush level, as documented in AbstractIOHandler.hpp. + * @param flushParams Flush params, as documented in AbstractIOHandler.hpp. * @param flushIOHandler Tasks will always be enqueued to the backend. * If this flag is true, tasks will be flushed to the backend. */ std::future flush_impl( iterations_iterator begin, iterations_iterator end, - FlushLevel level, + internal::FlushParams flushParams, bool flushIOHandler = true); - void flushFileBased(iterations_iterator begin, iterations_iterator end); + void flushFileBased( + iterations_iterator begin, + iterations_iterator end, + internal::FlushParams flushParams); /* * Group-based and variable-based iteration layouts share a lot of logic * (realistically, the variable-based iteration layout only throws out @@ -549,7 +552,10 @@ OPENPMD_private * As a convention, methods that deal with both layouts are called * .*GorVBased, short for .*GroupOrVariableBased */ - void flushGorVBased(iterations_iterator begin, iterations_iterator end); + void flushGorVBased( + iterations_iterator begin, + iterations_iterator end, + internal::FlushParams flushParams); void flushMeshesPath(); void flushParticlesPath(); void readFileBased(); diff --git a/include/openPMD/Span.hpp b/include/openPMD/Span.hpp index 48b24b02f9..93bc0d24f7 100644 --- a/include/openPMD/Span.hpp +++ b/include/openPMD/Span.hpp @@ -125,7 +125,7 @@ class DynamicMemoryView // might need to update m_recordComponent.IOHandler()->enqueue( IOTask(&m_recordComponent, m_param)); - m_recordComponent.IOHandler()->flush(); + m_recordComponent.IOHandler()->flush(internal::defaultFlushParams); } return Span{static_cast(m_param.out->ptr), m_size}; } diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 8b71e41a6e..58648c8b2a 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -273,9 +273,9 @@ OPENPMD_protected Iteration &containingIteration(); /** @} */ - void seriesFlush(FlushLevel); + void seriesFlush(internal::FlushParams); - void flushAttributes(); + void flushAttributes(internal::FlushParams const &); enum ReadMode { /** diff --git a/include/openPMD/backend/BaseRecord.hpp b/include/openPMD/backend/BaseRecord.hpp index 80946374f8..65ae298da9 100644 --- a/include/openPMD/backend/BaseRecord.hpp +++ b/include/openPMD/backend/BaseRecord.hpp @@ -141,8 +141,9 @@ class BaseRecord : public Container void readBase(); private: - void flush(std::string const &) final; - virtual void flush_impl(std::string const &) = 0; + void flush(std::string const &, internal::FlushParams const &) final; + virtual void + flush_impl(std::string const &, internal::FlushParams const &) = 0; virtual void read() = 0; /** @@ -250,7 +251,7 @@ BaseRecord::erase(key_type const &key) Parameter dDelete; dDelete.name = "."; this->IOHandler()->enqueue(IOTask(&rc, dDelete)); - this->IOHandler()->flush(); + this->IOHandler()->flush(internal::defaultFlushParams); } res = Container::erase(key); } @@ -280,7 +281,7 @@ BaseRecord::erase(iterator res) Parameter dDelete; dDelete.name = "."; this->IOHandler()->enqueue(IOTask(&rc, dDelete)); - this->IOHandler()->flush(); + this->IOHandler()->flush(internal::defaultFlushParams); } ret = Container::erase(res); } @@ -315,7 +316,7 @@ inline void BaseRecord::readBase() aRead.name = "unitDimension"; this->IOHandler()->enqueue(IOTask(this, aRead)); - this->IOHandler()->flush(); + this->IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::ARR_DBL_7) this->setAttribute( "unitDimension", @@ -340,7 +341,7 @@ inline void BaseRecord::readBase() aRead.name = "timeOffset"; this->IOHandler()->enqueue(IOTask(this, aRead)); - this->IOHandler()->flush(); + this->IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::FLOAT) this->setAttribute( "timeOffset", Attribute(*aRead.resource).template get()); @@ -353,7 +354,8 @@ inline void BaseRecord::readBase() } template -inline void BaseRecord::flush(std::string const &name) +inline void BaseRecord::flush( + std::string const &name, internal::FlushParams const &flushParams) { if (!this->written() && this->empty()) throw std::runtime_error( @@ -361,7 +363,7 @@ inline void BaseRecord::flush(std::string const &name) "RecordComponents: " + name); - this->flush_impl(name); + this->flush_impl(name, flushParams); // flush_impl must take care to correctly set the dirty() flag so this // method doesn't do it } diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 4973c05e15..9dd163ecae 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -388,7 +388,7 @@ class Container : public Attributable Parameter pDelete; pDelete.path = "."; IOHandler()->enqueue(IOTask(&res->second, pDelete)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); } return container().erase(key); } @@ -405,7 +405,7 @@ class Container : public Attributable Parameter pDelete; pDelete.path = "."; IOHandler()->enqueue(IOTask(&res->second, pDelete)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); } return container().erase(res); } @@ -436,7 +436,8 @@ OPENPMD_protected container().clear(); } - virtual void flush(std::string const &path) + virtual void + flush(std::string const &path, internal::FlushParams const &flushParams) { if (!written()) { @@ -445,7 +446,7 @@ OPENPMD_protected IOHandler()->enqueue(IOTask(this, pCreate)); } - flushAttributes(); + flushAttributes(flushParams); } // clang-format off diff --git a/include/openPMD/backend/PatchRecord.hpp b/include/openPMD/backend/PatchRecord.hpp index 27ddb0ba96..84d180bac5 100644 --- a/include/openPMD/backend/PatchRecord.hpp +++ b/include/openPMD/backend/PatchRecord.hpp @@ -41,7 +41,8 @@ class PatchRecord : public BaseRecord private: PatchRecord() = default; - void flush_impl(std::string const &) override; + void + flush_impl(std::string const &, internal::FlushParams const &) override; void read() override; }; // PatchRecord } // namespace openPMD diff --git a/include/openPMD/backend/PatchRecordComponent.hpp b/include/openPMD/backend/PatchRecordComponent.hpp index 5f5ed91952..280b674ceb 100644 --- a/include/openPMD/backend/PatchRecordComponent.hpp +++ b/include/openPMD/backend/PatchRecordComponent.hpp @@ -91,7 +91,7 @@ class PatchRecordComponent : public BaseRecordComponent OPENPMD_private // clang-format on - void flush(std::string const &); + void flush(std::string const &, internal::FlushParams const &); void read(); /** diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index 7006f3648d..8944d92dea 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -114,7 +114,7 @@ class Writable final OPENPMD_private // clang-format on - void seriesFlush(FlushLevel); + void seriesFlush(internal::FlushParams); /* * These members need to be shared pointers since distinct instances of * Writable may share them. diff --git a/src/IO/ADIOS/ADIOS1IOHandler.cpp b/src/IO/ADIOS/ADIOS1IOHandler.cpp index 3a6dc803d2..cde3b75156 100644 --- a/src/IO/ADIOS/ADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS1IOHandler.cpp @@ -330,7 +330,7 @@ ADIOS1IOHandler::ADIOS1IOHandler( ADIOS1IOHandler::~ADIOS1IOHandler() = default; -std::future ADIOS1IOHandler::flush() +std::future ADIOS1IOHandler::flush(internal::FlushParams const &) { return m_impl->flush(); } @@ -431,7 +431,7 @@ ADIOS1IOHandler::ADIOS1IOHandler(std::string path, Access at, json::TracingJSON) ADIOS1IOHandler::~ADIOS1IOHandler() = default; -std::future ADIOS1IOHandler::flush() +std::future ADIOS1IOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index b0e10f7a8e..5afa60628a 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -253,7 +253,8 @@ std::string ADIOS2IOHandlerImpl::fileSuffix() const } } -std::future ADIOS2IOHandlerImpl::flush() +std::future +ADIOS2IOHandlerImpl::flush(internal::FlushParams const &flushParams) { auto res = AbstractIOHandlerImpl::flush(); for (auto &p : m_fileData) @@ -261,7 +262,7 @@ std::future ADIOS2IOHandlerImpl::flush() if (m_dirty.find(p.first) != m_dirty.end()) { p.second->flush( - m_handler->m_flushLevel, /* writeAttributes = */ false); + flushParams.flushLevel, /* writeAttributes = */ false); } else { @@ -2869,9 +2870,10 @@ ADIOS2IOHandler::ADIOS2IOHandler( , m_impl{this, std::move(options), std::move(engineType)} {} -std::future ADIOS2IOHandler::flush() +std::future +ADIOS2IOHandler::flush(internal::FlushParams const &flushParams) { - return m_impl.flush(); + return m_impl.flush(flushParams); } #else // openPMD_HAVE_ADIOS2 @@ -2889,7 +2891,7 @@ ADIOS2IOHandler::ADIOS2IOHandler( : AbstractIOHandler(std::move(path), at) {} -std::future ADIOS2IOHandler::flush() +std::future ADIOS2IOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp index df6f817098..78ac9f4c59 100644 --- a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp @@ -349,7 +349,7 @@ ParallelADIOS1IOHandler::ParallelADIOS1IOHandler( ParallelADIOS1IOHandler::~ParallelADIOS1IOHandler() = default; -std::future ParallelADIOS1IOHandler::flush() +std::future ParallelADIOS1IOHandler::flush(internal::FlushParams const &) { return m_impl->flush(); } @@ -471,7 +471,7 @@ ParallelADIOS1IOHandler::ParallelADIOS1IOHandler( ParallelADIOS1IOHandler::~ParallelADIOS1IOHandler() = default; -std::future ParallelADIOS1IOHandler::flush() +std::future ParallelADIOS1IOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/DummyIOHandler.cpp b/src/IO/DummyIOHandler.cpp index f10cd50ac9..308f584ce4 100644 --- a/src/IO/DummyIOHandler.cpp +++ b/src/IO/DummyIOHandler.cpp @@ -32,7 +32,7 @@ DummyIOHandler::DummyIOHandler(std::string path, Access at) void DummyIOHandler::enqueue(IOTask const &) {} -std::future DummyIOHandler::flush() +std::future DummyIOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 0d855aa004..de541ab0f0 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -2292,7 +2292,7 @@ HDF5IOHandler::HDF5IOHandler( HDF5IOHandler::~HDF5IOHandler() = default; -std::future HDF5IOHandler::flush() +std::future HDF5IOHandler::flush(internal::FlushParams const &) { return m_impl->flush(); } @@ -2306,7 +2306,7 @@ HDF5IOHandler::HDF5IOHandler( HDF5IOHandler::~HDF5IOHandler() = default; -std::future HDF5IOHandler::flush() +std::future HDF5IOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/HDF5/ParallelHDF5IOHandler.cpp b/src/IO/HDF5/ParallelHDF5IOHandler.cpp index c0ce9e49e6..e0a17ed980 100644 --- a/src/IO/HDF5/ParallelHDF5IOHandler.cpp +++ b/src/IO/HDF5/ParallelHDF5IOHandler.cpp @@ -54,7 +54,7 @@ ParallelHDF5IOHandler::ParallelHDF5IOHandler( ParallelHDF5IOHandler::~ParallelHDF5IOHandler() = default; -std::future ParallelHDF5IOHandler::flush() +std::future ParallelHDF5IOHandler::flush(internal::FlushParams const &) { return m_impl->flush(); } @@ -196,7 +196,7 @@ ParallelHDF5IOHandler::ParallelHDF5IOHandler( ParallelHDF5IOHandler::~ParallelHDF5IOHandler() = default; -std::future ParallelHDF5IOHandler::flush() +std::future ParallelHDF5IOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/JSON/JSONIOHandler.cpp b/src/IO/JSON/JSONIOHandler.cpp index 158c5454ed..15d18194c7 100644 --- a/src/IO/JSON/JSONIOHandler.cpp +++ b/src/IO/JSON/JSONIOHandler.cpp @@ -29,7 +29,7 @@ JSONIOHandler::JSONIOHandler(std::string path, Access at) : AbstractIOHandler{path, at}, m_impl{JSONIOHandlerImpl{this}} {} -std::future JSONIOHandler::flush() +std::future JSONIOHandler::flush(internal::FlushParams const &) { return m_impl.flush(); } diff --git a/src/Iteration.cpp b/src/Iteration.cpp index eb5e43d189..73c8a46847 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -127,7 +127,7 @@ Iteration &Iteration::close(bool _flush) auto end = begin; ++end; - s.flush_impl(begin, end, FlushLevel::UserFlush); + s.flush_impl(begin, end, {FlushLevel::UserFlush}); } } else @@ -154,7 +154,8 @@ Iteration &Iteration::open() // figure out my iteration number auto begin = s.indexOf(*this); s.openIteration(begin->first, *this); - IOHandler()->flush(); + // @todo, maybe collective here + IOHandler()->flush(internal::defaultFlushParams); return *this; } @@ -191,7 +192,10 @@ bool Iteration::closedByWriter() const } } -void Iteration::flushFileBased(std::string const &filename, uint64_t i) +void Iteration::flushFileBased( + std::string const &filename, + uint64_t i, + internal::FlushParams const &flushParams) { /* Find the root point [Series] of this file, * meshesPath and particlesPath are stored there */ @@ -224,7 +228,7 @@ void Iteration::flushFileBased(std::string const &filename, uint64_t i) fOpen.name = filename; fOpen.encoding = IterationEncoding::fileBased; IOHandler()->enqueue(IOTask(&s.writable(), fOpen)); - flush(); + flush(flushParams); return; } @@ -234,10 +238,11 @@ void Iteration::flushFileBased(std::string const &filename, uint64_t i) s.openIteration(i, *this); } - flush(); + flush(flushParams); } -void Iteration::flushGroupBased(uint64_t i) +void Iteration::flushGroupBased( + uint64_t i, internal::FlushParams const &flushParams) { if (!written()) { @@ -247,10 +252,11 @@ void Iteration::flushGroupBased(uint64_t i) IOHandler()->enqueue(IOTask(this, pCreate)); } - flush(); + flush(flushParams); } -void Iteration::flushVariableBased(uint64_t i) +void Iteration::flushVariableBased( + uint64_t i, internal::FlushParams const &flushParams) { if (!written()) { @@ -261,17 +267,17 @@ void Iteration::flushVariableBased(uint64_t i) this->setAttribute("snapshot", i); } - flush(); + flush(flushParams); } -void Iteration::flush() +void Iteration::flush(internal::FlushParams const &flushParams) { if (IOHandler()->m_frontendAccess == Access::READ_ONLY) { for (auto &m : meshes) - m.second.flush(m.first); + m.second.flush(m.first, flushParams); for (auto &species : particles) - species.second.flush(species.first); + species.second.flush(species.first, flushParams); } else { @@ -286,9 +292,9 @@ void Iteration::flush() s.setMeshesPath("meshes/"); s.flushMeshesPath(); } - meshes.flush(s.meshesPath()); + meshes.flush(s.meshesPath(), flushParams); for (auto &m : meshes) - m.second.flush(m.first); + m.second.flush(m.first, flushParams); } else { @@ -302,16 +308,16 @@ void Iteration::flush() s.setParticlesPath("particles/"); s.flushParticlesPath(); } - particles.flush(s.particlesPath()); + particles.flush(s.particlesPath(), flushParams); for (auto &species : particles) - species.second.flush(species.first); + species.second.flush(species.first, flushParams); } else { particles.dirty() = false; } - flushAttributes(); + flushAttributes(flushParams); } } @@ -379,7 +385,7 @@ void Iteration::read_impl(std::string const &groupPath) aRead.name = "dt"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::FLOAT) setDt(Attribute(*aRead.resource).get()); else if (*aRead.dtype == DT::DOUBLE) @@ -391,7 +397,7 @@ void Iteration::read_impl(std::string const &groupPath) aRead.name = "time"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::FLOAT) setTime(Attribute(*aRead.resource).get()); else if (*aRead.dtype == DT::DOUBLE) @@ -403,7 +409,7 @@ void Iteration::read_impl(std::string const &groupPath) aRead.name = "timeUnitSI"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::DOUBLE) setTimeUnitSI(Attribute(*aRead.resource).get()); else @@ -421,7 +427,7 @@ void Iteration::read_impl(std::string const &groupPath) if (version == "1.0.0" || version == "1.0.1") { IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); hasMeshes = std::count( pList.paths->begin(), pList.paths->end(), @@ -450,7 +456,7 @@ void Iteration::read_impl(std::string const &groupPath) /* obtain all non-scalar meshes */ IOHandler()->enqueue(IOTask(&meshes, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter aList; for (auto const &mesh_name : *pList.paths) @@ -460,7 +466,7 @@ void Iteration::read_impl(std::string const &groupPath) aList.attributes->clear(); IOHandler()->enqueue(IOTask(&m, pOpen)); IOHandler()->enqueue(IOTask(&m, aList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); auto att_begin = aList.attributes->begin(); auto att_end = aList.attributes->end(); @@ -471,7 +477,7 @@ void Iteration::read_impl(std::string const &groupPath) MeshRecordComponent &mrc = m[MeshRecordComponent::SCALAR]; mrc.parent() = m.parent(); IOHandler()->enqueue(IOTask(&mrc, pOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); mrc.get().m_isConstant = true; } m.read(); @@ -480,7 +486,7 @@ void Iteration::read_impl(std::string const &groupPath) /* obtain all scalar meshes */ Parameter dList; IOHandler()->enqueue(IOTask(&meshes, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &mesh_name : *dList.datasets) @@ -488,11 +494,11 @@ void Iteration::read_impl(std::string const &groupPath) Mesh &m = map[mesh_name]; dOpen.name = mesh_name; IOHandler()->enqueue(IOTask(&m, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); MeshRecordComponent &mrc = m[MeshRecordComponent::SCALAR]; mrc.parent() = m.parent(); IOHandler()->enqueue(IOTask(&mrc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); mrc.written() = false; mrc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); mrc.written() = true; @@ -514,7 +520,7 @@ void Iteration::read_impl(std::string const &groupPath) /* obtain all particle species */ pList.paths->clear(); IOHandler()->enqueue(IOTask(&particles, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); internal::EraseStaleEntries map{particles}; for (auto const &species_name : *pList.paths) @@ -522,7 +528,7 @@ void Iteration::read_impl(std::string const &groupPath) ParticleSpecies &p = map[species_name]; pOpen.path = species_name; IOHandler()->enqueue(IOTask(&p, pOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); p.read(); } } diff --git a/src/Mesh.cpp b/src/Mesh.cpp index bdb6081148..b30c7e0f5d 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -213,12 +213,13 @@ template Mesh &Mesh::setTimeOffset(double); template Mesh &Mesh::setTimeOffset(float); -void Mesh::flush_impl(std::string const &name) +void Mesh::flush_impl( + std::string const &name, internal::FlushParams const &flushParams) { if (IOHandler()->m_frontendAccess == Access::READ_ONLY) { for (auto &comp : *this) - comp.second.flush(comp.first); + comp.second.flush(comp.first, flushParams); } else { @@ -228,8 +229,8 @@ void Mesh::flush_impl(std::string const &name) { MeshRecordComponent &mrc = at(RecordComponent::SCALAR); mrc.parent() = parent(); - mrc.flush(name); - IOHandler()->flush(); + mrc.flush(name, flushParams); + IOHandler()->flush(flushParams); writable().abstractFilePosition = mrc.writable().abstractFilePosition; written() = true; @@ -248,7 +249,7 @@ void Mesh::flush_impl(std::string const &name) { for (auto &comp : *this) { - comp.second.flush(name); + comp.second.flush(name, flushParams); writable().abstractFilePosition = comp.second.writable().abstractFilePosition; } @@ -256,10 +257,10 @@ void Mesh::flush_impl(std::string const &name) else { for (auto &comp : *this) - comp.second.flush(comp.first); + comp.second.flush(comp.first, flushParams); } - flushAttributes(); + flushAttributes(flushParams); } } @@ -272,7 +273,7 @@ void Mesh::read() aRead.name = "geometry"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { std::string tmpGeometry = Attribute(*aRead.resource).get(); @@ -293,7 +294,7 @@ void Mesh::read() aRead.name = "dataOrder"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::CHAR) setDataOrder( static_cast(Attribute(*aRead.resource).get())); @@ -313,7 +314,7 @@ void Mesh::read() aRead.name = "axisLabels"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::VEC_STRING || *aRead.dtype == DT::STRING) setAxisLabels( Attribute(*aRead.resource).get >()); @@ -323,7 +324,7 @@ void Mesh::read() aRead.name = "gridSpacing"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Attribute a = Attribute(*aRead.resource); if (*aRead.dtype == DT::VEC_FLOAT || *aRead.dtype == DT::FLOAT) setGridSpacing(a.get >()); @@ -338,7 +339,7 @@ void Mesh::read() aRead.name = "gridGlobalOffset"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::VEC_DOUBLE || *aRead.dtype == DT::DOUBLE) setGridGlobalOffset( Attribute(*aRead.resource).get >()); @@ -348,7 +349,7 @@ void Mesh::read() aRead.name = "gridUnitSI"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::DOUBLE) setGridUnitSI(Attribute(*aRead.resource).get()); else @@ -364,7 +365,7 @@ void Mesh::read() { Parameter pList; IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter pOpen; for (auto const &component : *pList.paths) @@ -378,7 +379,7 @@ void Mesh::read() Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &component : *dList.datasets) @@ -386,7 +387,7 @@ void Mesh::read() MeshRecordComponent &rc = map[component]; dOpen.name = component; IOHandler()->enqueue(IOTask(&rc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); rc.written() = false; rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); rc.written() = true; diff --git a/src/ParticlePatches.cpp b/src/ParticlePatches.cpp index 952aff37ac..76017bbf94 100644 --- a/src/ParticlePatches.cpp +++ b/src/ParticlePatches.cpp @@ -35,7 +35,7 @@ void ParticlePatches::read() { Parameter pList; IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter pOpen; for (auto const &record_name : *pList.paths) @@ -48,7 +48,7 @@ void ParticlePatches::read() Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &component_name : *dList.datasets) @@ -65,7 +65,7 @@ void ParticlePatches::read() dOpen.name = component_name; IOHandler()->enqueue(IOTask(&pr, dOpen)); IOHandler()->enqueue(IOTask(&prc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (determineDatatype() != *dOpen.dtype) throw std::runtime_error( diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 8b00413e80..fdc4fa6f88 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -38,7 +38,7 @@ void ParticleSpecies::read() /* obtain all non-scalar records */ Parameter pList; IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); internal::EraseStaleEntries map{*this}; @@ -61,7 +61,7 @@ void ParticleSpecies::read() aList.attributes->clear(); IOHandler()->enqueue(IOTask(&r, pOpen)); IOHandler()->enqueue(IOTask(&r, aList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); auto att_begin = aList.attributes->begin(); auto att_end = aList.attributes->end(); @@ -73,7 +73,7 @@ void ParticleSpecies::read() RecordComponent &rc = scalarMap[RecordComponent::SCALAR]; rc.parent() = r.parent(); IOHandler()->enqueue(IOTask(&rc, pOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); rc.get().m_isConstant = true; } r.read(); @@ -90,7 +90,7 @@ void ParticleSpecies::read() /* obtain all scalar records */ Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &record_name : *dList.datasets) @@ -100,12 +100,12 @@ void ParticleSpecies::read() Record &r = map[record_name]; dOpen.name = record_name; IOHandler()->enqueue(IOTask(&r, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); internal::EraseStaleEntries scalarMap(r); RecordComponent &rc = scalarMap[RecordComponent::SCALAR]; rc.parent() = r.parent(); IOHandler()->enqueue(IOTask(&rc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); rc.written() = false; rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); rc.written() = true; @@ -138,14 +138,15 @@ namespace } } // namespace -void ParticleSpecies::flush(std::string const &path) +void ParticleSpecies::flush( + std::string const &path, internal::FlushParams const &flushParams) { if (IOHandler()->m_frontendAccess == Access::READ_ONLY) { for (auto &record : *this) - record.second.flush(record.first); + record.second.flush(record.first, flushParams); for (auto &patch : particlePatches) - patch.second.flush(patch.first); + patch.second.flush(patch.first, flushParams); } else { @@ -156,16 +157,16 @@ void ParticleSpecies::flush(std::string const &path) if (it != end()) it->second.setUnitDimension({{UnitDimension::L, 1}}); - Container::flush(path); + Container::flush(path, flushParams); for (auto &record : *this) - record.second.flush(record.first); + record.second.flush(record.first, flushParams); if (flushParticlePatches(particlePatches)) { - particlePatches.flush("particlePatches"); + particlePatches.flush("particlePatches", flushParams); for (auto &patch : particlePatches) - patch.second.flush(patch.first); + patch.second.flush(patch.first, flushParams); } } } diff --git a/src/Record.cpp b/src/Record.cpp index ce18bdccc5..da57ebeef1 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -43,12 +43,13 @@ Record &Record::setUnitDimension(std::map const &udim) return *this; } -void Record::flush_impl(std::string const &name) +void Record::flush_impl( + std::string const &name, internal::FlushParams const &flushParams) { if (IOHandler()->m_frontendAccess == Access::READ_ONLY) { for (auto &comp : *this) - comp.second.flush(comp.first); + comp.second.flush(comp.first, flushParams); } else { @@ -58,8 +59,8 @@ void Record::flush_impl(std::string const &name) { RecordComponent &rc = at(RecordComponent::SCALAR); rc.parent() = parent(); - rc.flush(name); - IOHandler()->flush(); + rc.flush(name, flushParams); + IOHandler()->flush(flushParams); writable().abstractFilePosition = rc.writable().abstractFilePosition; written() = true; @@ -78,7 +79,7 @@ void Record::flush_impl(std::string const &name) { for (auto &comp : *this) { - comp.second.flush(name); + comp.second.flush(name, flushParams); writable().abstractFilePosition = comp.second.writable().abstractFilePosition; } @@ -86,10 +87,10 @@ void Record::flush_impl(std::string const &name) else { for (auto &comp : *this) - comp.second.flush(comp.first); + comp.second.flush(comp.first, flushParams); } - flushAttributes(); + flushAttributes(flushParams); } } @@ -104,7 +105,7 @@ void Record::read() { Parameter pList; IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter pOpen; for (auto const &component : *pList.paths) @@ -118,7 +119,7 @@ void Record::read() Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &component : *dList.datasets) @@ -126,7 +127,7 @@ void Record::read() RecordComponent &rc = (*this)[component]; dOpen.name = component; IOHandler()->enqueue(IOTask(&rc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); rc.written() = false; rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); rc.written() = true; diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index ad4ca07a83..c69c09e997 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -191,10 +191,11 @@ bool RecordComponent::empty() const return get().m_isEmpty; } -void RecordComponent::flush(std::string const &name) +void RecordComponent::flush( + std::string const &name, internal::FlushParams const &flushParams) { auto &rc = get(); - if (IOHandler()->m_flushLevel == FlushLevel::SkeletonOnly) + if (flushParams.flushLevel == FlushLevel::SkeletonOnly) { rc.m_name = name; return; @@ -273,7 +274,7 @@ void RecordComponent::flush(std::string const &name) rc.m_chunks.pop(); } - flushAttributes(); + flushAttributes(flushParams); } } @@ -292,7 +293,7 @@ void RecordComponent::readBase() { aRead.name = "value"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Attribute a(*aRead.resource); DT dtype = *aRead.dtype; @@ -357,7 +358,7 @@ void RecordComponent::readBase() aRead.name = "shape"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); a = Attribute(*aRead.resource); Extent e; @@ -383,7 +384,7 @@ void RecordComponent::readBase() aRead.name = "unitSI"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::DOUBLE) setUnitSI(Attribute(*aRead.resource).get()); else diff --git a/src/Series.cpp b/src/Series.cpp index 2913eeb9b2..68059e2af9 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -377,7 +377,7 @@ void Series::flush() flush_impl( series.iterations.begin(), series.iterations.end(), - FlushLevel::UserFlush); + {FlushLevel::UserFlush}); } std::unique_ptr Series::parseInput(std::string filepath) @@ -564,10 +564,9 @@ void Series::initDefaults(IterationEncoding ie) std::future Series::flush_impl( iterations_iterator begin, iterations_iterator end, - FlushLevel level, + internal::FlushParams flushParams, bool flushIOHandler) { - IOHandler()->m_flushLevel = level; auto &series = get(); series.m_lastFlushSuccessful = true; try @@ -576,34 +575,33 @@ std::future Series::flush_impl( { using IE = IterationEncoding; case IE::fileBased: - flushFileBased(begin, end); + flushFileBased(begin, end, flushParams); break; case IE::groupBased: case IE::variableBased: - flushGorVBased(begin, end); + flushGorVBased(begin, end, flushParams); break; } if (flushIOHandler) { - auto res = IOHandler()->flush(); - IOHandler()->m_flushLevel = FlushLevel::InternalFlush; - return res; + return IOHandler()->flush(flushParams); } else { - IOHandler()->m_flushLevel = FlushLevel::InternalFlush; return {}; } } catch (...) { - IOHandler()->m_flushLevel = FlushLevel::InternalFlush; series.m_lastFlushSuccessful = false; throw; } } -void Series::flushFileBased(iterations_iterator begin, iterations_iterator end) +void Series::flushFileBased( + iterations_iterator begin, + iterations_iterator end, + internal::FlushParams flushParams) { auto &series = get(); if (end == begin) @@ -618,7 +616,7 @@ void Series::flushFileBased(iterations_iterator begin, iterations_iterator end) { using IO = IterationOpened; case IO::HasBeenOpened: - it->second.flush(); + it->second.flush(flushParams); break; case IO::RemainsClosed: break; @@ -635,7 +633,7 @@ void Series::flushFileBased(iterations_iterator begin, iterations_iterator end) } // Phase 3 - IOHandler()->flush(); + IOHandler()->flush(flushParams); } else { @@ -656,12 +654,13 @@ void Series::flushFileBased(iterations_iterator begin, iterations_iterator end) dirty() |= it->second.dirty(); std::string filename = iterationFilename(it->first); - it->second.flushFileBased(filename, it->first); + it->second.flushFileBased(filename, it->first, flushParams); series.iterations.flush( - auxiliary::replace_first(basePath(), "%T/", "")); + auxiliary::replace_first(basePath(), "%T/", ""), + flushParams); - flushAttributes(); + flushAttributes(flushParams); break; } case IO::RemainsClosed: @@ -679,7 +678,7 @@ void Series::flushFileBased(iterations_iterator begin, iterations_iterator end) } // Phase 3 - IOHandler()->flush(); + IOHandler()->flush(flushParams); /* reset the dirty bit for every iteration (i.e. file) * otherwise only the first iteration will have updates attributes @@ -690,7 +689,10 @@ void Series::flushFileBased(iterations_iterator begin, iterations_iterator end) } } -void Series::flushGorVBased(iterations_iterator begin, iterations_iterator end) +void Series::flushGorVBased( + iterations_iterator begin, + iterations_iterator end, + internal::FlushParams flushParams) { auto &series = get(); if (IOHandler()->m_frontendAccess == Access::READ_ONLY) @@ -701,7 +703,7 @@ void Series::flushGorVBased(iterations_iterator begin, iterations_iterator end) { using IO = IterationOpened; case IO::HasBeenOpened: - it->second.flush(); + it->second.flush(flushParams); break; case IO::RemainsClosed: break; @@ -717,7 +719,7 @@ void Series::flushGorVBased(iterations_iterator begin, iterations_iterator end) } // Phase 3 - IOHandler()->flush(); + IOHandler()->flush(flushParams); } else { @@ -730,7 +732,7 @@ void Series::flushGorVBased(iterations_iterator begin, iterations_iterator end) } series.iterations.flush( - auxiliary::replace_first(basePath(), "%T/", "")); + auxiliary::replace_first(basePath(), "%T/", ""), flushParams); for (auto it = begin; it != end; ++it) { @@ -747,10 +749,10 @@ void Series::flushGorVBased(iterations_iterator begin, iterations_iterator end) { using IE = IterationEncoding; case IE::groupBased: - it->second.flushGroupBased(it->first); + it->second.flushGroupBased(it->first, flushParams); break; case IE::variableBased: - it->second.flushVariableBased(it->first); + it->second.flushVariableBased(it->first, flushParams); break; default: throw std::runtime_error( @@ -771,8 +773,8 @@ void Series::flushGorVBased(iterations_iterator begin, iterations_iterator end) } } - flushAttributes(); - IOHandler()->flush(); + flushAttributes(flushParams); + IOHandler()->flush(flushParams); } } @@ -846,7 +848,7 @@ void Series::readFileBased() iteration.runDeferredParseAccess(); Parameter fClose; iteration.IOHandler()->enqueue(IOTask(&iteration, fClose)); - iteration.IOHandler()->flush(); + iteration.IOHandler()->flush(internal::defaultFlushParams); iteration.get().m_closed = internal::CloseStatus::ClosedTemporarily; }; if (series.m_parseLazily) @@ -892,7 +894,7 @@ void Series::readOneIterationFileBased(std::string const &filePath) fOpen.name = filePath; IOHandler()->enqueue(IOTask(this, fOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); series.iterations.parent() = getWritable(this); readBase(); @@ -900,7 +902,7 @@ void Series::readOneIterationFileBased(std::string const &filePath) using DT = Datatype; aRead.name = "iterationEncoding"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { std::string encoding = Attribute(*aRead.resource).get(); @@ -937,7 +939,7 @@ void Series::readOneIterationFileBased(std::string const &filePath) aRead.name = "iterationFormat"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { written() = false; @@ -967,7 +969,7 @@ void Series::readGorVBased(bool do_init) fOpen.name = series.m_name; fOpen.encoding = iterationEncoding(); IOHandler()->enqueue(IOTask(this, fOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (do_init) { @@ -977,7 +979,7 @@ void Series::readGorVBased(bool do_init) Parameter aRead; aRead.name = "iterationEncoding"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { std::string encoding = @@ -1011,7 +1013,7 @@ void Series::readGorVBased(bool do_init) aRead.name = "iterationFormat"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { written() = false; @@ -1039,7 +1041,7 @@ void Series::readGorVBased(bool do_init) /* obtain all paths inside the basepath (i.e. all iterations) */ Parameter pList; IOHandler()->enqueue(IOTask(&series.iterations, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); auto readSingleIteration = [&series, &pOpen, this]( uint64_t index, @@ -1112,7 +1114,7 @@ void Series::readBase() aRead.name = "openPMD"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) setOpenPMD(Attribute(*aRead.resource).get()); else @@ -1120,7 +1122,7 @@ void Series::readBase() aRead.name = "openPMDextension"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == determineDatatype()) setOpenPMDextension(Attribute(*aRead.resource).get()); else @@ -1129,7 +1131,7 @@ void Series::readBase() aRead.name = "basePath"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) setAttribute("basePath", Attribute(*aRead.resource).get()); else @@ -1138,14 +1140,14 @@ void Series::readBase() Parameter aList; IOHandler()->enqueue(IOTask(this, aList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (std::count( aList.attributes->begin(), aList.attributes->end(), "meshesPath") == 1) { aRead.name = "meshesPath"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { /* allow setting the meshes path after completed IO */ @@ -1169,7 +1171,7 @@ void Series::readBase() { aRead.name = "particlesPath"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { /* allow setting the meshes path after completed IO */ @@ -1220,6 +1222,7 @@ AdvanceStatus Series::advance( iterations_iterator begin, Iteration &iteration) { + constexpr internal::FlushParams flushParams = {FlushLevel::UserFlush}; auto &series = get(); auto end = begin; ++end; @@ -1238,7 +1241,8 @@ AdvanceStatus Series::advance( itData.m_closed = internal::CloseStatus::Open; } - flush_impl(begin, end, FlushLevel::UserFlush, /* flushIOHandler = */ false); + // @todo really collective? + flush_impl(begin, end, flushParams, /* flushIOHandler = */ false); if (oldCloseStatus == internal::CloseStatus::ClosedInFrontend) { @@ -1309,17 +1313,7 @@ AdvanceStatus Series::advance( // We cannot call Series::flush now, since the IO handler is still filled // from calling flush(Group|File)based, but has not been emptied yet // Do that manually - IOHandler()->m_flushLevel = FlushLevel::UserFlush; - try - { - IOHandler()->flush(); - } - catch (...) - { - IOHandler()->m_flushLevel = FlushLevel::InternalFlush; - throw; - } - IOHandler()->m_flushLevel = FlushLevel::InternalFlush; + IOHandler()->flush(flushParams); return *param.status; } diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 52748e9806..d037ad68bc 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -66,7 +66,7 @@ bool Attributable::deleteAttribute(std::string const &key) Parameter aDelete; aDelete.name = key; IOHandler()->enqueue(IOTask(this, aDelete)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); attri.m_attributes.erase(it); return true; } @@ -206,14 +206,14 @@ auto Attributable::myPath() const -> MyPath return res; } -void Attributable::seriesFlush(FlushLevel level) +void Attributable::seriesFlush(internal::FlushParams flushParams) { - writable().seriesFlush(level); + writable().seriesFlush(flushParams); } -void Attributable::flushAttributes() +void Attributable::flushAttributes(internal::FlushParams const &flushParams) { - if (IOHandler()->m_flushLevel == FlushLevel::SkeletonOnly) + if (flushParams.flushLevel == FlushLevel::SkeletonOnly) { return; } @@ -237,7 +237,7 @@ void Attributable::readAttributes(ReadMode mode) auto &attri = get(); Parameter aList; IOHandler()->enqueue(IOTask(this, aList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); std::vector written_attributes = attributes(); /* std::set_difference requires sorted ranges */ @@ -277,7 +277,7 @@ void Attributable::readAttributes(ReadMode mode) IOHandler()->enqueue(IOTask(this, aRead)); try { - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); } catch (unsupported_data_error const &e) { diff --git a/src/backend/BaseRecordComponent.cpp b/src/backend/BaseRecordComponent.cpp index 1c86431e02..4460b7ede0 100644 --- a/src/backend/BaseRecordComponent.cpp +++ b/src/backend/BaseRecordComponent.cpp @@ -61,7 +61,7 @@ ChunkTable BaseRecordComponent::availableChunks() Parameter param; IOTask task(this, param); IOHandler()->enqueue(task); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); return std::move(*param.chunks); } diff --git a/src/backend/MeshRecordComponent.cpp b/src/backend/MeshRecordComponent.cpp index aa5afcce34..9602868a07 100644 --- a/src/backend/MeshRecordComponent.cpp +++ b/src/backend/MeshRecordComponent.cpp @@ -34,7 +34,7 @@ void MeshRecordComponent::read() aRead.name = "position"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Attribute a = Attribute(*aRead.resource); if (*aRead.dtype == DT::VEC_FLOAT || *aRead.dtype == DT::FLOAT) setPosition(a.get >()); diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 9c94241338..f31b135b62 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -36,19 +36,21 @@ PatchRecord::setUnitDimension(std::map const &udim) return *this; } -void PatchRecord::flush_impl(std::string const &path) +void PatchRecord::flush_impl( + std::string const &path, internal::FlushParams const &flushParams) { if (this->find(RecordComponent::SCALAR) == this->end()) { if (IOHandler()->m_frontendAccess != Access::READ_ONLY) Container::flush( - path); // warning (clang-tidy-10): bugprone-parent-virtual-call + path, flushParams); // warning (clang-tidy-10): + // bugprone-parent-virtual-call for (auto &comp : *this) - comp.second.flush(comp.first); + comp.second.flush(comp.first, flushParams); } else - this->operator[](RecordComponent::SCALAR).flush(path); - if (IOHandler()->m_flushLevel == FlushLevel::UserFlush) + this->operator[](RecordComponent::SCALAR).flush(path, flushParams); + if (flushParams.flushLevel == FlushLevel::UserFlush) { this->dirty() = false; } @@ -59,7 +61,7 @@ void PatchRecord::read() Parameter aRead; aRead.name = "unitDimension"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == Datatype::ARR_DBL_7 || *aRead.dtype == Datatype::VEC_DOUBLE) @@ -72,7 +74,7 @@ void PatchRecord::read() Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &component_name : *dList.datasets) @@ -80,7 +82,7 @@ void PatchRecord::read() PatchRecordComponent &prc = (*this)[component_name]; dOpen.name = component_name; IOHandler()->enqueue(IOTask(&prc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); /* allow all attributes to be set */ prc.written() = false; prc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); diff --git a/src/backend/PatchRecordComponent.cpp b/src/backend/PatchRecordComponent.cpp index 6eb696f174..ecc44625a2 100644 --- a/src/backend/PatchRecordComponent.cpp +++ b/src/backend/PatchRecordComponent.cpp @@ -80,7 +80,8 @@ PatchRecordComponent::PatchRecordComponent( : BaseRecordComponent{data}, m_patchRecordComponentData{std::move(data)} {} -void PatchRecordComponent::flush(std::string const &name) +void PatchRecordComponent::flush( + std::string const &name, internal::FlushParams const &flushParams) { auto &rc = get(); if (IOHandler()->m_frontendAccess == Access::READ_ONLY) @@ -109,7 +110,7 @@ void PatchRecordComponent::flush(std::string const &name) rc.m_chunks.pop(); } - flushAttributes(); + flushAttributes(flushParams); } } @@ -119,7 +120,7 @@ void PatchRecordComponent::read() aRead.name = "unitSI"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == Datatype::DOUBLE) setUnitSI(Attribute(*aRead.resource).get()); else diff --git a/src/backend/Writable.cpp b/src/backend/Writable.cpp index 7f3733904b..58ce2e0ad4 100644 --- a/src/backend/Writable.cpp +++ b/src/backend/Writable.cpp @@ -35,15 +35,15 @@ Writable::Writable(internal::AttributableData *a) void Writable::seriesFlush() { - seriesFlush(FlushLevel::UserFlush); + seriesFlush({FlushLevel::UserFlush}); } -void Writable::seriesFlush(FlushLevel level) +void Writable::seriesFlush(internal::FlushParams flushParams) { auto series = Attributable({attributable, [](auto const *) {}}).retrieveSeries(); series.flush_impl( - series.iterations.begin(), series.iterations.end(), level); + series.iterations.begin(), series.iterations.end(), flushParams); } } // namespace openPMD From 6b7c5dfe1ae2c56d64e4d0396e763c7349987b27 Mon Sep 17 00:00:00 2001 From: ToruNiina Date: Sun, 13 Mar 2022 18:23:10 -0700 Subject: [PATCH 05/70] toml11: v3.7.1 --- .../openPMD/thirdParty/toml11/CMakeLists.txt | 71 +- share/openPMD/thirdParty/toml11/README.md | 1967 +++++++++++++++++ share/openPMD/thirdParty/toml11/toml.hpp | 10 +- .../thirdParty/toml11/toml/combinator.hpp | 4 +- .../thirdParty/toml11/toml/comments.hpp | 2 +- .../thirdParty/toml11/toml/datetime.hpp | 20 +- share/openPMD/thirdParty/toml11/toml/get.hpp | 4 +- .../openPMD/thirdParty/toml11/toml/lexer.hpp | 51 +- .../thirdParty/toml11/toml/literal.hpp | 1 + .../openPMD/thirdParty/toml11/toml/parser.hpp | 327 ++- .../thirdParty/toml11/toml/serializer.hpp | 91 +- .../openPMD/thirdParty/toml11/toml/string.hpp | 8 +- .../openPMD/thirdParty/toml11/toml/traits.hpp | 19 +- .../openPMD/thirdParty/toml11/toml/types.hpp | 2 +- .../thirdParty/toml11/toml/utility.hpp | 9 +- .../openPMD/thirdParty/toml11/toml/value.hpp | 6 +- .../thirdParty/toml11/toml/version.hpp | 42 + 17 files changed, 2490 insertions(+), 144 deletions(-) create mode 100644 share/openPMD/thirdParty/toml11/README.md create mode 100644 share/openPMD/thirdParty/toml11/toml/version.hpp diff --git a/share/openPMD/thirdParty/toml11/CMakeLists.txt b/share/openPMD/thirdParty/toml11/CMakeLists.txt index 49b665d6a3..911874cbe2 100644 --- a/share/openPMD/thirdParty/toml11/CMakeLists.txt +++ b/share/openPMD/thirdParty/toml11/CMakeLists.txt @@ -1,17 +1,20 @@ cmake_minimum_required(VERSION 3.1) enable_testing() -project(toml11 VERSION 3.7.0) +project(toml11 VERSION 3.7.1) option(toml11_BUILD_TEST "Build toml tests" OFF) +option(toml11_INSTALL "Install CMake targets during install step." ON) option(toml11_TEST_WITH_ASAN "use LLVM address sanitizer" OFF) option(toml11_TEST_WITH_UBSAN "use LLVM undefined behavior sanitizer" OFF) include(CheckCXXCompilerFlag) if("${CMAKE_VERSION}" VERSION_GREATER 3.1) - set(CMAKE_CXX_STANDARD 11 CACHE STRING "The C++ standard whose features are requested to build all targets.") - set(CMAKE_CXX_STANDARD_REQUIRED ON CACHE BOOL "Boolean describing whether the value of CXX_STANDARD is a requirement.") set(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "Boolean specifying whether compiler specific extensions are requested.") + if(NOT DEFINED CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 11 CACHE STRING "The C++ standard whose features are requested to build all targets.") + endif() + set(CMAKE_CXX_STANDARD_REQUIRED ON CACHE BOOL "Boolean describing whether the value of CXX_STANDARD is a requirement.") else() # Manually check for C++11 compiler flag. CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11) @@ -38,7 +41,7 @@ else() endif() if(MSVC) - add_definitions("/Zc:__cplusplus") # define __cplusplus value correctly +# add_definitions("/Zc:__cplusplus") # define __cplusplus value correctly add_definitions("/utf-8") # enable to use u8"" literal if(MSVC_VERSION LESS 1910) message(STATUS "MSVC < 1910. DEFINE_CONVERSION_NON_INTRUSIVE is disabled") @@ -73,38 +76,40 @@ write_basic_package_version_file( COMPATIBILITY SameMajorVersion ) -configure_package_config_file( - cmake/toml11Config.cmake.in - ${toml11_config} - INSTALL_DESTINATION ${toml11_install_cmake_dir} - PATH_VARS toml11_install_cmake_dir -) +if (toml11_INSTALL) + configure_package_config_file( + cmake/toml11Config.cmake.in + ${toml11_config} + INSTALL_DESTINATION ${toml11_install_cmake_dir} + PATH_VARS toml11_install_cmake_dir + ) -# Install config files -install(FILES ${toml11_config} ${toml11_config_version} - DESTINATION ${toml11_install_cmake_dir} -) + # Install config files + install(FILES ${toml11_config} ${toml11_config_version} + DESTINATION ${toml11_install_cmake_dir} + ) -# Install header files -install( - FILES toml.hpp - DESTINATION "${toml11_install_include_dir}" -) -install( - DIRECTORY "toml" - DESTINATION "${toml11_install_include_dir}" - FILES_MATCHING PATTERN "*.hpp" -) + # Install header files + install( + FILES toml.hpp + DESTINATION "${toml11_install_include_dir}" + ) + install( + DIRECTORY "toml" + DESTINATION "${toml11_install_include_dir}" + FILES_MATCHING PATTERN "*.hpp" + ) -# Export targets and install them -install(TARGETS toml11 - EXPORT toml11Targets -) -install(EXPORT toml11Targets - FILE toml11Targets.cmake - DESTINATION ${toml11_install_cmake_dir} - NAMESPACE toml11:: -) + # Export targets and install them + install(TARGETS toml11 + EXPORT toml11Targets + ) + install(EXPORT toml11Targets + FILE toml11Targets.cmake + DESTINATION ${toml11_install_cmake_dir} + NAMESPACE toml11:: + ) +endif() if (toml11_BUILD_TEST) add_subdirectory(tests) diff --git a/share/openPMD/thirdParty/toml11/README.md b/share/openPMD/thirdParty/toml11/README.md new file mode 100644 index 0000000000..6471b2edb6 --- /dev/null +++ b/share/openPMD/thirdParty/toml11/README.md @@ -0,0 +1,1967 @@ +toml11 +====== + +[![Build Status on GitHub Actions](https://github.com/ToruNiina/toml11/workflows/build/badge.svg)](https://github.com/ToruNiina/toml11/actions) +[![Build Status on TravisCI](https://travis-ci.org/ToruNiina/toml11.svg?branch=master)](https://travis-ci.org/ToruNiina/toml11) +[![Build status on Appveyor](https://ci.appveyor.com/api/projects/status/m2n08a926asvg5mg/branch/master?svg=true)](https://ci.appveyor.com/project/ToruNiina/toml11/branch/master) +[![Build status on CircleCI](https://circleci.com/gh/ToruNiina/toml11/tree/master.svg?style=svg)](https://circleci.com/gh/ToruNiina/toml11/tree/master) +[![Version](https://img.shields.io/github/release/ToruNiina/toml11.svg?style=flat)](https://github.com/ToruNiina/toml11/releases) +[![License](https://img.shields.io/github/license/ToruNiina/toml11.svg?style=flat)](LICENSE) +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1209136.svg)](https://doi.org/10.5281/zenodo.1209136) + +toml11 is a C++11 (or later) header-only toml parser/encoder depending only on C++ standard library. + +- It is compatible to the latest version of [TOML v1.0.0](https://toml.io/en/v1.0.0). +- It is one of the most TOML standard compliant libraries, tested with [the language agnostic test suite for TOML parsers by BurntSushi](https://github.com/BurntSushi/toml-test). +- It shows highly informative error messages. You can see the error messages about invalid files at [CircleCI](https://circleci.com/gh/ToruNiina/toml11). +- It has configurable container. You can use any random-access containers and key-value maps as backend containers. +- It optionally preserves comments without any overhead. +- It has configurable serializer that supports comments, inline tables, literal strings and multiline strings. +- It supports user-defined type conversion from/into toml values. +- It correctly handles UTF-8 sequences, with or without BOM, both on posix and Windows. + +## Example + +```cpp +#include +#include + +int main() +{ + // ```toml + // title = "an example toml file" + // nums = [3, 1, 4, 1, 5] + // ``` + auto data = toml::parse("example.toml"); + + // find a value with the specified type from a table + std::string title = toml::find(data, "title"); + + // convert the whole array into any container automatically + std::vector nums = toml::find>(data, "nums"); + + // access with STL-like manner + if(!data.contains("foo")) + { + data["foo"] = "bar"; + } + + // pass a fallback + std::string name = toml::find_or(data, "name", "not found"); + + // width-dependent formatting + std::cout << std::setw(80) << data << std::endl; + + return 0; +} +``` + +## Table of Contents + +- [Integration](#integration) +- [Decoding a toml file](#decoding-a-toml-file) + - [In the case of syntax error](#in-the-case-of-syntax-error) + - [Invalid UTF-8 Codepoints](#invalid-utf-8-codepoints) +- [Finding a toml value](#finding-a-toml-value) + - [Finding a value in a table](#finding-a-value-in-a-table) + - [In case of error](#in-case-of-error) + - [Dotted keys](#dotted-keys) +- [Casting a toml value](#casting-a-toml-value) +- [Checking value type](#checking-value-type) +- [More about conversion](#more-about-conversion) + - [Converting an array](#converting-an-array) + - [Converting a table](#converting-a-table) + - [Getting an array of tables](#getting-an-array-of-tables) + - [Cost of conversion](#cost-of-conversion) + - [Converting datetime and its variants](#converting-datetime-and-its-variants) +- [Getting with a fallback](#getting-with-a-fallback) +- [Expecting conversion](#expecting-conversion) +- [Visiting a toml::value](#visiting-a-tomlvalue) +- [Constructing a toml::value](#constructing-a-tomlvalue) +- [Preserving Comments](#preserving-comments) +- [Customizing containers](#customizing-containers) +- [TOML literal](#toml-literal) +- [Conversion between toml value and arbitrary types](#conversion-between-toml-value-and-arbitrary-types) +- [Formatting user-defined error messages](#formatting-user-defined-error-messages) +- [Obtaining location information](#obtaining-location-information) +- [Exceptions](#exceptions) +- [Colorize Error Messages](#colorize-error-messages) +- [Serializing TOML data](#serializing-toml-data) +- [Underlying types](#underlying-types) +- [Unreleased TOML features](#unreleased-toml-features) +- [Breaking Changes from v2](#breaking-changes-from-v2) +- [Running Tests](#running-tests) +- [Contributors](#contributors) +- [Licensing Terms](#licensing-terms) + +## Integration + +Just include the file after adding it to the include path. + +```cpp +#include // that's all! now you can use it. +#include + +int main() +{ + const auto data = toml::parse("example.toml"); + const auto title = toml::find(data, "title"); + std::cout << "the title is " << title << std::endl; + return 0; +} +``` + +The convenient way is to add this repository as a git-submodule or to install +it in your system by CMake. + +Note for MSVC: We recommend to set `/Zc:__cplusplus` to detect C++ version correctly. + +## Decoding a toml file + +To parse a toml file, the only thing you have to do is +to pass a filename to the `toml::parse` function. + +```cpp +const std::string fname("sample.toml"); +const toml::value data = toml::parse(fname); +``` + +As required by the TOML specification, the top-level value is always a table. +You can find a value inside it, cast it into a table explicitly, and insert it as a value into other `toml::value`. + +If it encounters an error while opening a file, it will throw `std::runtime_error`. + +You can also pass a `std::istream` to the `toml::parse` function. +To show a filename in an error message, however, it is recommended to pass the +filename with the stream. + +```cpp +std::ifstream ifs("sample.toml", std::ios_base::binary); +assert(ifs.good()); +const auto data = toml::parse(ifs, /*optional -> */ "sample.toml"); +``` + +**Note**: When you are **on Windows, open a file in binary mode**. +If a file is opened in text-mode, CRLF ("\r\n") will automatically be +converted to LF ("\n") and this causes inconsistency between file size +and the contents that would be read. This causes weird error. + +### In the case of syntax error + +If there is a syntax error in a toml file, `toml::parse` will throw +`toml::syntax_error` that inherits `std::exception`. + +toml11 has clean and informative error messages inspired by Rust and +it looks like the following. + +```console +terminate called after throwing an instance of 'toml::syntax_error' + what(): [error] toml::parse_table: invalid line format # error description + --> example.toml # file name + 3 | a = 42 = true # line num and content + | ^------ expected newline, but got '='. # error reason +``` + +If you (mistakenly) duplicate tables and got an error, it is helpful to see +where they are. toml11 shows both at the same time like the following. + +```console +terminate called after throwing an instance of 'toml::syntax_error' + what(): [error] toml::insert_value: table ("table") already exists. + --> duplicate-table.toml + 1 | [table] + | ~~~~~~~ table already exists here + ... + 3 | [table] + | ~~~~~~~ table defined twice +``` + +When toml11 encounters a malformed value, it tries to detect what type it is. +Then it shows hints to fix the format. An error message while reading one of +the malformed files in [the language agnostic test suite](https://github.com/BurntSushi/toml-test). +is shown below. + +```console +what(): [error] bad time: should be HH:MM:SS.subsec + --> ./datetime-malformed-no-secs.toml + 1 | no-secs = 1987-07-05T17:45Z + | ^------- HH:MM:SS.subsec + | +Hint: pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999 +Hint: fail: 1979-05-27T7:32:00, 1979-05-27 17:32 +``` + +You can find other examples in a job named `output_result` on +[CircleCI](https://circleci.com/gh/ToruNiina/toml11). + +Since the error message generation is generally a difficult task, the current +status is not ideal. If you encounter a weird error message, please let us know +and contribute to improve the quality! + +### Invalid UTF-8 codepoints + +It throws `syntax_error` if a value of an escape sequence +representing unicode character is not a valid UTF-8 codepoint. + +```console + what(): [error] toml::read_utf8_codepoint: input codepoint is too large. + --> utf8.toml + 1 | exceeds_unicode = "\U0011FFFF example" + | ^--------- should be in [0x00..0x10FFFF] +``` + +## Finding a toml value + +After parsing successfully, you can obtain the values from the result of +`toml::parse` using `toml::find` function. + +```toml +# sample.toml +answer = 42 +pi = 3.14 +numbers = [1,2,3] +time = 1979-05-27T07:32:00Z +``` + +``` cpp +const auto data = toml::parse("sample.toml"); +const auto answer = toml::find(data, "answer"); +const auto pi = toml::find(data, "pi"); +const auto numbers = toml::find>(data, "numbers"); +const auto timepoint = toml::find(data, "time"); +``` + +By default, `toml::find` returns a `toml::value`. + +```cpp +const toml::value& answer = toml::find(data, "answer"); +``` + +When you pass an exact TOML type that does not require type conversion, +`toml::find` returns a reference without copying the value. + +```cpp +const auto data = toml::parse("sample.toml"); +const auto& answer = toml::find(data, "answer"); +``` + +If the specified type requires conversion, you can't take a reference to the value. +See also [underlying types](#underlying-types). + +**NOTE**: For some technical reason, automatic conversion between `integer` and +`floating` is not supported. If you want to get a floating value even if a value +has integer value, you need to convert it manually after obtaining a value, +like the following. + +```cpp +const auto vx = toml::find(data, "x"); +double x = vx.is_floating() ? vx.as_floating(std::nothrow) : + static_cast(vx.as_integer()); // it throws if vx is neither + // floating nor integer. +``` + +### Finding a value in a table + +There are several way to get a value defined in a table. +First, you can get a table as a normal value and find a value from the table. + +```toml +[fruit] +name = "apple" +[fruit.physical] +color = "red" +shape = "round" +``` + +``` cpp +const auto data = toml::parse("fruit.toml"); +const auto& fruit = toml::find(data, "fruit"); +const auto name = toml::find(fruit, "name"); + +const auto& physical = toml::find(fruit, "physical"); +const auto color = toml::find(physical, "color"); +const auto shape = toml::find(physical, "shape"); +``` + +Here, variable `fruit` is a `toml::value` and can be used as the first argument +of `toml::find`. + +Second, you can pass as many arguments as the number of subtables to `toml::find`. + +```cpp +const auto data = toml::parse("fruit.toml"); +const auto color = toml::find(data, "fruit", "physical", "color"); +const auto shape = toml::find(data, "fruit", "physical", "shape"); +``` + +### Finding a value in an array + +You can find n-th value in an array by `toml::find`. + +```toml +values = ["foo", "bar", "baz"] +``` + +``` cpp +const auto data = toml::parse("sample.toml"); +const auto values = toml::find(data, "values"); +const auto bar = toml::find(values, 1); +``` + +`toml::find` can also search array recursively. + +```cpp +const auto data = toml::parse("fruit.toml"); +const auto bar = toml::find(data, "values", 1); +``` + +Before calling `toml::find`, you can check if a value corresponding to a key +exists. You can use both `bool toml::value::contains(const key&) const` and +`std::size_t toml::value::count(const key&) const`. Those behaves like the +`std::map::contains` and `std::map::count`. + +```cpp +const auto data = toml::parse("fruit.toml"); +if(data.contains("fruit") && data.at("fruit").count("physical") != 0) +{ + // ... +} +``` + +### In case of error + +If the value does not exist, `toml::find` throws `std::out_of_range` with the +location of the table. + +```console +terminate called after throwing an instance of 'std::out_of_range' + what(): [error] key "answer" not found + --> example.toml + 6 | [tab] + | ~~~~~ in this table +``` + +---- + +If the specified type differs from the actual value contained, it throws +`toml::type_error` that inherits `std::exception`. + +Similar to the case of syntax error, toml11 also displays clean error messages. +The error message when you choose `int` to get `string` value would be like this. + +```console +terminate called after throwing an instance of 'toml::type_error' + what(): [error] toml::value bad_cast to integer + --> example.toml + 3 | title = "TOML Example" + | ~~~~~~~~~~~~~~ the actual type is string +``` + +**NOTE**: In order to show this kind of error message, all the toml values have +a pointer to represent its range in a file. The entire contents of a file is +shared by `toml::value`s and remains on the heap memory. It is recommended to +destruct all the `toml::value` classes after configuring your application +if you have a large TOML file compared to the memory resource. + +### Dotted keys + +TOML v0.5.0 has a new feature named "dotted keys". +You can chain keys to represent the structure of the data. + +```toml +physical.color = "orange" +physical.shape = "round" +``` + +This is equivalent to the following. + +```toml +[physical] +color = "orange" +shape = "round" +``` + +You can get both of the above tables with the same c++ code. + +```cpp +const auto physical = toml::find(data, "physical"); +const auto color = toml::find(physical, "color"); +``` + +The following code does not work for the above toml file. + +```cpp +// XXX this does not work! +const auto color = toml::find(data, "physical.color"); +``` + +The above code works with the following toml file. + +```toml +"physical.color" = "orange" +# equivalent to {"physical.color": "orange"}, +# NOT {"physical": {"color": "orange"}}. +``` + + +## Casting a toml value + +### `toml::get` + +`toml::parse` returns `toml::value`. `toml::value` is a union type that can +contain one of the following types. + +- `toml::boolean` (`bool`) +- `toml::integer` (`std::int64_t`) +- `toml::floating` (`double`) +- `toml::string` (a type convertible to std::string) +- `toml::local_date` +- `toml::local_time` +- `toml::local_datetime` +- `toml::offset_datetime` +- `toml::array` (by default, `std::vector`) + - It depends. See [customizing containers](#customizing-containers) for detail. +- `toml::table` (by default, `std::unordered_map`) + - It depends. See [customizing containers](#customizing-containers) for detail. + +To get a value inside, you can use `toml::get()`. The usage is the same as +`toml::find` (actually, `toml::find` internally uses `toml::get` after casting +a value to `toml::table`). + +``` cpp +const toml::value data = toml::parse("sample.toml"); +const toml::value answer_ = toml::get(data).at("answer"); +const std::int64_t answer = toml::get(answer_); +``` + +When you pass an exact TOML type that does not require type conversion, +`toml::get` returns a reference through which you can modify the content +(if the `toml::value` is `const`, it returns `const` reference). + +```cpp +toml::value data = toml::parse("sample.toml"); +toml::value answer_ = toml::get(data).at("answer"); +toml::integer& answer = toml::get(answer_); +answer = 6 * 9; // write to data.answer. now `answer_` contains 54. +``` + +If the specified type requires conversion, you can't take a reference to the value. +See also [underlying types](#underlying-types). + +It also throws a `toml::type_error` if the type differs. + +### `as_xxx` + +You can also use a member function to cast a value. + +```cpp +const std::int64_t answer = data.as_table().at("answer").as_integer(); +``` + +It also throws a `toml::type_error` if the type differs. If you are sure that +the value `v` contains a value of the specified type, you can suppress checking +by passing `std::nothrow`. + +```cpp +const auto& answer = data.as_table().at("answer"); +if(answer.is_integer() && answer.as_integer(std::nothrow) == 42) +{ + std::cout << "value is 42" << std::endl; +} +``` + +If `std::nothrow` is passed, the functions are marked as noexcept. + +By casting a `toml::value` into an array or a table, you can iterate over the +elements. + +```cpp +const auto data = toml::parse("example.toml"); +std::cout << "keys in the top-level table are the following: \n"; +for(const auto& [k, v] : data.as_table()) +{ + std::cout << k << '\n'; +} + +const auto& fruits = toml::find(data, "fruits"); +for(const auto& v : fruits.as_array()) +{ + std::cout << toml::find(v, "name") << '\n'; +} +``` + +The full list of the functions is below. + +```cpp +namespace toml { +class value { + // ... + const boolean& as_boolean() const&; + const integer& as_integer() const&; + const floating& as_floating() const&; + const string& as_string() const&; + const offset_datetime& as_offset_datetime() const&; + const local_datetime& as_local_datetime() const&; + const local_date& as_local_date() const&; + const local_time& as_local_time() const&; + const array& as_array() const&; + const table& as_table() const&; + // -------------------------------------------------------- + // non-const version + boolean& as_boolean() &; + // ditto... + // -------------------------------------------------------- + // rvalue version + boolean&& as_boolean() &&; + // ditto... + + // -------------------------------------------------------- + // noexcept versions ... + const boolean& as_boolean(const std::nothrow_t&) const& noexcept; + boolean& as_boolean(const std::nothrow_t&) & noexcept; + boolean&& as_boolean(const std::nothrow_t&) && noexcept; + // ditto... +}; +} // toml +``` + +### `at()` + +You can access to the element of a table and an array by `toml::basic_value::at`. + +```cpp +const toml::value v{1,2,3,4,5}; +std::cout << v.at(2).as_integer() << std::endl; // 3 + +const toml::value v{{"foo", 42}, {"bar", 3.14}}; +std::cout << v.at("foo").as_integer() << std::endl; // 42 +``` + +If an invalid key (integer for a table, string for an array), it throws +`toml::type_error` for the conversion. If the provided key is out-of-range, +it throws `std::out_of_range`. + +Note that, although `std::string` has `at()` member function, `toml::value::at` +throws if the contained type is a string. Because `std::string` does not +contain `toml::value`. + +### `operator[]` + +You can also access to the element of a table and an array by +`toml::basic_value::operator[]`. + +```cpp +const toml::value v{1,2,3,4,5}; +std::cout << v[2].as_integer() << std::endl; // 3 + +const toml::value v{{"foo", 42}, {"bar", 3.14}}; +std::cout << v["foo"].as_integer() << std::endl; // 42 +``` + +When you access to a `toml::value` that is not initialized yet via +`operator[](const std::string&)`, the `toml::value` will be a table, +just like the `std::map`. + +```cpp +toml::value v; // not initialized as a table. +v["foo"] = 42; // OK. `v` will be a table. +``` + +Contrary, if you access to a `toml::value` that contains an array via `operator[]`, +it does not check anything. It converts `toml::value` without type check and then +access to the n-th element without boundary check, just like the `std::vector::operator[]`. + +```cpp +toml::value v; // not initialized as an array +v[2] = 42; // error! UB +``` + +Please make sure that the `toml::value` has an array inside when you access to +its element via `operator[]`. + +## Checking value type + +You can check the type of a value by `is_xxx` function. + +```cpp +const toml::value v = /* ... */; +if(v.is_integer()) +{ + std::cout << "value is an integer" << std::endl; +} +``` + +The complete list of the functions is below. + +```cpp +namespace toml { +class value { + // ... + bool is_boolean() const noexcept; + bool is_integer() const noexcept; + bool is_floating() const noexcept; + bool is_string() const noexcept; + bool is_offset_datetime() const noexcept; + bool is_local_datetime() const noexcept; + bool is_local_date() const noexcept; + bool is_local_time() const noexcept; + bool is_array() const noexcept; + bool is_table() const noexcept; + bool is_uninitialized() const noexcept; + // ... +}; +} // toml +``` + +Also, you can get `enum class value_t` from `toml::value::type()`. + +```cpp +switch(data.at("something").type()) +{ + case toml::value_t::integer: /*do some stuff*/ ; break; + case toml::value_t::floating: /*do some stuff*/ ; break; + case toml::value_t::string : /*do some stuff*/ ; break; + default : throw std::runtime_error( + "unexpected type : " + toml::stringize(data.at("something").type())); +} +``` + +The complete list of the `enum`s can be found in the section +[underlying types](#underlying-types). + +The `enum`s can be used as a parameter of `toml::value::is` function like the following. + +```cpp +toml::value v = /* ... */; +if(v.is(toml::value_t::boolean)) // ... +``` + +## More about conversion + +Since `toml::find` internally uses `toml::get`, all the following examples work +with both `toml::get` and `toml::find`. + +### Converting an array + +You can get any kind of `container` class from a `toml::array` +except for `map`-like classes. + +``` cpp +// # sample.toml +// numbers = [1,2,3] + +const auto numbers = toml::find(data, "numbers"); + +const auto vc = toml::get >(numbers); +const auto ls = toml::get >(numbers); +const auto dq = toml::get >(numbers); +const auto ar = toml::get>(numbers); +// if the size of data.at("numbers") is larger than that of std::array, +// it will throw toml::type_error because std::array is not resizable. +``` + +Surprisingly, you can convert `toml::array` into `std::pair` and `std::tuple`. + +```cpp +// numbers = [1,2,3] +const auto tp = toml::get>(numbers); +``` + +This functionality is helpful when you have a toml file like the following. + +```toml +array_of_arrays = [[1, 2, 3], ["foo", "bar", "baz"]] # toml allows this +``` + +What is the corresponding C++ type? +Obviously, it is a `std::pair` of `std::vector`s. + +```cpp +const auto array_of_arrays = toml::find(data, "array_of_arrays"); +const auto aofa = toml::get< + std::pair, std::vector> + >(array_of_arrays); +``` + +If you don't know the type of the elements, you can use `toml::array`, +which is a `std::vector` of `toml::value`, instead. + +```cpp +const auto a_of_a = toml::get(array_of_arrays); +const auto first = toml::get>(a_of_a.at(0)); +``` + +You can change the implementation of `toml::array` with `std::deque` or some +other array-like container. See [Customizing containers](#customizing-containers) +for detail. + +### Converting a table + +When all the values of the table have the same type, toml11 allows you to +convert a `toml::table` to a `map` that contains the convertible type. + +```toml +[tab] +key1 = "foo" # all the values are +key2 = "bar" # toml String +``` + +```cpp +const auto data = toml::parse("sample.toml"); +const auto tab = toml::find>(data, "tab"); +std::cout << tab["key1"] << std::endl; // foo +std::cout << tab["key2"] << std::endl; // bar +``` + +But since `toml::table` is just an alias of `std::unordered_map`, +normally you don't need to convert it because it has all the functionalities that +`std::unordered_map` has (e.g. `operator[]`, `count`, and `find`). In most cases +`toml::table` is sufficient. + +```cpp +toml::table tab = toml::get(data); +if(data.count("title") != 0) +{ + data["title"] = std::string("TOML example"); +} +``` + +You can change the implementation of `toml::table` with `std::map` or some +other map-like container. See [Customizing containers](#customizing-containers) +for detail. + +### Getting an array of tables + +An array of tables is just an array of tables. +You can get it in completely the same way as the other arrays and tables. + +```toml +# sample.toml +array_of_inline_tables = [{key = "value1"}, {key = "value2"}, {key = "value3"}] + +[[array_of_tables]] +key = "value4" +[[array_of_tables]] +key = "value5" +[[array_of_tables]] +key = "value6" +``` + +```cpp +const auto data = toml::parse("sample.toml"); +const auto aot1 = toml::find>(data, "array_of_inline_tables"); +const auto aot2 = toml::find>(data, "array_of_tables"); +``` + +### Cost of conversion + +Although conversion through `toml::(get|find)` is convenient, it has additional +copy-cost because it copies data contained in `toml::value` to the +user-specified type. Of course in some cases this overhead is not ignorable. + +```cpp +// the following code constructs a std::vector. +// it requires heap allocation for vector and element conversion. +const auto array = toml::find>(data, "foo"); +``` + +By passing the exact types, `toml::get` returns reference that has no overhead. + +``` cpp +const auto& tab = toml::find(data, "tab"); +const auto& numbers = toml::find(data, "numbers"); +``` + +Also, `as_xxx` are zero-overhead because they always return a reference. + +``` cpp +const auto& tab = toml::find(data, "tab" ).as_table(); +const auto& numbers = toml::find(data, "numbers").as_array(); +``` + +In this case you need to call `toml::get` each time you access to +the element of `toml::array` because `toml::array` is an array of `toml::value`. + +```cpp +const auto& num0 = toml::get(numbers.at(0)); +const auto& num1 = toml::get(numbers.at(1)); +const auto& num2 = toml::get(numbers.at(2)); +``` + +### Converting datetime and its variants + +TOML v0.5.0 has 4 different datetime objects, `local_date`, `local_time`, +`local_datetime`, and `offset_datetime`. + +Since `local_date`, `local_datetime`, and `offset_datetime` represent a time +point, you can convert them to `std::chrono::system_clock::time_point`. + +Contrary, `local_time` does not represents a time point because they lack a +date information, but it can be converted to `std::chrono::duration` that +represents a duration from the beginning of the day, `00:00:00.000`. + +```toml +# sample.toml +date = 2018-12-23 +time = 12:30:00 +l_dt = 2018-12-23T12:30:00 +o_dt = 2018-12-23T12:30:00+09:30 +``` + +```cpp +const auto data = toml::parse("sample.toml"); + +const auto date = toml::get(data.at("date")); +const auto l_dt = toml::get(data.at("l_dt")); +const auto o_dt = toml::get(data.at("o_dt")); + +const auto time = toml::get(data.at("time")); // 12 * 60 + 30 min +``` + +`local_date` and `local_datetime` are assumed to be in the local timezone when +they are converted into `time_point`. On the other hand, `offset_datetime` only +uses the offset part of the data and it does not take local timezone into account. + +To contain datetime data, toml11 defines its own datetime types. +For more detail, you can see the definitions in [toml/datetime.hpp](toml/datetime.hpp). + +## Getting with a fallback + +`toml::find_or` returns a default value if the value is not found or has a +different type. + +```cpp +const auto data = toml::parse("example.toml"); +const auto num = toml::find_or(data, "num", 42); +``` + +It works recursively if you pass several keys for subtables. +In that case, the last argument is considered to be the optional value. +All other arguments between `toml::value` and the optinoal value are considered as keys. + +```cpp +// [fruit.physical] +// color = "red" +auto data = toml::parse("fruit.toml"); +auto color = toml::find_or(data, "fruit", "physical", "color", "red"); +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^ +// arguments optional value +``` + +Also, `toml::get_or` returns a default value if `toml::get` failed. + +```cpp +toml::value v("foo"); // v contains String +const int value = toml::get_or(v, 42); // conversion fails. it returns 42. +``` + +These functions automatically deduce what type you want to get +from the default value you passed. + +To get a reference through this function, take care about the default value. + +```cpp +toml::value v("foo"); // v contains String +toml::integer& i = toml::get_or(v, 42); // does not work because binding `42` + // to `integer&` is invalid +toml::integer opt = 42; +toml::integer& i = toml::get_or(v, opt); // this works. +``` + +## Expecting conversion + +By using `toml::expect`, you will get your expected value or an error message +without throwing `toml::type_error`. + +```cpp +const auto value = toml::expect(data.at("title")); +if(value.is_ok()) { + std::cout << value.unwrap() << std::endl; +} else { + std::cout << value.unwrap_err() << std::endl; +} +``` + +Also, you can pass a function object to modify the expected value. + +```cpp +const auto value = toml::expect(data.at("number")) + .map(// function that receives expected type (here, int) + [](const int number) -> double { + return number * 1.5 + 1.0; + }).unwrap_or(/*default value =*/ 3.14); +``` + +## Visiting a toml::value + +toml11 provides `toml::visit` to apply a function to `toml::value` in the +same way as `std::variant`. + +```cpp +const toml::value v(3.14); +toml::visit([](const auto& val) -> void { + std::cout << val << std::endl; + }, v); +``` + +The function object that would be passed to `toml::visit` must be able to +receive all the possible TOML types. Also, the result types should be the same +each other. + +## Constructing a toml::value + +`toml::value` can be constructed in various ways. + +```cpp +toml::value v(true); // boolean +toml::value v(42); // integer +toml::value v(3.14); // floating +toml::value v("foobar"); // string +toml::value v(toml::local_date(2019, toml::month_t::Apr, 1)); // date +toml::value v{1, 2, 3, 4, 5}; // array +toml::value v{{"foo", 42}, {"bar", 3.14}, {"baz", "qux"}}; // table +``` + +When constructing a string, you can choose to use either literal or basic string. +By default, it will be a basic string. + +```cpp +toml::value v("foobar", toml::string_t::basic ); +toml::value v("foobar", toml::string_t::literal); +``` + +Datetime objects can be constructed from `std::tm` and +`std::chrono::system_clock::time_point`. But you need to specify what type +you use to avoid ambiguity. + +```cpp +const auto now = std::chrono::system_clock::now(); +toml::value v(toml::local_date(now)); +toml::value v(toml::local_datetime(now)); +toml::value v(toml::offset_datetime(now)); +``` + +Since local time is not equivalent to a time point, because it lacks date +information, it will be constructed from `std::chrono::duration`. + +```cpp +toml::value v(toml::local_time(std::chrono::hours(10))); +``` + +You can construct an array object not only from `initializer_list`, but also +from STL containers. In that case, the element type must be convertible to +`toml::value`. + +```cpp +std::vector vec{1,2,3,4,5}; +toml::value v(vec); +``` + +When you construct an array value, all the elements of `initializer_list` +must be convertible into `toml::value`. + +If a `toml::value` has an array, you can `push_back` an element in it. + +```cpp +toml::value v{1,2,3,4,5}; +v.push_back(6); +``` + +`emplace_back` also works. + +## Preserving comments + +toml11 v3 or later allows you yo choose whether comments are preserved or not via template parameter + +```cpp +const auto data1 = toml::parse("example.toml"); +const auto data2 = toml::parse("example.toml"); +``` + +or macro definition. + +```cpp +#define TOML11_PRESERVE_COMMENTS_BY_DEFAULT +#include +``` + +This feature is controlled by template parameter in `toml::basic_value<...>`. +`toml::value` is an alias of `toml::basic_value<...>`. + +If template parameter is explicitly specified, the return value of `toml::parse` +will be `toml::basic_value`. +If the macro is defined, the alias `toml::value` will be +`toml::basic_value`. + +Comments related to a value can be obtained by `toml::value::comments()`. +The return value has the same interface as `std::vector`. + +```cpp +const auto& com = v.comments(); +for(const auto& c : com) +{ + std::cout << c << std::endl; +} +``` + +Comments just before and just after (within the same line) a value are kept in a value. + +```toml +# this is a comment for v1. +v1 = "foo" + +v2 = "bar" # this is a comment for v2. +# Note that this comment is NOT a comment for v2. + +# this comment is not related to any value +# because there are empty lines between v3. +# this comment will be ignored even if you set `preserve_comments`. + +# this is a comment for v3 +# this is also a comment for v3. +v3 = "baz" # ditto. +``` + +Each comment line becomes one element of a `std::vector`. + +Hash signs will be removed, but spaces after hash sign will not be removed. + +```cpp +v1.comments().at(0) == " this is a comment for v1."s; + +v2.comments().at(1) == " this is a comment for v1."s; + +v3.comments().at(0) == " this is a comment for v3."s; +v3.comments().at(1) == " this is also a comment for v3."s; +v3.comments().at(2) == " ditto."s; +``` + +Note that a comment just after an opening brace of an array will not be a +comment for the array. + +```toml +# this is a comment for a. +a = [ # this is not a comment for a. this will be ignored. + 1, 2, 3, + # this is a comment for `42`. + 42, # this is also a comment for `42`. + 5 +] # this is a comment for a. +``` + +You can also append and modify comments. +The interfaces are the same as `std::vector`. + +```cpp +toml::basic_value v(42); +v.comments().push_back(" add this comment."); +// # add this comment. +// i = 42 +``` + +Also, you can pass a `std::vector` when constructing a +`toml::basic_value`. + +```cpp +std::vector comments{"comment 1", "comment 2"}; +const toml::basic_value v1(42, std::move(comments)); +const toml::basic_value v2(42, {"comment 1", "comment 2"}); +``` + +When `toml::discard_comments` is chosen, comments will not be contained in a value. +`value::comments()` will always be kept empty. +All the modification on comments would be ignored. +All the element access in a `discard_comments` causes the same error as accessing +an element of an empty `std::vector`. + +The comments will also be serialized. If comments exist, those comments will be +added just before the values. + +__NOTE__: Result types from `toml::parse(...)` and +`toml::parse(...)` are different. + +## Customizing containers + +Actually, `toml::basic_value` has 3 template arguments. + +```cpp +template class Table = std::unordered_map, + template class Array = std::vector> +class basic_value; +``` + +This enables you to change the containers used inside. E.g. you can use +`std::map` to contain a table object instead of `std::unordered_map`. +And also can use `std::deque` as a array object instead of `std::vector`. + +You can set these parameters while calling `toml::parse` function. + +```cpp +const auto data = toml::parse< + toml::preserve_comments, std::map, std::deque + >("example.toml"); +``` + +Needless to say, the result types from `toml::parse(...)` and +`toml::parse(...)` are different (unless you specify the same +types as default). + +Note that, since `toml::table` and `toml::array` is an alias for a table and an +array of a default `toml::value`, so it is different from the types actually +contained in a `toml::basic_value` when you customize containers. +To get the actual type in a generic way, use +`typename toml::basic_type::table_type` and +`typename toml::basic_type::array_type`. + +## TOML literal + +toml11 supports `"..."_toml` literal. +It accept both a bare value and a file content. + +```cpp +using namespace toml::literals::toml_literals; + +// `_toml` can convert a bare value without key +const toml::value v = u8"0xDEADBEEF"_toml; +// v is an Integer value containing 0xDEADBEEF. + +// raw string literal (`R"(...)"` is useful for this purpose) +const toml::value t = u8R"( + title = "this is TOML literal" + [table] + key = "value" +)"_toml; +// the literal will be parsed and the result will be contained in t +``` + +The literal function is defined in the same way as the standard library literals +such as `std::literals::string_literals::operator""s`. + +```cpp +namespace toml +{ +inline namespace literals +{ +inline namespace toml_literals +{ +toml::value operator"" _toml(const char* str, std::size_t len); +} // toml_literals +} // literals +} // toml +``` + +Access to the operator can be gained with `using namespace toml::literals;`, +`using namespace toml::toml_literals`, and `using namespace toml::literals::toml_literals`. + +Note that a key that is composed only of digits is allowed in TOML. +And, unlike the file parser, toml-literal allows a bare value without a key. +Thus it is difficult to distinguish arrays having integers and definitions of +tables that are named as digits. +Currently, literal `[1]` becomes a table named "1". +To ensure a literal to be considered as an array with one element, you need to +add a comma after the first element (like `[1,]`). + +```cpp +"[1,2,3]"_toml; // This is an array +"[table]"_toml; // This is a table that has an empty table named "table" inside. +"[[1,2,3]]"_toml; // This is an array of arrays +"[[table]]"_toml; // This is a table that has an array of tables inside. + +"[[1]]"_toml; // This literal is ambiguous. + // Currently, it becomes a table that has array of table "1". +"1 = [{}]"_toml; // This is a table that has an array of table named 1. +"[[1,]]"_toml; // This is an array of arrays. +"[[1],]"_toml; // ditto. +``` + +NOTE: `_toml` literal returns a `toml::value` that does not have comments. + +## Conversion between toml value and arbitrary types + +You can also use `toml::get` and other related functions with the types +you defined after you implement a way to convert it. + +```cpp +namespace ext +{ +struct foo +{ + int a; + double b; + std::string c; +}; +} // ext + +const auto data = toml::parse("example.toml"); + +// to do this +const foo f = toml::find(data, "foo"); +``` + +There are 3 ways to use `toml::get` with the types that you defined. + +The first one is to implement `from_toml(const toml::value&)` member function. + +```cpp +namespace ext +{ +struct foo +{ + int a; + double b; + std::string c; + + void from_toml(const toml::value& v) + { + this->a = toml::find(v, "a"); + this->b = toml::find(v, "b"); + this->c = toml::find(v, "c"); + return; + } +}; +} // ext +``` + +In this way, because `toml::get` first constructs `foo` without arguments, +the type should be default-constructible. + +The second is to implement `constructor(const toml::value&)`. + +```cpp +namespace ext +{ +struct foo +{ + explicit foo(const toml::value& v) + : a(toml::find(v, "a")), b(toml::find(v, "b")), + c(toml::find(v, "c")) + {} + + int a; + double b; + std::string c; +}; +} // ext +``` + +Note that implicit default constructor declaration will be suppressed +when a constructor is defined. If you want to use the struct (here, `foo`) +in a container (e.g. `std::vector`), you may need to define default +constructor explicitly. + +The third is to implement specialization of `toml::from` for your type. + +```cpp +namespace ext +{ +struct foo +{ + int a; + double b; + std::string c; +}; +} // ext + +namespace toml +{ +template<> +struct from +{ + static ext::foo from_toml(const value& v) + { + ext::foo f; + f.a = find(v, "a"); + f.b = find(v, "b"); + f.c = find(v, "c"); + return f; + } +}; +} // toml +``` + +In this way, since the conversion function is defined outside of the class, +you can add conversion between `toml::value` and classes defined in another library. + +In some cases, a class has a templatized constructor that takes a template, `T`. +It confuses `toml::get/find` because it makes the class "constructible" from +`toml::value`. To avoid this problem, `toml::from` and `from_toml` always +precede constructor. It makes easier to implement conversion between +`toml::value` and types defined in other libraries because it skips constructor. + +But, importantly, you cannot define `toml::from` and `T.from_toml` at the same +time because it causes ambiguity in the overload resolution of `toml::get` and `toml::find`. + +So the precedence is `toml::from` == `T.from_toml()` > `T(toml::value)`. + +If you want to convert any versions of `toml::basic_value`, +you need to templatize the conversion function as follows. + +```cpp +struct foo +{ + template class M, template class A> + void from_toml(const toml::basic_value& v) + { + this->a = toml::find(v, "a"); + this->b = toml::find(v, "b"); + this->c = toml::find(v, "c"); + return; + } +}; +// or +namespace toml +{ +template<> +struct from +{ + template class M, template class A> + static ext::foo from_toml(const basic_value& v) + { + ext::foo f; + f.a = find(v, "a"); + f.b = find(v, "b"); + f.c = find(v, "c"); + return f; + } +}; +} // toml +``` + +---- + +The opposite direction is also supported in a similar way. You can directly +pass your type to `toml::value`'s constructor by introducing `into_toml` or +`toml::into`. + +```cpp +namespace ext +{ +struct foo +{ + int a; + double b; + std::string c; + + toml::value into_toml() const // you need to mark it const. + { + return toml::value{{"a", this->a}, {"b", this->b}, {"c", this->c}}; + } +}; +} // ext + +ext::foo f{42, 3.14, "foobar"}; +toml::value v(f); +``` + +The definition of `toml::into` is similar to `toml::from`. + +```cpp +namespace ext +{ +struct foo +{ + int a; + double b; + std::string c; +}; +} // ext + +namespace toml +{ +template<> +struct into +{ + static toml::value into_toml(const ext::foo& f) + { + return toml::value{{"a", f.a}, {"b", f.b}, {"c", f.c}}; + } +}; +} // toml + +ext::foo f{42, 3.14, "foobar"}; +toml::value v(f); +``` + +Any type that can be converted to `toml::value`, e.g. `int`, `toml::table` and +`toml::array` are okay to return from `into_toml`. + +You can also return a custom `toml::basic_value` from `toml::into`. + +```cpp +namespace toml +{ +template<> +struct into +{ + static toml::basic_value into_toml(const ext::foo& f) + { + toml::basic_value v{{"a", f.a}, {"b", f.b}, {"c", f.c}}; + v.comments().push_back(" comment"); + return v; + } +}; +} // toml +``` + +But note that, if this `basic_value` would be assigned into other `toml::value` +that discards `comments`, the comments would be dropped. + +### Macro to automatically define conversion functions + +There is a helper macro that automatically generates conversion functions `from` and `into` for a simple struct. + +```cpp +namespace foo +{ +struct Foo +{ + std::string s; + double d; + int i; +}; +} // foo + +TOML11_DEFINE_CONVERSION_NON_INTRUSIVE(foo::Foo, s, d, i) + +int main() +{ + const auto file = toml::parse("example.toml"); + auto f = toml::find(file, "foo"); +} +``` + +And then you can use `toml::find(file, "foo");` + +**Note** that, because of a slight difference in implementation of preprocessor between gcc/clang and MSVC, [you need to define `/Zc:preprocessor`](https://github.com/ToruNiina/toml11/issues/139#issuecomment-803683682) to use it in MSVC (Thank you @glebm !). + +## Formatting user-defined error messages + +When you encounter an error after you read the toml value, you may want to +show the error with the value. + +toml11 provides you a function that formats user-defined error message with +related values. With a code like the following, + +```cpp +const auto value = toml::find(data, "num"); +if(value < 0) +{ + std::cerr << toml::format_error("[error] value should be positive", + data.at("num"), "positive number required") + << std::endl; +} +``` + +you will get an error message like this. + +```console +[error] value should be positive + --> example.toml + 3 | num = -42 + | ~~~ positive number required +``` + +When you pass two values to `toml::format_error`, + +```cpp +const auto min = toml::find(range, "min"); +const auto max = toml::find(range, "max"); +if(max < min) +{ + std::cerr << toml::format_error("[error] max should be larger than min", + data.at("min"), "minimum number here", + data.at("max"), "maximum number here"); + << std::endl; +} +``` + +you will get an error message like this. + +```console +[error] max should be larger than min + --> example.toml + 3 | min = 54 + | ~~ minimum number here + ... + 4 | max = 42 + | ~~ maximum number here +``` + +You can print hints at the end of the message. + +```cpp +std::vector hints; +hints.push_back("positive number means n >= 0."); +hints.push_back("negative number is not positive."); +std::cerr << toml::format_error("[error] value should be positive", + data.at("num"), "positive number required", hints) + << std::endl; +``` + +```console +[error] value should be positive + --> example.toml + 2 | num = 42 + | ~~ positive number required + | +Hint: positive number means n >= 0. +Hint: negative number is not positive. +``` + +## Obtaining location information + +You can also format error messages in your own way by using `source_location`. + +```cpp +struct source_location +{ + std::uint_least32_t line() const noexcept; + std::uint_least32_t column() const noexcept; + std::uint_least32_t region() const noexcept; + std::string const& file_name() const noexcept; + std::string const& line_str() const noexcept; +}; +// +-- line() +--- length of the region (here, region() == 9) +// v .---+---. +// 12 | value = "foo bar" <- line_str() returns the line itself. +// ^-------- column() points here +``` + +You can get this by +```cpp +const toml::value v = /*...*/; +const toml::source_location loc = v.location(); +``` + +## Exceptions + +The following `exception` classes inherits `toml::exception` that inherits +`std::exception`. + +```cpp +namespace toml { +struct exception : public std::exception {/**/}; +struct syntax_error : public toml::exception {/**/}; +struct type_error : public toml::exception {/**/}; +struct internal_error : public toml::exception {/**/}; +} // toml +``` + +`toml::exception` has `toml::exception::location()` member function that returns +`toml::source_location`, in addition to `what()`. + +```cpp +namespace toml { +struct exception : public std::exception +{ + // ... + source_location const& location() const noexcept; +}; +} // toml +``` + +It represents where the error occurs. + +`syntax_error` will be thrown from `toml::parse` and `_toml` literal. +`type_error` will be thrown from `toml::get/find`, `toml::value::as_xxx()`, and +other functions that takes a content inside of `toml::value`. + +Note that, currently, from `toml::value::at()` and `toml::find(value, key)` +may throw an `std::out_of_range` that does not inherits `toml::exception`. + +Also, in some cases, most likely in the file open error, it will throw an +`std::runtime_error`. + +## Colorize Error Messages + +By defining `TOML11_COLORIZE_ERROR_MESSAGE`, the error messages from +`toml::parse` and `toml::find|get` will be colorized. By default, this feature +is turned off. + +With the following toml file taken from `toml-lang/toml/tests/hard_example.toml`, + +```toml +[error] +array = [ + "This might most likely happen in multiline arrays", + Like here, + "or here, + and here" + ] End of array comment, forgot the # +``` + +the error message would be like this. + +![error-message-1](https://github.com/ToruNiina/toml11/blob/misc/misc/toml11-err-msg-1.png) + +With the following, + +```toml +[error] +# array = [ +# "This might most likely happen in multiline arrays", +# Like here, +# "or here, +# and here" +# ] End of array comment, forgot the # +number = 3.14 pi <--again forgot the # +``` + +the error message would be like this. + +![error-message-2](https://github.com/ToruNiina/toml11/blob/misc/misc/toml11-err-msg-2.png) + +The message would be messy when it is written to a file, not a terminal because +it uses [ANSI escape code](https://en.wikipedia.org/wiki/ANSI_escape_code). + +Without `TOML11_COLORIZE_ERROR_MESSAGE`, you can still colorize user-defined +error message by passing `true` to the `toml::format_error` function. +If you define `TOML11_COLORIZE_ERROR_MESSAGE`, the value is `true` by default. +If not, the default value would be `false`. + +```cpp +std::cerr << toml::format_error("[error] value should be positive", + data.at("num"), "positive number required", + hints, /*colorize = */ true) << std::endl; +``` + +Note: It colorize `[error]` in red. That means that it detects `[error]` prefix +at the front of the error message. If there is no `[error]` prefix, +`format_error` adds it to the error message. + +## Serializing TOML data + +toml11 enables you to serialize data into toml format. + +```cpp +const toml::value data{{"foo", 42}, {"bar", "baz"}}; +std::cout << data << std::endl; +// bar = "baz" +// foo = 42 +``` + +toml11 automatically makes a small table and small array inline. +You can specify the width to make them inline by `std::setw` for streams. + +```cpp +const toml::value data{ + {"qux", {{"foo", 42}, {"bar", "baz"}}}, + {"quux", {"small", "array", "of", "strings"}}, + {"foobar", {"this", "array", "of", "strings", "is", "too", "long", + "to", "print", "into", "single", "line", "isn't", "it?"}}, +}; + +// the threshold becomes 80. +std::cout << std::setw(80) << data << std::endl; +// foobar = [ +// "this","array","of","strings","is","too","long","to","print","into", +// "single","line","isn't","it?", +// ] +// quux = ["small","array","of","strings"] +// qux = {bar="baz",foo=42} + + +// the width is 0. nothing become inline. +std::cout << std::setw(0) << data << std::endl; +// foobar = [ +// "this", +// ... (snip) +// "it?", +// ] +// quux = [ +// "small", +// "array", +// "of", +// "strings", +// ] +// [qux] +// bar = "baz" +// foo = 42 +``` + +It is recommended to set width before printing data. Some I/O functions changes +width to 0, and it makes all the stuff (including `toml::array`) multiline. +The resulting files becomes too long. + +To control the precision of floating point numbers, you need to pass +`std::setprecision` to stream. + +```cpp +const toml::value data{ + {"pi", 3.141592653589793}, + {"e", 2.718281828459045} +}; +std::cout << std::setprecision(17) << data << std::endl; +// e = 2.7182818284590451 +// pi = 3.1415926535897931 +std::cout << std::setprecision( 7) << data << std::endl; +// e = 2.718282 +// pi = 3.141593 +``` + +There is another way to format toml values, `toml::format()`. +It returns `std::string` that represents a value. + +```cpp +const toml::value v{{"a", 42}}; +const std::string fmt = toml::format(v); +// a = 42 +``` + +Note that since `toml::format` formats a value, the resulting string may lack +the key value. + +```cpp +const toml::value v{3.14}; +const std::string fmt = toml::format(v); +// 3.14 +``` + +To control the width and precision, `toml::format` receives optional second and +third arguments to set them. By default, the width is 80 and the precision is +`std::numeric_limits::max_digit10`. + +```cpp +const auto serial = toml::format(data, /*width = */ 0, /*prec = */ 17); +``` + +When you pass a comment-preserving-value, the comment will also be serialized. +An array or a table containing a value that has a comment would not be inlined. + +## Underlying types + +The toml types (can be used as `toml::*` in this library) and corresponding `enum` names are listed in the table below. + +| TOML type | underlying c++ type | enum class | +| -------------- | ---------------------------------- | -------------------------------- | +| Boolean | `bool` | `toml::value_t::boolean` | +| Integer | `std::int64_t` | `toml::value_t::integer` | +| Float | `double` | `toml::value_t::floating` | +| String | `toml::string` | `toml::value_t::string` | +| LocalDate | `toml::local_date` | `toml::value_t::local_date` | +| LocalTime | `toml::local_time` | `toml::value_t::local_time` | +| LocalDatetime | `toml::local_datetime` | `toml::value_t::local_datetime` | +| OffsetDatetime | `toml::offset_datetime` | `toml::value_t::offset_datetime` | +| Array | `array-like` | `toml::value_t::array` | +| Table | `map-like` | `toml::value_t::table` | + +`array-like` and `map-like` are the STL containers that works like a `std::vector` and +`std::unordered_map`, respectively. By default, `std::vector` and `std::unordered_map` +are used. See [Customizing containers](#customizing-containers) for detail. + +`toml::string` is effectively the same as `std::string` but has an additional +flag that represents a kind of a string, `string_t::basic` and `string_t::literal`. +Although `std::string` is not an exact toml type, still you can get a reference +that points to internal `std::string` by using `toml::get()` for convenience. +The most important difference between `std::string` and `toml::string` is that +`toml::string` will be formatted as a TOML string when outputted with `ostream`. +This feature is introduced to make it easy to write a custom serializer. + +`Datetime` variants are `struct` that are defined in this library. +Because `std::chrono::system_clock::time_point` is a __time point__, +not capable of representing a Local Time independent from a specific day. + +## Unreleased TOML features + +Since TOML v1.0.0-rc.1 has been released, those features are now activated by +default. We no longer need to define `TOML11_USE_UNRELEASED_FEATURES`. + +- Leading zeroes in exponent parts of floats are permitted. + - e.g. `1.0e+01`, `5e+05` + - [toml-lang/toml/PR/656](https://github.com/toml-lang/toml/pull/656) +- Allow raw tab characters in basic strings and multi-line basic strings. + - [toml-lang/toml/PR/627](https://github.com/toml-lang/toml/pull/627) +- Allow heterogeneous arrays + - [toml-lang/toml/PR/676](https://github.com/toml-lang/toml/pull/676) + +## Note about heterogeneous arrays + +Although `toml::parse` allows heterogeneous arrays, constructor of `toml::value` +does not. Here the reason is explained. + +```cpp +// this won't be compiled +toml::value v{ + "foo", 3.14, 42, {1,2,3,4,5}, {{"key", "value"}} +} +``` + +There is a workaround for this. By explicitly converting values into +`toml::value`, you can initialize `toml::value` with a heterogeneous array. +Also, you can first initialize a `toml::value` with an array and then +`push_back` into it. + +```cpp +// OK! +toml::value v{ + toml::value("foo"), toml::value(3.14), toml::value(42), + toml::value{1,2,3,4,5}, toml::value{{"key", "value"}} +} + +// OK! +toml::value v(toml::array{}); +v.push_back("foo"); +v.push_back(3.14); + +// OK! +toml::array a; +a.push_back("foo"); +a.push_back(3.14); +toml::value v(std::move(a)); +``` + +The reason why the first example is not allowed is the following. +Let's assume that you are initializing a `toml::value` with a table. + +```cpp + // # expecting TOML table. +toml::value v{ // [v] + {"answer", 42}, // answer = 42 + {"pi", 3.14}, // pi = 3.14 + {"foo", "bar"} // foo = "bar" +}; +``` + +This is indistinguishable from a (heterogeneous) TOML array definition. + +```toml +v = [ + ["answer", 42], + ["pi", 3.14], + ["foo", "bar"], +] +``` + +This means that the above C++ code makes constructor's overload resolution +ambiguous. So a constructor that allows both "table as an initializer-list" and +"heterogeneous array as an initializer-list" cannot be implemented. + +Thus, although it is painful, we need to explicitly cast values into +`toml::value` when you initialize heterogeneous array in a C++ code. + +```cpp +toml::value v{ + toml::value("foo"), toml::value(3.14), toml::value(42), + toml::value{1,2,3,4,5}, toml::value{{"key", "value"}} +}; +``` + +## Breaking Changes from v2 + +Although toml11 is relatively new library (it's three years old now), it had +some confusing and inconvenient user-interfaces because of historical reasons. + +Between v2 and v3, those interfaces are rearranged. + +- `toml::parse` now returns a `toml::value`, not `toml::table`. +- `toml::value` is now an alias of `toml::basic_value`. + - See [Customizing containers](#customizing-containers) for detail. +- The elements of `toml::value_t` are renamed as `snake_case`. + - See [Underlying types](#underlying-types) for detail. +- Supports for the CamelCaseNames are dropped. + - See [Underlying types](#underlying-types) for detail. +- `(is|as)_float` has been removed to make the function names consistent with others. + - Since `float` is a keyword, toml11 named a float type as `toml::floating`. + - Also a `value_t` corresponds to `toml::floating` is named `value_t::floating`. + - So `(is|as)_floating` is introduced and `is_float` has been removed. + - See [Casting a toml::value](#casting-a-tomlvalue) and [Checking value type](#checking-value-type) for detail. +- An overload of `toml::find` for `toml::table` has been dropped. Use `toml::value` version instead. + - Because type conversion between a table and a value causes ambiguity while overload resolution + - Since `toml::parse` now returns a `toml::value`, this feature becomes less important. + - Also because `toml::table` is a normal STL container, implementing utility function is easy. + - See [Finding a toml::value](#finding-a-toml-value) for detail. +- An overload of `operator<<` and `toml::format` for `toml::table`s are dropped. + - Use `toml::value` instead. + - See [Serializing TOML data](#serializing-toml-data) for detail. +- Interface around comments. + - See [Preserving Comments](#preserving-comments) for detail. +- An ancient `from_toml/into_toml` has been removed. Use arbitrary type conversion support. + - See [Conversion between toml value and arbitrary types](#conversion-between-toml-value-and-arbitrary-types) for detail. + +Such a big change will not happen in the coming years. + +## Running Tests + +After cloning this repository, run the following command (thank you @jwillikers +for automating test set fetching!). + +```sh +$ mkdir build +$ cd build +$ cmake .. -Dtoml11_BUILD_TEST=ON +$ make +$ make test +``` + +To run the language agnostic test suite, you need to compile +`tests/check_toml_test.cpp` and pass it to the tester. + +## Contributors + +I appreciate the help of the contributors who introduced the great feature to this library. + +- Guillaume Fraux (@Luthaf) + - Windows support and CI on Appvayor + - Intel Compiler support +- Quentin Khan (@xaxousis) + - Found & Fixed a bug around ODR + - Improved error messages for invalid keys to show the location where the parser fails +- Petr Beneš (@wbenny) + - Fixed warnings on MSVC +- Ivan Shynkarenka (@chronoxor) + - Fixed Visual Studio 2019 warnings +- Khoi Dinh Trinh (@khoitd1997) + - Fixed warnings while type conversion +- @KerstinKeller + - Added installation script to CMake +- J.C. Moyer (@jcmoyer) + - Fixed an example code in the documentation +- Jt Freeman (@blockparty-sh) + - Fixed feature test macro around `localtime_s` + - Suppress warnings in Debug mode +- OGAWA Kenichi (@kenichiice) + - Suppress warnings on intel compiler +- Jordan Williams (@jwillikers) + - Fixed clang range-loop-analysis warnings + - Fixed feature test macro to suppress -Wundef + - Use cache variables in CMakeLists.txt + - Automate test set fetching, update and refactor CMakeLists.txt +- Scott McCaskill + - Parse 9 digits (nanoseconds) of fractional seconds in a `local_time` +- Shu Wang (@halfelf) + - fix "Finding a value in an array" example in README +- @maass-tv and @SeverinLeonhardt + - Fix MSVC warning C4866 +- OGAWA KenIchi (@kenichiice) + - Fix include path in README +- Mohammed Alyousef (@MoAlyousef) + - Made testing optional in CMake +- Ivan Shynkarenka (@chronoxor) + - Fix compilation error in `` with MinGW +- Alex Merry (@amerry) + - Add missing include files +- sneakypete81 (@sneakypete81) + - Fix typo in error message +- Oliver Kahrmann (@founderio) + - Fix missing filename in error message if parsed file is empty +- Karl Nilsson (@karl-nilsson) + - Fix many spelling errors +- ohdarling88 (@ohdarling) + - Fix a bug in a constructor of serializer +- estshorter (@estshorter) + - Fix MSVC warning C26478 +- Philip Top (@phlptp) + - Improve checking standard library feature availability check +- Louis Marascio (@marascio) + - Fix free-nonheap-object warning +- Axel Huebl (@ax3l) + - Make installation optional if the library is embedded + +## Licensing terms + +This product is licensed under the terms of the [MIT License](LICENSE). + +- Copyright (c) 2017-2022 Toru Niina + +All rights reserved. diff --git a/share/openPMD/thirdParty/toml11/toml.hpp b/share/openPMD/thirdParty/toml11/toml.hpp index f34cfcccaf..6b0ca1ed4d 100644 --- a/share/openPMD/thirdParty/toml11/toml.hpp +++ b/share/openPMD/thirdParty/toml11/toml.hpp @@ -25,17 +25,9 @@ #ifndef TOML_FOR_MODERN_CPP #define TOML_FOR_MODERN_CPP -#ifndef __cplusplus -# error "__cplusplus is not defined" -#endif - -#if __cplusplus < 201103L && _MSC_VER < 1900 -# error "toml11 requires C++11 or later." -#endif - #define TOML11_VERSION_MAJOR 3 #define TOML11_VERSION_MINOR 7 -#define TOML11_VERSION_PATCH 0 +#define TOML11_VERSION_PATCH 1 #include "toml/parser.hpp" #include "toml/literal.hpp" diff --git a/share/openPMD/thirdParty/toml11/toml/combinator.hpp b/share/openPMD/thirdParty/toml11/toml/combinator.hpp index e250188fe2..33ecca1eb5 100644 --- a/share/openPMD/thirdParty/toml11/toml/combinator.hpp +++ b/share/openPMD/thirdParty/toml11/toml/combinator.hpp @@ -29,7 +29,7 @@ namespace detail // to output character as an error message. inline std::string show_char(const char c) { - // It supress an error that occurs only in Debug mode of MSVC++ on Windows. + // It suppresses an error that occurs only in Debug mode of MSVC++ on Windows. // I'm not completely sure but they check the value of char to be in the // range [0, 256) and some of the COMPLETELY VALID utf-8 character sometimes // has negative value (if char has sign). So here it re-interprets c as @@ -154,7 +154,7 @@ struct sequence invoke(location& loc) { const auto first = loc.iter(); - const auto rslt = Head::invoke(loc); + auto rslt = Head::invoke(loc); if(rslt.is_err()) { loc.reset(first); diff --git a/share/openPMD/thirdParty/toml11/toml/comments.hpp b/share/openPMD/thirdParty/toml11/toml/comments.hpp index b9ce8061fb..ec25041175 100644 --- a/share/openPMD/thirdParty/toml11/toml/comments.hpp +++ b/share/openPMD/thirdParty/toml11/toml/comments.hpp @@ -346,7 +346,7 @@ operator+(const empty_iterator& lhs, typename empty_iterator::differ // // Why this is chose as the default type is because the last version (2.x.y) // does not contain any comments in a value. To minimize the impact on the -// efficiency, this is choosed as a default. +// efficiency, this is chosen as a default. // // To reduce the memory footprint, later we can try empty base optimization (EBO). struct discard_comments diff --git a/share/openPMD/thirdParty/toml11/toml/datetime.hpp b/share/openPMD/thirdParty/toml11/toml/datetime.hpp index d8127c150a..36db1e101c 100644 --- a/share/openPMD/thirdParty/toml11/toml/datetime.hpp +++ b/share/openPMD/thirdParty/toml11/toml/datetime.hpp @@ -21,34 +21,34 @@ namespace toml namespace detail { // TODO: find more sophisticated way to handle this -#if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 1) || defined(_XOPEN_SOURCE) || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(_POSIX_SOURCE) +#if defined(_MSC_VER) inline std::tm localtime_s(const std::time_t* src) { std::tm dst; - const auto result = ::localtime_r(src, &dst); - if (!result) { throw std::runtime_error("localtime_r failed."); } + const auto result = ::localtime_s(&dst, src); + if (result) { throw std::runtime_error("localtime_s failed."); } return dst; } inline std::tm gmtime_s(const std::time_t* src) { std::tm dst; - const auto result = ::gmtime_r(src, &dst); - if (!result) { throw std::runtime_error("gmtime_r failed."); } + const auto result = ::gmtime_s(&dst, src); + if (result) { throw std::runtime_error("gmtime_s failed."); } return dst; } -#elif defined(_MSC_VER) +#elif (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 1) || defined(_XOPEN_SOURCE) || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(_POSIX_SOURCE) inline std::tm localtime_s(const std::time_t* src) { std::tm dst; - const auto result = ::localtime_s(&dst, src); - if (result) { throw std::runtime_error("localtime_s failed."); } + const auto result = ::localtime_r(src, &dst); + if (!result) { throw std::runtime_error("localtime_r failed."); } return dst; } inline std::tm gmtime_s(const std::time_t* src) { std::tm dst; - const auto result = ::gmtime_s(&dst, src); - if (result) { throw std::runtime_error("gmtime_s failed."); } + const auto result = ::gmtime_r(src, &dst); + if (!result) { throw std::runtime_error("gmtime_r failed."); } return dst; } #else // fallback. not threadsafe diff --git a/share/openPMD/thirdParty/toml11/toml/get.hpp b/share/openPMD/thirdParty/toml11/toml/get.hpp index 6669e3402d..901790b551 100644 --- a/share/openPMD/thirdParty/toml11/toml/get.hpp +++ b/share/openPMD/thirdParty/toml11/toml/get.hpp @@ -140,7 +140,7 @@ get(basic_value&& v) // ============================================================================ // std::string_view -#if __cplusplus >= 201703L +#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 template class M, template class V> inline detail::enable_if_t::value, std::string_view> @@ -215,6 +215,7 @@ template, // T is a container detail::negation>, // w/o push_back(...) + detail::negation>, // T does not have special conversion detail::negation< // not toml::array detail::is_exact_toml_type>> >::value, T> @@ -324,6 +325,7 @@ template, // T is a container detail::negation>, // w/o push_back + detail::negation>, // T does not have special conversion detail::negation< // T is not toml::array detail::is_exact_toml_type>> >::value, T> diff --git a/share/openPMD/thirdParty/toml11/toml/lexer.hpp b/share/openPMD/thirdParty/toml11/toml/lexer.hpp index 2eea996ebf..ea5050b8dd 100644 --- a/share/openPMD/thirdParty/toml11/toml/lexer.hpp +++ b/share/openPMD/thirdParty/toml11/toml/lexer.hpp @@ -119,8 +119,8 @@ using lex_local_time = lex_partial_time; // =========================================================================== using lex_quotation_mark = character<'"'>; -using lex_basic_unescaped = exclude, // 0x09 (tab) - in_range<0x0a, 0x1F>, // is allowed +using lex_basic_unescaped = exclude, // 0x09 (tab) is allowed + in_range<0x0A, 0x1F>, character<0x22>, character<0x5C>, character<0x7F>>>; @@ -166,7 +166,7 @@ using lex_basic_string = sequence, maybe >; -using lex_ml_basic_unescaped = exclude, // 0x09 - in_range<0x0a, 0x1F>, // is tab +using lex_ml_basic_unescaped = exclude, // 0x09 is tab + in_range<0x0A, 0x1F>, character<0x5C>, // backslash character<0x7F>, // DEL lex_ml_basic_string_delim>>; @@ -196,8 +196,8 @@ using lex_ml_basic_string = sequence; -using lex_literal_char = exclude, - in_range<0x10, 0x19>, character<0x27>>>; +using lex_literal_char = exclude, in_range<0x0A, 0x1F>, + character<0x7F>, character<0x27>>>; using lex_apostrophe = character<'\''>; using lex_literal_string = sequence, @@ -212,7 +212,7 @@ using lex_ml_literal_string_close = sequence< >; using lex_ml_literal_char = exclude, - in_range<0x10, 0x1F>, + in_range<0x0A, 0x1F>, character<0x7F>, lex_ml_literal_string_delim>>; using lex_ml_literal_body = repeat, @@ -225,12 +225,6 @@ using lex_string = either; // =========================================================================== - -using lex_comment_start_symbol = character<'#'>; -using lex_non_eol = either, exclude>>; -using lex_comment = sequence>; - using lex_dot_sep = sequence, character<'.'>, maybe>; using lex_unquoted_key = repeat, lex_array_table_close>; +using lex_utf8_1byte = in_range<0x00, 0x7F>; +using lex_utf8_2byte = sequence< + in_range(0xC2), static_cast(0xDF)>, + in_range(0x80), static_cast(0xBF)> + >; +using lex_utf8_3byte = sequence(0xE0)>, in_range(0xA0), static_cast(0xBF)>>, + sequence(0xE1), static_cast(0xEC)>, in_range(0x80), static_cast(0xBF)>>, + sequence(0xED)>, in_range(0x80), static_cast(0x9F)>>, + sequence(0xEE), static_cast(0xEF)>, in_range(0x80), static_cast(0xBF)>> + >, in_range(0x80), static_cast(0xBF)>>; +using lex_utf8_4byte = sequence(0xF0)>, in_range(0x90), static_cast(0xBF)>>, + sequence(0xF1), static_cast(0xF3)>, in_range(0x80), static_cast(0xBF)>>, + sequence(0xF4)>, in_range(0x80), static_cast(0x8F)>> + >, in_range(0x80), static_cast(0xBF)>, + in_range(0x80), static_cast(0xBF)>>; +using lex_utf8_code = either< + lex_utf8_1byte, + lex_utf8_2byte, + lex_utf8_3byte, + lex_utf8_4byte + >; + +using lex_comment_start_symbol = character<'#'>; +using lex_non_eol_ascii = either, in_range<0x20, 0x7E>>; +using lex_comment = sequence, unlimited>>; + } // detail } // toml #endif // TOML_LEXER_HPP diff --git a/share/openPMD/thirdParty/toml11/toml/literal.hpp b/share/openPMD/thirdParty/toml11/toml/literal.hpp index 1d338b7efc..04fbbc13e1 100644 --- a/share/openPMD/thirdParty/toml11/toml/literal.hpp +++ b/share/openPMD/thirdParty/toml11/toml/literal.hpp @@ -87,6 +87,7 @@ operator"" _toml(const char* str, std::size_t len) ::toml::detail::location loc( std::string("TOML literal encoded in a C++ code"), std::vector(str, str + len)); + // literal length does not include the null character at the end. return literal_internal_impl(std::move(loc)); } diff --git a/share/openPMD/thirdParty/toml11/toml/parser.hpp b/share/openPMD/thirdParty/toml11/toml/parser.hpp index f6d6d83d32..bfa5531f0d 100644 --- a/share/openPMD/thirdParty/toml11/toml/parser.hpp +++ b/share/openPMD/thirdParty/toml11/toml/parser.hpp @@ -364,6 +364,19 @@ inline result parse_escape_sequence(location& loc) return err(msg); } +inline std::ptrdiff_t check_utf8_validity(const std::string& reg) +{ + location loc("tmp", reg); + const auto u8 = repeat::invoke(loc); + if(!u8 || loc.iter() != loc.end()) + { + const auto error_location = std::distance(loc.begin(), loc.iter()); + assert(0 <= error_location); + return error_location; + } + return -1; +} + inline result, std::string> parse_ml_basic_string(location& loc) { @@ -432,7 +445,21 @@ parse_ml_basic_string(location& loc) source_location(inner_loc)); } } - return ok(std::make_pair(toml::string(retval), token.unwrap())); + + const auto err_loc = check_utf8_validity(token.unwrap().str()); + if(err_loc == -1) + { + return ok(std::make_pair(toml::string(retval), token.unwrap())); + } + else + { + inner_loc.reset(first); + inner_loc.advance(err_loc); + throw syntax_error(format_underline( + "parse_ml_basic_string: invalid utf8 sequence found", + {{source_location(inner_loc), "here"}}), + source_location(inner_loc)); + } } else { @@ -484,7 +511,21 @@ parse_basic_string(location& loc) } quot = lex_quotation_mark::invoke(inner_loc); } - return ok(std::make_pair(toml::string(retval), token.unwrap())); + + const auto err_loc = check_utf8_validity(token.unwrap().str()); + if(err_loc == -1) + { + return ok(std::make_pair(toml::string(retval), token.unwrap())); + } + else + { + inner_loc.reset(first); + inner_loc.advance(err_loc); + throw syntax_error(format_underline( + "parse_ml_basic_string: invalid utf8 sequence found", + {{source_location(inner_loc), "here"}}), + source_location(inner_loc)); + } } else { @@ -545,8 +586,22 @@ parse_ml_literal_string(location& loc) source_location(inner_loc)); } } - return ok(std::make_pair(toml::string(retval, toml::string_t::literal), - token.unwrap())); + + const auto err_loc = check_utf8_validity(token.unwrap().str()); + if(err_loc == -1) + { + return ok(std::make_pair(toml::string(retval, toml::string_t::literal), + token.unwrap())); + } + else + { + inner_loc.reset(first); + inner_loc.advance(err_loc); + throw syntax_error(format_underline( + "parse_ml_basic_string: invalid utf8 sequence found", + {{source_location(inner_loc), "here"}}), + source_location(inner_loc)); + } } else { @@ -584,9 +639,23 @@ parse_literal_string(location& loc) {{source_location(inner_loc), "should be '"}}), source_location(inner_loc)); } - return ok(std::make_pair( - toml::string(body.unwrap().str(), toml::string_t::literal), - token.unwrap())); + + const auto err_loc = check_utf8_validity(token.unwrap().str()); + if(err_loc == -1) + { + return ok(std::make_pair( + toml::string(body.unwrap().str(), toml::string_t::literal), + token.unwrap())); + } + else + { + inner_loc.reset(first); + inner_loc.advance(err_loc); + throw syntax_error(format_underline( + "parse_ml_basic_string: invalid utf8 sequence found", + {{source_location(inner_loc), "here"}}), + source_location(inner_loc)); + } } else { @@ -662,12 +731,36 @@ parse_local_date(location& loc) {{source_location(inner_loc), "here"}}), source_location(inner_loc)); } - return ok(std::make_pair(local_date( - static_cast(from_string(y.unwrap().str(), 0)), - static_cast( - static_cast(from_string(m.unwrap().str(), 0)-1)), - static_cast(from_string(d.unwrap().str(), 0))), - token.unwrap())); + + const auto year = static_cast(from_string(y.unwrap().str(), 0)); + const auto month = static_cast(from_string(m.unwrap().str(), 0)); + const auto day = static_cast(from_string(d.unwrap().str(), 0)); + + // We briefly check whether the input date is valid or not. But here, we + // only check if the RFC3339 compliance. + // Actually there are several special date that does not exist, + // because of historical reasons, such as 1582/10/5-1582/10/14 (only in + // several countries). But here, we do not care about such a complicated + // rule. It makes the code complicated and there is only low probability + // that such a specific date is needed in practice. If someone need to + // validate date accurately, that means that the one need a specialized + // library for their purpose in a different layer. + { + const bool is_leap = (year % 4 == 0) && ((year % 100 != 0) || (year % 400 == 0)); + const auto max_day = (month == 2) ? (is_leap ? 29 : 28) : + ((month == 4 || month == 6 || month == 9 || month == 11) ? 30 : 31); + + if((month < 1 || 12 < month) || (day < 1 || max_day < day)) + { + throw syntax_error(format_underline("toml::parse_date: " + "invalid date: it does not conform RFC3339.", {{ + source_location(loc), "month should be 01-12, day should be" + " 01-28,29,30,31, depending on month/year." + }}), source_location(inner_loc)); + } + } + return ok(std::make_pair(local_date(year, static_cast(month - 1), day), + token.unwrap())); } else { @@ -711,10 +804,22 @@ parse_local_time(location& loc) {{source_location(inner_loc), "here"}}), source_location(inner_loc)); } - local_time time( - from_string(h.unwrap().str(), 0), - from_string(m.unwrap().str(), 0), - from_string(s.unwrap().str(), 0), 0, 0); + + const int hour = from_string(h.unwrap().str(), 0); + const int minute = from_string(m.unwrap().str(), 0); + const int second = from_string(s.unwrap().str(), 0); + + if((hour < 0 || 23 < hour) || (minute < 0 || 59 < minute) || + (second < 0 || 60 < second)) // it may be leap second + { + throw syntax_error(format_underline("toml::parse_time: " + "invalid time: it does not conform RFC3339.", {{ + source_location(loc), "hour should be 00-23, minute should be" + " 00-59, second should be 00-60 (depending on the leap" + " second rules.)"}}), source_location(inner_loc)); + } + + local_time time(hour, minute, second, 0, 0); const auto before_secfrac = inner_loc.iter(); if(const auto secfrac = lex_time_secfrac::invoke(inner_loc)) @@ -794,7 +899,7 @@ parse_local_datetime(location& loc) { throw internal_error(format_underline( "toml::parse_local_datetime: invalid datetime format", - {{source_location(inner_loc), "invalid time fomrat"}}), + {{source_location(inner_loc), "invalid time format"}}), source_location(inner_loc)); } return ok(std::make_pair( @@ -828,15 +933,26 @@ parse_offset_datetime(location& loc) if(const auto ofs = lex_time_numoffset::invoke(inner_loc)) { const auto str = ofs.unwrap().str(); + + const auto hour = from_string(str.substr(1,2), 0); + const auto minute = from_string(str.substr(4,2), 0); + + if((hour < 0 || 23 < hour) || (minute < 0 || 59 < minute)) + { + throw syntax_error(format_underline("toml::parse_offset_datetime: " + "invalid offset: it does not conform RFC3339.", {{ + source_location(loc), "month should be 01-12, day should be" + " 01-28,29,30,31, depending on month/year." + }}), source_location(inner_loc)); + } + if(str.front() == '+') { - offset = time_offset(from_string(str.substr(1,2), 0), - from_string(str.substr(4,2), 0)); + offset = time_offset(hour, minute); } else { - offset = time_offset(-from_string(str.substr(1,2), 0), - -from_string(str.substr(4,2), 0)); + offset = time_offset(-hour, -minute); } } else if(*inner_loc.iter() != 'Z' && *inner_loc.iter() != 'z') @@ -1141,6 +1257,9 @@ std::string format_dotted_keys(InputIterator first, const InputIterator last) // forward decl for is_valid_forward_table_definition result, region>, std::string> parse_table_key(location& loc); +template +result, std::string> +parse_inline_table(location& loc); // The following toml file is allowed. // ```toml @@ -1166,16 +1285,81 @@ parse_table_key(location& loc); // of the key. If the key region points deeper node, it would be allowed. // Otherwise, the key points the same node. It would be rejected. template -bool is_valid_forward_table_definition(const Value& fwd, +bool is_valid_forward_table_definition(const Value& fwd, const Value& inserting, Iterator key_first, Iterator key_curr, Iterator key_last) { + // ------------------------------------------------------------------------ + // check type of the value to be inserted/merged + + std::string inserting_reg = ""; + if(const auto ptr = detail::get_region(inserting)) + { + inserting_reg = ptr->str(); + } + location inserting_def("internal", std::move(inserting_reg)); + if(const auto inlinetable = parse_inline_table(inserting_def)) + { + // check if we are overwriting existing table. + // ```toml + // # NG + // a.b = 42 + // a = {d = 3.14} + // ``` + // Inserting an inline table to a existing super-table is not allowed in + // any case. If we found it, we can reject it without further checking. + return false; + } + + // Valid and invalid cases when inserting to the [a.b] table: + // + // ## Invalid + // + // ```toml + // # invalid + // [a] + // b.c.d = "foo" + // [a.b] # a.b is already defined and closed + // d = "bar" + // ``` + // ```toml + // # invalid + // a = {b.c.d = "foo"} + // [a.b] # a is already defined and inline table is closed + // d = "bar" + // ``` + // ```toml + // # invalid + // a.b.c.d = "foo" + // [a.b] # a.b is already defined and dotted-key table is closed + // d = "bar" + // ``` + // + // ## Valid + // + // ```toml + // # OK. a.b is defined, but is *overwritable* + // [a.b.c] + // d = "foo" + // [a.b] + // d = "bar" + // ``` + // ```toml + // # OK. a.b is defined, but is *overwritable* + // [a] + // b.c.d = "foo" + // b.e = "bar" + // ``` + + // ------------------------------------------------------------------------ + // check table defined before + std::string internal = ""; if(const auto ptr = detail::get_region(fwd)) { internal = ptr->str(); } location def("internal", std::move(internal)); - if(const auto tabkeys = parse_table_key(def)) + if(const auto tabkeys = parse_table_key(def)) // [table.key] { // table keys always contains all the nodes from the root. const auto& tks = tabkeys.unwrap().first; @@ -1188,7 +1372,7 @@ bool is_valid_forward_table_definition(const Value& fwd, // the keys are not equivalent. it is allowed. return true; } - if(const auto dotkeys = parse_key(def)) + if(const auto dotkeys = parse_key(def)) // a.b.c = "foo" { // consider the following case. // [a] @@ -1196,6 +1380,18 @@ bool is_valid_forward_table_definition(const Value& fwd, // [a.b.c] // e = 2.71 // this defines the table [a.b.c] twice. no? + if(const auto reopening_dotkey_by_table = parse_table_key(inserting_def)) + { + // re-opening a dotkey-defined table by a table is invalid. + // only dotkey can append a key-val. Like: + // ```toml + // a.b.c = "foo" + // a.b.d = "bar" # OK. reopen `a.b` by dotkey + // [a.b] + // e = "bar" # Invalid. re-opening `a.b` by [a.b] is not allowed. + // ``` + return false; + } // a dotted key starts from the node representing a table in which the // dotted key belongs to. @@ -1268,7 +1464,10 @@ insert_nested_key(typename Value::table_type& root, const Value& v, } // the above if-else-if checks tab->at(k) is an array auto& a = tab->at(k).as_array(); - if(!(a.front().is_table())) + // If table element is defined as [[array_of_tables]], it + // cannot be an empty array. If an array of tables is + // defined as `aot = []`, it cannot be appended. + if(a.empty() || !(a.front().is_table())) { throw syntax_error(format_underline(concat_to_string( "toml::insert_value: array of table (\"", @@ -1288,7 +1487,7 @@ insert_nested_key(typename Value::table_type& root, const Value& v, // b = 54 // ``` // Here, from the type information, these cannot be detected - // bacause inline table is also a table. + // because inline table is also a table. // But toml v0.5.0 explicitly says it is invalid. The above // array-of-tables has a static size and appending to the // array is invalid. @@ -1338,7 +1537,7 @@ insert_nested_key(typename Value::table_type& root, const Value& v, // ```toml // # comment 1 // aot = [ - // # coment 2 + // # comment 2 // {foo = "bar"}, // ] // ``` @@ -1362,7 +1561,7 @@ insert_nested_key(typename Value::table_type& root, const Value& v, if(tab->at(k).is_table() && v.is_table()) { if(!is_valid_forward_table_definition( - tab->at(k), first, iter, last)) + tab->at(k), v, first, iter, last)) { throw syntax_error(format_underline(concat_to_string( "toml::insert_value: table (\"", @@ -1380,6 +1579,16 @@ insert_nested_key(typename Value::table_type& root, const Value& v, auto& t = tab->at(k).as_table(); for(const auto& kv : v.as_table()) { + if(tab->at(k).contains(kv.first)) + { + throw syntax_error(format_underline(concat_to_string( + "toml::insert_value: value (\"", + format_dotted_keys(first, last), + "\") already exists."), { + {t.at(kv.first).location(), "already exists here"}, + {v.location(), "this defined twice"} + }), v.location()); + } t[kv.first] = kv.second; } detail::change_region(tab->at(k), key_reg); @@ -1494,22 +1703,24 @@ parse_inline_table(location& loc) {{source_location(loc), "the next token is not an inline table"}})); } loc.advance(); + + // check if the inline table is an empty table = { } + maybe::invoke(loc); + if(loc.iter() != loc.end() && *loc.iter() == '}') + { + loc.advance(); // skip `}` + return ok(std::make_pair(retval, region(loc, first, loc.iter()))); + } + // it starts from "{". it should be formatted as inline-table while(loc.iter() != loc.end()) { - maybe::invoke(loc); - if(loc.iter() != loc.end() && *loc.iter() == '}') - { - loc.advance(); // skip `}` - return ok(std::make_pair(retval, - region(loc, first, loc.iter()))); - } - const auto kv_r = parse_key_value_pair(loc); if(!kv_r) { return err(kv_r.unwrap_err()); } + const auto& kvpair = kv_r.unwrap(); const std::vector& keys = kvpair.first.first; const auto& key_reg = kvpair.first.second; @@ -1526,10 +1737,19 @@ parse_inline_table(location& loc) using lex_table_separator = sequence, character<','>>; const auto sp = lex_table_separator::invoke(loc); + if(!sp) { maybe::invoke(loc); - if(loc.iter() != loc.end() && *loc.iter() == '}') + + if(loc.iter() == loc.end()) + { + throw syntax_error(format_underline( + "toml::parse_inline_table: missing table separator `}` ", + {{source_location(loc), "should be `}`"}}), + source_location(loc)); + } + else if(*loc.iter() == '}') { loc.advance(); // skip `}` return ok(std::make_pair( @@ -1550,6 +1770,18 @@ parse_inline_table(location& loc) source_location(loc)); } } + else // `,` is found + { + maybe::invoke(loc); + if(loc.iter() != loc.end() && *loc.iter() == '}') + { + throw syntax_error(format_underline( + "toml::parse_inline_table: trailing comma is not allowed in" + " an inline table", + {{source_location(loc), "should be `}`"}}), + source_location(loc)); + } + } } loc.reset(first); throw syntax_error(format_underline("toml::parse_inline_table: " @@ -1802,7 +2034,7 @@ parse_table_key(location& loc) source_location(inner_loc)); } - // after [table.key], newline or EOF(empty table) requried. + // after [table.key], newline or EOF(empty table) required. if(loc.iter() != loc.end()) { using lex_newline_after_table_key = @@ -1859,7 +2091,7 @@ parse_array_table_key(location& loc) source_location(inner_loc)); } - // after [[table.key]], newline or EOF(empty table) requried. + // after [[table.key]], newline or EOF(empty table) required. if(loc.iter() != loc.end()) { using lex_newline_after_table_key = @@ -2022,7 +2254,7 @@ result parse_toml_file(location& loc) table_type data; // root object is also a table, but without [tablename] - if(auto tab = parse_ml_table(loc)) + if(const auto tab = parse_ml_table(loc)) { data = std::move(tab.unwrap()); } @@ -2097,11 +2329,16 @@ parse(std::istream& is, const std::string& fname = "unknown file") std::vector letters(static_cast(fsize)); is.read(letters.data(), fsize); - while(!letters.empty() && letters.back() == '\0') + // append LF. + // Although TOML does not require LF at the EOF, to make parsing logic + // simpler, we "normalize" the content by adding LF if it does not exist. + // It also checks if the last char is CR, to avoid changing the meaning. + // This is not the *best* way to deal with the last character, but is a + // simple and quick fix. + if(!letters.empty() && letters.back() != '\n' && letters.back() != '\r') { - letters.pop_back(); + letters.push_back('\n'); } - assert(letters.empty() || letters.back() != '\0'); detail::location loc(std::move(fname), std::move(letters)); @@ -2150,7 +2387,7 @@ basic_value parse(const std::string& fname) // Without this, both parse(std::string) and parse(std::filesystem::path) // matches to parse("filename.toml"). This breaks the existing code. // -// This function exactly matches to the invokation with c-string. +// This function exactly matches to the invocation with c-string. // So this function is preferred than others and the ambiguity disappears. template class Table = std::unordered_map, diff --git a/share/openPMD/thirdParty/toml11/toml/serializer.hpp b/share/openPMD/thirdParty/toml11/toml/serializer.hpp index b27abfd9a0..88ae775a83 100644 --- a/share/openPMD/thirdParty/toml11/toml/serializer.hpp +++ b/share/openPMD/thirdParty/toml11/toml/serializer.hpp @@ -2,6 +2,7 @@ // Distributed under the MIT License. #ifndef TOML11_SERIALIZER_HPP #define TOML11_SERIALIZER_HPP +#include #include #include @@ -28,6 +29,11 @@ template std::basic_string format_key(const std::basic_string& k) { + if(k.empty()) + { + return std::string("\"\""); + } + // check the key can be a bare (unquoted) key detail::location loc(k, std::vector(k.begin(), k.end())); detail::lex_unquoted_key::invoke(loc); @@ -60,9 +66,12 @@ template std::basic_string format_keys(const std::vector>& keys) { - std::basic_string serialized; - if(keys.empty()) {return serialized;} + if(keys.empty()) + { + return std::string("\"\""); + } + std::basic_string serialized; for(const auto& ky : keys) { serialized += format_key(ky); @@ -115,6 +124,29 @@ struct serializer } std::string operator()(const floating_type f) const { + if(std::isnan(f)) + { + if(std::signbit(f)) + { + return std::string("-nan"); + } + else + { + return std::string("nan"); + } + } + else if(!std::isfinite(f)) + { + if(std::signbit(f)) + { + return std::string("-inf"); + } + else + { + return std::string("inf"); + } + } + const auto fmt = "%.*g"; const auto bsz = std::snprintf(nullptr, 0, fmt, this->float_prec_, f); // +1 for null character(\0) @@ -146,8 +178,9 @@ struct serializer { if(s.kind == string_t::basic) { - if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || - std::find(s.str.cbegin(), s.str.cend(), '\"') != s.str.cend()) + if((std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || + std::find(s.str.cbegin(), s.str.cend(), '\"') != s.str.cend()) && + this->width_ != (std::numeric_limits::max)()) { // if linefeed or double-quote is contained, // make it multiline basic string. @@ -309,7 +342,17 @@ struct serializer continue; } std::string next_elem; - next_elem += toml::visit(*this, item); + if(item.is_table()) + { + serializer ser(*this); + ser.can_be_inlined_ = true; + ser.width_ = (std::numeric_limits::max)(); + next_elem += toml::visit(ser, item); + } + else + { + next_elem += toml::visit(*this, item); + } // comma before newline. if(!next_elem.empty() && next_elem.back() == '\n') {next_elem.pop_back();} @@ -398,7 +441,19 @@ struct serializer case '\f': {retval += "\\f"; break;} case '\n': {retval += "\\n"; break;} case '\r': {retval += "\\r"; break;} - default : {retval += c; break;} + default : + { + if((0x00 <= c && c <= 0x08) || (0x0A <= c && c <= 0x1F) || c == 0x7F) + { + retval += "\\u00"; + retval += char(48 + (c / 16)); + retval += char((c % 16 < 10 ? 48 : 55) + (c % 16)); + } + else + { + retval += c; + } + } } } return retval; @@ -432,7 +487,21 @@ struct serializer } break; } - default: {retval += *i; break;} + default : + { + const auto c = *i; + if((0x00 <= c && c <= 0x08) || (0x0A <= c && c <= 0x1F) || c == 0x7F) + { + retval += "\\u00"; + retval += char(48 + (c / 16)); + retval += char((c % 16 < 10 ? 48 : 55) + (c % 16)); + } + else + { + retval += c; + } + } + } } // Only 1 or 2 consecutive `"`s are allowed in multiline basic string. @@ -764,7 +833,7 @@ format(const basic_value& v, std::size_t w = 80u, oss << v.comments(); oss << '\n'; // to split the file comment from the first element } - const auto serialized = visit(serializer(w, fprec, no_comment, false), v); + const auto serialized = visit(serializer(w, fprec, false, no_comment), v); oss << serialized; return oss.str(); } @@ -785,7 +854,7 @@ template std::basic_ostream& nocomment(std::basic_ostream& os) { - // by default, it is zero. and by defalut, it shows comments. + // by default, it is zero. and by default, it shows comments. os.iword(detail::comment_index(os)) = 1; return os; } @@ -794,7 +863,7 @@ template std::basic_ostream& showcomment(std::basic_ostream& os) { - // by default, it is zero. and by defalut, it shows comments. + // by default, it is zero. and by default, it shows comments. os.iword(detail::comment_index(os)) = 0; return os; } @@ -811,7 +880,7 @@ operator<<(std::basic_ostream& os, const basic_value& v) const int fprec = static_cast(os.precision()); os.width(0); - // by defualt, iword is initialized byl 0. And by default, toml11 outputs + // by default, iword is initialized by 0. And by default, toml11 outputs // comments. So `0` means showcomment. 1 means nocommnet. const bool no_comment = (1 == os.iword(detail::comment_index(os))); diff --git a/share/openPMD/thirdParty/toml11/toml/string.hpp b/share/openPMD/thirdParty/toml11/toml/string.hpp index a6ed0801c8..def3e57c38 100644 --- a/share/openPMD/thirdParty/toml11/toml/string.hpp +++ b/share/openPMD/thirdParty/toml11/toml/string.hpp @@ -2,13 +2,17 @@ // Distributed under the MIT License. #ifndef TOML11_STRING_HPP #define TOML11_STRING_HPP + +#include "version.hpp" + #include #include #include -#if __cplusplus >= 201703L +#if TOML11_CPLUSPLUS_STANDARD_VERSION >= 201703L #if __has_include() +#define TOML11_USING_STRING_VIEW 1 #include #endif #endif @@ -53,7 +57,7 @@ struct string string& operator+=(const std::string& rhs) {str += rhs; return *this;} string& operator+=(const string& rhs) {str += rhs.str; return *this;} -#if __cplusplus >= 201703L +#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 explicit string(std::string_view s): kind(string_t::basic), str(s){} string(std::string_view s, string_t k): kind(k), str(s){} diff --git a/share/openPMD/thirdParty/toml11/toml/traits.hpp b/share/openPMD/thirdParty/toml11/toml/traits.hpp index 0064e374a5..255d9e888c 100644 --- a/share/openPMD/thirdParty/toml11/toml/traits.hpp +++ b/share/openPMD/thirdParty/toml11/toml/traits.hpp @@ -5,6 +5,7 @@ #include "from.hpp" #include "into.hpp" +#include "version.hpp" #include #include @@ -13,7 +14,7 @@ #include #include -#if __cplusplus >= 201703L +#if TOML11_CPLUSPLUS_STANDARD_VERSION >= 201703L #if __has_include() #include #endif // has_include() @@ -146,7 +147,7 @@ struct has_specialized_into : decltype(has_specialized_into_impl::check(nullp // --------------------------------------------------------------------------- // C++17 and/or/not -#if __cplusplus >= 201703L +#if TOML11_CPLUSPLUS_STANDARD_VERSION >= 201703L using std::conjunction; using std::disjunction; @@ -208,8 +209,10 @@ template struct is_container : conjunction< negation>, // not a map negation>, // not a std::string -#if __cplusplus >= 201703L +#if TOML11_CPLUSPLUS_STANDARD_VERSION >= 201703L +#if __has_include() negation>, // not a std::string_view +#endif // has_include() #endif has_iterator, // has T::iterator has_value_type // has T::value_type @@ -231,7 +234,7 @@ struct is_basic_value<::toml::basic_value>: std::true_type{}; // --------------------------------------------------------------------------- // C++14 index_sequence -#if __cplusplus >= 201402L +#if TOML11_CPLUSPLUS_STANDARD_VERSION >= 201402L using std::index_sequence; using std::make_index_sequence; @@ -261,12 +264,12 @@ struct index_sequence_maker<0> template using make_index_sequence = typename index_sequence_maker::type; -#endif // __cplusplus >= 2014 +#endif // cplusplus >= 2014 // --------------------------------------------------------------------------- // C++14 enable_if_t -#if __cplusplus >= 201402L +#if TOML11_CPLUSPLUS_STANDARD_VERSION >= 201402L using std::enable_if_t; @@ -275,12 +278,12 @@ using std::enable_if_t; template using enable_if_t = typename std::enable_if::type; -#endif // __cplusplus >= 2014 +#endif // cplusplus >= 2014 // --------------------------------------------------------------------------- // return_type_of_t -#if __cplusplus >= 201703L +#if TOML11_CPLUSPLUS_STANDARD_VERSION >= 201703L && defined(__cpp_lib_is_invocable) && __cpp_lib_is_invocable>=201703 template using return_type_of_t = std::invoke_result_t; diff --git a/share/openPMD/thirdParty/toml11/toml/types.hpp b/share/openPMD/thirdParty/toml11/toml/types.hpp index 7bf4b2e82f..1e420e7fd3 100644 --- a/share/openPMD/thirdParty/toml11/toml/types.hpp +++ b/share/openPMD/thirdParty/toml11/toml/types.hpp @@ -28,7 +28,7 @@ using key = std::string; using boolean = bool; using integer = std::int64_t; -using floating = double; // "float" is a keyward, cannot use it here. +using floating = double; // "float" is a keyword, cannot use it here. // the following stuffs are structs defined here, so aliases are not needed. // - string // - offset_datetime diff --git a/share/openPMD/thirdParty/toml11/toml/utility.hpp b/share/openPMD/thirdParty/toml11/toml/utility.hpp index 4a6b4309d7..53a18b944a 100644 --- a/share/openPMD/thirdParty/toml11/toml/utility.hpp +++ b/share/openPMD/thirdParty/toml11/toml/utility.hpp @@ -7,8 +7,9 @@ #include #include "traits.hpp" +#include "version.hpp" -#if __cplusplus >= 201402L +#if TOML11_CPLUSPLUS_STANDARD_VERSION >= 201402L # define TOML11_MARK_AS_DEPRECATED(msg) [[deprecated(msg)]] #elif defined(__GNUC__) # define TOML11_MARK_AS_DEPRECATED(msg) __attribute__((deprecated(msg))) @@ -21,7 +22,7 @@ namespace toml { -#if __cplusplus >= 201402L +#if TOML11_CPLUSPLUS_STANDARD_VERSION >= 201402L using std::make_unique; @@ -33,7 +34,7 @@ inline std::unique_ptr make_unique(Ts&& ... args) return std::unique_ptr(new T(std::forward(args)...)); } -#endif // __cplusplus >= 2014 +#endif // TOML11_CPLUSPLUS_STANDARD_VERSION >= 2014 namespace detail { @@ -91,7 +92,7 @@ T from_string(const std::string& str, T opt) namespace detail { -#if __cplusplus >= 201402L +#if TOML11_CPLUSPLUS_STANDARD_VERSION >= 201402L template decltype(auto) last_one(T&& tail) noexcept { diff --git a/share/openPMD/thirdParty/toml11/toml/value.hpp b/share/openPMD/thirdParty/toml11/toml/value.hpp index 7c68514d89..1b43db8d4c 100644 --- a/share/openPMD/thirdParty/toml11/toml/value.hpp +++ b/share/openPMD/thirdParty/toml11/toml/value.hpp @@ -101,7 +101,7 @@ throw_key_not_found_error(const Value& v, const key& ky) // ```toml // a = {b = "c"} // ``` - // toml11 consideres the inline table body as the table region. Here, + // toml11 considers the inline table body as the table region. Here, // `{b = "c"}` is the region of the table "a". The size of the region // is 9, not 1. The shotest inline table still has two characters, `{` // and `}`. The size cannot be 1. @@ -110,7 +110,7 @@ throw_key_not_found_error(const Value& v, const key& ky) // ```toml // [a] // ``` - // toml11 consideres the whole table key as the table region. Here, + // toml11 considers the whole table key as the table region. Here, // `[a]` is the table region. The size is 3, not 1. // throw std::out_of_range(format_underline(concat_to_string( @@ -624,7 +624,7 @@ class basic_value assigner(this->string_, toml::string(std::string(s), kind)); } -#if __cplusplus >= 201703L +#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 basic_value(std::string_view s) : type_(value_t::string), region_info_(std::make_shared(region_base{})) diff --git a/share/openPMD/thirdParty/toml11/toml/version.hpp b/share/openPMD/thirdParty/toml11/toml/version.hpp new file mode 100644 index 0000000000..9cbfa39be0 --- /dev/null +++ b/share/openPMD/thirdParty/toml11/toml/version.hpp @@ -0,0 +1,42 @@ +#ifndef TOML11_VERSION_HPP +#define TOML11_VERSION_HPP + +// This file checks C++ version. + +#ifndef __cplusplus +# error "__cplusplus is not defined" +#endif + +// Since MSVC does not define `__cplusplus` correctly unless you pass +// `/Zc:__cplusplus` when compiling, the workaround macros are added. +// Those enables you to define version manually or to use MSVC specific +// version macro automatically. +// +// The value of `__cplusplus` macro is defined in the C++ standard spec, but +// MSVC ignores the value, maybe because of backward compatibility. Instead, +// MSVC defines _MSVC_LANG that has the same value as __cplusplus defined in +// the C++ standard. First we check the manual version definition, and then +// we check if _MSVC_LANG is defined. If neither, use normal `__cplusplus`. +// +// FYI: https://docs.microsoft.com/en-us/cpp/build/reference/zc-cplusplus?view=msvc-170 +// https://docs.microsoft.com/en-us/cpp/preprocessor/predefined-macros?view=msvc-170 +// +#if defined(TOML11_ENFORCE_CXX11) +# define TOML11_CPLUSPLUS_STANDARD_VERSION 201103L +#elif defined(TOML11_ENFORCE_CXX14) +# define TOML11_CPLUSPLUS_STANDARD_VERSION 201402L +#elif defined(TOML11_ENFORCE_CXX17) +# define TOML11_CPLUSPLUS_STANDARD_VERSION 201703L +#elif defined(TOML11_ENFORCE_CXX20) +# define TOML11_CPLUSPLUS_STANDARD_VERSION 202002L +#elif defined(_MSVC_LANG) && defined(_MSC_VER) && 1910 <= _MSC_VER +# define TOML11_CPLUSPLUS_STANDARD_VERSION _MSVC_LANG +#else +# define TOML11_CPLUSPLUS_STANDARD_VERSION __cplusplus +#endif + +#if TOML11_CPLUSPLUS_STANDARD_VERSION < 201103L && _MSC_VER < 1900 +# error "toml11 requires C++11 or later." +#endif + +#endif// TOML11_VERSION_HPP From cf564b0d002c2b0664fc15d7f7cfc4b75f6fbb2c Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sun, 13 Mar 2022 18:20:53 -0700 Subject: [PATCH 06/70] CMake & Docs: toml11 v3.7.1 --- CHANGELOG.rst | 2 +- CMakeLists.txt | 5 +++-- NEWS.rst | 2 +- README.md | 4 ++-- docs/source/dev/buildoptions.rst | 2 +- docs/source/dev/dependencies.rst | 2 +- 6 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a34edc1581..ad38af5cfd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,7 +17,7 @@ Changes to "0.14.0" Features """""""" -- include internally shipped toml11 v3.7.0 #1148 +- include internally shipped toml11 v3.7.1 #1148 #1227 - pybind11: require version 2.9.1+ #1220 Bug Fixes diff --git a/CMakeLists.txt b/CMakeLists.txt index c776fbc47a..17ec82d33d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -211,10 +211,11 @@ target_link_libraries(openPMD::thirdparty::nlohmann_json # external library: toml11 if(openPMD_USE_INTERNAL_TOML11) + set(toml11_INSTALL OFF CACHE INTERNAL "") add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/toml11") - message(STATUS "toml11: Using INTERNAL version '3.7.0'") + message(STATUS "toml11: Using INTERNAL version '3.7.1'") else() - find_package(toml11 3.7.0 CONFIG REQUIRED) + find_package(toml11 3.7.1 CONFIG REQUIRED) message(STATUS "toml11: Found version '${toml11_VERSION}'") endif() add_library(openPMD::thirdparty::toml11 INTERFACE IMPORTED) diff --git a/NEWS.rst b/NEWS.rst index baf0caf8f8..264225f275 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -10,7 +10,7 @@ Building openPMD-api now requires a compiler that supports C++17 or newer. ``MPark.Variant`` is not a dependency anymore (kudos and thanks for the great time!). Python 3.10 is now supported. -openPMD-api now depends on `toml11 `__ 3.7.0+. +openPMD-api now depends on `toml11 `__ 3.7.1+. pybind11 2.9.1 is now the minimally supported version for Python support. The following backend-specific members of the ``Dataset`` class have been removed: ``Dataset::setChunkSize()``, ``Dataset::setCompression()``, ``Dataset::setCustomTransform()``, ``Dataset::chunkSize``, ``Dataset::compression``, ``Dataset::transform``. diff --git a/README.md b/README.md index 5710039873..eabdc58d10 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ Shipped internally in `share/openPMD/thirdParty/`: * [Catch2](https://github.com/catchorg/Catch2) 2.13.4+ ([BSL-1.0](https://github.com/catchorg/Catch2/blob/master/LICENSE.txt)) * [pybind11](https://github.com/pybind/pybind11) 2.9.1+ ([new BSD](https://github.com/pybind/pybind11/blob/master/LICENSE)) * [NLohmann-JSON](https://github.com/nlohmann/json) 3.9.1+ ([MIT](https://github.com/nlohmann/json/blob/develop/LICENSE.MIT)) -* [toml11](https://github.com/ToruNiina/toml11) 3.7.0+ ([MIT](https://github.com/ToruNiina/toml11/blob/master/LICENSE)) +* [toml11](https://github.com/ToruNiina/toml11) 3.7.1+ ([MIT](https://github.com/ToruNiina/toml11/blob/master/LICENSE)) I/O backends: * [JSON](https://en.wikipedia.org/wiki/JSON) @@ -273,7 +273,7 @@ The following options allow to switch to external installs: | `openPMD_USE_INTERNAL_CATCH` | **ON**/OFF | Catch2 | 2.13.4+ | | `openPMD_USE_INTERNAL_PYBIND11` | **ON**/OFF | pybind11 | 2.9.1+ | | `openPMD_USE_INTERNAL_JSON` | **ON**/OFF | NLohmann-JSON | 3.9.1+ | -| `openPMD_USE_INTERNAL_TOML11` | **ON**/OFF | toml11 | 3.7.0+ | +| `openPMD_USE_INTERNAL_TOML11` | **ON**/OFF | toml11 | 3.7.1+ | By default, this will build as a shared library (`libopenPMD.[so|dylib|dll]`) and installs also its headers. In order to build a static library, append `-DBUILD_SHARED_LIBS=OFF` to the `cmake` command. diff --git a/docs/source/dev/buildoptions.rst b/docs/source/dev/buildoptions.rst index 61ee43e95e..d22b6117bf 100644 --- a/docs/source/dev/buildoptions.rst +++ b/docs/source/dev/buildoptions.rst @@ -71,7 +71,7 @@ CMake Option Values Installs Library Version ``openPMD_USE_INTERNAL_CATCH`` **ON**/OFF No Catch2 2.13.4+ ``openPMD_USE_INTERNAL_PYBIND11`` **ON**/OFF No pybind11 2.9.1+ ``openPMD_USE_INTERNAL_JSON`` **ON**/OFF No NLohmann-JSON 3.9.1+ -``openPMD_USE_INTERNAL_TOML11`` **ON**/OFF No toml11 3.7.0+ +``openPMD_USE_INTERNAL_TOML11`` **ON**/OFF No toml11 3.7.1+ ================================= =========== ======== ============= ======== diff --git a/docs/source/dev/dependencies.rst b/docs/source/dev/dependencies.rst index 01f80698a4..243b74649e 100644 --- a/docs/source/dev/dependencies.rst +++ b/docs/source/dev/dependencies.rst @@ -20,7 +20,7 @@ The following libraries are shipped internally in ``share/openPMD/thirdParty/`` * `Catch2 `_ 2.13.4+ (`BSL-1.0 `__) * `pybind11 `_ 2.9.1+ (`new BSD `_) * `NLohmann-JSON `_ 3.9.1+ (`MIT `_) -* `toml11 `_ 3.7.0+ (`MIT `__) +* `toml11 `_ 3.7.1+ (`MIT `__) Optional: I/O backends ---------------------- From 509cdede7bb1c31539a3428b9a5ed9f423058536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 14 Mar 2022 04:34:37 +0100 Subject: [PATCH 07/70] Don't safeguard empty strings while reading (#1223) * Don't safeguard empty strings while reading * Update TODO Comment * Add missing update for VEC_CLONG_DOUBLE Co-authored-by: Axel Huebl --- include/openPMD/backend/Attributable.hpp | 65 +++++++-- src/backend/Attributable.cpp | 176 ++++++++++++++++++----- 2 files changed, 194 insertions(+), 47 deletions(-) diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 58648c8b2a..bba3eb409c 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -90,21 +90,37 @@ namespace internal A_MAP m_attributes; }; + enum class SetAttributeMode : char + { + WhileReadingAttributes, + FromPublicAPICall + }; + /** Verify values of attributes in the frontend * * verify string attributes are not empty (backend restriction, e.g., HDF5) */ template - inline void attr_value_check(std::string const /* key */, T /* value */) + inline void attr_value_check( + std::string const /* key */, T /* value */, SetAttributeMode) {} template <> - inline void attr_value_check(std::string const key, std::string const value) + inline void attr_value_check( + std::string const key, std::string const value, SetAttributeMode mode) { - if (value.empty()) - throw std::runtime_error( - "[setAttribute] Value for string attribute '" + key + - "' must not be empty!"); + switch (mode) + { + case SetAttributeMode::FromPublicAPICall: + if (value.empty()) + throw std::runtime_error( + "[setAttribute] Value for string attribute '" + key + + "' must not be empty!"); + break; + case SetAttributeMode::WhileReadingAttributes: + // no checks while reading + break; + } } template @@ -276,6 +292,13 @@ OPENPMD_protected void seriesFlush(internal::FlushParams); void flushAttributes(internal::FlushParams const &); + + template + bool setAttributeImpl( + std::string const &key, T value, internal::SetAttributeMode); + bool setAttributeImpl( + std::string const &key, char const value[], internal::SetAttributeMode); + enum ReadMode { /** @@ -410,11 +433,29 @@ OPENPMD_protected virtual void linkHierarchy(Writable &w); }; // Attributable -// TODO explicitly instantiate Attributable::setAttribute for all T in Datatype template inline bool Attributable::setAttribute(std::string const &key, T value) { - internal::attr_value_check(key, value); + return setAttributeImpl( + key, std::move(value), internal::SetAttributeMode::FromPublicAPICall); +} + +inline bool +Attributable::setAttribute(std::string const &key, char const value[]) +{ + return setAttributeImpl( + key, value, internal::SetAttributeMode::FromPublicAPICall); +} + +// note: we explicitly instantiate Attributable::setAttributeImpl for all T in +// Datatype in Attributable.cpp +template +inline bool Attributable::setAttributeImpl( + std::string const &key, + T value, + internal::SetAttributeMode setAttributeMode) +{ + internal::attr_value_check(key, value, setAttributeMode); auto &attri = get(); if (IOHandler() && Access::READ_ONLY == IOHandler()->m_frontendAccess) @@ -442,10 +483,12 @@ inline bool Attributable::setAttribute(std::string const &key, T value) } } -inline bool -Attributable::setAttribute(std::string const &key, char const value[]) +inline bool Attributable::setAttributeImpl( + std::string const &key, + char const value[], + internal::SetAttributeMode setAttributeMode) { - return this->setAttribute(key, std::string(value)); + return this->setAttributeImpl(key, std::string(value), setAttributeMode); } template diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index d037ad68bc..44238b161d 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -299,96 +299,183 @@ void Attributable::readAttributes(ReadMode mode) } std::array arr; std::copy_n(vector.begin(), 7, arr.begin()); - setAttribute(key, std::move(arr)); + setAttributeImpl( + key, + std::move(arr), + internal::SetAttributeMode::WhileReadingAttributes); } else { - setAttribute(key, std::move(vector)); + setAttributeImpl( + key, + std::move(vector), + internal::SetAttributeMode::WhileReadingAttributes); } }; switch (*aRead.dtype) { case DT::CHAR: - setAttribute(att, a.get()); + setAttributeImpl( + att, + a.get(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::UCHAR: - setAttribute(att, a.get()); + setAttributeImpl( + att, + a.get(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::SHORT: - setAttribute(att, a.get()); + setAttributeImpl( + att, + a.get(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::INT: - setAttribute(att, a.get()); + setAttributeImpl( + att, + a.get(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::LONG: - setAttribute(att, a.get()); + setAttributeImpl( + att, + a.get(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::LONGLONG: - setAttribute(att, a.get()); + setAttributeImpl( + att, + a.get(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::USHORT: - setAttribute(att, a.get()); + setAttributeImpl( + att, + a.get(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::UINT: - setAttribute(att, a.get()); + setAttributeImpl( + att, + a.get(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::ULONG: - setAttribute(att, a.get()); + setAttributeImpl( + att, + a.get(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::ULONGLONG: - setAttribute(att, a.get()); + setAttributeImpl( + att, + a.get(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::FLOAT: - setAttribute(att, a.get()); + setAttributeImpl( + att, + a.get(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::DOUBLE: - setAttribute(att, a.get()); + setAttributeImpl( + att, + a.get(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::LONG_DOUBLE: - setAttribute(att, a.get()); + setAttributeImpl( + att, + a.get(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::CFLOAT: - setAttribute(att, a.get >()); + setAttributeImpl( + att, + a.get >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::CDOUBLE: - setAttribute(att, a.get >()); + setAttributeImpl( + att, + a.get >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::CLONG_DOUBLE: - setAttribute(att, a.get >()); + setAttributeImpl( + att, + a.get >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::STRING: - setAttribute(att, a.get()); + setAttributeImpl( + att, + a.get(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::VEC_CHAR: - setAttribute(att, a.get >()); + setAttributeImpl( + att, + a.get >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::VEC_SHORT: - setAttribute(att, a.get >()); + setAttributeImpl( + att, + a.get >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::VEC_INT: - setAttribute(att, a.get >()); + setAttributeImpl( + att, + a.get >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::VEC_LONG: - setAttribute(att, a.get >()); + setAttributeImpl( + att, + a.get >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::VEC_LONGLONG: - setAttribute(att, a.get >()); + setAttributeImpl( + att, + a.get >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::VEC_UCHAR: - setAttribute(att, a.get >()); + setAttributeImpl( + att, + a.get >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::VEC_USHORT: - setAttribute(att, a.get >()); + setAttributeImpl( + att, + a.get >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::VEC_UINT: - setAttribute(att, a.get >()); + setAttributeImpl( + att, + a.get >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::VEC_ULONG: - setAttribute(att, a.get >()); + setAttributeImpl( + att, + a.get >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::VEC_ULONGLONG: - setAttribute(att, a.get >()); + setAttributeImpl( + att, + a.get >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::VEC_FLOAT: guardUnitDimension(att, a.get >()); @@ -400,23 +487,40 @@ void Attributable::readAttributes(ReadMode mode) guardUnitDimension(att, a.get >()); break; case DT::VEC_CFLOAT: - setAttribute(att, a.get > >()); + setAttributeImpl( + att, + a.get > >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::VEC_CDOUBLE: - setAttribute(att, a.get > >()); + setAttributeImpl( + att, + a.get > >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::VEC_CLONG_DOUBLE: - setAttribute( - att, a.get > >()); + setAttributeImpl( + att, + a.get > >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::VEC_STRING: - setAttribute(att, a.get >()); + setAttributeImpl( + att, + a.get >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::ARR_DBL_7: - setAttribute(att, a.get >()); + setAttributeImpl( + att, + a.get >(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::BOOL: - setAttribute(att, a.get()); + setAttributeImpl( + att, + a.get(), + internal::SetAttributeMode::WhileReadingAttributes); break; case DT::UNDEFINED: throw std::runtime_error("Invalid Attribute datatype during read"); From 5a65f6df28a1f510d0ae242293d2d410aa32cda5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Mar 2022 10:31:14 -0700 Subject: [PATCH 08/70] [pre-commit.ci] pre-commit autoupdate (#1230) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/hadialqattan/pycln: v1.2.4 → v1.2.5](https://github.com/hadialqattan/pycln/compare/v1.2.4...v1.2.5) - [github.com/mgedmin/check-manifest: 0.47 → 0.48](https://github.com/mgedmin/check-manifest/compare/0.47...0.48) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c36ffc186a..07254e9c8f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -71,7 +71,7 @@ repos: # Autoremoves unused Python imports - repo: https://github.com/hadialqattan/pycln - rev: v1.2.4 + rev: v1.2.5 hooks: - id: pycln name: pycln (python) @@ -109,7 +109,7 @@ repos: # Checks the manifest for missing files (native support) - repo: https://github.com/mgedmin/check-manifest - rev: "0.47" + rev: "0.48" hooks: - id: check-manifest # This is a slow hook, so only run this if --hook-stage manual is passed From b6845588dcd7e28b6a2f0acf1d36eb772c3858f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 29 Mar 2022 08:24:03 +0200 Subject: [PATCH 09/70] ps_make_timer_name_: add memory leak suppression (#1235) --- .github/ci/sanitizer/clang/Leak.supp | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ci/sanitizer/clang/Leak.supp b/.github/ci/sanitizer/clang/Leak.supp index 3c482e7069..81e8ee7f75 100644 --- a/.github/ci/sanitizer/clang/Leak.supp +++ b/.github/ci/sanitizer/clang/Leak.supp @@ -15,3 +15,4 @@ leak:adios_inq_var # ADIOS2 leak:adios2::core::engine::SstReader::* leak:adios2::core::engine::SstWriter::* +leak:ps_make_timer_name_ From 7e3b667444bf156b2d221d4bddf7bb8584eb599c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 29 Mar 2022 08:41:04 +0200 Subject: [PATCH 10/70] Clearly fail when users select a wrong backend (#1214) * Clearly fail when users select a wrong backend * ICC: Unreachable end of function * CI fixes --- src/IO/AbstractIOHandlerHelper.cpp | 79 ++++++++++++--------- test/CoreTest.cpp | 107 +++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 31 deletions(-) diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index c05eec7c19..22fa6af60f 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -20,6 +20,7 @@ */ #include "openPMD/IO/AbstractIOHandlerHelper.hpp" +#include "openPMD/Error.hpp" #include "openPMD/IO/ADIOS/ADIOS1IOHandler.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" #include "openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp" @@ -29,8 +30,33 @@ #include "openPMD/IO/JSON/JSONIOHandler.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" +#include +#include + namespace openPMD { + +namespace +{ + template + std::shared_ptr + constructIOHandler(std::string const &backendName, Args &&...args) + { + if constexpr (enabled) + { + return std::make_shared(std::forward(args)...); + } + else + { + throw error::WrongAPIUsage( + "openPMD-api built without support for " + "backend '" + + backendName + "'."); + } + throw "Unreachable"; + } +} // namespace + #if openPMD_HAVE_MPI template <> std::shared_ptr createIOHandler( @@ -44,24 +70,20 @@ std::shared_ptr createIOHandler( switch (format) { case Format::HDF5: - return std::make_shared( - path, access, comm, std::move(options)); + return constructIOHandler( + "HDF5", path, access, comm, std::move(options)); case Format::ADIOS1: -#if openPMD_HAVE_ADIOS1 - return std::make_shared( - path, access, std::move(options), comm); -#else - throw std::runtime_error("openPMD-api built without ADIOS1 support"); -#endif + return constructIOHandler( + "ADIOS1", path, access, std::move(options), comm); case Format::ADIOS2: - return std::make_shared( - path, access, comm, std::move(options), "bp4"); + return constructIOHandler( + "ADIOS2", path, access, comm, std::move(options), "bp4"); case Format::ADIOS2_SST: - return std::make_shared( - path, access, comm, std::move(options), "sst"); + return constructIOHandler( + "ADIOS2", path, access, comm, std::move(options), "sst"); case Format::ADIOS2_SSC: - return std::make_shared( - path, access, comm, std::move(options), "ssc"); + return constructIOHandler( + "ADIOS2", path, access, comm, std::move(options), "ssc"); default: throw std::runtime_error( "Unknown file format! Did you specify a file ending?"); @@ -77,28 +99,23 @@ std::shared_ptr createIOHandler( switch (format) { case Format::HDF5: - return std::make_shared( - path, access, std::move(options)); + return constructIOHandler( + "HDF5", path, access, std::move(options)); case Format::ADIOS1: -#if openPMD_HAVE_ADIOS1 - return std::make_shared( - path, access, std::move(options)); -#else - throw std::runtime_error("openPMD-api built without ADIOS1 support"); -#endif -#if openPMD_HAVE_ADIOS2 + return constructIOHandler( + "ADIOS1", path, access, std::move(options)); case Format::ADIOS2: - return std::make_shared( - path, access, std::move(options), "bp4"); + return constructIOHandler( + "ADIOS2", path, access, std::move(options), "bp4"); case Format::ADIOS2_SST: - return std::make_shared( - path, access, std::move(options), "sst"); + return constructIOHandler( + "ADIOS2", path, access, std::move(options), "sst"); case Format::ADIOS2_SSC: - return std::make_shared( - path, access, std::move(options), "ssc"); -#endif // openPMD_HAVE_ADIOS2 + return constructIOHandler( + "ADIOS2", path, access, std::move(options), "ssc"); case Format::JSON: - return std::make_shared(path, access); + return constructIOHandler( + "JSON", path, access); default: throw std::runtime_error( "Unknown file format! Did you specify a file ending?"); diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index b05787fbe7..f737141bf4 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -1296,3 +1296,110 @@ TEST_CASE("DoConvert_single_value_to_vector", "[core]") std::vector{0, 1, 2, 3, 4, 5, 6}); } } + +TEST_CASE("unavailable_backend", "[core]") +{ +#if !openPMD_HAVE_ADIOS1 + { + auto fail = []() { + Series( + "unavailable.bp", Access::CREATE, R"({"backend": "ADIOS1"})"); + }; + REQUIRE_THROWS_WITH( + fail(), + "Wrong API usage: openPMD-api built without support for backend " + "'ADIOS1'."); + } +#endif +#if !openPMD_HAVE_ADIOS2 + { + auto fail = []() { + Series( + "unavailable.bp", Access::CREATE, R"({"backend": "ADIOS2"})"); + }; + REQUIRE_THROWS_WITH( + fail(), + "Wrong API usage: openPMD-api built without support for backend " + "'ADIOS2'."); + } +#endif +#if !openPMD_HAVE_ADIOS1 && !openPMD_HAVE_ADIOS2 + { + auto fail = []() { Series("unavailable.bp", Access::CREATE); }; + REQUIRE_THROWS_WITH( + fail(), + "Wrong API usage: openPMD-api built without support for backend " + "'ADIOS2'."); + } +#endif +#if !openPMD_HAVE_HDF5 + { + auto fail = []() { + Series("unavailable.h5", Access::CREATE, R"({"backend": "HDF5"})"); + }; + REQUIRE_THROWS_WITH( + fail(), + "Wrong API usage: openPMD-api built without support for backend " + "'HDF5'."); + } +#endif + +#if openPMD_HAVE_MPI +#if !openPMD_HAVE_ADIOS1 + { + auto fail = []() { + Series( + "unavailable.bp", + Access::CREATE, + MPI_COMM_WORLD, + R"({"backend": "ADIOS1"})"); + }; + REQUIRE_THROWS_WITH( + fail(), + "Wrong API usage: openPMD-api built without support for backend " + "'ADIOS1'."); + } +#endif +#if !openPMD_HAVE_ADIOS2 + { + auto fail = []() { + Series( + "unavailable.bp", + Access::CREATE, + MPI_COMM_WORLD, + R"({"backend": "ADIOS2"})"); + }; + REQUIRE_THROWS_WITH( + fail(), + "Wrong API usage: openPMD-api built without support for backend " + "'ADIOS2'."); + } +#endif +#if !openPMD_HAVE_ADIOS1 && !openPMD_HAVE_ADIOS2 + { + auto fail = []() { + Series("unavailable.bp", Access::CREATE, MPI_COMM_WORLD); + }; + REQUIRE_THROWS_WITH( + fail(), + "Wrong API usage: openPMD-api built without support for backend " + "'ADIOS2'."); + } +#endif +#if !openPMD_HAVE_HDF5 + { + auto fail = []() { + Series( + "unavailable.h5", + Access::CREATE, + MPI_COMM_WORLD, + R"({"backend": "HDF5"})"); + }; + REQUIRE_THROWS_WITH( + fail(), + "Wrong API usage: openPMD-api built without support for backend " + "'HDF5'."); + } +#endif +#endif +} From 3c57afbb1e0ce4a597a9977961a43f66c6a5119f Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 29 Mar 2022 01:07:12 -0700 Subject: [PATCH 11/70] Fix: Missing HDF5 Include (#1236) Add explicit HDF5 include in HDF5IOHandler.cpp --- src/IO/HDF5/HDF5IOHandler.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index de541ab0f0..d0725395f7 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -31,6 +31,8 @@ #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/backend/Attribute.hpp" + +#include #endif #include From 8b4a58a8c4604a84f2d17d5df11a24ded5a0c069 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Fri, 1 Apr 2022 02:03:14 -0700 Subject: [PATCH 12/70] added support in 8a to use encodings (#1231) * added support in 8a to use encodings * Vector: use ::at() * use writeIterations instead of iterations (Franz's suggestions) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fixed typo * typo! * removed helper function * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Co-authored-by: Axel Huebl Co-authored-by: Junmin Gu Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/source/usage/benchmarks.rst | 3 ++ examples/8a_benchmark_write_parallel.cpp | 38 +++++++++++++++++------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/docs/source/usage/benchmarks.rst b/docs/source/usage/benchmarks.rst index 148ceb2253..34af2f4e78 100644 --- a/docs/source/usage/benchmarks.rst +++ b/docs/source/usage/benchmarks.rst @@ -85,6 +85,9 @@ Without specifying this parameter, a default will be applied. This parameter does not expected to impact the performance of writing, it will likely make a difference for certain reading patterns if the underlying storage is using subfiles. +Optional input parameter: encoding +The supported iteration encodings are either f(ile), g(roup), v(ariable). By default, we use variable encoding. + Reading ^^^^^^^ diff --git a/examples/8a_benchmark_write_parallel.cpp b/examples/8a_benchmark_write_parallel.cpp index af2f283d12..1c3df5f7d2 100644 --- a/examples/8a_benchmark_write_parallel.cpp +++ b/examples/8a_benchmark_write_parallel.cpp @@ -219,6 +219,7 @@ class AbstractPattern { public: AbstractPattern(const TestInput &input); + virtual bool setLayOut(int step) = 0; unsigned long getNthMeshExtent(unsigned int n, Offset &offset, Extent &count); @@ -352,6 +353,8 @@ class TestInput int m_Steps = 1; //!< num of iterations std::string m_Backend = ".bp"; //!< I/O backend by file ending bool m_Unbalance = false; //! load is different among processors + openPMD::IterationEncoding m_Encoding = + openPMD::IterationEncoding::variableBased; int m_Ratio = 1; //! particle:mesh ratio unsigned long m_XFactor = 0; // if not overwritten, use m_MPISize @@ -393,6 +396,21 @@ void parse(TestInput &input, std::string line) return; } + if (vec.at(0).compare("encoding") == 0) + { + if (vec.at(1).compare("f") == 0) + input.m_Encoding = openPMD::IterationEncoding::fileBased; + else if (vec.at(1).compare("g") == 0) + input.m_Encoding = openPMD::IterationEncoding::groupBased; +#if openPMD_HAVE_ADIOS2 + // BP5 must be matched with a stream engine. + if (auxiliary::getEnvString("OPENPMD_ADIOS2_ENGINE", "BP4") == "BP5") + input.m_Encoding = openPMD::IterationEncoding::variableBased; +#endif + + return; + } + if (vec[0].compare("ratio") == 0) { input.m_Ratio = atoi(vec[1].c_str()); @@ -619,6 +637,7 @@ void AbstractPattern::run() if (m_Input.m_Unbalance) balance = "u"; + if (m_Input.m_Encoding == openPMD::IterationEncoding::fileBased) { // file based std::ostringstream s; s << m_Input.m_Prefix << "/8a_parallel_" << m_GlobalMesh.size() << "D" @@ -627,7 +646,7 @@ void AbstractPattern::run() std::string filename = s.str(); { - std::string tag = "Writing: " + filename; + std::string tag = "Writing filebased: " + filename; Timer kk(tag.c_str(), m_Input.m_MPIRank); for (int step = 1; step <= m_Input.m_Steps; step++) @@ -635,26 +654,26 @@ void AbstractPattern::run() setLayOut(step); Series series = Series(filename, Access::CREATE, MPI_COMM_WORLD); + series.setIterationEncoding(m_Input.m_Encoding); series.setMeshesPath("fields"); store(series, step); } } } -#ifdef NEVER // runs into error for ADIOS. so temporarily disabled - { // group based + { // group/var based std::ostringstream s; s << m_Input.m_Prefix << "/8a_parallel_" << m_GlobalMesh.size() << "D" << balance << m_Input.m_Backend; std::string filename = s.str(); { - std::string tag = "Writing: " + filename; + std::string tag = "Writing a single file:" + filename; Timer kk(tag.c_str(), m_Input.m_MPIRank); Series series = Series(filename, Access::CREATE, MPI_COMM_WORLD); + series.setIterationEncoding(m_Input.m_Encoding); series.setMeshesPath("fields"); - for (int step = 1; step <= m_Input.m_Steps; step++) { setLayOut(step); @@ -662,7 +681,6 @@ void AbstractPattern::run() } } } -#endif } // run() /* @@ -687,10 +705,11 @@ void AbstractPattern::store(Series &series, int step) std::string scalar = openPMD::MeshRecordComponent::SCALAR; storeMesh(series, step, field_rho, scalar); - ParticleSpecies &currSpecies = series.iterations[step].particles["ion"]; + ParticleSpecies &currSpecies = + series.writeIterations()[step].particles["ion"]; storeParticles(currSpecies, step); - series.iterations[step].close(); + series.writeIterations()[step].close(); } /* @@ -709,8 +728,7 @@ void AbstractPattern::storeMesh( const std::string &compName) { MeshRecordComponent compA = - series.iterations[step].meshes[fieldName][compName]; - + series.writeIterations()[step].meshes[fieldName][compName]; Datatype datatype = determineDatatype(); Dataset dataset = Dataset(datatype, m_GlobalMesh); From 29bb474837288ace5d6d6bf68854a2d96624795d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 1 Apr 2022 15:20:22 -0700 Subject: [PATCH 13/70] CI: NVHPC New Apt Repo (#1241) Update the NVHPC install instructions to the latest and greatest. Fix failing CI (dependency install). Also upgrade to 21.11 as on Perlmutter. --- .../{install_nvhpc21-5.sh => install_nvhpc21-11.sh} | 11 +++++------ .github/workflows/nvidia.yml | 8 ++++---- 2 files changed, 9 insertions(+), 10 deletions(-) rename .github/workflows/dependencies/{install_nvhpc21-5.sh => install_nvhpc21-11.sh} (50%) diff --git a/.github/workflows/dependencies/install_nvhpc21-5.sh b/.github/workflows/dependencies/install_nvhpc21-11.sh similarity index 50% rename from .github/workflows/dependencies/install_nvhpc21-5.sh rename to .github/workflows/dependencies/install_nvhpc21-11.sh index ee966fef1b..f333d7121b 100755 --- a/.github/workflows/dependencies/install_nvhpc21-5.sh +++ b/.github/workflows/dependencies/install_nvhpc21-11.sh @@ -14,14 +14,13 @@ sudo apt-get install -y \ pkg-config \ wget -wget -q https://developer.download.nvidia.com/hpc-sdk/21.5/nvhpc-21-5_21.5_amd64.deb \ - https://developer.download.nvidia.com/hpc-sdk/21.5/nvhpc-2021_21.5_amd64.deb -sudo apt-get update -sudo apt-get install -y ./nvhpc-21-5_21.5_amd64.deb ./nvhpc-2021_21.5_amd64.deb -rm -rf ./nvhpc-21-5_21.5_amd64.deb ./nvhpc-2021_21.5_amd64.deb +echo 'deb [trusted=yes] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | \ + sudo tee /etc/apt/sources.list.d/nvhpc.list +sudo apt-get update -y +sudo apt-get install -y --no-install-recommends nvhpc-21-11 # things should reside in /opt/nvidia/hpc_sdk now # activation via: # source /etc/profile.d/modules.sh -# module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/20.7 +# module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/21.11 diff --git a/.github/workflows/nvidia.yml b/.github/workflows/nvidia.yml index 8fed676da8..d978022457 100644 --- a/.github/workflows/nvidia.yml +++ b/.github/workflows/nvidia.yml @@ -32,8 +32,8 @@ jobs: cmake --build build --parallel 2 ctest --test-dir build --output-on-failure - tests-nvhpc21-5-nvcc: - name: NVHPC@21.5 + tests-nvhpc21-11-nvcc: + name: NVHPC@21.11 runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false # Catch warnings: @@ -42,11 +42,11 @@ jobs: steps: - uses: actions/checkout@v2 - name: Dependencies - run: .github/workflows/dependencies/install_nvhpc21-5.sh + run: .github/workflows/dependencies/install_nvhpc21-11.sh - name: Build & Install run: | source /etc/profile.d/modules.sh - module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/21.5 + module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/21.11 which nvcc || echo "nvcc not in PATH!" which nvc++ || echo "nvc++ not in PATH!" From 9c4173a8b5d8ee9485b0df834ae5b0ee6433cbd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 5 Apr 2022 19:50:39 +0200 Subject: [PATCH 14/70] Increase reference count also in other load_chunk overload (#1225) * Increase reference count also in other load_chunk overload Don't know if it is necessary, but looks like we forgot it earlier * Add comment Co-authored-by: Axel Huebl --- src/binding/python/RecordComponent.cpp | 62 +++++++++++++------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/src/binding/python/RecordComponent.cpp b/src/binding/python/RecordComponent.cpp index 2006f7cf6a..272b6b27cd 100644 --- a/src/binding/python/RecordComponent.cpp +++ b/src/binding/python/RecordComponent.cpp @@ -539,53 +539,55 @@ void load_chunk( } } + // here, we increase a reference on the user-passed data so that + // temporary and lost-scope variables stay alive until we flush + // note: this does not yet prevent the user, as in C++, to build + // a race condition by manipulating the data they passed + auto load_data = + [&r, &buffer, &buffer_info, &offset, &extent](auto cxxtype) { + using CXXType = decltype(cxxtype); + buffer.inc_ref(); + // buffer_info.inc_ref(); + void *data = buffer_info.ptr; + std::shared_ptr shared( + (CXXType *)data, [buffer](CXXType *) { buffer.dec_ref(); }); + r.loadChunk(std::move(shared), offset, extent); + }; + if (r.getDatatype() == Datatype::CHAR) - r.loadChunk(shareRaw((char *)buffer_info.ptr), offset, extent); + load_data((char)0); else if (r.getDatatype() == Datatype::UCHAR) - r.loadChunk( - shareRaw((unsigned char *)buffer_info.ptr), offset, extent); + load_data((unsigned char)0); else if (r.getDatatype() == Datatype::SHORT) - r.loadChunk(shareRaw((short *)buffer_info.ptr), offset, extent); + load_data((short)0); else if (r.getDatatype() == Datatype::INT) - r.loadChunk(shareRaw((int *)buffer_info.ptr), offset, extent); + load_data((int)0); else if (r.getDatatype() == Datatype::LONG) - r.loadChunk(shareRaw((long *)buffer_info.ptr), offset, extent); + load_data((long)0); else if (r.getDatatype() == Datatype::LONGLONG) - r.loadChunk( - shareRaw((long long *)buffer_info.ptr), offset, extent); + load_data((long long)0); else if (r.getDatatype() == Datatype::USHORT) - r.loadChunk( - shareRaw((unsigned short *)buffer_info.ptr), offset, extent); + load_data((unsigned short)0); else if (r.getDatatype() == Datatype::UINT) - r.loadChunk( - shareRaw((unsigned int *)buffer_info.ptr), offset, extent); + load_data((unsigned int)0); else if (r.getDatatype() == Datatype::ULONG) - r.loadChunk( - shareRaw((unsigned long *)buffer_info.ptr), offset, extent); + load_data((unsigned long)0); else if (r.getDatatype() == Datatype::ULONGLONG) - r.loadChunk( - shareRaw((unsigned long long *)buffer_info.ptr), offset, extent); + load_data((unsigned long long)0); else if (r.getDatatype() == Datatype::LONG_DOUBLE) - r.loadChunk( - shareRaw((long double *)buffer_info.ptr), offset, extent); + load_data((long double)0); else if (r.getDatatype() == Datatype::DOUBLE) - r.loadChunk( - shareRaw((double *)buffer_info.ptr), offset, extent); + load_data((double)0); else if (r.getDatatype() == Datatype::FLOAT) - r.loadChunk(shareRaw((float *)buffer_info.ptr), offset, extent); + load_data((float)0); else if (r.getDatatype() == Datatype::CLONG_DOUBLE) - r.loadChunk>( - shareRaw((std::complex *)buffer_info.ptr), - offset, - extent); + load_data((std::complex)0); else if (r.getDatatype() == Datatype::CDOUBLE) - r.loadChunk>( - shareRaw((std::complex *)buffer_info.ptr), offset, extent); + load_data((std::complex)0); else if (r.getDatatype() == Datatype::CFLOAT) - r.loadChunk>( - shareRaw((std::complex *)buffer_info.ptr), offset, extent); + load_data((std::complex)0); else if (r.getDatatype() == Datatype::BOOL) - r.loadChunk(shareRaw((bool *)buffer_info.ptr), offset, extent); + load_data((bool)0); else throw std::runtime_error( std::string("Datatype not known in 'loadChunk'!")); From 54b34f4dc46bc8950ad9d487bddf37de3cd44591 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Apr 2022 13:23:47 -0700 Subject: [PATCH 15/70] [pre-commit.ci] pre-commit autoupdate (#1243) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.1.0 → v4.2.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.1.0...v4.2.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 07254e9c8f..01619b8a60 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ exclude: '^share/openPMD/thirdParty' # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.2.0 hooks: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] From e0b381fba9f46f763b2684d20edeed8f0437193d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 11 Apr 2022 13:28:18 -0700 Subject: [PATCH 16/70] CI: Resilver Spack Cache --- .github/workflows/linux.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 315e8683b6..2a3e27a621 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v2 - name: Spack Cache uses: actions/cache@v2 - with: {path: /opt/spack, key: clang6_nopy_nompi_h5_libcpp } + with: {path: /opt/spack, key: clang6_nopy_nompi_h5_libcpp_v2 } - name: Install run: | sudo apt-get update @@ -47,7 +47,7 @@ jobs: - uses: actions/checkout@v2 - name: Spack Cache uses: actions/cache@v2 - with: {path: /opt/spack, key: clang6_nopy_ompi_h5_ad1_ad2_bp3_libcpp } + with: {path: /opt/spack, key: clang6_nopy_ompi_h5_ad1_ad2_bp3_libcpp_v2 } - name: Install run: | sudo apt-get update @@ -87,7 +87,7 @@ jobs: - uses: actions/checkout@v2 - name: Spack Cache uses: actions/cache@v2 - with: {path: /opt/spack, key: clang7_nopy_ompi_h5_ad1_ad2 } + with: {path: /opt/spack, key: clang7_nopy_ompi_h5_ad1_ad2_v2 } - name: Install run: | sudo apt-get update @@ -122,7 +122,7 @@ jobs: - uses: actions/checkout@v2 - name: Spack Cache uses: actions/cache@v2 - with: {path: /opt/spack, key: clang12_py38_nompi_h5_ad1_ad2 } + with: {path: /opt/spack, key: clang12_py38_nompi_h5_ad1_ad2_v2 } - name: Install run: | sudo apt-get update @@ -154,7 +154,7 @@ jobs: - uses: actions/checkout@v2 - name: Spack Cache uses: actions/cache@v2 - with: {path: /opt/spack, key: clang8_py38_mpich_h5_ad1_ad2_newLayout } + with: {path: /opt/spack, key: clang8_py38_mpich_h5_ad1_ad2_newLayout_v2 } - name: Install run: | sudo apt-get update @@ -200,7 +200,7 @@ jobs: - uses: actions/checkout@v2 - name: Spack Cache uses: actions/cache@v2 - with: {path: /opt/spack, key: gcc7_py36_ompi_h5_ad1_ad2 } + with: {path: /opt/spack, key: gcc7_py36_ompi_h5_ad1_ad2_v2 } - name: Install run: | sudo apt-get update From 73695f499cf674f294d5647791ae0f6d77bb85bd Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 11 Apr 2022 18:47:51 -0700 Subject: [PATCH 17/70] CI: Fix Spack (#1244) * Spack: Remove Old Files * Spack: v0.17.1 * Update Env Files --- .../clang12_py38_nompi_h5_ad1_ad2/spack.yaml | 11 +- .../clang6_nopy_nompi_h5_libcpp/spack.yaml | 5 +- .../spack.yaml | 13 +- .../clang7_nopy_ompi_h5_ad1_ad2/spack.yaml | 13 +- .../clang8_py38_mpich_h5_ad1_ad2/spack.yaml | 13 +- .../clangtidy_nopy_ompi_h5_ad1_ad2/spack.yaml | 13 +- .../gcc7_py36_ompi_h5_ad1_ad2/spack.yaml | 13 +- .github/ci/spack/compilers.yaml | 200 ------------------ .github/ci/spack/config.yaml | 2 - .github/ci/spack/packages.yaml | 155 -------------- .github/workflows/dependencies/install_spack | 15 +- .github/workflows/linux.yml | 8 + 12 files changed, 66 insertions(+), 395 deletions(-) delete mode 100644 .github/ci/spack/compilers.yaml delete mode 100644 .github/ci/spack/config.yaml delete mode 100644 .github/ci/spack/packages.yaml diff --git a/.github/ci/spack-envs/clang12_py38_nompi_h5_ad1_ad2/spack.yaml b/.github/ci/spack-envs/clang12_py38_nompi_h5_ad1_ad2/spack.yaml index 076314528d..13ff80e2a8 100644 --- a/.github/ci/spack-envs/clang12_py38_nompi_h5_ad1_ad2/spack.yaml +++ b/.github/ci/spack-envs/clang12_py38_nompi_h5_ad1_ad2/spack.yaml @@ -19,21 +19,21 @@ spack: variants: ~mpi ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 cmake: externals: - - spec: "cmake" + - spec: cmake@3.23.0 prefix: /usr buildable: False perl: externals: - - spec: "perl" + - spec: perl@5.26.1 prefix: /usr buildable: False python: externals: - - spec: "python" + - spec: python@3.8.0 prefix: /usr buildable: False all: - target: ['x86_64'] + target: [x86_64] variants: ~fortran providers: mpi: [openmpi] @@ -56,3 +56,6 @@ spack: config: build_jobs: 2 + + mirrors: + E4S: https://cache.e4s.io diff --git a/.github/ci/spack-envs/clang6_nopy_nompi_h5_libcpp/spack.yaml b/.github/ci/spack-envs/clang6_nopy_nompi_h5_libcpp/spack.yaml index 98a9a4c476..41c6744eab 100644 --- a/.github/ci/spack-envs/clang6_nopy_nompi_h5_libcpp/spack.yaml +++ b/.github/ci/spack-envs/clang6_nopy_nompi_h5_libcpp/spack.yaml @@ -10,7 +10,7 @@ spack: packages: all: - target: ['x86_64'] + target: [x86_64] variants: ~mpi ~fortran compiler: [clang@6.0.0] @@ -33,3 +33,6 @@ spack: config: build_jobs: 2 + + mirrors: + E4S: https://cache.e4s.io diff --git a/.github/ci/spack-envs/clang6_nopy_ompi_h5_ad1_ad2_bp3_libcpp/spack.yaml b/.github/ci/spack-envs/clang6_nopy_ompi_h5_ad1_ad2_bp3_libcpp/spack.yaml index 4ca32b2894..bae2afb9f8 100644 --- a/.github/ci/spack-envs/clang6_nopy_ompi_h5_ad1_ad2_bp3_libcpp/spack.yaml +++ b/.github/ci/spack-envs/clang6_nopy_ompi_h5_ad1_ad2_bp3_libcpp/spack.yaml @@ -18,26 +18,26 @@ spack: variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 cmake: externals: - - spec: "cmake" + - spec: cmake@3.23.0 prefix: /usr buildable: False openmpi: externals: - - spec: "openmpi" + - spec: openmpi@2.1.1 prefix: /usr buildable: False perl: externals: - - spec: "perl" + - spec: perl@5.26.1 prefix: /usr buildable: False python: externals: - - spec: "python" + - spec: python@3.8.0 prefix: /usr buildable: False all: - target: ['x86_64'] + target: [x86_64] variants: ~fortran providers: mpi: [openmpi] @@ -62,3 +62,6 @@ spack: config: build_jobs: 2 + + mirrors: + E4S: https://cache.e4s.io diff --git a/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad1_ad2/spack.yaml b/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad1_ad2/spack.yaml index ecb40b6e56..e5a71811e5 100644 --- a/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad1_ad2/spack.yaml +++ b/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad1_ad2/spack.yaml @@ -18,26 +18,26 @@ spack: variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 cmake: externals: - - spec: "cmake" + - spec: cmake@3.23.0 prefix: /usr buildable: False openmpi: externals: - - spec: "openmpi" + - spec: openmpi@2.1.1 prefix: /usr buildable: False perl: externals: - - spec: "perl" + - spec: perl@5.26.1 prefix: /usr buildable: False python: externals: - - spec: "python" + - spec: python@3.8.0 prefix: /usr buildable: False all: - target: ['x86_64'] + target: [x86_64] variants: ~fortran providers: mpi: [openmpi] @@ -60,3 +60,6 @@ spack: config: build_jobs: 2 + + mirrors: + E4S: https://cache.e4s.io diff --git a/.github/ci/spack-envs/clang8_py38_mpich_h5_ad1_ad2/spack.yaml b/.github/ci/spack-envs/clang8_py38_mpich_h5_ad1_ad2/spack.yaml index fd63cb5a8c..4fccfe9d0b 100644 --- a/.github/ci/spack-envs/clang8_py38_mpich_h5_ad1_ad2/spack.yaml +++ b/.github/ci/spack-envs/clang8_py38_mpich_h5_ad1_ad2/spack.yaml @@ -18,26 +18,26 @@ spack: variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 cmake: externals: - - spec: "cmake" + - spec: cmake@3.23.0 prefix: /usr buildable: False mpich: externals: - - spec: "mpich" + - spec: mpich@3.3 prefix: /usr buildable: False perl: externals: - - spec: "perl" + - spec: perl@5.26.1 prefix: /usr buildable: False python: externals: - - spec: "python" + - spec: python@3.8.0 prefix: /usr buildable: False all: - target: ['x86_64'] + target: [x86_64] variants: ~fortran providers: mpi: [mpich] @@ -60,3 +60,6 @@ spack: config: build_jobs: 2 + + mirrors: + E4S: https://cache.e4s.io diff --git a/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad1_ad2/spack.yaml b/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad1_ad2/spack.yaml index 716195350d..98acde6e62 100644 --- a/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad1_ad2/spack.yaml +++ b/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad1_ad2/spack.yaml @@ -18,26 +18,26 @@ spack: variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 cmake: externals: - - spec: "cmake" + - spec: cmake@3.23.0 prefix: /usr buildable: False openmpi: externals: - - spec: "openmpi" + - spec: openmpi@2.1.1 prefix: /usr buildable: False perl: externals: - - spec: "perl" + - spec: perl@5.26.1 prefix: /usr buildable: False python: externals: - - spec: "python" + - spec: python@3.8.0 prefix: /usr buildable: False all: - target: ['x86_64'] + target: [x86_64] variants: ~fortran providers: mpi: [openmpi] @@ -60,3 +60,6 @@ spack: config: build_jobs: 2 + + mirrors: + E4S: https://cache.e4s.io diff --git a/.github/ci/spack-envs/gcc7_py36_ompi_h5_ad1_ad2/spack.yaml b/.github/ci/spack-envs/gcc7_py36_ompi_h5_ad1_ad2/spack.yaml index 5fdbd7393e..7662512e8e 100644 --- a/.github/ci/spack-envs/gcc7_py36_ompi_h5_ad1_ad2/spack.yaml +++ b/.github/ci/spack-envs/gcc7_py36_ompi_h5_ad1_ad2/spack.yaml @@ -18,26 +18,26 @@ spack: variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 cmake: externals: - - spec: "cmake" + - spec: cmake@3.23.0 prefix: /usr buildable: False openmpi: externals: - - spec: "openmpi" + - spec: openmpi@2.1.1 prefix: /usr buildable: False perl: externals: - - spec: "perl" + - spec: perl@5.26.1 prefix: /usr buildable: False python: externals: - - spec: "python" + - spec: python@3.6.3 prefix: /usr buildable: False all: - target: ['x86_64'] + target: [x86_64] variants: ~fortran compiler: [gcc@7.0.0] @@ -58,3 +58,6 @@ spack: config: build_jobs: 2 + + mirrors: + E4S: https://cache.e4s.io diff --git a/.github/ci/spack/compilers.yaml b/.github/ci/spack/compilers.yaml deleted file mode 100644 index ab971b0fec..0000000000 --- a/.github/ci/spack/compilers.yaml +++ /dev/null @@ -1,200 +0,0 @@ -compilers: -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: highsierra - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: null - fc: null - spec: apple-clang@9.1.0 - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: { - cxxflags: "-stdlib=libc++" - } - modules: [] - operating_system: highsierra - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: null - fc: null - spec: apple-clang@10.0.0 - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: mojave - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: null - fc: null - spec: apple-clang@11.0.0 - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: ubuntu16.04 - paths: - cc: /usr/lib/llvm-5.0/bin/clang - cxx: /usr/lib/llvm-5.0/bin/clang++ - f77: null - fc: null - spec: clang@5.0.0 - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: { - cxxflags: "-stdlib=libc++" - } - modules: [] - operating_system: ubuntu16.04 - paths: - cc: /usr/bin/clang-6.0 - cxx: /usr/bin/clang++-6.0 - f77: null - fc: null - spec: clang@6.0.0 - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: ubuntu16.04 - paths: - cc: /usr/local/clang-7.0.0/bin/clang - cxx: /usr/local/clang-7.0.0/bin/clang++ - f77: /usr/bin/gfortran-4.9 - fc: /usr/bin/gfortran-4.9 - spec: clang@7.0.0 - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: ubuntu14.04 - paths: - cc: /usr/bin/gcc-4.8 - cxx: /usr/bin/g++-4.8 - f77: /usr/bin/gfortran-4.8 - fc: /usr/bin/gfortran-4.8 - spec: gcc@4.8.5 - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: ubuntu14.04 - paths: - cc: /usr/bin/gcc-4.9 - cxx: /usr/bin/g++-4.9 - f77: /usr/bin/gfortran-4.9 - fc: /usr/bin/gfortran-4.9 - spec: gcc@4.9.4 - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: ubuntu14.04 - paths: - cc: /usr/bin/gcc-6 - cxx: /usr/bin/g++-6 - f77: /usr/bin/gfortran-6 - fc: /usr/bin/gfortran-6 - spec: gcc@6.5.0 - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: ubuntu14.04 - paths: - cc: /usr/bin/gcc-7 - cxx: /usr/bin/g++-7 - f77: /usr/bin/gfortran-7 - fc: /usr/bin/gfortran-7 - spec: gcc@7.4.0 - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: ubuntu16.04 - paths: - cc: /usr/bin/gcc-8 - cxx: /usr/bin/g++-8 - f77: /usr/bin/gfortran-8 - fc: /usr/bin/gfortran-8 - spec: gcc@8.1.0 - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: ubuntu18.04 - paths: - cc: /usr/bin/gcc-7 - cxx: /usr/bin/g++-7 - f77: /usr/bin/gfortran-7 - fc: /usr/bin/gfortran-7 - spec: gcc@7.4.0 - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: ubuntu18.04 - paths: - cc: /usr/bin/clang-8 - cxx: /usr/bin/clang++-8 - f77: null - fc: null - spec: clang@8.0.0 - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: ubuntu18.04 - paths: - cc: /usr/bin/clang-10 - cxx: /usr/bin/clang++-10 - f77: null - fc: null - spec: clang@10.0.0 - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: ubuntu18.04 - paths: - cc: /usr/bin/gcc-9 - cxx: /usr/bin/g++-9 - f77: /usr/bin/gfortran-9 - fc: /usr/bin/gfortran-9 - spec: gcc@9.3.0 - target: x86_64 diff --git a/.github/ci/spack/config.yaml b/.github/ci/spack/config.yaml deleted file mode 100644 index 77a2e80adb..0000000000 --- a/.github/ci/spack/config.yaml +++ /dev/null @@ -1,2 +0,0 @@ -config: - build_jobs: 2 diff --git a/.github/ci/spack/packages.yaml b/.github/ci/spack/packages.yaml deleted file mode 100644 index 838554c359..0000000000 --- a/.github/ci/spack/packages.yaml +++ /dev/null @@ -1,155 +0,0 @@ -packages: - perl: - version: [5.14.0, 5.18.2, 5.22.4, 5.26.2] - externals: - - spec: "perl@5.14.0%gcc@4.8.5 arch=linux-ubuntu14-x86_64" - prefix: /usr - - spec: "perl@5.14.0%gcc@4.9.4 arch=linux-ubuntu14-x86_64" - prefix: /usr - - spec: "perl@5.14.0%gcc@6.5.0 arch=linux-ubuntu14-x86_64" - prefix: /usr - - spec: "perl@5.14.0%gcc@7.4.0 arch=linux-ubuntu14-x86_64" - prefix: /usr - - spec: "perl@5.22.4%gcc@8.1.0 arch=linux-ubuntu16-x86_64" - prefix: /usr - - spec: "perl@5.26.2%gcc@9.3.0 arch=linux-ubuntu18-x86_64" - prefix: /usr - - spec: "perl@5.22.4%clang@5.0.0 arch=linux-ubuntu16-x86_64" - prefix: /usr - - spec: "perl@5.22.4%clang@6.0.0 arch=linux-ubuntu16-x86_64" - prefix: /usr - - spec: "perl@5.22.4%clang@7.0.0 arch=linux-ubuntu16-x86_64" - prefix: /usr - - spec: "perl@5.26.1%clang@8.0.0 arch=linux-ubuntu18-x86_64" - prefix: /usr - - spec: "perl@5.26.1%clang@10.0.0 arch=linux-ubuntu18-x86_64" - prefix: /usr - - spec: "perl@5.18.2%apple-clang@9.1.0 arch=darwin-highsierra-x86_64" - prefix: /usr - - spec: "perl@5.18.2%apple-clang@10.0.0 arch=darwin-highsierra-x86_64" - prefix: /usr - - spec: "perl@5.18.2%apple-clang@11.0.0 arch=darwin-mojave-x86_64" - prefix: /usr - buildable: False - cmake: - version: [3.12.0] - externals: - - spec: "cmake@3.12.0%gcc@4.8.5 arch=linux-ubuntu14-x86_64" - prefix: /home/travis/.cache/cmake-3.12.0 - - spec: "cmake@3.12.0%gcc@4.9.4 arch=linux-ubuntu14-x86_64" - prefix: /home/travis/.cache/cmake-3.12.0 - - spec: "cmake@3.12.0%gcc@6.5.0 arch=linux-ubuntu14-x86_64" - prefix: /home/travis/.cache/cmake-3.12.0 - - spec: "cmake@3.12.0%gcc@7.4.0 arch=linux-ubuntu14-x86_64" - prefix: /home/travis/.cache/cmake-3.12.0 - - spec: "cmake@3.12.0%gcc@8.1.0 arch=linux-ubuntu16-x86_64" - prefix: /home/travis/.cache/cmake-3.12.0 - - spec: "cmake@3.12.0%gcc@9.3.0 arch=linux-ubuntu18-x86_64" - prefix: /home/travis/.cache/cmake-3.12.0 - - spec: "cmake@3.12.0%clang@5.0.0 arch=linux-ubuntu16-x86_64" - prefix: /home/travis/.cache/cmake-3.12.0 - - spec: "cmake@3.12.0%clang@6.0.0 arch=linux-ubuntu16-x86_64" - prefix: /home/travis/.cache/cmake-3.12.0 - - spec: "cmake@3.12.0%clang@7.0.0 arch=linux-ubuntu16-x86_64" - prefix: /home/travis/.cache/cmake-3.12.0 - - spec: "cmake@3.12.0%clang@8.0.0 arch=linux-ubuntu18-x86_64" - prefix: /home/travis/.cache/cmake-3.12.0 - - spec: "cmake@3.12.0%clang@10.0.0 arch=linux-ubuntu18-x86_64" - prefix: /home/travis/.cache/cmake-3.12.0 - - spec: "cmake@3.12.0%apple-clang@9.1.0 arch=darwin-highsierra-x86_64" - prefix: /Applications/CMake.app/Contents/ - - spec: "cmake@3.12.0%apple-clang@10.0.0 arch=darwin-highsierra-x86_64" - prefix: /Applications/CMake.app/Contents/ - - spec: "cmake@3.12.0%apple-clang@11.0.0 arch=darwin-mojave-x86_64" - prefix: /Applications/CMake.app/Contents/ - buildable: False - openmpi: - version: [1.6.5, 1.10.2, 2.1.1] - externals: - - spec: "openmpi@1.6.5%gcc@4.8.5 ~wrapper-rpath arch=linux-ubuntu14-x86_64" - prefix: /usr - - spec: "openmpi@1.6.5%gcc@4.9.4 ~wrapper-rpath arch=linux-ubuntu14-x86_64" - prefix: /usr - - spec: "openmpi@1.6.5%gcc@6.5.0 ~wrapper-rpath arch=linux-ubuntu14-x86_64" - prefix: /usr - - spec: "openmpi@1.6.5%gcc@7.4.0 ~wrapper-rpath arch=linux-ubuntu14-x86_64" - prefix: /usr - - spec: "openmpi@1.10.2%gcc@8.1.0 arch=linux-ubuntu16-x86_64" - prefix: /usr - - spec: "openmpi@2.1.1%gcc@9.3.0 arch=linux-ubuntu18-x86_64" - prefix: /usr - - spec: "openmpi@1.10.2%clang@5.0.0 arch=linux-ubuntu16-x86_64" - prefix: /usr - - spec: "openmpi@1.10.2%clang@7.0.0 arch=linux-ubuntu16-x86_64" - prefix: /usr - - spec: "openmpi@2.1.1%clang@10.0.0 arch=linux-ubuntu18-x86_64" - prefix: /usr - buildable: False - mpich: - version: [3.3] - externals: - - spec: "mpich@3.3%clang@8.0.0 arch=linux-ubuntu18-x86_64" - prefix:: /usr - buildable: False - hdf5: - version: [1.10.1, 1.8.13] - adios: - variants: ~zfp ~sz ~lz4 ~blosc - adios2: - variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 - # ~shared is a work-around macOS dylib rpath issue for ADIOS2 - # https://github.com/ornladios/ADIOS2/issues/2316 - # https://spack.readthedocs.io/en/latest/config_yaml.html - # ~bzip2 - # Library not loaded: @rpath/libbz2.1.dylib - python: - version: [3.5.5, 3.6.3, 3.7.1, 3.7.2, 3.8.0] - externals: - - spec: "python@3.8.0%clang@10.0.0 arch=linux-ubuntu18-x86_64" - prefix: /home/travis/virtualenv/python3.8 - - spec: "python@3.8.0%clang@8.0.0 arch=linux-ubuntu18-x86_64" - prefix: /home/travis/virtualenv/python3.8 - - spec: "python@3.8.0%clang@7.0.0 arch=linux-ubuntu16-x86_64" - prefix: /home/travis/virtualenv/python3.8 - - spec: "python@3.7.1%clang@7.0.0 arch=linux-ubuntu16-x86_64" - prefix: /home/travis/virtualenv/python3.7 - - spec: "python@3.6.3%clang@6.0.0 arch=linux-ubuntu16-x86_64" - prefix: /home/travis/virtualenv/python3.6 - - spec: "python@3.6.3%clang@5.0.0 arch=linux-ubuntu16-x86_64" - prefix: /home/travis/virtualenv/python3.6 - - spec: "python@3.8.0%gcc@9.3.0 arch=linux-ubuntu18-x86_64" - prefix: /home/travis/virtualenv/python3.8 - - spec: "python@3.6.3%gcc@7.4.0 arch=linux-ubuntu14-x86_64" - prefix: /home/travis/virtualenv/python3.6 - - spec: "python@3.5.5%gcc@6.5.0 arch=linux-ubuntu14-x86_64" - prefix: /home/travis/virtualenv/python3.5 - - spec: "python@3.6.3%gcc@4.9.4 arch=linux-ubuntu14-x86_64" - prefix: /home/travis/virtualenv/python3.6 - - spec: "python@3.5.5%gcc@4.8.5 arch=linux-ubuntu14-x86_64" - prefix: /home/travis/virtualenv/python3.5 - - spec: "python@3.6.3%gcc@6.5.0 arch=linux-ubuntu14-x86_64" - prefix: /home/travis/virtualenv/python3.6 - - spec: "python@3.7.1%gcc@8.1.0 arch=linux-ubuntu16-x86_64" - prefix: /home/travis/virtualenv/python3.7 - - spec: "python@3.7.2%apple-clang@9.1.0 arch=darwin-highsierra-x86_64" - prefix: /usr/local/opt/python - - spec: "python@3.7.2%apple-clang@10.0.0 arch=darwin-highsierra-x86_64" - prefix: /usr/local/opt/python - - spec: "python@3.7.2%apple-clang@11.0.0 arch=darwin-mojave-x86_64" - prefix: /usr/local/opt/python - buildable: False - - # speed up builds of dependencies - ncurses: - variants: ~termlib - gettext: - variants: ~curses ~libxml2 ~git ~tar ~bzip2 ~xz - py-numpy: - variants: ~blas ~lapack - - # set generic binary generation, mpi providers and compiler versions - all: - target: ['x86_64'] - providers: - mpi: [openmpi, mpich] - compiler: [clang@5.0.0, clang@6.0.0, clang@7.0.0, clang@8.0.0, clang@10.0.0, apple-clang@9.1.0, apple-clang@10.0.0, apple-clang@11.0.0, gcc@4.8.5, gcc@4.9.4, gcc@6.5.0, gcc@7.4.0, gcc@8.1.0, gcc@9.3.0] diff --git a/.github/workflows/dependencies/install_spack b/.github/workflows/dependencies/install_spack index 9c54127010..30d7d06a92 100755 --- a/.github/workflows/dependencies/install_spack +++ b/.github/workflows/dependencies/install_spack @@ -3,7 +3,7 @@ set -eu -o pipefail -spack_ver="2b6f896ca744081a38579573a52824bf334fb54b" +spack_ver="0.17.1" cd /opt if [[ -d spack && ! -f spack_${spack_ver} ]] @@ -13,8 +13,8 @@ fi if [ ! -d spack ] then # download - curl -sOL https://github.com/spack/spack/archive/${spack_ver}.tar.gz - tar -xf ${spack_ver}.tar.gz && rm ${spack_ver}.tar.gz + curl -sOL https://github.com/spack/spack/archive/refs/tags/v${spack_ver}.tar.gz + tar -xf v${spack_ver}.tar.gz && rm v${spack_ver}.tar.gz mv spack-${spack_ver} spack touch spack_${spack_ver} fi @@ -23,14 +23,13 @@ fi ln -s /opt/spack/bin/spack /usr/bin/spack # add binary mirror -spack mirror add E4S https://cache.e4s.io -curl -sOL https://oaciss.uoregon.edu/e4s/e4s.pub -spack gpg trust e4s.pub +#spack mirror add E4S https://cache.e4s.io +#spack buildcache keys -it # find compilers & external packages -spack compiler find +#spack compiler find #spack external find # accessible by regular CI user chmod a+rwx -R /opt/spack -chmod a+rwx $HOME/.spack/ +#chmod a+rwx -R $HOME/.spack/ diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 2a3e27a621..98f8958475 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -59,6 +59,10 @@ jobs: # false positive on src/auxiliary/Filesystem.cpp # [[maybe_unused]] MPI_Datatype const MPI_Types< unsigned >::value = MPI_UNSIGNED; run: | + cmake --version + mpiexec --version + perl --version + python --version eval $(spack env activate --sh .github/ci/spack-envs/clang6_nopy_ompi_h5_ad1_ad2_bp3_libcpp/) spack install @@ -163,6 +167,10 @@ jobs: - name: Build env: {CC: clang-8, CXX: clang++-8, CXXFLAGS: -Werror -Wno-deprecated-declarations, OPENPMD2_ADIOS2_SCHEMA: 20210209} run: | + cmake --version + mpiexec --version + perl --version + python --version eval $(spack env activate --sh .github/ci/spack-envs/clang8_py38_mpich_h5_ad1_ad2/) spack install From cbb076fb9de60b06df4f26791bbff483fe0fb369 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Fri, 15 Apr 2022 10:07:57 -0700 Subject: [PATCH 18/70] use CUDA ptrs in 8a when it is enabled (#1240) * use GPU ptrs in 8a when CUDA is enabled * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * use CUDA_TESTS instead of CUDA default is OFF checked during build examples only * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * modified cuda options * Unify Style * CMake Option: Update Description for CUDA Tests * CMake: Update openPMD_USE_CUDA_EXAMPLES Rename build option. * Simplify CMake * Fix define: openPMD_HAVE_CUDA_EXAMPLES Renamed Co-authored-by: Junmin Gu Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl --- CMakeLists.txt | 46 +++++++++++++++++---- README.md | 12 +++--- docs/source/dev/buildoptions.rst | 15 +++---- docs/source/dev/dependencies.rst | 2 + examples/8a_benchmark_write_parallel.cpp | 52 +++++++++++++++++++++--- include/openPMD/config.hpp.in | 4 ++ 6 files changed, 104 insertions(+), 27 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 17ec82d33d..991ce6a845 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -144,6 +144,7 @@ if(NOT DEFINED BUILD_EXAMPLES) endif() option(openPMD_BUILD_CLI_TOOLS "Build the command line tools" ${BUILD_CLI_TOOLS}) option(openPMD_BUILD_EXAMPLES "Build the examples" ${BUILD_EXAMPLES}) +openpmd_option(CUDA_EXAMPLES "Use CUDA in examples" OFF) # Helper Functions ############################################################ @@ -157,6 +158,15 @@ function(openpmd_cxx_required target) ) endfunction() +# CUDA C++ standard: requirements for a target +function(openpmd_cuda_required target) + target_compile_features(${target} PUBLIC cuda_std_17) + set_target_properties(${target} PROPERTIES + CUDA_SEPARABLE_COMPILATION ON + CUDA_EXTENSIONS OFF + CUDA_STANDARD_REQUIRED ON) +endfunction() + # Dependencies ################################################################ # @@ -223,6 +233,21 @@ target_link_libraries(openPMD::thirdparty::toml11 INTERFACE toml11::toml11) +# external: CUDA (optional) +if(openPMD_BUILD_EXAMPLES) # currently only used in examples + if(openPMD_USE_CUDA_EXAMPLES STREQUAL AUTO) + find_package(CUDAToolkit) + elseif(openPMD_USE_CUDA_EXAMPLES) + find_package(CUDAToolkit REQUIRED) + endif() +endif() +if(CUDAToolkit_FOUND) + enable_language(CUDA) + set(openPMD_HAVE_CUDA_EXAMPLES TRUE) +else() + set(openPMD_HAVE_CUDA_EXAMPLES FALSE) +endif() + # external library: HDF5 (optional) # note: in the new hdf5-cmake.config files, major releases like # 1.8, 1.10 and 1.12 are not marked compatible versions @@ -866,17 +891,20 @@ endif() if(openPMD_BUILD_EXAMPLES) foreach(examplename ${openPMD_EXAMPLE_NAMES}) - if(${examplename} MATCHES ".+parallel$") - if(openPMD_HAVE_MPI) - add_executable(${examplename} examples/${examplename}.cpp) - openpmd_cxx_required(${examplename}) - target_link_libraries(${examplename} PRIVATE openPMD) - endif() + if(${examplename} MATCHES ".+parallel$" AND NOT openPMD_HAVE_MPI) + # skip parallel test + continue() + endif() + + add_executable(${examplename} examples/${examplename}.cpp) + if (openPMD_HAVE_CUDA_EXAMPLES) + set_source_files_properties(examples/${examplename}.cpp + PROPERTIES LANGUAGE CUDA) + openpmd_cuda_required(${examplename}) else() - add_executable(${examplename} examples/${examplename}.cpp) - openpmd_cxx_required(${examplename}) - target_link_libraries(${examplename} PRIVATE openPMD) + openpmd_cxx_required(${examplename}) endif() + target_link_libraries(${examplename} PRIVATE openPMD) endforeach() endif() diff --git a/README.md b/README.md index eabdc58d10..9486a3e9a1 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,7 @@ Optional language bindings: * mpi4py 2.1+ (optional, for MPI) * pandas 1.0+ (optional, for dataframes) * dask 2021+ (optional, for dask dataframes) +* CUDA C++ (optional, currently used only in tests) ## Installation @@ -285,11 +286,12 @@ In order to build with debug symbols, pass `-DCMAKE_BUILD_TYPE=Debug` to your `c By default, tests, examples and command line tools are built. In order to skip building those, pass ``OFF`` to these ``cmake`` options: -| CMake Option | Values | Description | -|---------------------------|------------|--------------------------| -| `openPMD_BUILD_TESTING` | **ON**/OFF | Build tests | -| `openPMD_BUILD_EXAMPLES` | **ON**/OFF | Build examples | -| `openPMD_BUILD_CLI_TOOLS` | **ON**/OFF | Build command-line tools | +| CMake Option | Values | Description | +|-------------------------------|------------|--------------------------| +| `openPMD_BUILD_TESTING` | **ON**/OFF | Build tests | +| `openPMD_BUILD_EXAMPLES` | **ON**/OFF | Build examples | +| `openPMD_BUILD_CLI_TOOLS` | **ON**/OFF | Build command-line tools | +| `openPMD_USE_CUDA_EXAMPLES` | ON/**OFF** | Use CUDA in examples | ## Linking to your project diff --git a/docs/source/dev/buildoptions.rst b/docs/source/dev/buildoptions.rst index d22b6117bf..a897e78e77 100644 --- a/docs/source/dev/buildoptions.rst +++ b/docs/source/dev/buildoptions.rst @@ -81,10 +81,11 @@ Tests, Examples and Command Line Tools By default, tests, examples and command line tools are built. In order to skip building those, pass ``OFF`` to these ``cmake`` options: -============================== =============== ================================================== -CMake Option Values Description -============================== =============== ================================================== -``openPMD_BUILD_TESTING`` **ON**/OFF Build tests -``openPMD_BUILD_EXAMPLES`` **ON**/OFF Build examples -``openPMD_BUILD_CLI_TOOLS`` **ON**/OFF Build command-line tools -============================== =============== ================================================== +=============================== =============== ================================================== +CMake Option Values Description +=============================== =============== ================================================== +``openPMD_BUILD_TESTING`` **ON**/OFF Build tests +``openPMD_BUILD_EXAMPLES`` **ON**/OFF Build examples +``openPMD_BUILD_CLI_TOOLS`` **ON**/OFF Build command-line tools +``openPMD_USE_CUDA_EXAMPLES`` ON/**OFF** Use CUDA in examples +=============================== =============== ================================================== diff --git a/docs/source/dev/dependencies.rst b/docs/source/dev/dependencies.rst index 243b74649e..c5da7454bb 100644 --- a/docs/source/dev/dependencies.rst +++ b/docs/source/dev/dependencies.rst @@ -46,6 +46,8 @@ Optional: language bindings * pandas 1.0+ (optional, for dataframes) * dask 2021+ (optional, for dask dataframes) +* CUDA C++ (optional, currently used only in tests) + Quick Install with Spack ------------------------ diff --git a/examples/8a_benchmark_write_parallel.cpp b/examples/8a_benchmark_write_parallel.cpp index 1c3df5f7d2..f28676e143 100644 --- a/examples/8a_benchmark_write_parallel.cpp +++ b/examples/8a_benchmark_write_parallel.cpp @@ -36,6 +36,11 @@ #include #endif +#if openPMD_HAVE_CUDA_EXAMPLES +#include +#include +#endif + using std::cout; using namespace openPMD; @@ -163,8 +168,9 @@ class Timer int m_Rank = 0; }; -/** createData - * generate a shared ptr of given size with given type & default value +/** createDataCPU + * generate a shared ptr of given size with given type & default value on + * CPU * * @param T data type * @param size data size @@ -175,14 +181,13 @@ class Timer template std::shared_ptr -createData(const unsigned long &size, const T &val, const T &increment) +createDataCPU(const unsigned long &size, const T &val, const T &increment) { auto E = std::shared_ptr{new T[size], [](T *d) { delete[] d; }}; for (unsigned long i = 0ul; i < size; i++) { if (increment != 0) - // E.get()[i] = val+i; E.get()[i] = val + i * increment; else E.get()[i] = val; @@ -190,6 +195,43 @@ createData(const unsigned long &size, const T &val, const T &increment) return E; } +#if openPMD_HAVE_CUDA_EXAMPLES +template +std::shared_ptr +createDataGPU(const unsigned long &size, const T &val, const T &increment) +{ + auto myCudaMalloc = [](size_t mySize) { + void *ptr; + cudaMalloc((void **)&ptr, mySize); + return ptr; + }; + auto deleter = [](T *ptr) { cudaFree(ptr); }; + auto E = std::shared_ptr{(T *)myCudaMalloc(size * sizeof(T)), deleter}; + + T *data = new T[size]; + for (unsigned long i = 0ul; i < size; i++) + { + if (increment != 0) + data[i] = val + i * increment; + else + data[i] = val; + } + cudaMemcpy(E.get(), data, size * sizeof(T), cudaMemcpyHostToDevice); + return E; +} +#endif + +template +std::shared_ptr +createData(const unsigned long &size, const T &val, const T &increment) +{ +#if openPMD_HAVE_CUDA_EXAMPLES + return createDataGPU(size, val, increment); +#else + return createDataCPU(size, val, increment); +#endif +} + /** Find supported backends * (looking for ADIOS2 or H5) * @@ -782,8 +824,6 @@ void AbstractPattern::storeParticles(ParticleSpecies &currSpecies, int &step) { unsigned long offset = 0, count = 0; getNthParticleExtent(n, offset, count); - // std::cout< 0) { auto ids = createData(count, offset, 1); diff --git a/include/openPMD/config.hpp.in b/include/openPMD/config.hpp.in index 63be511355..1989dfcbca 100644 --- a/include/openPMD/config.hpp.in +++ b/include/openPMD/config.hpp.in @@ -41,3 +41,7 @@ #ifndef openPMD_HAVE_ADIOS2 # cmakedefine01 openPMD_HAVE_ADIOS2 #endif + +#ifndef openPMD_HAVE_CUDA_EXAMPLES +# cmakedefine01 openPMD_HAVE_CUDA_EXAMPLES +#endif From 29c587bf46514aadd978eebf34ab68be986e3825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 15 Apr 2022 19:14:27 +0200 Subject: [PATCH 19/70] Fixes for BP5 engine (#1215) * Enable Span-based API in BP5 engine * Bp5 fixes * Add Test for BP5 dataset without steps --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 9 +- src/IO/ADIOS/ADIOS2IOHandler.cpp | 89 ++++++++++---- test/SerialIOTest.cpp | 119 +++++++++++++++++++ 3 files changed, 193 insertions(+), 24 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 8b94d3b51b..eece83eb14 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -1075,9 +1075,15 @@ namespace detail * @brief Begin or end an ADIOS step. * * @param mode Whether to begin or end a step. + * @param calledExplicitly True if called due to a public API call. + * False if called from requireActiveStep. + * Some engines (BP5) require that every interaction happens within + * an active step, meaning that we need to call advance() + * implicitly at times. When doing that, do not tag the dataset + * with __openPMD_internal/useSteps (yet). * @return AdvanceStatus */ - AdvanceStatus advance(AdvanceMode mode); + AdvanceStatus advance(AdvanceMode mode, bool calledExplicitly); /* * Delete all buffered actions without running them. @@ -1201,7 +1207,6 @@ namespace detail Undecided }; StreamStatus streamStatus = StreamStatus::OutsideOfStep; - adios2::StepStatus m_lastStepStatus = adios2::StepStatus::OK; /** * See documentation for StreamStatus::Parsing. diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 5afa60628a..c9f8526b3f 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -605,6 +605,7 @@ void ADIOS2IOHandlerImpl::writeAttribute( auto prefix = filePositionToString(pos); auto &filedata = getFileData(file, IfFileNotOpen::ThrowError); + filedata.requireActiveStep(); filedata.invalidateAttributesMap(); m_dirty.emplace(std::move(file)); @@ -703,7 +704,13 @@ void ADIOS2IOHandlerImpl::getBufferView( Writable *writable, Parameter ¶meters) { // @todo check access mode - if (m_engineType != "bp4") + std::string optInEngines[] = {"bp4", "bp5", "file", "filestream"}; + if (std::none_of( + begin(optInEngines), + end(optInEngines), + [this](std::string const &engine) { + return engine == this->m_engineType; + })) { parameters.out->backendManagedBuffer = false; return; @@ -764,6 +771,7 @@ void ADIOS2IOHandlerImpl::readAttribute( auto file = refreshFileFromParent(writable, /* preferParentFile = */ false); auto pos = setAndGetFilePosition(writable); detail::BufferedActions &ba = getFileData(file, IfFileNotOpen::ThrowError); + ba.requireActiveStep(); switch (attributeLayout()) { using AL = AttributeLayout; @@ -1006,7 +1014,8 @@ void ADIOS2IOHandlerImpl::advance( { auto file = m_files[writable]; auto &ba = getFileData(file, IfFileNotOpen::ThrowError); - *parameters.status = ba.advance(parameters.mode); + *parameters.status = + ba.advance(parameters.mode, /* calledExplicitly = */ true); } void ADIOS2IOHandlerImpl::closePath( @@ -1470,6 +1479,7 @@ namespace detail auto &filedata = impl->getFileData( file, ADIOS2IOHandlerImpl::IfFileNotOpen::ThrowError); + filedata.requireActiveStep(); filedata.invalidateAttributesMap(); adios2::IO IO = filedata.m_IO; impl->m_dirty.emplace(std::move(file)); @@ -2161,9 +2171,18 @@ namespace detail { (void)impl; static std::set streamingEngines = { - "sst", "insitumpi", "inline", "staging", "nullcore", "ssc"}; + "sst", + "insitumpi", + "inline", + "staging", + "nullcore", + "ssc", + "filestream", + "bp5"}; + // diskStreamingEngines is a subset of streamingEngines + static std::set diskStreamingEngines{"bp5", "filestream"}; static std::set fileEngines = { - "bp5", "bp4", "bp3", "hdf5", "file"}; + "bp4", "bp3", "hdf5", "file"}; // step/variable-based iteration encoding requires the new schema if (m_impl->m_iterationEncoding == IterationEncoding::variableBased) @@ -2189,7 +2208,14 @@ namespace detail { isStreaming = true; optimizeAttributesStreaming = - schema() == SupportedSchema::s_0000_00_00; + // Optimizing attributes in streaming mode is not needed in + // the variable-based ADIOS2 schema + schema() == SupportedSchema::s_0000_00_00 && + // Also, it should only be done when truly streaming, not + // when using a disk-based engine that behaves like a + // streaming engine (otherwise attributes might vanish) + diskStreamingEngines.find(m_engineType) == + diskStreamingEngines.end(); streamStatus = StreamStatus::OutsideOfStep; } else @@ -2206,7 +2232,6 @@ namespace detail * file being read. */ streamStatus = StreamStatus::Undecided; - // @todo no?? should be default in both modes delayOpeningTheFirstStep = true; break; case adios2::Mode::Write: @@ -2511,9 +2536,22 @@ namespace detail adios2::Engine &BufferedActions::requireActiveStep() { adios2::Engine &eng = getEngine(); + /* + * If streamStatus is Parsing, do NOT open the step. + */ if (streamStatus == StreamStatus::OutsideOfStep) { - m_lastStepStatus = eng.BeginStep(); + switch ( + advance(AdvanceMode::BEGINSTEP, /* calledExplicitly = */ false)) + { + case AdvanceStatus::OVER: + throw std::runtime_error( + "[ADIOS2] Operation requires active step but no step is " + "left."); + case AdvanceStatus::OK: + // pass + break; + } if (m_mode == adios2::Mode::Read && attributeLayout() == AttributeLayout::ByAdiosVariables) { @@ -2665,7 +2703,8 @@ namespace detail /* flushUnconditionally = */ false); } - AdvanceStatus BufferedActions::advance(AdvanceMode mode) + AdvanceStatus + BufferedActions::advance(AdvanceMode mode, bool calledExplicitly) { if (streamStatus == StreamStatus::Undecided) { @@ -2681,8 +2720,20 @@ namespace detail return AdvanceStatus::OK; } - m_IO.DefineAttribute( - ADIOS2Defaults::str_usesstepsAttribute, 1); + /* + * If advance() is called implicitly (by requireActiveStep()), the + * Series is not necessarily using steps (logically). + * But in some ADIOS2 engines, at least one step must be opened + * (physically) to do anything. + * The usessteps tag should only be set when the Series is *logically* + * using steps. + */ + if (calledExplicitly) + { + m_IO.DefineAttribute( + ADIOS2Defaults::str_usesstepsAttribute, 1); + } + switch (mode) { case AdvanceMode::ENDSTEP: { @@ -2715,21 +2766,11 @@ namespace detail return AdvanceStatus::OK; } case AdvanceMode::BEGINSTEP: { - adios2::StepStatus adiosStatus = m_lastStepStatus; + adios2::StepStatus adiosStatus{}; - // Step might have been opened implicitly already - // by requireActiveStep() - // In that case, streamStatus is DuringStep and Adios - // return status is stored in m_lastStepStatus if (streamStatus != StreamStatus::DuringStep) { - flush( - FlushLevel::UserFlush, - [&adiosStatus](BufferedActions &, adios2::Engine &engine) { - adiosStatus = engine.BeginStep(); - }, - /* writeAttributes = */ false, - /* flushUnconditionally = */ true); + adiosStatus = getEngine().BeginStep(); if (adiosStatus == adios2::StepStatus::OK && m_mode == adios2::Mode::Read && attributeLayout() == AttributeLayout::ByAdiosVariables) @@ -2737,6 +2778,10 @@ namespace detail preloadAttributes.preloadAttributes(m_IO, m_engine.value()); } } + else + { + adiosStatus = adios2::StepStatus::OK; + } AdvanceStatus res = AdvanceStatus::OK; switch (adiosStatus) { diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 6209f47d8c..a53d22632b 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -5145,6 +5145,26 @@ TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") backend.extension, false, backend.jsonBaseConfig()); +#if openPMD_HAVE_ADIOS2 && \ + ADIOS2_VERSION_MAJOR * 1000000000 + ADIOS2_VERSION_MINOR * 100000000 + \ + ADIOS2_VERSION_PATCH * 1000000 + ADIOS2_VERSION_TWEAK >= \ + 2701001223 + if (backend.extension == "bp") + { + iterate_nonstreaming_series( + "../samples/iterate_nonstreaming_series_filebased_bp5_%T." + + backend.extension, + false, + json::merge( + backend.jsonBaseConfig(), "adios2.engine = \"bp5\"")); + iterate_nonstreaming_series( + "../samples/iterate_nonstreaming_series_groupbased_bp5." + + backend.extension, + false, + json::merge( + backend.jsonBaseConfig(), "adios2.engine = \"bp5\"")); + } +#endif } #if openPMD_HAVE_ADIOS2 iterate_nonstreaming_series( @@ -5154,6 +5174,105 @@ TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") #endif } +#if openPMD_HAVE_ADIOS2 && \ + ADIOS2_VERSION_MAJOR * 1000000000 + ADIOS2_VERSION_MINOR * 100000000 + \ + ADIOS2_VERSION_PATCH * 1000000 + ADIOS2_VERSION_TWEAK >= \ + 2701001223 +void adios2_bp5_no_steps(bool usesteps) +{ + std::string const config = R"END( +{ + "adios2": + { + "new_attribute_layout": true, + "schema": 20210209 + } +})END"; + { + adios2::ADIOS adios; + auto IO = adios.DeclareIO("IO"); + IO.SetEngine("bp5"); + auto engine = + IO.Open("../samples/bp5_no_steps.bp", adios2::Mode::Write); + if (usesteps) + { + engine.BeginStep(); + } + + // write default openPMD attributes + auto writeAttribute = [&IO](std::string const &name, auto value) { + IO.DefineAttribute(name, value); + }; + IO.DefineAttribute("/basePath", std::string("/data/%T/")); + IO.DefineAttribute("/date", std::string("2021-02-22 11:14:00 +0000")); + IO.DefineAttribute("/iterationEncoding", std::string("groupBased")); + IO.DefineAttribute("/iterationFormat", std::string("/data/%T/")); + IO.DefineAttribute("/meshesPath", std::string("meshes/")); + IO.DefineAttribute("/openPMD", std::string("1.1.0")); + IO.DefineAttribute("/openPMDextension", uint32_t(0)); + IO.DefineAttribute("/software", std::string("openPMD-api")); + IO.DefineAttribute("/softwareVersion", std::string("0.15.0-dev")); + + IO.DefineAttribute("/data/0/dt", double(1)); + IO.DefineAttribute( + "/data/0/meshes/theta/axisLabels", + std::vector{"x"}.data(), + 1); + IO.DefineAttribute("/data/0/meshes/theta/dataOrder", std::string("C")); + IO.DefineAttribute( + "/data/0/meshes/theta/geometry", std::string("cartesian")); + IO.DefineAttribute("/data/0/meshes/theta/gridGlobalOffset", double(0)); + IO.DefineAttribute("/data/0/meshes/theta/gridSpacing", double(1)); + IO.DefineAttribute("/data/0/meshes/theta/gridUnitSI", double(1)); + IO.DefineAttribute("/data/0/meshes/theta/position", double(0)); + IO.DefineAttribute("/data/0/meshes/theta/timeOffset", double(0)); + IO.DefineAttribute( + "/data/0/meshes/theta/unitDimension", + std::vector(7, 0).data(), + 7); + IO.DefineAttribute("/data/0/meshes/theta/unitSI", double(1)); + IO.DefineAttribute("/data/0/time", double(0)); + IO.DefineAttribute("/data/0/timeUnitSI", double(1)); + + IO.DefineAttribute( + "__openPMD_internal/openPMD2_adios2_schema", 0); + IO.DefineAttribute("__openPMD_internal/useSteps", 0); + + std::vector data{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + engine.Put( + IO.DefineVariable("/data/0/meshes/theta", {10}, {0}, {10}), + data.data()); + + if (usesteps) + { + engine.EndStep(); + } + engine.Close(); + } + + { + Series read( + "../samples/bp5_no_steps.bp", + Access::READ_ONLY, + "adios2.engine.type = \"bp5\""); + auto data = read.iterations[0] + .meshes["theta"][RecordComponent::SCALAR] + .loadChunk({0}, {10}); + read.flush(); + for (size_t i = 0; i < 10; ++i) + { + REQUIRE(data.get()[i] == int(i)); + } + } +} + +TEST_CASE("adios2_bp5_no_steps", "[serial][adios2]") +{ + adios2_bp5_no_steps(/* usesteps = */ false); + adios2_bp5_no_steps(/* usesteps = */ true); +} +#endif + void extendDataset(std::string const &ext, std::string const &jsonConfig) { std::string filename = "../samples/extendDataset." + ext; From 88c3ad1d28fa5e5dd6f30fe8a689853e4c9a0fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 15 Apr 2022 19:17:54 +0200 Subject: [PATCH 20/70] Fix GCC 11 Compiler Warnings (#1213) * Compile warnings in Attribute.hpp G++ 11 does not understand that these variables are not primitive types and complains about possibly uninitialized values. Use the {} constructor to fix this. * Ignore deprecated AccessType in SerialIOTest * ICC issue: statement unreachable * Put throw statement into an else branch ICC seems to resolve reachability of statements after resolving constexpr if * Add pragma for ICPC * Push pragma only in G++ and Clang * Two fixes 1. Reset to default warning behavior after disabling 2. Use __GNUC_MINOR__ instead of __GNUC__ --- include/openPMD/backend/Attribute.hpp | 59 +++++++++++++++++++-------- test/SerialIOTest.cpp | 13 ++++++ 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/include/openPMD/backend/Attribute.hpp b/include/openPMD/backend/Attribute.hpp index 87ffc8a0c0..f83b99a2ae 100644 --- a/include/openPMD/backend/Attribute.hpp +++ b/include/openPMD/backend/Attribute.hpp @@ -122,6 +122,7 @@ class Attribute template auto doConvert(T *pv) -> U { + (void)pv; if constexpr (std::is_convertible_v) { return static_cast(*pv); @@ -132,13 +133,15 @@ auto doConvert(T *pv) -> U typename T::value_type, typename U::value_type>) { - U res; + U res{}; res.reserve(pv->size()); std::copy(pv->begin(), pv->end(), std::back_inserter(res)); return res; } - - throw std::runtime_error("getCast: no vector cast possible."); + else + { + throw std::runtime_error("getCast: no vector cast possible."); + } } // conversion cast: array to vector // if a backend reports a std::array<> for something where @@ -149,14 +152,16 @@ auto doConvert(T *pv) -> U typename T::value_type, typename U::value_type>) { - U res; + U res{}; res.reserve(pv->size()); std::copy(pv->begin(), pv->end(), std::back_inserter(res)); return res; } - - throw std::runtime_error( - "getCast: no array to vector conversion possible."); + else + { + throw std::runtime_error( + "getCast: no array to vector conversion possible."); + } } // conversion cast: vector to array // if a backend reports a std::vector<> for something where @@ -167,7 +172,7 @@ auto doConvert(T *pv) -> U typename T::value_type, typename U::value_type>) { - U res; + U res{}; if (res.size() != pv->size()) { throw std::runtime_error( @@ -180,28 +185,46 @@ auto doConvert(T *pv) -> U } return res; } - - throw std::runtime_error( - "getCast: no vector to array conversion possible."); + else + { + throw std::runtime_error( + "getCast: no vector to array conversion possible."); + } } // conversion cast: turn a single value into a 1-element vector else if constexpr (auxiliary::IsVector_v) { if constexpr (std::is_convertible_v) { - U res; + U res{}; res.reserve(1); res.push_back(static_cast(*pv)); return res; } - - throw std::runtime_error( - "getCast: no scalar to vector conversion possible."); + else + { + throw std::runtime_error( + "getCast: no scalar to vector conversion possible."); + } } - - (void)pv; - throw std::runtime_error("getCast: no cast possible."); + else + { + throw std::runtime_error("getCast: no cast possible."); + } +#if defined(__INTEL_COMPILER) +/* + * ICPC has trouble with if constexpr, thinking that return statements are + * missing afterwards. Deactivate the warning. + * Note that putting a statement here will not help to fix this since it will + * then complain about unreachable code. + * https://community.intel.com/t5/Intel-C-Compiler/quot-if-constexpr-quot-and-quot-missing-return-statement-quot-in/td-p/1154551 + */ +#pragma warning(disable : 1011) +} +#pragma warning(default : 1011) +#else } +#endif /** Retrieve a stored specific Attribute and cast if convertible. * diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index a53d22632b..ddf7e3d31f 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -414,6 +414,13 @@ TEST_CASE("available_chunks_test_json", "[serial][json]") TEST_CASE("multiple_series_handles_test", "[serial]") { + /* + * clang also understands these pragmas. + */ +#if defined(__GNUC__MINOR__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif /* * First test: No premature flushes through destructor when another copy * is still around @@ -456,6 +463,12 @@ TEST_CASE("multiple_series_handles_test", "[serial]") */ series_ptr->flush(); } +#if defined(__INTEL_COMPILER) +#pragma warning(disable : 2282) +#endif +#if defined(__GNUC__MINOR__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif } void close_iteration_test(std::string file_ending) From abe49a2931423045f98902e9f847fedfc699d6aa Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 15 Apr 2022 10:26:40 -0700 Subject: [PATCH 21/70] Python Iteration: Fix __repr__ (time) (#1242) Small numbers, as common for iterations, were flushed to zero in `std::to_string(double)` of the representation of `Iteration` variables: ``` step __repr__ 100 was: 8.687655225973454e-14 * 1.0 ``` --- src/binding/python/Iteration.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/binding/python/Iteration.cpp b/src/binding/python/Iteration.cpp index 6454789b1e..480de17e7e 100644 --- a/src/binding/python/Iteration.cpp +++ b/src/binding/python/Iteration.cpp @@ -23,6 +23,8 @@ #include "openPMD/Iteration.hpp" +#include +#include #include namespace py = pybind11; @@ -36,10 +38,10 @@ void init_Iteration(py::module &m) .def( "__repr__", [](Iteration const &it) { - return ""; + std::stringstream ss; + ss << ""; + return ss.str(); }) .def_property( From 036a276782c35f025b0513e06f1116eb2d5818a0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Apr 2022 13:20:02 -0700 Subject: [PATCH 22/70] [pre-commit.ci] pre-commit autoupdate (#1247) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-clang-format: v13.0.1 → v14.0.1](https://github.com/pre-commit/mirrors-clang-format/compare/v13.0.1...v14.0.1) - [github.com/hadialqattan/pycln: v1.2.5 → v1.3.1](https://github.com/hadialqattan/pycln/compare/v1.2.5...v1.3.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 01619b8a60..0a1d959474 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,13 +65,13 @@ repos: # clang-format v13 # to run manually, use .github/workflows/clang-format/clang-format.sh - repo: https://github.com/pre-commit/mirrors-clang-format - rev: v13.0.1 + rev: v14.0.1 hooks: - id: clang-format # Autoremoves unused Python imports - repo: https://github.com/hadialqattan/pycln - rev: v1.2.5 + rev: v1.3.1 hooks: - id: pycln name: pycln (python) From cf7d8dcd81d2f128fbde429947a5664344d1126a Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 20 Apr 2022 16:56:31 -0700 Subject: [PATCH 23/70] Fix: Unused Lambda in SerialIOTest (#1251) Saw this as an unused variable warning in conda. --- test/SerialIOTest.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index ddf7e3d31f..5cf48f8eb8 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -5213,9 +5213,6 @@ void adios2_bp5_no_steps(bool usesteps) } // write default openPMD attributes - auto writeAttribute = [&IO](std::string const &name, auto value) { - IO.DefineAttribute(name, value); - }; IO.DefineAttribute("/basePath", std::string("/data/%T/")); IO.DefineAttribute("/date", std::string("2021-02-22 11:14:00 +0000")); IO.DefineAttribute("/iterationEncoding", std::string("groupBased")); From c2c2718130d195a64b58c38ad430edda1c33490e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 25 Apr 2022 20:30:40 -0700 Subject: [PATCH 24/70] SerialIOTest: Fix GCC Pragma Check (#1255) There was a typo in `__GNUC_MINOR__`, causing the test not to work on all GCC versions. --- test/SerialIOTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 5cf48f8eb8..aa3de84b43 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -417,7 +417,7 @@ TEST_CASE("multiple_series_handles_test", "[serial]") /* * clang also understands these pragmas. */ -#if defined(__GNUC__MINOR__) || defined(__clang__) +#if defined(__GNUC_MINOR__) || defined(__clang__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif From 51810387027af3ec89fad7e29d822685f87d9763 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 25 Apr 2022 20:32:16 -0700 Subject: [PATCH 25/70] Fix: forgot one in last --- test/SerialIOTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index aa3de84b43..af63b21557 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -466,7 +466,7 @@ TEST_CASE("multiple_series_handles_test", "[serial]") #if defined(__INTEL_COMPILER) #pragma warning(disable : 2282) #endif -#if defined(__GNUC__MINOR__) || defined(__clang__) +#if defined(__GNUC_MINOR__) || defined(__clang__) #pragma GCC diagnostic pop #endif } From 04078886fd028cf8f29474b351493ed8afdfa8d6 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 28 Apr 2022 16:54:14 -0700 Subject: [PATCH 26/70] Doc: Iterate Loop reference (#1254) The previous construct is valid, but creates a clang-tidy warning for `performance-for-range-copy`. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9486a3e9a1..91e1a7550f 100644 --- a/README.md +++ b/README.md @@ -46,16 +46,16 @@ Writing & reading through those backends and their associated files are supporte auto s = openPMD::Series("samples/git-sample/data%T.h5", openPMD::Access::READ_ONLY); -for( auto const [step, it] : s.iterations ) { +for( auto const & [step, it] : s.iterations ) { std::cout << "Iteration: " << step << "\n"; - for( auto const [name, mesh] : it.meshes ) { + for( auto const & [name, mesh] : it.meshes ) { std::cout << " Mesh '" << name << "' attributes:\n"; for( auto const& val : mesh.attributes() ) std::cout << " " << val << '\n'; } - for( auto const [name, species] : it.particles ) { + for( auto const & [name, species] : it.particles ) { std::cout << " Particle species '" << name << "' attributes:\n"; for( auto const& val : species.attributes() ) std::cout << " " << val << '\n'; From 139a3aaa162597ab7065683af61b171c1ff33741 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 28 Apr 2022 16:55:13 -0700 Subject: [PATCH 27/70] CI: Update CUDA repo key (#1256) Nvidia has made changes in the signing keys. https://forums.developer.nvidia.com/t/notice-cuda-linux-repository-key-rotation/212771 --- .github/workflows/dependencies/install_nvcc11.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/dependencies/install_nvcc11.sh b/.github/workflows/dependencies/install_nvcc11.sh index e56a5a8695..9b0948926b 100755 --- a/.github/workflows/dependencies/install_nvcc11.sh +++ b/.github/workflows/dependencies/install_nvcc11.sh @@ -15,8 +15,7 @@ sudo apt-get install -y \ pkg-config \ wget -sudo wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub -sudo apt-key add 7fa2af80.pub +sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 /" \ | sudo tee /etc/apt/sources.list.d/cuda.list sudo apt-get update From 9eb83fede7c3526c43aef4130a3c4a1c7e26fc35 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 28 Apr 2022 17:49:29 -0700 Subject: [PATCH 28/70] Revert SerialIOTest: Fix GCC Pragma Check (#1259) * Revert "SerialIOTest: Fix GCC Pragma Check (#1255)" This reverts commit c2c2718130d195a64b58c38ad430edda1c33490e. * Revert "Fix: forgot one in last" This reverts commit 51810387027af3ec89fad7e29d822685f87d9763. --- test/SerialIOTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index af63b21557..5cf48f8eb8 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -417,7 +417,7 @@ TEST_CASE("multiple_series_handles_test", "[serial]") /* * clang also understands these pragmas. */ -#if defined(__GNUC_MINOR__) || defined(__clang__) +#if defined(__GNUC__MINOR__) || defined(__clang__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif @@ -466,7 +466,7 @@ TEST_CASE("multiple_series_handles_test", "[serial]") #if defined(__INTEL_COMPILER) #pragma warning(disable : 2282) #endif -#if defined(__GNUC_MINOR__) || defined(__clang__) +#if defined(__GNUC__MINOR__) || defined(__clang__) #pragma GCC diagnostic pop #endif } From 87b27477a015296cdfa3ba2beb0701703c9a59b4 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 28 Apr 2022 17:54:04 -0700 Subject: [PATCH 29/70] CI: Fix ADIOS 2.7.1 on Windows (#1258) Because BP5 is a conditional compile variant in ADIOS 2.8.0 --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 55a5a78f70..4575d7e935 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -88,7 +88,7 @@ install: # Configure the VM. - cmd: conda install -n root --quiet --yes numpy cmake hdf5 python=%CONDA_PY% # ADIOS2 build only for 64bit Windows - - cmd: if "%TARGET_ARCH%"=="x64" conda install -n root --quiet --yes adios2 python=%CONDA_PY% + - cmd: if "%TARGET_ARCH%"=="x64" conda install -n root --quiet --yes adios2==2.7.1 python=%CONDA_PY% before_build: - cmd: cd C:\projects\openpmd-api From d0a98dd42743fb86ca7b4341e1bae37127c92101 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 28 Apr 2022 19:56:46 -0700 Subject: [PATCH 30/70] CI: Switch to Mamba (#1261) --- .github/workflows/linux.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 98f8958475..b7997741f5 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -303,12 +303,13 @@ jobs: - name: Install shell: bash -eo pipefail -l {0} run: | - conda env create --file conda.yml + conda install -c conda-forge -y mamba + mamba env create --file conda.yml - name: Build shell: bash -eo pipefail -l {0} env: {CXXFLAGS: -Werror -Wno-deprecated-declarations} run: | - conda activate openpmd-api-dev + source activate openpmd-api-dev share/openPMD/download_samples.sh build chmod u-w build/samples/git-sample/*.h5 From 861f71951f3cadfd7a49dd40405e8bf786ed49db Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 28 Apr 2022 21:45:07 -0700 Subject: [PATCH 31/70] SerialIOTest: Fix GCC Pragma Check (#1260) There was a typo in `__GNUC_MINOR__`, causing the test not to work on all GCC versions. Follow-up to #1213 --- test/SerialIOTest.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 5cf48f8eb8..92de22aa43 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -417,9 +417,11 @@ TEST_CASE("multiple_series_handles_test", "[serial]") /* * clang also understands these pragmas. */ -#if defined(__GNUC__MINOR__) || defined(__clang__) +#if defined(__GNUC_MINOR__) && !defined(__INTEL_COMPILER) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#elif defined(__clang__) +#pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif /* * First test: No premature flushes through destructor when another copy @@ -463,11 +465,10 @@ TEST_CASE("multiple_series_handles_test", "[serial]") */ series_ptr->flush(); } -#if defined(__INTEL_COMPILER) -#pragma warning(disable : 2282) -#endif -#if defined(__GNUC__MINOR__) || defined(__clang__) +#if defined(__GNUC_MINOR__) && !defined(__INTEL_COMPILER) #pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop #endif } From e40f7c49fdaed67928cee054910b619ef3b50627 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 28 Apr 2022 23:15:21 -0700 Subject: [PATCH 32/70] Fix Zero Pattern Issue (#1253) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Test: Demonstrate Pattern Issue As in #1173 * JSON: Improve File Open Error Message Include path to file * Upon parsing, store each iteration's filename If the padding is inconsistent, a later Iteration::open() needs the original filename. Trying to compute the filename from the expansion pattern will lead to wrong filenames. * Fix: Clang-Tidy performance-for-range-copy Co-authored-by: Franz Pöschel --- include/openPMD/Iteration.hpp | 8 ++++++ src/IO/JSON/JSONIOHandlerImpl.cpp | 2 +- src/Iteration.cpp | 1 + src/Series.cpp | 26 +++++++++++++++--- test/SerialIOTest.cpp | 44 +++++++++++++++++++++++++++++++ 5 files changed, 77 insertions(+), 4 deletions(-) diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 3d8ccb4567..964319c6ea 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -102,6 +102,14 @@ namespace internal * Otherwise empty. */ std::optional m_deferredParseAccess{}; + + /** + * Upon reading a file, set this field to the used file name. + * In inconsistent iteration paddings, we must remember the name of the + * file since it cannot be reconstructed from the filename pattern + * alone. + */ + std::optional m_overrideFilebasedFilename{}; }; } // namespace internal /** @brief Logical compilation of data from one snapshot (e.g. a single diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index c225155da1..61f8d239d5 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -916,7 +916,7 @@ JSONIOHandlerImpl::getFilehandle(File fileName, Access access) fs->open(path, std::ios_base::in); break; } - VERIFY(fs->good(), "[JSON] Failed opening a file"); + VERIFY(fs->good(), "[JSON] Failed opening a file '" + path + "'"); return fs; } diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 73c8a46847..057d47c1bb 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -364,6 +364,7 @@ void Iteration::readFileBased( auto series = retrieveSeries(); series.readOneIterationFileBased(filePath); + get().m_overrideFilebasedFilename = filePath; read_impl(groupPath); } diff --git a/src/Series.cpp b/src/Series.cpp index 68059e2af9..e240c9da52 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1191,14 +1191,34 @@ void Series::readBase() std::string Series::iterationFilename(uint64_t i) { + /* + * The filename might have been overridden at the Series level or at the + * Iteration level. See the struct members' documentation for the reasons. + */ auto &series = get(); if (series.m_overrideFilebasedFilename.has_value()) { return series.m_overrideFilebasedFilename.value(); } - std::stringstream iteration(""); - iteration << std::setw(series.m_filenamePadding) << std::setfill('0') << i; - return series.m_filenamePrefix + iteration.str() + series.m_filenamePostfix; + else if (auto iteration = iterations.find(i); // + iteration != iterations.end() && + iteration->second.get().m_overrideFilebasedFilename.has_value()) + { + return iteration->second.get().m_overrideFilebasedFilename.value(); + } + else + { + + /* + * If no filename has been explicitly stored, we use the filename + * pattern to compute it. + */ + std::stringstream iterationNr(""); + iterationNr << std::setw(series.m_filenamePadding) << std::setfill('0') + << i; + return series.m_filenamePrefix + iterationNr.str() + + series.m_filenamePostfix; + } } Series::iterations_iterator Series::indexOf(Iteration const &iteration) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 92de22aa43..ac7a7e4d1c 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -5659,3 +5659,47 @@ TEST_CASE("late_setting_of_iterationencoding", "[serial]") REQUIRE( auxiliary::file_exists("../samples/change_name_and_encoding_10.json")); } + +void varying_pattern(std::string const file_ending) +{ + { + std::string filename = "../samples/varying_pattern_%06T." + file_ending; + ::openPMD::Series series = + ::openPMD::Series(filename, ::openPMD::Access::CREATE); + + for (auto i : {0, 8000, 10000, 100000, 2000000}) + { + auto it = series.iterations[i]; + it.setAttribute("my_step", i); + } + series.flush(); + } + { + std::string filename = "../samples/varying_pattern_%T." + file_ending; + ::openPMD::Series series = + ::openPMD::Series(filename, ::openPMD::Access::READ_ONLY); + + REQUIRE(series.iterations.size() == 5); + for (auto const &[step, it] : series.iterations) + { + std::cout << "Iteration: " << step << "\n"; + REQUIRE(it.getAttribute("my_step").get() == int(step)); + } + + helper::listSeries(series, true, std::cout); + + for (auto i : {0, 8000, 10000, 100000, 2000000}) + { + auto it = series.iterations[i]; + REQUIRE(it.getAttribute("my_step").get() == i); + } + } +} + +TEST_CASE("varying_zero_pattern", "[serial]") +{ + for (auto const &t : testedFileExtensions()) + { + varying_pattern(t); + } +} From e5bc793cd2a701bfb3258231629af85187d73b1b Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 28 Apr 2022 23:15:49 -0700 Subject: [PATCH 33/70] Doc: Update HDF5 Coll. Metadata Versions (#1250) * Doc: Update HDF5 Coll. Metadata Versions Update the documentation about HDFFV-11260 and the newly released fixes in release lines. * HDF5 Coll. MD Reads: Simplify Wording * Tiny wording improvement and a + --- docs/source/backends/hdf5.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/source/backends/hdf5.rst b/docs/source/backends/hdf5.rst index 5d6c21c424..4387b5dcf9 100644 --- a/docs/source/backends/hdf5.rst +++ b/docs/source/backends/hdf5.rst @@ -97,10 +97,9 @@ Known Issues .. warning:: Jul 23th, 2021 (`HDFFV-11260 `__): - Collective HDF5 metadata reads became broken in 1.10.5. - Consider using 1.10.4 if you plan to enable the collective HDF5 metadata operations optimization in openPMD (``OPENPMD_HDF5_COLLECTIVE_METADATA=ON``). - Enabling this feature with a newer version will make HDF5 fall back to the individual metadata operations. - HDF5 plans to fix the issue in the upcoming 1.10.8+ and 1.12.2+ releases, but visit the issue tracker above to see the status of the bug fix. + Collective HDF5 metadata reads (``OPENPMD_HDF5_COLLECTIVE_METADATA=ON``) broke in 1.10.5, falling back to individual metadata operations. + HDF5 releases 1.10.4 and earlier are not affected; versions 1.10.9+, 1.12.2+ and 1.13.1+ fixed the issue. + Selected References ------------------- From 7b8a31ec1fd41e21b1dc4046fe99255014935734 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 2 May 2022 16:48:20 -0700 Subject: [PATCH 34/70] CI: Warn Deprecations (#1246) Remove `-Wno-deprecated-declarations` where possible. --- .github/workflows/intel.yml | 4 ++-- .github/workflows/linux.yml | 18 +++++++++--------- .github/workflows/macos.yml | 2 +- .github/workflows/nvidia.yml | 2 +- .github/workflows/tooling.yml | 4 ++-- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index e682dd3bd0..243ac4208b 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -17,7 +17,7 @@ jobs: run: | sudo .github/workflows/dependencies/install_icc - name: Build - env: {CXXFLAGS: -Werror -Wno-deprecated-declarations} + env: {CXXFLAGS: -Werror} run: | set +e; source /opt/intel/oneapi/setvars.sh; set -e share/openPMD/download_samples.sh build @@ -40,7 +40,7 @@ jobs: run: | sudo .github/workflows/dependencies/install_icx - name: Build - env: {CXXFLAGS: -Werror -Wno-deprecated-declarations} + env: {CXXFLAGS: -Werror} run: | set +e; source /opt/intel/oneapi/setvars.sh; set -e share/openPMD/download_samples.sh build diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index b7997741f5..0835877427 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -21,7 +21,7 @@ jobs: sudo apt-get install clang-6.0 libc++-dev libc++abi-dev sudo .github/workflows/dependencies/install_spack - name: Build - env: {CC: clang-6.0, CXX: clang++-6.0, CXXFLAGS: -stdlib=libc++ -Werror -Wno-deprecated-declarations -Wno-ignored-attributes -Wno-unused-const-variable} + env: {CC: clang-6.0, CXX: clang++-6.0, CXXFLAGS: -stdlib=libc++ -Werror -Wno-ignored-attributes -Wno-unused-const-variable} # -Wno-ignored-attributes -Wno-unused-const-variable: clang-6 has a # false positive on src/auxiliary/Filesystem.cpp # [[maybe_unused]] MPI_Datatype const MPI_Types< unsigned >::value = MPI_UNSIGNED; @@ -54,7 +54,7 @@ jobs: sudo apt-get install clang-6.0 libc++-dev libc++abi-dev gfortran libopenmpi-dev python3 sudo .github/workflows/dependencies/install_spack - name: Build - env: {CC: clang-6.0, CXX: clang++-6.0, CXXFLAGS: -stdlib=libc++ -Werror -Wno-deprecated-declarations -Wno-ignored-attributes -Wno-unused-const-variable} + env: {CC: clang-6.0, CXX: clang++-6.0, CXXFLAGS: -stdlib=libc++ -Werror -Wno-ignored-attributes -Wno-unused-const-variable} # -Wno-ignored-attributes -Wno-unused-const-variable: clang-6 has a # false positive on src/auxiliary/Filesystem.cpp # [[maybe_unused]] MPI_Datatype const MPI_Types< unsigned >::value = MPI_UNSIGNED; @@ -98,7 +98,7 @@ jobs: sudo apt-get install clang-7 gfortran libopenmpi-dev python3 sudo .github/workflows/dependencies/install_spack - name: Build - env: {CC: clang-7, CXX: clang++-7, CXXFLAGS: -Werror -Wno-deprecated-declarations, OPENPMD2_ADIOS2_SCHEMA: 20210209} + env: {CC: clang-7, CXX: clang++-7, CXXFLAGS: -Werror, OPENPMD2_ADIOS2_SCHEMA: 20210209} run: | eval $(spack env activate --sh .github/ci/spack-envs/clang7_nopy_ompi_h5_ad1_ad2/) spack install @@ -133,7 +133,7 @@ jobs: sudo apt-get install clang-12 libhdf5-dev python3 python3-pip python3-numpy python3-pandas sudo .github/workflows/dependencies/install_spack - name: Build - env: {CC: clang-12, CXX: clang++-12, CXXFLAGS: -Werror -Wno-deprecated-declarations} + env: {CC: clang-12, CXX: clang++-12, CXXFLAGS: -Werror} run: | eval $(spack env activate --sh .github/ci/spack-envs/clang12_py38_nompi_h5_ad1_ad2/) spack install @@ -165,7 +165,7 @@ jobs: sudo apt-get install clang-8 gfortran libmpich-dev python3 sudo .github/workflows/dependencies/install_spack - name: Build - env: {CC: clang-8, CXX: clang++-8, CXXFLAGS: -Werror -Wno-deprecated-declarations, OPENPMD2_ADIOS2_SCHEMA: 20210209} + env: {CC: clang-8, CXX: clang++-8, CXXFLAGS: -Werror, OPENPMD2_ADIOS2_SCHEMA: 20210209} run: | cmake --version mpiexec --version @@ -221,7 +221,7 @@ jobs: python3 -m pip install -U dask python3 -m pip install -U pyarrow - name: Build - env: {CC: gcc-7, CXX: g++-7, CXXFLAGS: -Werror -Wno-deprecated-declarations} + env: {CC: gcc-7, CXX: g++-7, CXXFLAGS: -Werror} run: | eval $(spack env activate --sh .github/ci/spack-envs/gcc7_py36_ompi_h5_ad1_ad2/) spack install @@ -249,7 +249,7 @@ jobs: sudo apt-get install g++ libopenmpi-dev libhdf5-openmpi-dev libadios-dev python3 python3-numpy python3-mpi4py python3-pandas # TODO ADIOS1 (.pc file broken?) ADIOS2 - name: Build - env: {CXXFLAGS: -Werror -Wno-deprecated-declarations, PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig} + env: {CXXFLAGS: -Werror, PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig} run: | share/openPMD/download_samples.sh build chmod u-w build/samples/git-sample/*.h5 @@ -274,7 +274,7 @@ jobs: apk add hdf5-dev python3.10 -m pip install numpy - name: Build - env: {CXXFLAGS: -Werror -Wno-deprecated-declarations} + env: {CXXFLAGS: -Werror} run: | share/openPMD/download_samples.sh build chmod u-w build/samples/git-sample/*.h5 @@ -307,7 +307,7 @@ jobs: mamba env create --file conda.yml - name: Build shell: bash -eo pipefail -l {0} - env: {CXXFLAGS: -Werror -Wno-deprecated-declarations} + env: {CXXFLAGS: -Werror} run: | source activate openpmd-api-dev diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 182c2228ae..f0b2d6bda5 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -27,7 +27,7 @@ jobs: brew install python || true set -e - name: Build - env: {CXXFLAGS: -Werror -Wno-deprecated-declarations -DTOML11_DISABLE_STD_FILESYSTEM, MACOSX_DEPLOYMENT_TARGET: 10.13} + env: {CXXFLAGS: -Werror -DTOML11_DISABLE_STD_FILESYSTEM, MACOSX_DEPLOYMENT_TARGET: 10.13} # C++11 & 14 support in macOS 10.9+ # C++17 support in macOS 10.13+/10.14+ # https://cibuildwheel.readthedocs.io/en/stable/cpp_standards/#macos-and-deployment-target-versions diff --git a/.github/workflows/nvidia.yml b/.github/workflows/nvidia.yml index d978022457..847d40e3ac 100644 --- a/.github/workflows/nvidia.yml +++ b/.github/workflows/nvidia.yml @@ -11,7 +11,7 @@ jobs: name: CTK@11.2 runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false - env: {CXX: nvcc, CXXFLAGS: "--forward-unknown-to-host-compiler -Xcompiler -Werror -Wno-deprecated-declarations"} + env: {CXX: nvcc, CXXFLAGS: "--forward-unknown-to-host-compiler -Xcompiler -Werror"} steps: - uses: actions/checkout@v2 - name: Dependencies diff --git a/.github/workflows/tooling.yml b/.github/workflows/tooling.yml index d8ade42186..9a1bb08977 100644 --- a/.github/workflows/tooling.yml +++ b/.github/workflows/tooling.yml @@ -22,7 +22,7 @@ jobs: sudo apt-get install clang clang-tidy gfortran libopenmpi-dev python sudo .github/workflows/dependencies/install_spack - name: Build - env: {CC: clang, CXX: clang++, CXXFLAGS: -Wno-deprecated-declarations} + env: {CC: clang, CXX: clang++} run: | eval $(spack env activate --sh .github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad1_ad2/) spack install @@ -53,7 +53,7 @@ jobs: python3 -m pip install -U numpy sudo .github/workflows/dependencies/install_spack - name: Build - env: {CC: mpicc, CXX: mpic++, OMPI_CC: clang-10, OMPI_CXX: clang++-10, CXXFLAGS: -Werror -Wno-deprecated-declarations, OPENPMD_HDF5_CHUNKS: none, OPENPMD_TEST_NFILES_MAX: 100} + env: {CC: mpicc, CXX: mpic++, OMPI_CC: clang-10, OMPI_CXX: clang++-10, CXXFLAGS: -Werror, OPENPMD_HDF5_CHUNKS: none, OPENPMD_TEST_NFILES_MAX: 100} run: | eval $(spack env activate --sh .github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad1_ad2/) spack install From 661209804fcd5282be1b93487658e12d8923da04 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 May 2022 16:48:39 -0700 Subject: [PATCH 35/70] [pre-commit.ci] pre-commit autoupdate (#1265) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/hadialqattan/pycln: v1.3.1 → v1.3.2](https://github.com/hadialqattan/pycln/compare/v1.3.1...v1.3.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0a1d959474..10aa070cf9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -71,7 +71,7 @@ repos: # Autoremoves unused Python imports - repo: https://github.com/hadialqattan/pycln - rev: v1.3.1 + rev: v1.3.2 hooks: - id: pycln name: pycln (python) From d12bd7a1aa0a4e1065b506fc00385731e52dea36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 4 May 2022 01:11:41 +0200 Subject: [PATCH 36/70] Fix lines for Operation enum (#1267) --- docs/source/dev/design.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/dev/design.rst b/docs/source/dev/design.rst index 547bb59e38..4b6131d14d 100644 --- a/docs/source/dev/design.rst +++ b/docs/source/dev/design.rst @@ -23,7 +23,7 @@ Therefore, enabling users to handle hierarchical, self-describing file formats w .. literalinclude:: IOTask.hpp :language: cpp - :lines: 51-77 + :lines: 44-60 Every task is designed to be a fully self-contained description of one such atomic operation. By describing a required minimal step of work (without any side-effect), these operations are the foundation of the unified handling mechanism across suitable file formats. The actual low-level exchange of data is implemented in ``IOHandlers``, one per file format (possibly two if handlingi MPI-parallel work is possible and requires different behaviour). From feefb0e395518282bcd4fd389fde7e3c17ba5ad6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 4 May 2022 01:12:36 +0200 Subject: [PATCH 37/70] BP5 tests: Only run them if ADIOS2 has BP5 (#1262) * Revert "CI: Fix ADIOS 2.7.1 on Windows (#1258)" This reverts commit 87b27477a015296cdfa3ba2beb0701703c9a59b4. * Use ADIOS2_HAVE_BP5 to enable BP5 tests --- .appveyor.yml | 2 +- test/SerialIOTest.cpp | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 4575d7e935..55a5a78f70 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -88,7 +88,7 @@ install: # Configure the VM. - cmd: conda install -n root --quiet --yes numpy cmake hdf5 python=%CONDA_PY% # ADIOS2 build only for 64bit Windows - - cmd: if "%TARGET_ARCH%"=="x64" conda install -n root --quiet --yes adios2==2.7.1 python=%CONDA_PY% + - cmd: if "%TARGET_ARCH%"=="x64" conda install -n root --quiet --yes adios2 python=%CONDA_PY% before_build: - cmd: cd C:\projects\openpmd-api diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index ac7a7e4d1c..6b79b707ee 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -5159,10 +5159,7 @@ TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") backend.extension, false, backend.jsonBaseConfig()); -#if openPMD_HAVE_ADIOS2 && \ - ADIOS2_VERSION_MAJOR * 1000000000 + ADIOS2_VERSION_MINOR * 100000000 + \ - ADIOS2_VERSION_PATCH * 1000000 + ADIOS2_VERSION_TWEAK >= \ - 2701001223 +#if openPMD_HAVE_ADIOS2 && defined(ADIOS2_HAVE_BP5) if (backend.extension == "bp") { iterate_nonstreaming_series( @@ -5188,10 +5185,7 @@ TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") #endif } -#if openPMD_HAVE_ADIOS2 && \ - ADIOS2_VERSION_MAJOR * 1000000000 + ADIOS2_VERSION_MINOR * 100000000 + \ - ADIOS2_VERSION_PATCH * 1000000 + ADIOS2_VERSION_TWEAK >= \ - 2701001223 +#if openPMD_HAVE_ADIOS2 && defined(ADIOS2_HAVE_BP5) void adios2_bp5_no_steps(bool usesteps) { std::string const config = R"END( From 1f467a3b7724e730e8625be0b5451970e466fcdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 4 May 2022 02:07:48 +0200 Subject: [PATCH 38/70] Fixes for opening an iteration (#1239) * Test * Don't flush when opening an iteration * CI fixes * FlushLevel: Use default base class in NVC++ * clang-tidy: Define member defaults of Writable in-class --- include/openPMD/IO/AbstractIOHandler.hpp | 10 ++- include/openPMD/Iteration.hpp | 19 +++-- include/openPMD/backend/Writable.hpp | 12 +-- src/IO/ADIOS/ADIOS2IOHandler.cpp | 1 + src/Iteration.cpp | 101 ++++++++++++++++------- src/ReadIterations.cpp | 10 ++- src/Series.cpp | 38 +++++++-- src/WriteIterations.cpp | 2 +- src/backend/Attributable.cpp | 8 +- src/backend/Writable.cpp | 8 +- test/SerialIOTest.cpp | 18 ++++ 11 files changed, 161 insertions(+), 66 deletions(-) diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index ff8b2fcccb..a7239a3375 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -61,7 +61,9 @@ class unsupported_data_error : public std::runtime_error * @brief Determine what items should be flushed upon Series::flush() * */ -enum class FlushLevel : unsigned char +// do not write `enum class FlushLevel : unsigned char` here since NVHPC +// does not compile it correctly +enum class FlushLevel { /** * Flush operation that was triggered by user code. @@ -84,7 +86,11 @@ enum class FlushLevel : unsigned char * CREATE_DATASET tasks. * Attributes may or may not be flushed yet. */ - SkeletonOnly + SkeletonOnly, + /** + * Only creates/opens files, nothing more + */ + CreateOrOpenFiles }; namespace internal diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 964319c6ea..844aaea12f 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -72,6 +72,7 @@ namespace internal * containing this iteration. */ std::string filename; + bool beginStep = false; }; class IterationData : public AttributableData @@ -257,14 +258,16 @@ class Iteration : public Attributable void flush(internal::FlushParams const &); void deferParseAccess(internal::DeferredParseAccess); /* - * Control flow for read(), readFileBased(), readGroupBased() and - * read_impl(): - * read() is called as the entry point. File-based and group-based + * Control flow for runDeferredParseAccess(), readFileBased(), + * readGroupBased() and read_impl(): + * runDeferredParseAccess() is called as the entry point. + * File-based and group-based * iteration layouts need to be parsed slightly differently: * In file-based iteration layout, each iteration's file also contains * attributes for the /data group. In group-based layout, those have * already been parsed during opening of the Series. - * Hence, read() will call either readFileBased() or readGroupBased() to + * Hence, runDeferredParseAccess() will call either readFileBased() or + * readGroupBased() to * allow for those different control flows. * Finally, read_impl() is called which contains the common parsing * logic for an iteration. @@ -273,10 +276,10 @@ class Iteration : public Attributable * Calling it on an Iteration not yet parsed is an error. * */ - void read(); void reread(std::string const &path); - void readFileBased(std::string filePath, std::string const &groupPath); - void readGorVBased(std::string const &groupPath); + void readFileBased( + std::string filePath, std::string const &groupPath, bool beginStep); + void readGorVBased(std::string const &groupPath, bool beginStep); void read_impl(std::string const &groupPath); /** @@ -286,7 +289,7 @@ class Iteration : public Attributable * * @return AdvanceStatus */ - AdvanceStatus beginStep(); + AdvanceStatus beginStep(bool reread); /** * @brief End an IO step on the IO file (or file-like object) diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index 8944d92dea..cf0fed6429 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -119,11 +119,11 @@ OPENPMD_private * These members need to be shared pointers since distinct instances of * Writable may share them. */ - std::shared_ptr abstractFilePosition; - std::shared_ptr IOHandler; - internal::AttributableData *attributable; - Writable *parent; - bool dirty; + std::shared_ptr abstractFilePosition = nullptr; + std::shared_ptr IOHandler = nullptr; + internal::AttributableData *attributable = nullptr; + Writable *parent = nullptr; + bool dirty = true; /** * If parent is not null, then this is a vector of keys such that: * &(*parent)[key_1]...[key_n] == this @@ -146,6 +146,6 @@ OPENPMD_private * Writable and its meaning within the current dataset. * */ - bool written; + bool written = false; }; } // namespace openPMD diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index c9f8526b3f..e1c1bfe920 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -2652,6 +2652,7 @@ namespace detail case FlushLevel::InternalFlush: case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: /* * Tasks have been given to ADIOS2, but we don't flush them * yet. So, move everything to m_alreadyEnqueued to avoid diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 057d47c1bb..8991f0ebad 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -148,13 +148,12 @@ Iteration &Iteration::open() if (it.m_closed == CloseStatus::ParseAccessDeferred) { it.m_closed = CloseStatus::Open; + runDeferredParseAccess(); } - runDeferredParseAccess(); Series s = retrieveSeries(); // figure out my iteration number auto begin = s.indexOf(*this); s.openIteration(begin->first, *this); - // @todo, maybe collective here IOHandler()->flush(internal::defaultFlushParams); return *this; } @@ -238,7 +237,16 @@ void Iteration::flushFileBased( s.openIteration(i, *this); } - flush(flushParams); + switch (flushParams.flushLevel) + { + case FlushLevel::CreateOrOpenFiles: + break; + case FlushLevel::SkeletonOnly: + case FlushLevel::InternalFlush: + case FlushLevel::UserFlush: + flush(flushParams); + break; + } } void Iteration::flushGroupBased( @@ -252,7 +260,16 @@ void Iteration::flushGroupBased( IOHandler()->enqueue(IOTask(this, pCreate)); } - flush(flushParams); + switch (flushParams.flushLevel) + { + case FlushLevel::CreateOrOpenFiles: + break; + case FlushLevel::SkeletonOnly: + case FlushLevel::InternalFlush: + case FlushLevel::UserFlush: + flush(flushParams); + break; + } } void Iteration::flushVariableBased( @@ -267,7 +284,16 @@ void Iteration::flushVariableBased( this->setAttribute("snapshot", i); } - flush(flushParams); + switch (flushParams.flushLevel) + { + case FlushLevel::CreateOrOpenFiles: + break; + case FlushLevel::SkeletonOnly: + case FlushLevel::InternalFlush: + case FlushLevel::UserFlush: + flush(flushParams); + break; + } } void Iteration::flush(internal::FlushParams const &flushParams) @@ -327,26 +353,6 @@ void Iteration::deferParseAccess(DeferredParseAccess dr) std::make_optional(std::move(dr)); } -void Iteration::read() -{ - auto &it = get(); - if (!it.m_deferredParseAccess.has_value()) - { - return; - } - auto const &deferred = it.m_deferredParseAccess.value(); - if (deferred.fileBased) - { - readFileBased(deferred.filename, deferred.path); - } - else - { - readGorVBased(deferred.path); - } - // reset this thing - it.m_deferredParseAccess = std::optional(); -} - void Iteration::reread(std::string const &path) { if (get().m_deferredParseAccess.has_value()) @@ -359,8 +365,15 @@ void Iteration::reread(std::string const &path) } void Iteration::readFileBased( - std::string filePath, std::string const &groupPath) + std::string filePath, std::string const &groupPath, bool doBeginStep) { + if (doBeginStep) + { + /* + * beginStep() must take care to open files + */ + beginStep(/* reread = */ false); + } auto series = retrieveSeries(); series.readOneIterationFileBased(filePath); @@ -369,9 +382,15 @@ void Iteration::readFileBased( read_impl(groupPath); } -void Iteration::readGorVBased(std::string const &groupPath) +void Iteration::readGorVBased(std::string const &groupPath, bool doBeginStep) { - + if (doBeginStep) + { + /* + * beginStep() must take care to open files + */ + beginStep(/* reread = */ false); + } read_impl(groupPath); } @@ -541,7 +560,7 @@ void Iteration::read_impl(std::string const &groupPath) readAttributes(ReadMode::FullyReread); } -AdvanceStatus Iteration::beginStep() +AdvanceStatus Iteration::beginStep(bool reread) { using IE = IterationEncoding; auto series = retrieveSeries(); @@ -566,7 +585,8 @@ AdvanceStatus Iteration::beginStep() } // re-read -> new datasets might be available - if ((series.iterationEncoding() == IE::groupBased || + if (reread && + (series.iterationEncoding() == IE::groupBased || series.iterationEncoding() == IE::variableBased) && (this->IOHandler()->m_frontendAccess == Access::READ_ONLY || this->IOHandler()->m_frontendAccess == Access::READ_WRITE)) @@ -680,18 +700,37 @@ void Iteration::runDeferredParseAccess() { return; } + + auto &it = get(); + if (!it.m_deferredParseAccess.has_value()) + { + return; + } + auto const &deferred = it.m_deferredParseAccess.value(); + auto oldAccess = IOHandler()->m_frontendAccess; auto newAccess = const_cast(&IOHandler()->m_frontendAccess); *newAccess = Access::READ_WRITE; try { - read(); + if (deferred.fileBased) + { + readFileBased(deferred.filename, deferred.path, deferred.beginStep); + } + else + { + readGorVBased(deferred.path, deferred.beginStep); + } } catch (...) { + // reset this thing + it.m_deferredParseAccess = std::optional(); *newAccess = oldAccess; throw; } + // reset this thing + it.m_deferredParseAccess = std::optional(); *newAccess = oldAccess; } diff --git a/src/ReadIterations.cpp b/src/ReadIterations.cpp index bd70a91366..b677d97535 100644 --- a/src/ReadIterations.cpp +++ b/src/ReadIterations.cpp @@ -63,7 +63,7 @@ SeriesIterator::SeriesIterator(Series series) : m_series(std::move(series)) * the step after parsing the file is ok. */ openIteration(); - status = it->second.beginStep(); + status = it->second.beginStep(/* reread = */ true); break; case IterationEncoding::groupBased: case IterationEncoding::variableBased: @@ -72,7 +72,7 @@ SeriesIterator::SeriesIterator(Series series) : m_series(std::move(series)) * access to the file until now. Better to begin a step right away, * otherwise we might get another step's data. */ - status = it->second.beginStep(); + status = it->second.beginStep(/* reread = */ true); openIteration(); break; } @@ -107,7 +107,8 @@ SeriesIterator &SeriesIterator::operator++() case IE::variableBased: { // since we are in group-based iteration layout, it does not // matter which iteration we begin a step upon - AdvanceStatus status = currentIteration.beginStep(); + AdvanceStatus status{}; + status = currentIteration.beginStep(/* reread = */ true); if (status == AdvanceStatus::OVER) { *this = end(); @@ -142,7 +143,8 @@ SeriesIterator &SeriesIterator::operator++() using IE = IterationEncoding; case IE::fileBased: { auto &iteration = series.iterations[m_currentIteration]; - AdvanceStatus status = iteration.beginStep(); + AdvanceStatus status{}; + status = iteration.beginStep(/* reread = */ true); if (status == AdvanceStatus::OVER) { *this = end(); diff --git a/src/Series.cpp b/src/Series.cpp index e240c9da52..919ce4e77d 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1046,7 +1046,8 @@ void Series::readGorVBased(bool do_init) auto readSingleIteration = [&series, &pOpen, this]( uint64_t index, std::string path, - bool guardAgainstRereading) { + bool guardAgainstRereading, + bool beginStep) { if (series.iterations.contains(index)) { // maybe re-read @@ -1068,7 +1069,7 @@ void Series::readGorVBased(bool do_init) { // parse for the first time, resp. delay the parsing process Iteration &i = series.iterations[index]; - i.deferParseAccess({path, index, false, ""}); + i.deferParseAccess({path, index, false, "", beginStep}); if (!series.m_parseLazily) { i.runDeferredParseAccess(); @@ -1091,7 +1092,12 @@ void Series::readGorVBased(bool do_init) for (auto const &it : *pList.paths) { uint64_t index = std::stoull(it); - readSingleIteration(index, it, true); + /* + * For now: parse a Series in RandomAccess mode. + * (beginStep = false) + * A streaming read mode might come in a future API addition. + */ + readSingleIteration(index, it, true, false); } break; case IterationEncoding::variableBased: { @@ -1100,7 +1106,11 @@ void Series::readGorVBased(bool do_init) { index = series.iterations.getAttribute("snapshot").get(); } - readSingleIteration(index, "", false); + /* + * Variable-based iteration encoding relies on steps, so parsing must + * happen after opening the first step. + */ + readSingleIteration(index, "", false, true); break; } } @@ -1261,8 +1271,24 @@ AdvanceStatus Series::advance( itData.m_closed = internal::CloseStatus::Open; } - // @todo really collective? - flush_impl(begin, end, flushParams, /* flushIOHandler = */ false); + switch (mode) + { + case AdvanceMode::ENDSTEP: + flush_impl(begin, end, flushParams, /* flushIOHandler = */ false); + break; + case AdvanceMode::BEGINSTEP: + /* + * When beginning a step, there is nothing to flush yet. + * Data is not written in between steps. + * So only make sure that files are accessed. + */ + flush_impl( + begin, + end, + {FlushLevel::CreateOrOpenFiles}, + /* flushIOHandler = */ false); + break; + } if (oldCloseStatus == internal::CloseStatus::ClosedInFrontend) { diff --git a/src/WriteIterations.cpp b/src/WriteIterations.cpp index be2e72f47f..597298b80e 100644 --- a/src/WriteIterations.cpp +++ b/src/WriteIterations.cpp @@ -68,7 +68,7 @@ WriteIterations::mapped_type &WriteIterations::operator[](key_type &&key) auto &res = shared->iterations[std::move(key)]; if (res.getStepStatus() == StepStatus::NoStep) { - res.beginStep(); + res.beginStep(/* reread = */ false); res.setStepStatus(StepStatus::DuringStep); } return res; diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 44238b161d..b73c850409 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -213,9 +213,15 @@ void Attributable::seriesFlush(internal::FlushParams flushParams) void Attributable::flushAttributes(internal::FlushParams const &flushParams) { - if (flushParams.flushLevel == FlushLevel::SkeletonOnly) + switch (flushParams.flushLevel) { + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: return; + case FlushLevel::InternalFlush: + case FlushLevel::UserFlush: + // pass + break; } if (dirty()) { diff --git a/src/backend/Writable.cpp b/src/backend/Writable.cpp index 58ce2e0ad4..fae7630795 100644 --- a/src/backend/Writable.cpp +++ b/src/backend/Writable.cpp @@ -24,13 +24,7 @@ namespace openPMD { -Writable::Writable(internal::AttributableData *a) - : abstractFilePosition{nullptr} - , IOHandler{nullptr} - , attributable{a} - , parent{nullptr} - , dirty{true} - , written{false} +Writable::Writable(internal::AttributableData *a) : attributable{a} {} void Writable::seriesFlush() diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 6b79b707ee..dc366fa773 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -4688,6 +4688,12 @@ void variableBasedSeries(std::string const &file) std::vector data(1000, i); E_x.storeChunk(data, {0}, {1000}); + if (i > 2) + { + iteration.setAttribute( + "iteration_is_larger_than_two", "it truly is"); + } + // this tests changing extents and dimensionalities // across iterations auto E_y = iteration.meshes["E"]["y"]; @@ -4723,6 +4729,18 @@ void variableBasedSeries(std::string const &file) size_t last_iteration_index = 0; for (auto iteration : readSeries.readIterations()) { + if (iteration.iterationIndex > 2) + { + REQUIRE( + iteration.getAttribute("iteration_is_larger_than_two") + .get() == "it truly is"); + } + else + { + REQUIRE_FALSE(iteration.containsAttribute( + "iteration_is_larger_than_two")); + } + auto E_x = iteration.meshes["E"]["x"]; REQUIRE(E_x.getDimensionality() == 1); REQUIRE(E_x.getExtent()[0] == extent); From 36075341ca26a5a19865046183526c415b5134f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 6 May 2022 18:09:56 +0200 Subject: [PATCH 39/70] Avoid copying std::string in loop (#1268) --- src/IO/IOTask.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/IOTask.cpp b/src/IO/IOTask.cpp index 36c4f9c786..0b43cc3835 100644 --- a/src/IO/IOTask.cpp +++ b/src/IO/IOTask.cpp @@ -42,7 +42,7 @@ void Parameter::warnUnusedParameters< * Fake-read non-backend-specific options. Some backends don't read those * and we don't want to have warnings for them. */ - for (std::string const &key : {"resizable"}) + for (char const *key : {"resizable"}) { config[key]; } From 1d32922f59d683b0dfb16ce6f05b141daca3e936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 6 May 2022 18:16:54 +0200 Subject: [PATCH 40/70] Fix premature flushing (performance bug) (#1264) * No premature flushes in Series.cpp * Avoid flushing for scalar components * Document new IO task --- CMakeLists.txt | 1 + docs/source/dev/design.rst | 2 +- include/openPMD/IO/AbstractIOHandlerImpl.hpp | 14 +++++++ include/openPMD/IO/IOTask.hpp | 43 ++++++++++++++------ include/openPMD/Series.hpp | 6 ++- include/openPMD/backend/Writable.hpp | 1 + src/IO/ADIOS/ADIOS1IOHandler.cpp | 7 ++++ src/IO/ADIOS/ParallelADIOS1IOHandler.cpp | 7 ++++ src/IO/AbstractIOHandlerImpl.cpp | 33 +++++++++++++++ src/Mesh.cpp | 7 ++-- src/Record.cpp | 7 ++-- src/Series.cpp | 31 ++++++++++---- 12 files changed, 127 insertions(+), 32 deletions(-) create mode 100644 src/IO/AbstractIOHandlerImpl.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 991ce6a845..574344c5db 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -469,6 +469,7 @@ set(CORE_SOURCE src/benchmark/mpi/OneDimensionalBlockSlicer.cpp src/helper/list_series.cpp) set(IO_SOURCE + src/IO/AbstractIOHandlerImpl.cpp src/IO/AbstractIOHandlerHelper.cpp src/IO/DummyIOHandler.cpp src/IO/IOTask.cpp diff --git a/docs/source/dev/design.rst b/docs/source/dev/design.rst index 4b6131d14d..e01816b612 100644 --- a/docs/source/dev/design.rst +++ b/docs/source/dev/design.rst @@ -23,7 +23,7 @@ Therefore, enabling users to handle hierarchical, self-describing file formats w .. literalinclude:: IOTask.hpp :language: cpp - :lines: 44-60 + :lines: 44-62 Every task is designed to be a fully self-contained description of one such atomic operation. By describing a required minimal step of work (without any side-effect), these operations are the foundation of the unified handling mechanism across suitable file formats. The actual low-level exchange of data is implemented in ``IOHandlers``, one per file format (possibly two if handlingi MPI-parallel work is possible and requires different behaviour). diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index 170cf4b81a..e171115f95 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -190,6 +190,12 @@ class AbstractIOHandlerImpl deref_dynamic_cast >( i.parameter.get())); break; + case O::KEEP_SYNCHRONOUS: + keepSynchronous( + i.writable, + deref_dynamic_cast >( + i.parameter.get())); + break; } } catch (...) @@ -523,6 +529,14 @@ class AbstractIOHandlerImpl virtual void listAttributes(Writable *, Parameter &) = 0; + /** Treat the current Writable as equivalent to that in the parameter object + * + * Using the default implementation (which copies the abstractFilePath + * into the current writable) should be enough for all backends. + */ + void + keepSynchronous(Writable *, Parameter param); + AbstractIOHandler *m_handler; }; // AbstractIOHandlerImpl } // namespace openPMD diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 8d748c184c..f8d08ab821 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -44,18 +44,20 @@ Writable *getWritable(Attributable *); /** Type of IO operation between logical and persistent data. */ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){ - CREATE_FILE, OPEN_FILE, CLOSE_FILE, DELETE_FILE, + CREATE_FILE, OPEN_FILE, CLOSE_FILE, DELETE_FILE, - CREATE_PATH, CLOSE_PATH, OPEN_PATH, DELETE_PATH, + CREATE_PATH, CLOSE_PATH, OPEN_PATH, DELETE_PATH, LIST_PATHS, - CREATE_DATASET, EXTEND_DATASET, OPEN_DATASET, DELETE_DATASET, - WRITE_DATASET, READ_DATASET, LIST_DATASETS, GET_BUFFER_VIEW, + CREATE_DATASET, EXTEND_DATASET, OPEN_DATASET, DELETE_DATASET, + WRITE_DATASET, READ_DATASET, LIST_DATASETS, GET_BUFFER_VIEW, - DELETE_ATT, WRITE_ATT, READ_ATT, LIST_ATTS, + DELETE_ATT, WRITE_ATT, READ_ATT, LIST_ATTS, ADVANCE, - AVAILABLE_CHUNKS //!< Query chunks that can be loaded in a dataset + AVAILABLE_CHUNKS, //!< Query chunks that can be loaded in a dataset + KEEP_SYNCHRONOUS //!< Keep two items in the object model synchronous with + //!< each other }; // note: if you change the enum members here, please update // docs/source/dev/design.rst @@ -257,8 +259,8 @@ struct OPENPMDAPI_EXPORT Parameter new Parameter(*this)); } - std::shared_ptr > paths = - std::make_shared >(); + std::shared_ptr> paths = + std::make_shared>(); }; template <> @@ -434,8 +436,8 @@ struct OPENPMDAPI_EXPORT Parameter new Parameter(*this)); } - std::shared_ptr > datasets = - std::make_shared >(); + std::shared_ptr> datasets = + std::make_shared>(); }; template <> @@ -569,8 +571,8 @@ struct OPENPMDAPI_EXPORT Parameter new Parameter(*this)); } - std::shared_ptr > attributes = - std::make_shared >(); + std::shared_ptr> attributes = + std::make_shared>(); }; template <> @@ -619,6 +621,23 @@ struct OPENPMDAPI_EXPORT Parameter std::shared_ptr chunks = std::make_shared(); }; +template <> +struct OPENPMDAPI_EXPORT Parameter + : public AbstractParameter +{ + Parameter() = default; + Parameter(Parameter const &p) + : AbstractParameter(), otherWritable(p.otherWritable) + {} + + std::unique_ptr clone() const override + { + return std::make_unique>(*this); + } + + Writable *otherWritable; +}; + /** @brief Self-contained description of a single IO operation. * * Contained are diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 0649a9919b..a5af2be601 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -544,7 +544,8 @@ OPENPMD_private void flushFileBased( iterations_iterator begin, iterations_iterator end, - internal::FlushParams flushParams); + internal::FlushParams flushParams, + bool flushIOHandler = true); /* * Group-based and variable-based iteration layouts share a lot of logic * (realistically, the variable-based iteration layout only throws out @@ -555,7 +556,8 @@ OPENPMD_private void flushGorVBased( iterations_iterator begin, iterations_iterator end, - internal::FlushParams flushParams); + internal::FlushParams flushParams, + bool flushIOHandler = true); void flushMeshesPath(); void flushParticlesPath(); void readFileBased(); diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index cf0fed6429..1ed7054a6f 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -75,6 +75,7 @@ class Writable final friend class ParticleSpecies; friend class Series; friend class Record; + friend class AbstractIOHandlerImpl; template friend class CommonADIOS1IOHandlerImpl; friend class ADIOS1IOHandlerImpl; diff --git a/src/IO/ADIOS/ADIOS1IOHandler.cpp b/src/IO/ADIOS/ADIOS1IOHandler.cpp index cde3b75156..14ea1aeaaa 100644 --- a/src/IO/ADIOS/ADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS1IOHandler.cpp @@ -134,6 +134,12 @@ std::future ADIOS1IOHandlerImpl::flush() deref_dynamic_cast >( i.parameter.get())); break; + case O::KEEP_SYNCHRONOUS: + keepSynchronous( + i.writable, + deref_dynamic_cast >( + i.parameter.get())); + break; default: VERIFY( false, @@ -345,6 +351,7 @@ void ADIOS1IOHandler::enqueue(IOTask const &i) case Operation::CREATE_DATASET: case Operation::OPEN_FILE: case Operation::WRITE_ATT: + case Operation::KEEP_SYNCHRONOUS: m_setup.push(i); return; default: diff --git a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp index 78ac9f4c59..3fbc9aa395 100644 --- a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp @@ -156,6 +156,12 @@ std::future ParallelADIOS1IOHandlerImpl::flush() deref_dynamic_cast >( i.parameter.get())); break; + case O::KEEP_SYNCHRONOUS: + keepSynchronous( + i.writable, + deref_dynamic_cast >( + i.parameter.get())); + break; default: VERIFY( false, @@ -364,6 +370,7 @@ void ParallelADIOS1IOHandler::enqueue(IOTask const &i) case Operation::CREATE_DATASET: case Operation::OPEN_FILE: case Operation::WRITE_ATT: + case Operation::KEEP_SYNCHRONOUS: m_setup.push(i); return; default: diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp new file mode 100644 index 0000000000..d762df6a1f --- /dev/null +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -0,0 +1,33 @@ +/* Copyright 2022 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/backend/Writable.hpp" + +namespace openPMD +{ +void AbstractIOHandlerImpl::keepSynchronous( + Writable *writable, Parameter param) +{ + writable->abstractFilePosition = param.otherWritable->abstractFilePosition; + writable->written = true; +} +} // namespace openPMD diff --git a/src/Mesh.cpp b/src/Mesh.cpp index b30c7e0f5d..761a9a304a 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -230,10 +230,9 @@ void Mesh::flush_impl( MeshRecordComponent &mrc = at(RecordComponent::SCALAR); mrc.parent() = parent(); mrc.flush(name, flushParams); - IOHandler()->flush(flushParams); - writable().abstractFilePosition = - mrc.writable().abstractFilePosition; - written() = true; + Parameter pSynchronize; + pSynchronize.otherWritable = &mrc.writable(); + IOHandler()->enqueue(IOTask(this, pSynchronize)); } else { diff --git a/src/Record.cpp b/src/Record.cpp index da57ebeef1..90b0c61a91 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -60,10 +60,9 @@ void Record::flush_impl( RecordComponent &rc = at(RecordComponent::SCALAR); rc.parent() = parent(); rc.flush(name, flushParams); - IOHandler()->flush(flushParams); - writable().abstractFilePosition = - rc.writable().abstractFilePosition; - written() = true; + Parameter pSynchronize; + pSynchronize.otherWritable = &rc.writable(); + IOHandler()->enqueue(IOTask(this, pSynchronize)); } else { diff --git a/src/Series.cpp b/src/Series.cpp index 919ce4e77d..6adeefad55 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -575,11 +575,11 @@ std::future Series::flush_impl( { using IE = IterationEncoding; case IE::fileBased: - flushFileBased(begin, end, flushParams); + flushFileBased(begin, end, flushParams, flushIOHandler); break; case IE::groupBased: case IE::variableBased: - flushGorVBased(begin, end, flushParams); + flushGorVBased(begin, end, flushParams, flushIOHandler); break; } if (flushIOHandler) @@ -601,7 +601,8 @@ std::future Series::flush_impl( void Series::flushFileBased( iterations_iterator begin, iterations_iterator end, - internal::FlushParams flushParams) + internal::FlushParams flushParams, + bool flushIOHandler) { auto &series = get(); if (end == begin) @@ -633,7 +634,10 @@ void Series::flushFileBased( } // Phase 3 - IOHandler()->flush(flushParams); + if (flushIOHandler) + { + IOHandler()->flush(flushParams); + } } else { @@ -678,8 +682,10 @@ void Series::flushFileBased( } // Phase 3 - IOHandler()->flush(flushParams); - + if (flushIOHandler) + { + IOHandler()->flush(flushParams); + } /* reset the dirty bit for every iteration (i.e. file) * otherwise only the first iteration will have updates attributes */ @@ -692,7 +698,8 @@ void Series::flushFileBased( void Series::flushGorVBased( iterations_iterator begin, iterations_iterator end, - internal::FlushParams flushParams) + internal::FlushParams flushParams, + bool flushIOHandler) { auto &series = get(); if (IOHandler()->m_frontendAccess == Access::READ_ONLY) @@ -719,7 +726,10 @@ void Series::flushGorVBased( } // Phase 3 - IOHandler()->flush(flushParams); + if (flushIOHandler) + { + IOHandler()->flush(flushParams); + } } else { @@ -774,7 +784,10 @@ void Series::flushGorVBased( } flushAttributes(flushParams); - IOHandler()->flush(flushParams); + if (flushIOHandler) + { + IOHandler()->flush(flushParams); + } } } From d9b1ca3e07cbe6cd67d92142bf77b122258c9943 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 6 May 2022 18:19:10 +0200 Subject: [PATCH 41/70] Remove deprecated debug parameter in ADIOS2 (#1269) --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 2 -- src/IO/ADIOS/ADIOS2IOHandler.cpp | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index eece83eb14..aed11094e3 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -124,8 +124,6 @@ class ADIOS2IOHandlerImpl friend struct detail::BufferedActions; friend struct detail::BufferedAttributeRead; - static constexpr bool ADIOS2_DEBUG_MODE = false; - public: #if openPMD_HAVE_MPI diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index e1c1bfe920..3d55598f25 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -71,7 +71,7 @@ ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( json::TracingJSON cfg, std::string engineType) : AbstractIOHandlerImplCommon(handler) - , m_ADIOS{communicator, ADIOS2_DEBUG_MODE} + , m_ADIOS{communicator} , m_engineType(std::move(engineType)) { init(std::move(cfg)); @@ -82,7 +82,7 @@ ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( AbstractIOHandler *handler, json::TracingJSON cfg, std::string engineType) : AbstractIOHandlerImplCommon(handler) - , m_ADIOS{ADIOS2_DEBUG_MODE} + , m_ADIOS{} , m_engineType(std::move(engineType)) { init(std::move(cfg)); From 33a189c7aa7c5d620ee8815bae40792e4bcd8769 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 15:13:07 -0700 Subject: [PATCH 42/70] [pre-commit.ci] pre-commit autoupdate (#1270) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-clang-format: v14.0.1 → v14.0.3](https://github.com/pre-commit/mirrors-clang-format/compare/v14.0.1...v14.0.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 10aa070cf9..8716c33bf5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,7 +65,7 @@ repos: # clang-format v13 # to run manually, use .github/workflows/clang-format/clang-format.sh - repo: https://github.com/pre-commit/mirrors-clang-format - rev: v14.0.1 + rev: v14.0.3 hooks: - id: clang-format From f62a3296def8eba07068bac0c750fa040a0058e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 10 May 2022 00:31:53 +0200 Subject: [PATCH 43/70] ADIOS2 Append Mode (#1007) * error::Internal * Frontend implementation of appending Also use switch statement in many frontend places for the Access enum. With 4 variants, if statements become increasingly unreadable and it's good to have compiler warnings if a case is forgotten. Try to avoid default branches. Also refactor `autoDetectPadding()` into a separate function since it is now needed in two places. * Backend implementation 1) Make backends aware of Append mode 2) In ADIOS1, fix use of namespaces, only use #include statements outside of namespaces * Documentation and testing * Extend tests: also use variable-based encoding * Allow overwriting old datasets in HDF5 * Test that RW-mode in group-based mode still works * Document extended meaning of createFile task * Apply suggestions from code review Co-authored-by: Axel Huebl * Remove special test paths for ADIOS2 * C++17 and other CI fixes * Add missing throw * Apply suggestions from code review Co-authored-by: Axel Huebl * Use H5Lexists to be more efficient in lookup Co-authored-by: Axel Huebl --- docs/source/usage/workflow.rst | 34 ++ include/openPMD/Error.hpp | 11 + include/openPMD/IO/AbstractIOHandler.hpp | 18 + include/openPMD/IO/AbstractIOHandlerImpl.hpp | 19 +- include/openPMD/IO/Access.hpp | 3 +- src/Error.cpp | 7 + src/IO/ADIOS/ADIOS2IOHandler.cpp | 23 +- src/IO/ADIOS/CommonADIOS1IOHandler.cpp | 10 + src/IO/HDF5/HDF5IOHandler.cpp | 77 ++++- src/IO/JSON/JSONIOHandlerImpl.cpp | 20 +- src/Iteration.cpp | 99 +++--- src/Mesh.cpp | 11 +- src/ParticleSpecies.cpp | 11 +- src/Record.cpp | 11 +- src/RecordComponent.cpp | 12 +- src/Series.cpp | 161 +++++++-- src/backend/PatchRecordComponent.cpp | 11 +- src/binding/python/Error.cpp | 1 + test/SerialIOTest.cpp | 341 ++++++++++++++++++- 19 files changed, 747 insertions(+), 133 deletions(-) diff --git a/docs/source/usage/workflow.rst b/docs/source/usage/workflow.rst index febcebb41e..7cf0e232d7 100644 --- a/docs/source/usage/workflow.rst +++ b/docs/source/usage/workflow.rst @@ -1,5 +1,39 @@ .. _workflow: +Access modes +============ + +The openPMD-api distinguishes between a number of different access modes: + +* **Create mode**: Used for creating a new Series from scratch. + Any file possibly existing in the specified location will be overwritten. +* **Read-only mode**: Used for reading from an existing Series. + No modifications will be made. +* **Read/Write mode**: Creates a new Series if not existing, otherwise opens an existing Series for reading and writing. + New datasets and iterations will be inserted as needed. + Not fully supported by all backends: + + * ADIOS1: Automatically coerced to *Create* mode if the file does not exist yet and to *Read-only* mode if it exists. + * ADIOS2: Automatically coerced to *Create* mode if the file does not exist yet and to *Read-only* mode if it exists. + Since this happens on a per-file level, this mode allows to read from existing iterations and write to new iterations at the same time in file-based iteration encoding. +* **Append mode**: Restricted mode for appending new iterations to an existing Series that is supported by all backends at least in file-based iteration encoding, and by all but ADIOS1 in other encodings. + The API is equivalent to that of the *Create* mode, meaning that no reading is supported whatsoever. + If the Series does not exist yet, this behaves equivalently to the *Create* mode. + Existing iterations will not be deleted, newly-written iterations will be inserted. + + **Warning:** When writing an iteration that already exists, the behavior is implementation-defined and depends on the chosen backend and iteration encoding: + + * The new iteration might fully replace the old one. + * The new iteration might be merged into the old one. + * (To be removed in a future update) The old and new iteration might coexist in the resulting dataset. + + We suggest to fully define iterations when using Append mode (i.e. as if using Create mode) to avoid implementation-specific behavior. + Appending to an openPMD Series is only supported on a per-iteration level. + + **Warning:** There is no reading involved in using Append mode. + It is a user's responsibility to ensure that the appended dataset and the appended-to dataset are compatible with each other. + The results of using incompatible backend configurations are undefined. + Workflow ======== diff --git a/include/openPMD/Error.hpp b/include/openPMD/Error.hpp index 4fcda11eeb..e172670bcc 100644 --- a/include/openPMD/Error.hpp +++ b/include/openPMD/Error.hpp @@ -69,5 +69,16 @@ namespace error BackendConfigSchema(std::vector, std::string what); }; + + /** + * @brief Internal errors that should not happen. Please report. + * + * Example: A nullpointer is observed somewhere. + */ + class Internal : public Error + { + public: + Internal(std::string const &what); + }; } // namespace error } // namespace openPMD diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index a7239a3375..80616e62bc 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -121,6 +121,23 @@ namespace internal */ class AbstractIOHandler { + friend class Series; + +private: + void setIterationEncoding(IterationEncoding encoding) + { + /* + * In file-based iteration encoding, the APPEND mode is handled entirely + * by the frontend, the backend should just treat it as CREATE mode + */ + if (encoding == IterationEncoding::fileBased && + m_backendAccess == Access::APPEND) + { + // do we really want to have those as const members..? + *const_cast(&m_backendAccess) = Access::CREATE; + } + } + public: #if openPMD_HAVE_MPI AbstractIOHandler(std::string path, Access at, MPI_Comm) @@ -153,6 +170,7 @@ class AbstractIOHandler virtual std::string backendName() const = 0; std::string const directory; + // why do these need to be separate? Access const m_backendAccess; Access const m_frontendAccess; std::queue m_work; diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index e171115f95..518b9dac0a 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -266,14 +266,17 @@ class AbstractIOHandlerImpl * file. * * The operation should fail if m_handler->m_frontendAccess is - * Access::READ_ONLY. The new file should be located in - * m_handler->directory. The new file should have the filename - * parameters.name. The filename should include the correct corresponding - * filename extension. Any existing file should be overwritten if - * m_handler->m_frontendAccess is Access::CREATE. The Writables file - * position should correspond to the root group "/" of the hierarchy. The - * Writable should be marked written when the operation completes - * successfully. + * Access::READ_ONLY. If m_handler->m_frontendAccess is Access::APPEND, a + * possibly existing file should not be overwritten. Instead, written + * updates should then either occur in-place or in form of new IO steps. + * Support for reading is not necessary in Append mode. + * The new file should be located in m_handler->directory. + * The new file should have the filename parameters.name. + * The filename should include the correct corresponding filename extension. + * Any existing file should be overwritten if m_handler->m_frontendAccess is + * Access::CREATE. The Writables file position should correspond to the root + * group "/" of the hierarchy. The Writable should be marked written when + * the operation completes successfully. */ virtual void createFile(Writable *, Parameter const &) = 0; diff --git a/include/openPMD/IO/Access.hpp b/include/openPMD/IO/Access.hpp index 93ba662241..2b5d37f260 100644 --- a/include/openPMD/IO/Access.hpp +++ b/include/openPMD/IO/Access.hpp @@ -28,7 +28,8 @@ enum class Access { READ_ONLY, //!< open series as read-only, fails if series is not found READ_WRITE, //!< open existing series as writable - CREATE //!< create new series and truncate existing (files) + CREATE, //!< create new series and truncate existing (files) + APPEND //!< write new iterations to an existing series without reading }; // Access // deprecated name (used prior to 0.12.0) diff --git a/src/Error.cpp b/src/Error.cpp index aa6723e63f..a0331948c8 100644 --- a/src/Error.cpp +++ b/src/Error.cpp @@ -45,5 +45,12 @@ namespace error concatVector(errorLocation_in) + "': " + std::move(what)) , errorLocation(std::move(errorLocation_in)) {} + + Internal::Internal(std::string const &what) + : Error( + "Internal error: " + what + + "\nThis is a bug. Please report at ' " + "https://github.com/openPMD/openPMD-api/issues'.") + {} } // namespace error } // namespace openPMD diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 3d55598f25..072c5cd285 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -315,9 +315,6 @@ void ADIOS2IOHandlerImpl::createFile( m_iterationEncoding = parameters.encoding; associateWithFile(writable, shared_name); this->m_dirty.emplace(shared_name); - getFileData(shared_name, IfFileNotOpen::OpenImplicitly).m_mode = - adios2::Mode::Write; // WORKAROUND - // ADIOS2 does not yet implement ReadWrite Mode writable->written = true; writable->abstractFilePosition = std::make_shared(); @@ -1074,21 +1071,16 @@ adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode(std::string const &fullPath) if (auxiliary::directory_exists(fullPath) || auxiliary::file_exists(fullPath)) { - std::cerr << "ADIOS2 does currently not yet implement ReadWrite " - "(Append) mode. " - << "Replacing with Read mode." << std::endl; return adios2::Mode::Read; } else { - std::cerr << "ADIOS2 does currently not yet implement ReadWrite " - "(Append) mode. " - << "Replacing with Write mode." << std::endl; return adios2::Mode::Write; } - default: - return adios2::Mode::Undefined; + case Access::APPEND: + return adios2::Mode::Append; } + throw std::runtime_error("Unreachable!"); } json::TracingJSON ADIOS2IOHandlerImpl::nullvalue = { @@ -2235,6 +2227,7 @@ namespace detail delayOpeningTheFirstStep = true; break; case adios2::Mode::Write: + case adios2::Mode::Append: /* * File engines, write mode: * Default for old layout is no steps. @@ -2442,6 +2435,7 @@ namespace detail { switch (m_mode) { + case adios2::Mode::Append: case adios2::Mode::Write: { // usesSteps attribute only written upon ::advance() // this makes sure that the attribute is only put in case @@ -2686,17 +2680,14 @@ namespace detail switch (ba.m_mode) { case adios2::Mode::Write: + case adios2::Mode::Append: eng.PerformPuts(); break; case adios2::Mode::Read: eng.PerformGets(); break; - case adios2::Mode::Append: - // TODO order? - eng.PerformGets(); - eng.PerformPuts(); - break; default: + throw error::Internal("[ADIOS2] Unexpected access mode."); break; } }, diff --git a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp index e50207a74d..5e30ecabb3 100644 --- a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp @@ -20,6 +20,7 @@ */ #include "openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp" +#include "openPMD/Error.hpp" #if openPMD_HAVE_ADIOS1 @@ -406,6 +407,15 @@ void CommonADIOS1IOHandlerImpl::createFile( if (!auxiliary::ends_with(name, ".bp")) name += ".bp"; + if (m_handler->m_backendAccess == Access::APPEND && + auxiliary::file_exists(name)) + { + throw error::OperationUnsupportedInBackend( + "ADIOS1", + "Appending to existing file on disk (use Access::CREATE to " + "overwrite)"); + } + writable->written = true; writable->abstractFilePosition = std::make_shared("/"); diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index d0725395f7..51a013fca8 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -237,13 +237,41 @@ void HDF5IOHandlerImpl::createFile( std::string name = m_handler->directory + parameters.name; if (!auxiliary::ends_with(name, ".h5")) name += ".h5"; - unsigned flags; - if (m_handler->m_backendAccess == Access::CREATE) + unsigned flags{}; + switch (m_handler->m_backendAccess) + { + case Access::CREATE: flags = H5F_ACC_TRUNC; - else + break; + case Access::APPEND: + if (auxiliary::file_exists(name)) + { + flags = H5F_ACC_RDWR; + } + else + { + flags = H5F_ACC_TRUNC; + } + break; + case Access::READ_WRITE: flags = H5F_ACC_EXCL; - hid_t id = - H5Fcreate(name.c_str(), flags, H5P_DEFAULT, m_fileAccessProperty); + break; + case Access::READ_ONLY: + // condition has been checked above + throw std::runtime_error( + "[HDF5] Control flow error in createFile backend access mode."); + } + + hid_t id{}; + if (flags == H5F_ACC_RDWR) + { + id = H5Fopen(name.c_str(), flags, m_fileAccessProperty); + } + else + { + id = H5Fcreate( + name.c_str(), flags, H5P_DEFAULT, m_fileAccessProperty); + } VERIFY(id >= 0, "[HDF5] Internal error: Failed to create HDF5 file"); writable->written = true; @@ -409,6 +437,36 @@ void HDF5IOHandlerImpl::createDataset( "[HDF5] Internal error: Failed to open HDF5 group during dataset " "creation"); + if (m_handler->m_backendAccess == Access::APPEND) + { + // The dataset might already exist in the file from a previous run + // We delete it, otherwise we could not create it again with + // possibly different parameters. + if (htri_t link_id = H5Lexists(node_id, name.c_str(), H5P_DEFAULT); + link_id > 0) + { + // This only unlinks, but does not delete the dataset + // Deleting the actual dataset physically is now up to HDF5: + // > when removing an object with H5Ldelete, the HDF5 library + // > should be able to detect and recycle the file space when no + // > other reference to the deleted object exists + // https://github.com/openPMD/openPMD-api/pull/1007#discussion_r867223316 + herr_t status = H5Ldelete(node_id, name.c_str(), H5P_DEFAULT); + VERIFY( + status == 0, + "[HDF5] Internal error: Failed to delete old dataset '" + + name + "' from group for overwriting."); + } + else if (link_id < 0) + { + throw std::runtime_error( + "[HDF5] Internal error: Failed to check for link existence " + "of '" + + name + "' inside group for overwriting."); + } + // else: link_id == 0: Link does not exist, nothing to do + } + Datatype d = parameters.dtype; if (d == Datatype::UNDEFINED) { @@ -702,7 +760,14 @@ void HDF5IOHandlerImpl::openFile( Access at = m_handler->m_backendAccess; if (at == Access::READ_ONLY) flags = H5F_ACC_RDONLY; - else if (at == Access::READ_WRITE || at == Access::CREATE) + /* + * Within the HDF5 backend, APPEND and READ_WRITE mode are + * equivalent, but the openPMD frontend exposes no reading + * functionality in APPEND mode. + */ + else if ( + at == Access::READ_WRITE || at == Access::CREATE || + at == Access::APPEND) flags = H5F_ACC_RDWR; else throw std::runtime_error("[HDF5] Unknown file Access"); diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 61f8d239d5..6f8666fae5 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -117,7 +117,7 @@ void JSONIOHandlerImpl::createFile( file.invalidate(); } - std::string const dir(m_handler->directory); + std::string const &dir(m_handler->directory); if (!auxiliary::directory_exists(dir)) { auto success = auxiliary::create_directories(dir); @@ -126,8 +126,14 @@ void JSONIOHandlerImpl::createFile( associateWithFile(writable, shared_name); this->m_dirty.emplace(shared_name); - // make sure to overwrite! - this->m_jsonVals[shared_name] = std::make_shared(); + + if (m_handler->m_backendAccess != Access::APPEND) + { + // make sure to overwrite! + this->m_jsonVals[shared_name] = std::make_shared(); + } + // else: the JSON value is not available in m_jsonVals and will be + // read from the file later on before overwriting writable->written = true; writable->abstractFilePosition = std::make_shared(); @@ -910,6 +916,14 @@ JSONIOHandlerImpl::getFilehandle(File fileName, Access access) { case Access::CREATE: case Access::READ_WRITE: + case Access::APPEND: + /* + * Always truncate when writing, we alway write entire JSON + * datasets, never partial ones. + * Within the JSON backend, APPEND and READ_WRITE mode are + * equivalent, but the openPMD frontend exposes no reading + * functionality in APPEND mode. + */ fs->open(path, std::ios_base::out | std::ios_base::trunc); break; case Access::READ_ONLY: diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 8991f0ebad..1cf77ec180 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -298,15 +298,18 @@ void Iteration::flushVariableBased( void Iteration::flush(internal::FlushParams const &flushParams) { - if (IOHandler()->m_frontendAccess == Access::READ_ONLY) + switch (IOHandler()->m_frontendAccess) { + case Access::READ_ONLY: { for (auto &m : meshes) m.second.flush(m.first, flushParams); for (auto &species : particles) species.second.flush(species.first, flushParams); + break; } - else - { + case Access::READ_WRITE: + case Access::CREATE: + case Access::APPEND: { /* Find the root point [Series] of this file, * meshesPath and particlesPath are stored there */ Series s = retrieveSeries(); @@ -344,6 +347,8 @@ void Iteration::flush(internal::FlushParams const &flushParams) } flushAttributes(flushParams); + break; + } } } @@ -591,15 +596,26 @@ AdvanceStatus Iteration::beginStep(bool reread) (this->IOHandler()->m_frontendAccess == Access::READ_ONLY || this->IOHandler()->m_frontendAccess == Access::READ_WRITE)) { - bool previous = series.iterations.written(); - series.iterations.written() = false; - auto oldType = this->IOHandler()->m_frontendAccess; - auto newType = - const_cast(&this->IOHandler()->m_frontendAccess); - *newType = Access::READ_WRITE; - series.readGorVBased(false); - *newType = oldType; - series.iterations.written() = previous; + switch (IOHandler()->m_frontendAccess) + { + case Access::READ_ONLY: + case Access::READ_WRITE: { + bool previous = series.iterations.written(); + series.iterations.written() = false; + auto oldType = this->IOHandler()->m_frontendAccess; + auto newType = + const_cast(&this->IOHandler()->m_frontendAccess); + *newType = Access::READ_WRITE; + series.readGorVBased(false); + *newType = oldType; + series.iterations.written() = previous; + break; + } + case Access::CREATE: + case Access::APPEND: + // no re-reading necessary + break; + } } return status; @@ -696,42 +712,49 @@ void Iteration::linkHierarchy(Writable &w) void Iteration::runDeferredParseAccess() { - if (IOHandler()->m_frontendAccess == Access::CREATE) + switch (IOHandler()->m_frontendAccess) { - return; - } - - auto &it = get(); - if (!it.m_deferredParseAccess.has_value()) - { - return; - } - auto const &deferred = it.m_deferredParseAccess.value(); + case Access::READ_ONLY: + case Access::READ_WRITE: { + auto &it = get(); + if (!it.m_deferredParseAccess.has_value()) + { + return; + } + auto const &deferred = it.m_deferredParseAccess.value(); - auto oldAccess = IOHandler()->m_frontendAccess; - auto newAccess = const_cast(&IOHandler()->m_frontendAccess); - *newAccess = Access::READ_WRITE; - try - { - if (deferred.fileBased) + auto oldAccess = IOHandler()->m_frontendAccess; + auto newAccess = const_cast(&IOHandler()->m_frontendAccess); + *newAccess = Access::READ_WRITE; + try { - readFileBased(deferred.filename, deferred.path, deferred.beginStep); + if (deferred.fileBased) + { + readFileBased( + deferred.filename, deferred.path, deferred.beginStep); + } + else + { + readGorVBased(deferred.path, deferred.beginStep); + } } - else + catch (...) { - readGorVBased(deferred.path, deferred.beginStep); + // reset this thing + it.m_deferredParseAccess = std::optional(); + *newAccess = oldAccess; + throw; } - } - catch (...) - { // reset this thing it.m_deferredParseAccess = std::optional(); *newAccess = oldAccess; - throw; + break; + } + case Access::CREATE: + case Access::APPEND: + // no parsing in those modes + return; } - // reset this thing - it.m_deferredParseAccess = std::optional(); - *newAccess = oldAccess; } template float Iteration::time() const; diff --git a/src/Mesh.cpp b/src/Mesh.cpp index 761a9a304a..6f9b891635 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -216,13 +216,16 @@ template Mesh &Mesh::setTimeOffset(float); void Mesh::flush_impl( std::string const &name, internal::FlushParams const &flushParams) { - if (IOHandler()->m_frontendAccess == Access::READ_ONLY) + switch (IOHandler()->m_frontendAccess) { + case Access::READ_ONLY: { for (auto &comp : *this) comp.second.flush(comp.first, flushParams); + break; } - else - { + case Access::READ_WRITE: + case Access::CREATE: + case Access::APPEND: { if (!written()) { if (scalar()) @@ -260,6 +263,8 @@ void Mesh::flush_impl( } flushAttributes(flushParams); + break; + } } } diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index fdc4fa6f88..24ebadadec 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -141,15 +141,18 @@ namespace void ParticleSpecies::flush( std::string const &path, internal::FlushParams const &flushParams) { - if (IOHandler()->m_frontendAccess == Access::READ_ONLY) + switch (IOHandler()->m_frontendAccess) { + case Access::READ_ONLY: { for (auto &record : *this) record.second.flush(record.first, flushParams); for (auto &patch : particlePatches) patch.second.flush(patch.first, flushParams); + break; } - else - { + case Access::READ_WRITE: + case Access::CREATE: + case Access::APPEND: { auto it = find("position"); if (it != end()) it->second.setUnitDimension({{UnitDimension::L, 1}}); @@ -168,6 +171,8 @@ void ParticleSpecies::flush( for (auto &patch : particlePatches) patch.second.flush(patch.first, flushParams); } + break; + } } } diff --git a/src/Record.cpp b/src/Record.cpp index 90b0c61a91..bbd74ed0f5 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -46,13 +46,16 @@ Record &Record::setUnitDimension(std::map const &udim) void Record::flush_impl( std::string const &name, internal::FlushParams const &flushParams) { - if (IOHandler()->m_frontendAccess == Access::READ_ONLY) + switch (IOHandler()->m_frontendAccess) { + case Access::READ_ONLY: { for (auto &comp : *this) comp.second.flush(comp.first, flushParams); + break; } - else - { + case Access::READ_WRITE: + case Access::CREATE: + case Access::APPEND: { if (!written()) { if (scalar()) @@ -90,6 +93,8 @@ void Record::flush_impl( } flushAttributes(flushParams); + break; + } } } diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index c69c09e997..9e7a93e6ea 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -200,16 +200,18 @@ void RecordComponent::flush( rc.m_name = name; return; } - if (IOHandler()->m_frontendAccess == Access::READ_ONLY) + switch (IOHandler()->m_frontendAccess) { + case Access::READ_ONLY: while (!rc.m_chunks.empty()) { IOHandler()->enqueue(rc.m_chunks.front()); rc.m_chunks.pop(); } - } - else - { + break; + case Access::READ_WRITE: + case Access::CREATE: + case Access::APPEND: { /* * This catches when a user forgets to use resetDataset. */ @@ -275,6 +277,8 @@ void RecordComponent::flush( } flushAttributes(flushParams); + break; + } } } diff --git a/src/Series.cpp b/src/Series.cpp index 6adeefad55..36a7bd90d6 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -301,6 +301,7 @@ Series &Series::setIterationEncoding(IterationEncoding ie) setAttribute("iterationEncoding", std::string("variableBased")); break; } + IOHandler()->setIterationEncoding(ie); return *this; } @@ -469,6 +470,56 @@ bool Series::reparseExpansionPattern(std::string filenameWithExtension) return true; } +namespace +{ + /* + * Negative return values: + * -1: No padding detected, just keep the default from the file name + * -2: Contradicting paddings detected + */ + template + int autoDetectPadding( + std::function isPartOfSeries, + std::string const &directory, + MappingFunction &&mappingFunction) + { + bool isContained; + int padding; + uint64_t iterationIndex; + std::set paddings; + for (auto const &entry : auxiliary::list_directory(directory)) + { + std::tie(isContained, padding, iterationIndex) = + isPartOfSeries(entry); + if (isContained) + { + paddings.insert(padding); + // no std::forward as this is called repeatedly + mappingFunction(iterationIndex, entry); + } + } + if (paddings.size() == 1u) + return *paddings.begin(); + else if (paddings.empty()) + return -1; + else + return -2; + } + + int autoDetectPadding( + std::function isPartOfSeries, + std::string const &directory) + { + return autoDetectPadding( + std::move(isPartOfSeries), + directory, + [](uint64_t index, std::string const &filename) { + (void)index; + (void)filename; + }); + } +} // namespace + void Series::init( std::shared_ptr ioHandler, std::unique_ptr input) @@ -501,9 +552,10 @@ Given file pattern: ')END" << series.m_name << "'" << std::endl; } - if (IOHandler()->m_frontendAccess == Access::READ_ONLY || - IOHandler()->m_frontendAccess == Access::READ_WRITE) + switch (IOHandler()->m_frontendAccess) { + case Access::READ_ONLY: + case Access::READ_WRITE: { /* Allow creation of values in Containers and setting of Attributes * Would throw for Access::READ_ONLY */ auto oldType = IOHandler()->m_frontendAccess; @@ -528,11 +580,43 @@ Given file pattern: ')END" } *newType = oldType; + break; } - else - { + case Access::CREATE: { + initDefaults(input->iterationEncoding); + setIterationEncoding(input->iterationEncoding); + break; + } + case Access::APPEND: { initDefaults(input->iterationEncoding); setIterationEncoding(input->iterationEncoding); + if (input->iterationEncoding != IterationEncoding::fileBased) + { + break; + } + int padding = autoDetectPadding( + matcher( + series.m_filenamePrefix, + series.m_filenamePadding, + series.m_filenamePostfix, + series.m_format), + IOHandler()->directory); + switch (padding) + { + case -2: + throw std::runtime_error( + "Cannot write to a series with inconsistent iteration padding. " + "Please specify '%0T' or open as read-only."); + case -1: + std::cerr << "No matching iterations found: " << name() + << std::endl; + break; + default: + series.m_filenamePadding = padding; + break; + } + break; + } } series.m_lastFlushSuccessful = true; } @@ -609,7 +693,9 @@ void Series::flushFileBased( throw std::runtime_error( "fileBased output can not be written with no iterations."); - if (IOHandler()->m_frontendAccess == Access::READ_ONLY) + switch (IOHandler()->m_frontendAccess) + { + case Access::READ_ONLY: for (auto it = begin; it != end; ++it) { // Phase 1 @@ -639,8 +725,10 @@ void Series::flushFileBased( IOHandler()->flush(flushParams); } } - else - { + break; + case Access::READ_WRITE: + case Access::CREATE: + case Access::APPEND: { bool allDirty = dirty(); for (auto it = begin; it != end; ++it) { @@ -692,6 +780,8 @@ void Series::flushFileBased( dirty() = allDirty; } dirty() = false; + break; + } } } @@ -702,7 +792,9 @@ void Series::flushGorVBased( bool flushIOHandler) { auto &series = get(); - if (IOHandler()->m_frontendAccess == Access::READ_ONLY) + switch (IOHandler()->m_frontendAccess) + { + case Access::READ_ONLY: for (auto it = begin; it != end; ++it) { // Phase 1 @@ -731,8 +823,10 @@ void Series::flushGorVBased( IOHandler()->flush(flushParams); } } - else - { + break; + case Access::READ_WRITE: + case Access::CREATE: + case Access::APPEND: { if (!written()) { Parameter fCreate; @@ -788,6 +882,8 @@ void Series::flushGorVBased( { IOHandler()->flush(flushParams); } + break; + } } } @@ -827,23 +923,15 @@ void Series::readFileBased() series.m_filenamePadding, series.m_filenamePostfix, series.m_format); - bool isContained; - int padding; - uint64_t iterationIndex; - std::set paddings; - for (auto const &entry : auxiliary::list_directory(IOHandler()->directory)) - { - std::tie(isContained, padding, iterationIndex) = isPartOfSeries(entry); - if (isContained) - { - Iteration &i = series.iterations[iterationIndex]; - i.deferParseAccess( - {std::to_string(iterationIndex), iterationIndex, true, entry}); - // TODO skip if the padding is exact the number of chars in an - // iteration? - paddings.insert(padding); - } - } + + int padding = autoDetectPadding( + std::move(isPartOfSeries), + IOHandler()->directory, + // foreach found file with `filename` and `index`: + [&series](uint64_t index, std::string const &filename) { + Iteration &i = series.iterations[index]; + i.deferParseAccess({std::to_string(index), index, true, filename}); + }); if (series.iterations.empty()) { @@ -885,14 +973,15 @@ void Series::readFileBased() } } - if (paddings.size() == 1u) - series.m_filenamePadding = *paddings.begin(); + if (padding > 0) + series.m_filenamePadding = padding; - /* Frontend access type might change during Series::read() to allow - * parameter modification. Backend access type stays unchanged for the - * lifetime of a Series. */ - if (paddings.size() > 1u && - IOHandler()->m_backendAccess == Access::READ_WRITE) + /* Frontend access type might change during SeriesInterface::read() to allow + parameter modification. + * Backend access type stays unchanged for the lifetime of a Series. + autoDetectPadding() announces contradicting paddings with return status + -2. */ + if (padding == -2 && IOHandler()->m_backendAccess == Access::READ_WRITE) throw std::runtime_error( "Cannot write to a series with inconsistent iteration padding. " "Please specify '%0T' or open as read-only."); @@ -1639,6 +1728,10 @@ namespace internal Series impl{{this, [](auto const *) {}}}; impl.flush(); } + if (m_writeIterations.has_value()) + { + m_writeIterations = std::optional(); + } } catch (std::exception const &ex) { diff --git a/src/backend/PatchRecordComponent.cpp b/src/backend/PatchRecordComponent.cpp index ecc44625a2..a5178a9911 100644 --- a/src/backend/PatchRecordComponent.cpp +++ b/src/backend/PatchRecordComponent.cpp @@ -84,16 +84,19 @@ void PatchRecordComponent::flush( std::string const &name, internal::FlushParams const &flushParams) { auto &rc = get(); - if (IOHandler()->m_frontendAccess == Access::READ_ONLY) + switch (IOHandler()->m_frontendAccess) { + case Access::READ_ONLY: { while (!rc.m_chunks.empty()) { IOHandler()->enqueue(rc.m_chunks.front()); rc.m_chunks.pop(); } + break; } - else - { + case Access::READ_WRITE: + case Access::CREATE: + case Access::APPEND: { if (!written()) { Parameter dCreate; @@ -111,6 +114,8 @@ void PatchRecordComponent::flush( } flushAttributes(flushParams); + break; + } } } diff --git a/src/binding/python/Error.cpp b/src/binding/python/Error.cpp index ea5712296b..3b81ebe923 100644 --- a/src/binding/python/Error.cpp +++ b/src/binding/python/Error.cpp @@ -14,6 +14,7 @@ void init_Error(py::module &m) m, "ErrorWrongAPIUsage", baseError); py::register_exception( m, "ErrorBackendConfigSchema", baseError); + py::register_exception(m, "ErrorInternal", baseError); #ifndef NDEBUG m.def("test_throw", [](std::string description) { diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index dc366fa773..155d4fc58d 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1917,9 +1917,9 @@ inline void fileBased_write_test(const std::string &backend) REQUIRE(o.iterations[5].time() == 5.0); } - // extend existing series with new step and auto-detection of iteration - // padding { + // extend existing series with new step and auto-detection of iteration + // padding Series o = Series( "../samples/subdir/serial_fileBased_write%T." + backend, Access::READ_WRITE); @@ -2009,15 +2009,13 @@ inline void fileBased_write_test(const std::string &backend) } // write with auto-detection and in-consistent padding from step 10 - { - REQUIRE_THROWS_WITH( - Series( - "../samples/subdir/serial_fileBased_write%T." + backend, - Access::READ_WRITE), - Catch::Equals( - "Cannot write to a series with inconsistent iteration padding. " - "Please specify '%0T' or open as read-only.")); - } + REQUIRE_THROWS_WITH( + Series( + "../samples/subdir/serial_fileBased_write%T." + backend, + Access::READ_WRITE), + Catch::Equals( + "Cannot write to a series with inconsistent iteration padding. " + "Please specify '%0T' or open as read-only.")); // read back with fixed padding { @@ -5715,3 +5713,324 @@ TEST_CASE("varying_zero_pattern", "[serial]") varying_pattern(t); } } + +void append_mode( + std::string const &extension, + bool variableBased, + std::string jsonConfig = "{}") +{ + + std::string filename = (variableBased ? "../samples/append_variablebased." + : "../samples/append_groupbased.") + + extension; + std::vector data(10, 0); + auto writeSomeIterations = [&data]( + WriteIterations &&writeIterations, + std::vector indices) { + for (auto index : indices) + { + auto it = writeIterations[index]; + auto dataset = it.meshes["E"]["x"]; + dataset.resetDataset({Datatype::INT, {10}}); + dataset.storeChunk(data, {0}, {10}); + // test that it works without closing too + it.close(); + } + }; + { + Series write(filename, Access::CREATE, jsonConfig); + if (variableBased) + { + if (write.backend() != "ADIOS2") + { + return; + } + write.setIterationEncoding(IterationEncoding::variableBased); + } + writeSomeIterations( + write.writeIterations(), std::vector{0, 1}); + } + { + Series write(filename, Access::APPEND, jsonConfig); + if (variableBased) + { + write.setIterationEncoding(IterationEncoding::variableBased); + } + if (write.backend() == "ADIOS1") + { + REQUIRE_THROWS_WITH( + write.flush(), + Catch::Equals( + "Operation unsupported in ADIOS1: Appending to existing " + "file on disk (use Access::CREATE to overwrite)")); + // destructor will be noisy now + return; + } + + writeSomeIterations( + write.writeIterations(), std::vector{2, 3}); + write.flush(); + } + { + Series write(filename, Access::APPEND, jsonConfig); + if (variableBased) + { + write.setIterationEncoding(IterationEncoding::variableBased); + } + if (write.backend() == "ADIOS1") + { + REQUIRE_THROWS_WITH( + write.flush(), + Catch::Equals( + "Operation unsupported in ADIOS1: Appending to existing " + "file on disk (use Access::CREATE to overwrite)")); + // destructor will be noisy now + return; + } + + writeSomeIterations( + write.writeIterations(), std::vector{4, 3}); + write.flush(); + } + { + Series read(filename, Access::READ_ONLY); + if (variableBased) + { + // in variable-based encodings, iterations are not parsed ahead of + // time but as they go + unsigned counter = 0; + for (auto const &iteration : read.readIterations()) + { + REQUIRE(iteration.iterationIndex == counter); + ++counter; + } + REQUIRE(counter == 5); + } + else + { + REQUIRE(read.iterations.size() == 5); + } + /* + * Roadmap: for now, reading this should work by ignoring the last + * duplicate iteration. + * After merging https://github.com/openPMD/openPMD-api/pull/949, we + * should see both instances when reading. + * Final goal: Read only the last instance. + */ + helper::listSeries(read); + } +} + +TEST_CASE("append_mode", "[serial]") +{ + for (auto const &t : testedFileExtensions()) + { + if (t == "bp") + { + std::string jsonConfigOld = R"END( +{ + "adios2": + { + "schema": 0, + "engine": + { + "usesteps" : true + } + } +})END"; + std::string jsonConfigNew = R"END( +{ + "adios2": + { + "schema": 20210209, + "engine": + { + "usesteps" : true + } + } +})END"; + append_mode(t, false, jsonConfigOld); + append_mode(t, false, jsonConfigNew); + append_mode(t, true, jsonConfigOld); + append_mode(t, true, jsonConfigNew); + } + else + { + append_mode(t, false); + } + } +} + +void append_mode_filebased(std::string const &extension) +{ + std::string jsonConfig = R"END( +{ + "adios2": + { + "schema": 20210209, + "engine": + { + "usesteps" : true + } + } +})END"; + auto writeSomeIterations = [](WriteIterations &&writeIterations, + std::vector indices) { + for (auto index : indices) + { + auto it = writeIterations[index]; + auto dataset = it.meshes["E"]["x"]; + dataset.resetDataset({Datatype::INT, {1}}); + dataset.makeConstant(0); + // test that it works without closing too + it.close(); + } + }; + if (auxiliary::directory_exists("../samples/append")) + { + auxiliary::remove_directory("../samples/append"); + } + { + Series write( + "../samples/append/append_%T." + extension, + Access::CREATE, + jsonConfig); + writeSomeIterations( + write.writeIterations(), std::vector{0, 1}); + } + { + Series write( + "../samples/append/append_%T." + extension, + Access::APPEND, + jsonConfig); + writeSomeIterations( + write.writeIterations(), std::vector{4, 5}); + write.flush(); + } + { + Series write( + "../samples/append/append_%T." + extension, + Access::APPEND, + jsonConfig); + writeSomeIterations( + write.writeIterations(), std::vector{2, 3}); + write.flush(); + } + { + Series write( + "../samples/append/append_%T." + extension, + Access::APPEND, + jsonConfig); + // overwrite a previous iteration + writeSomeIterations( + write.writeIterations(), std::vector{4, 123}); + write.flush(); + } + { + Series read( + "../samples/append/append_%T." + extension, Access::READ_ONLY); + REQUIRE(read.iterations.size() == 7); + } +} + +TEST_CASE("append_mode_filebased", "[serial]") +{ + for (auto const &t : testedFileExtensions()) + { + append_mode_filebased(t); + } +} + +void groupbased_read_write(std::string const &ext) +{ + int data = 0; + Dataset ds(Datatype::INT, {1}); + std::string filename = "../samples/groupbased_read_write." + ext; + + { + Series write(filename, Access::CREATE); + auto E_x = write.iterations[0].meshes["E"]["x"]; + auto E_y = write.iterations[0].meshes["E"]["y"]; + E_x.resetDataset(ds); + E_y.resetDataset(ds); + E_x.storeChunk(shareRaw(&data), {0}, {1}); + E_y.storeChunk(shareRaw(&data), {0}, {1}); + + E_x.setAttribute("updated_in_run", 0); + E_y.setAttribute("updated_in_run", 0); + } + + { + Series write(filename, Access::READ_WRITE); + // create a new iteration + auto E_x = write.iterations[1].meshes["E"]["x"]; + E_x.resetDataset(ds); + + // overwrite old dataset + auto E_y = write.iterations[0].meshes["E"]["y"]; + + data = 1; + + E_x.storeChunk(shareRaw(&data), {0}, {1}); + E_y.storeChunk(shareRaw(&data), {0}, {1}); + + E_x.setAttribute("updated_in_run", 1); + E_y.setAttribute("updated_in_run", 1); + } + + { + Series read(filename, Access::READ_ONLY); + auto E_x_0_fromRun0 = read.iterations[0].meshes["E"]["x"]; + auto E_x_1_fromRun1 = read.iterations[1].meshes["E"]["x"]; + auto E_y_0_fromRun1 = read.iterations[0].meshes["E"]["y"]; + + REQUIRE(E_x_0_fromRun0.getAttribute("updated_in_run").get() == 0); + REQUIRE(E_x_1_fromRun1.getAttribute("updated_in_run").get() == 1); + REQUIRE(E_y_0_fromRun1.getAttribute("updated_in_run").get() == 1); + + auto chunk_E_x_0_fromRun0 = E_x_0_fromRun0.loadChunk({0}, {1}); + auto chunk_E_x_1_fromRun1 = E_x_1_fromRun1.loadChunk({0}, {1}); + auto chunk_E_y_0_fromRun1 = E_y_0_fromRun1.loadChunk({0}, {1}); + + read.flush(); + + REQUIRE(*chunk_E_x_0_fromRun0 == 0); + REQUIRE(*chunk_E_x_1_fromRun1 == 1); + REQUIRE(*chunk_E_y_0_fromRun1 == 1); + } + + // check that truncation works correctly + { + Series write(filename, Access::CREATE); + // create a new iteration + auto E_x = write.iterations[2].meshes["E"]["x"]; + E_x.resetDataset(ds); + + data = 2; + + E_x.storeChunk(shareRaw(&data), {0}, {1}); + E_x.setAttribute("updated_in_run", 2); + } + + { + Series read(filename, Access::READ_ONLY); + REQUIRE(read.iterations.size() == 1); + REQUIRE(read.iterations.count(2) == 1); + } +} + +TEST_CASE("groupbased_read_write", "[serial]") +{ + constexpr char const *supportsGroupbasedRW[] = {"h5", "json"}; + for (auto const &t : testedFileExtensions()) + { + for (auto const supported : supportsGroupbasedRW) + { + if (t == supported) + { + groupbased_read_write(t); + break; + } + } + } +} From 8357943a2ceed822eee5b81070b17267a0ac5334 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 11 May 2022 03:19:05 -0700 Subject: [PATCH 44/70] CI: ADIOS 2.7.1 (#1271) Make sure we continue to build a ADIOS 2.7.1 variant in CI as long as we support it. Package managers otherwise will over time all transition to pick the latest ADIOS2 release. --- .github/ci/spack-envs/clang8_py38_mpich_h5_ad1_ad2/spack.yaml | 2 +- .github/workflows/linux.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/ci/spack-envs/clang8_py38_mpich_h5_ad1_ad2/spack.yaml b/.github/ci/spack-envs/clang8_py38_mpich_h5_ad1_ad2/spack.yaml index 4fccfe9d0b..4a6ab7f74e 100644 --- a/.github/ci/spack-envs/clang8_py38_mpich_h5_ad1_ad2/spack.yaml +++ b/.github/ci/spack-envs/clang8_py38_mpich_h5_ad1_ad2/spack.yaml @@ -7,7 +7,7 @@ spack: specs: - adios - - adios2 + - adios2@2.7.1 - hdf5 - mpich diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 0835877427..9fd23b4803 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -151,6 +151,7 @@ jobs: cmake --build build --parallel 2 ctest --test-dir build --output-on-failure + # ADIOS2 v2.7.1 clang8_py38_mpich_h5_ad1_ad2_newLayout: runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false From 0b4bb53e32ea46f32adcfd194a1060e4229ec562 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 11 May 2022 11:32:13 -0700 Subject: [PATCH 45/70] CI: Expand Read-Only Permission Tests (#1272) * CI: Expand Read-Only Permission Tests Expand to: - thetaMode h5 files - ADIOS bp files * CI: Simplify Logic (chmod in download) `chmod` is now in the sample download step, so we can remove the line in CI scripts. --- .github/workflows/intel.yml | 2 -- .github/workflows/linux.yml | 9 --------- .github/workflows/macos.yml | 1 - .github/workflows/nvidia.yml | 2 -- .github/workflows/tooling.yml | 2 -- share/openPMD/download_samples.sh | 8 ++++++++ 6 files changed, 8 insertions(+), 16 deletions(-) diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index 243ac4208b..2b3cea6233 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -21,7 +21,6 @@ jobs: run: | set +e; source /opt/intel/oneapi/setvars.sh; set -e share/openPMD/download_samples.sh build - chmod u-w build/samples/git-sample/*.h5 cmake -S . -B build \ -DCMAKE_C_COMPILER=$(which icc) \ -DCMAKE_CXX_COMPILER=$(which icpc) \ @@ -44,7 +43,6 @@ jobs: run: | set +e; source /opt/intel/oneapi/setvars.sh; set -e share/openPMD/download_samples.sh build - chmod u-w build/samples/git-sample/*.h5 cmake -S . -B build \ -DCMAKE_C_COMPILER=$(which icx) \ -DCMAKE_CXX_COMPILER=$(which icpx) \ diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 9fd23b4803..6c35b3aab1 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -30,7 +30,6 @@ jobs: spack install share/openPMD/download_samples.sh build - chmod u-w build/samples/git-sample/*.h5 cmake -S . -B build \ -DopenPMD_USE_PYTHON=OFF \ -DopenPMD_USE_MPI=OFF \ @@ -67,7 +66,6 @@ jobs: spack install share/openPMD/download_samples.sh build - chmod u-w build/samples/git-sample/*.h5 cmake -S . -B build \ -DopenPMD_USE_PYTHON=OFF \ -DopenPMD_USE_MPI=ON \ @@ -104,7 +102,6 @@ jobs: spack install share/openPMD/download_samples.sh build - chmod u-w build/samples/git-sample/*.h5 cmake -S . -B build \ -DopenPMD_USE_PYTHON=OFF \ -DopenPMD_USE_MPI=ON \ @@ -139,7 +136,6 @@ jobs: spack install share/openPMD/download_samples.sh build - chmod u-w build/samples/git-sample/*.h5 cmake -S . -B build \ -DopenPMD_USE_PYTHON=ON \ -DopenPMD_USE_MPI=OFF \ @@ -176,7 +172,6 @@ jobs: spack install share/openPMD/download_samples.sh build - chmod u-w build/samples/git-sample/*.h5 cmake -S . -B build \ -DopenPMD_USE_PYTHON=OFF \ -DopenPMD_USE_MPI=ON \ @@ -228,7 +223,6 @@ jobs: spack install share/openPMD/download_samples.sh build - chmod u-w build/samples/git-sample/*.h5 cmake -S . -B build \ -DopenPMD_USE_PYTHON=ON \ -DopenPMD_USE_MPI=ON \ @@ -253,7 +247,6 @@ jobs: env: {CXXFLAGS: -Werror, PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig} run: | share/openPMD/download_samples.sh build - chmod u-w build/samples/git-sample/*.h5 cmake -S . -B build \ -DopenPMD_USE_PYTHON=ON \ -DopenPMD_USE_MPI=ON \ @@ -278,7 +271,6 @@ jobs: env: {CXXFLAGS: -Werror} run: | share/openPMD/download_samples.sh build - chmod u-w build/samples/git-sample/*.h5 cmake -S . -B build \ -DopenPMD_USE_PYTHON=ON \ -DopenPMD_USE_MPI=OFF \ @@ -313,7 +305,6 @@ jobs: source activate openpmd-api-dev share/openPMD/download_samples.sh build - chmod u-w build/samples/git-sample/*.h5 cmake -S . -B build \ -DopenPMD_USE_PYTHON=ON \ -DopenPMD_USE_MPI=ON \ diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index f0b2d6bda5..40528ae36e 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -34,7 +34,6 @@ jobs: # std::filesystem needs macOS 10.15 run: | share/openPMD/download_samples.sh build - chmod u-w build/samples/git-sample/*.h5 cmake -S . -B build \ -DopenPMD_USE_PYTHON=ON \ -DopenPMD_USE_MPI=ON \ diff --git a/.github/workflows/nvidia.yml b/.github/workflows/nvidia.yml index 847d40e3ac..446872426b 100644 --- a/.github/workflows/nvidia.yml +++ b/.github/workflows/nvidia.yml @@ -23,7 +23,6 @@ jobs: which nvcc || echo "nvcc not in PATH!" share/openPMD/download_samples.sh build - chmod u-w build/samples/git-sample/*.h5 cmake -S . -B build \ -DopenPMD_USE_PYTHON=OFF \ -DopenPMD_USE_MPI=OFF \ @@ -59,7 +58,6 @@ jobs: cmake --version share/openPMD/download_samples.sh build - chmod u-w build/samples/git-sample/*.h5 cmake -S . -B build \ -DCMAKE_C_COMPILER=$(which nvc) \ -DCMAKE_CXX_COMPILER=$(which nvc++) \ diff --git a/.github/workflows/tooling.yml b/.github/workflows/tooling.yml index 9a1bb08977..65cdc05780 100644 --- a/.github/workflows/tooling.yml +++ b/.github/workflows/tooling.yml @@ -28,7 +28,6 @@ jobs: spack install share/openPMD/download_samples.sh build - chmod u-w build/samples/git-sample/*.h5 cmake -S . -B build \ -DCMAKE_CXX_CLANG_TIDY="$(which clang-tidy);-system-headers=0" \ -DopenPMD_USE_INVASIVE_TESTS=ON @@ -59,7 +58,6 @@ jobs: spack install SOURCEPATH="$(pwd)" share/openPMD/download_samples.sh build - chmod u-w build/samples/git-sample/*.h5 export LDFLAGS="${LDFLAGS} -fsanitize=address,undefined -shared-libsan" export CXXFLAGS="${CXXFLAGS} -fsanitize=address,undefined -shared-libsan" cmake -S . -B build \ diff --git a/share/openPMD/download_samples.sh b/share/openPMD/download_samples.sh index 199943681b..aef491ca1f 100755 --- a/share/openPMD/download_samples.sh +++ b/share/openPMD/download_samples.sh @@ -40,4 +40,12 @@ unzip diags.zip mv diags/hdf5/data00000050.h5 samples/issue-sample/empty_alternate_fbpic_00000050.h5 rm -rf diags.zip diags +# make sure we do not need write access when reading data +chmod u-w samples/git-sample/*.h5 +chmod u-w samples/git-sample/thetaMode/*.h5 +chmod u-w samples/samples/issue-sample/*.h5 +chmod u-w samples/samples/issue-sample/no_fields/*.h5 +chmod u-w samples/samples/issue-sample/no_particles/*.h5 +find samples/git-sample/3d-bp4 -type f -exec chmod u-w {} \; + cd ${orgdir} From f803cf36d4983998ef38dfea20dd7b49a4f686cb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 19 May 2022 16:41:07 +0200 Subject: [PATCH 46/70] [pre-commit.ci] pre-commit autoupdate (#1276) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/Lucas-C/pre-commit-hooks: v1.1.13 → v1.1.14](https://github.com/Lucas-C/pre-commit-hooks/compare/v1.1.13...v1.1.14) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8716c33bf5..6dc9b346bc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,7 +48,7 @@ repos: # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.1.13 + rev: v1.1.14 hooks: - id: remove-tabs From b10fd69ece7eeb619d12b9a43f8987c87e72e3b5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 23 May 2022 22:15:00 -0700 Subject: [PATCH 47/70] [pre-commit.ci] pre-commit autoupdate (#1279) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/Lucas-C/pre-commit-hooks: v1.1.14 → v1.2.0](https://github.com/Lucas-C/pre-commit-hooks/compare/v1.1.14...v1.2.0) - [github.com/hadialqattan/pycln: v1.3.2 → v1.3.3](https://github.com/hadialqattan/pycln/compare/v1.3.2...v1.3.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6dc9b346bc..eff6613ea5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,7 +48,7 @@ repos: # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.1.14 + rev: v1.2.0 hooks: - id: remove-tabs @@ -71,7 +71,7 @@ repos: # Autoremoves unused Python imports - repo: https://github.com/hadialqattan/pycln - rev: v1.3.2 + rev: v1.3.3 hooks: - id: pycln name: pycln (python) From edafd4fa90cedb49bfcdbc543f531553cf8bdc4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 May 2022 11:01:41 -0700 Subject: [PATCH 48/70] Build(deps): Bump s-weigand/setup-conda from 1.1.0 to 1.1.1 (#1284) Bumps [s-weigand/setup-conda](https://github.com/s-weigand/setup-conda) from 1.1.0 to 1.1.1. - [Release notes](https://github.com/s-weigand/setup-conda/releases) - [Commits](https://github.com/s-weigand/setup-conda/compare/v1.1.0...v1.1.1) --- updated-dependencies: - dependency-name: s-weigand/setup-conda dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/source.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/source.yml b/.github/workflows/source.yml index 6aa8a58290..bad03d9dec 100644 --- a/.github/workflows/source.yml +++ b/.github/workflows/source.yml @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - - uses: s-weigand/setup-conda@v1.1.0 + - uses: s-weigand/setup-conda@v1.1.1 with: update-conda: true conda-channels: conda-forge From 20e271b1353caba349a6c7081c0ba84d4723e2a2 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 8 Jun 2022 15:12:21 -0700 Subject: [PATCH 49/70] ADIOS1: Link Own `IOTask.cpp` (#1288) Related to #1287: We use a function of `IOTask.cpp` in the separate ADIOS1 libs that we wrap. --- CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 574344c5db..dcebf9e12e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -488,13 +488,15 @@ set(IO_ADIOS1_SEQUENTIAL_SOURCE src/auxiliary/Filesystem.cpp src/ChunkInfo.cpp src/IO/ADIOS/CommonADIOS1IOHandler.cpp - src/IO/ADIOS/ADIOS1IOHandler.cpp) + src/IO/ADIOS/ADIOS1IOHandler.cpp + src/IO/IOTask.cpp) set(IO_ADIOS1_SOURCE src/Error.cpp src/auxiliary/Filesystem.cpp src/ChunkInfo.cpp src/IO/ADIOS/CommonADIOS1IOHandler.cpp - src/IO/ADIOS/ParallelADIOS1IOHandler.cpp) + src/IO/ADIOS/ParallelADIOS1IOHandler.cpp + src/IO/IOTask.cpp) # library if(openPMD_BUILD_SHARED_LIBS) From faa5bf40d4bb301d5bda4709989f2b56fa78b0f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 17 Jun 2022 10:58:25 +0200 Subject: [PATCH 50/70] Distinguish all char types in ADIOS2 backend (#1275) * basicDatatype and toVectorType generic over datatype * Introduce ADIOS2Datatype to properly support BP5 * CI fixes * Compatibility with ADIOS 2.7.0 * Remove complex long double datatypes * Remove ADIOS2Datatype, add SCHAR to openPMD::Datatype * Add missing copyright header --- include/openPMD/Datatype.hpp | 56 ++++----- include/openPMD/DatatypeHelpers.hpp | 7 ++ include/openPMD/Datatype_internal.hpp | 114 +++++++++++++++++ include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp | 2 + include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp | 16 ++- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 14 +-- .../IO/ADIOS/ADIOS2PreloadAttributes.hpp | 8 +- include/openPMD/Iteration.hpp | 2 - include/openPMD/auxiliary/Memory.hpp | 5 + include/openPMD/backend/Attribute.hpp | 4 +- include/openPMD/binding/python/Numpy.hpp | 2 + src/Datatype.cpp | 119 ++++++++++-------- src/IO/ADIOS/ADIOS2Auxiliary.cpp | 19 +-- src/IO/ADIOS/ADIOS2IOHandler.cpp | 53 ++++---- src/IO/ADIOS/CommonADIOS1IOHandler.cpp | 16 +++ src/IO/HDF5/HDF5Auxiliary.cpp | 10 ++ src/IO/HDF5/HDF5IOHandler.cpp | 12 ++ src/RecordComponent.cpp | 71 +++-------- src/backend/Attributable.cpp | 12 ++ src/binding/python/PatchRecordComponent.cpp | 54 +++----- src/binding/python/RecordComponent.cpp | 4 + test/SerialIOTest.cpp | 20 ++- 22 files changed, 380 insertions(+), 240 deletions(-) create mode 100644 include/openPMD/Datatype_internal.hpp diff --git a/include/openPMD/Datatype.hpp b/include/openPMD/Datatype.hpp index 66133881c8..39be86ab43 100644 --- a/include/openPMD/Datatype.hpp +++ b/include/openPMD/Datatype.hpp @@ -42,7 +42,8 @@ namespace openPMD enum class Datatype : int { CHAR, - UCHAR, // SCHAR, + UCHAR, + SCHAR, SHORT, INT, LONG, @@ -74,6 +75,7 @@ enum class Datatype : int VEC_CFLOAT, VEC_CDOUBLE, VEC_CLONG_DOUBLE, + VEC_SCHAR, VEC_STRING, ARR_DBL_7, @@ -124,6 +126,10 @@ inline constexpr Datatype determineDatatype() { return DT::UCHAR; } + else if (decay_equiv::value) + { + return DT::SCHAR; + } else if (decay_equiv::value) { return DT::SHORT; @@ -208,6 +214,10 @@ inline constexpr Datatype determineDatatype() { return DT::VEC_UCHAR; } + else if (decay_equiv>::value) + { + return DT::VEC_SCHAR; + } else if (decay_equiv>::value) { return DT::VEC_USHORT; @@ -276,6 +286,10 @@ inline constexpr Datatype determineDatatype(std::shared_ptr) { return DT::UCHAR; } + else if (decay_equiv::value) + { + return DT::SCHAR; + } else if (decay_equiv::value) { return DT::SHORT; @@ -360,6 +374,10 @@ inline constexpr Datatype determineDatatype(std::shared_ptr) { return DT::VEC_UCHAR; } + else if (decay_equiv>::value) + { + return DT::VEC_SCHAR; + } else if (decay_equiv>::value) { return DT::VEC_USHORT; @@ -434,9 +452,9 @@ inline size_t toBytes(Datatype d) case DT::UCHAR: case DT::VEC_UCHAR: return sizeof(unsigned char); - // case DT::SCHAR: - // case DT::VEC_SCHAR: - // return sizeof(signed char); + case DT::SCHAR: + case DT::VEC_SCHAR: + return sizeof(signed char); case DT::SHORT: case DT::VEC_SHORT: return sizeof(short); @@ -776,36 +794,6 @@ inline bool isSame(openPMD::Datatype const d, openPMD::Datatype const e) return false; } -namespace detail -{ - template - struct BasicDatatypeHelper - { - Datatype m_dt = determineDatatype(); - }; - - template - struct BasicDatatypeHelper> - { - Datatype m_dt = BasicDatatypeHelper{}.m_dt; - }; - - template - struct BasicDatatypeHelper> - { - Datatype m_dt = BasicDatatypeHelper{}.m_dt; - }; - - struct BasicDatatype - { - template - static Datatype call(); - - template - static Datatype call(); - }; -} // namespace detail - /** * @brief basicDatatype Strip openPMD Datatype of std::vector, std::array et. * al. diff --git a/include/openPMD/DatatypeHelpers.hpp b/include/openPMD/DatatypeHelpers.hpp index f9332d2ddb..07f3c7fc37 100644 --- a/include/openPMD/DatatypeHelpers.hpp +++ b/include/openPMD/DatatypeHelpers.hpp @@ -110,6 +110,8 @@ auto switchType(Datatype dt, Args &&...args) case Datatype::UCHAR: return Action::template call( std::forward(args)...); + case Datatype::SCHAR: + return Action::template call(std::forward(args)...); case Datatype::SHORT: return Action::template call(std::forward(args)...); case Datatype::INT: @@ -164,6 +166,9 @@ auto switchType(Datatype dt, Args &&...args) case Datatype::VEC_UCHAR: return Action::template call >( std::forward(args)...); + case Datatype::VEC_SCHAR: + return Action::template call >( + std::forward(args)...); case Datatype::VEC_USHORT: return Action::template call >( std::forward(args)...); @@ -241,6 +246,8 @@ auto switchNonVectorType(Datatype dt, Args &&...args) case Datatype::UCHAR: return Action::template call( std::forward(args)...); + case Datatype::SCHAR: + return Action::template call(std::forward(args)...); case Datatype::SHORT: return Action::template call(std::forward(args)...); case Datatype::INT: diff --git a/include/openPMD/Datatype_internal.hpp b/include/openPMD/Datatype_internal.hpp new file mode 100644 index 0000000000..64e9061c54 --- /dev/null +++ b/include/openPMD/Datatype_internal.hpp @@ -0,0 +1,114 @@ +/* Copyright 2022 Franz Poeschel, Axel Huebl + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/auxiliary/TypeTraits.hpp" + +#include +#include +#include + +namespace openPMD +{ +namespace detail +{ + template + struct BasicDatatype + { + using DT = typename DoDetermineDatatype::DT_enum; + + template + constexpr static DT call() + { + if constexpr (auxiliary::IsVector_v) + { + return DoDetermineDatatype::template call< + typename T::value_type>(); + } + else if constexpr (auxiliary::IsArray_v) + { + return DoDetermineDatatype::template call< + typename T::value_type>(); + } + else + { + return DoDetermineDatatype::template call(); + } +#if defined(__INTEL_COMPILER) +/* + * ICPC has trouble with if constexpr, thinking that return statements are + * missing afterwards. Deactivate the warning. + * Note that putting a statement here will not help to fix this since it will + * then complain about unreachable code. + * https://community.intel.com/t5/Intel-C-Compiler/quot-if-constexpr-quot-and-quot-missing-return-statement-quot-in/td-p/1154551 + */ +#pragma warning(disable : 1011) + } +#pragma warning(default : 1011) +#else + } +#endif + + constexpr static char const *errorMsg = + "basicDatatype: received unknown datatype."; + }; + + template + struct ToVectorType + { + using DT = typename DoDetermineDatatype::DT_enum; + + template + constexpr static DT call() + { + if constexpr (auxiliary::IsVector_v) + { + return DoDetermineDatatype::template call(); + } + else if constexpr (auxiliary::IsArray_v) + { + return DoDetermineDatatype::template call< + std::vector>(); + } + else + { + return DoDetermineDatatype::template call>(); + } +#if defined(__INTEL_COMPILER) +/* + * ICPC has trouble with if constexpr, thinking that return statements are + * missing afterwards. Deactivate the warning. + * Note that putting a statement here will not help to fix this since it will + * then complain about unreachable code. + * https://community.intel.com/t5/Intel-C-Compiler/quot-if-constexpr-quot-and-quot-missing-return-statement-quot-in/td-p/1154551 + */ +#pragma warning(disable : 1011) + } +#pragma warning(default : 1011) +#else + } +#endif + + constexpr static char const *errorMsg = + "toVectorType: received unknown datatype."; + }; +} // namespace detail +} // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp index 838725da94..c4e99d773f 100644 --- a/include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp @@ -86,6 +86,8 @@ inline ADIOS_DATATYPES getBP1DataType(Datatype dtype) { case DT::CHAR: case DT::VEC_CHAR: + case DT::SCHAR: + case DT::VEC_SCHAR: return adios_byte; case DT::UCHAR: case DT::VEC_UCHAR: diff --git a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp index 593c937eba..87a06f1e27 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp @@ -50,13 +50,13 @@ namespace detail }; template - struct ToDatatypeHelper > + struct ToDatatypeHelper> { static std::string type(); }; template - struct ToDatatypeHelper > + struct ToDatatypeHelper> { static std::string type(); }; @@ -149,6 +149,8 @@ auto switchAdios2AttributeType(Datatype dt, Args &&...args) case Datatype::UCHAR: return Action::template call( std::forward(args)...); + case Datatype::SCHAR: + return Action::template call(std::forward(args)...); case Datatype::SHORT: return Action::template call(std::forward(args)...); case Datatype::INT: @@ -175,10 +177,10 @@ auto switchAdios2AttributeType(Datatype dt, Args &&...args) case Datatype::LONG_DOUBLE: return Action::template call(std::forward(args)...); case Datatype::CFLOAT: - return Action::template call >( + return Action::template call>( std::forward(args)...); case Datatype::CDOUBLE: - return Action::template call >( + return Action::template call>( std::forward(args)...); // missing std::complex< long double > type in ADIOS2 v2.6.0 // case Datatype::CLONG_DOUBLE: @@ -227,6 +229,8 @@ auto switchAdios2VariableType(Datatype dt, Args &&...args) case Datatype::UCHAR: return Action::template call( std::forward(args)...); + case Datatype::SCHAR: + return Action::template call(std::forward(args)...); case Datatype::SHORT: return Action::template call(std::forward(args)...); case Datatype::INT: @@ -253,10 +257,10 @@ auto switchAdios2VariableType(Datatype dt, Args &&...args) case Datatype::LONG_DOUBLE: return Action::template call(std::forward(args)...); case Datatype::CFLOAT: - return Action::template call >( + return Action::template call>( std::forward(args)...); case Datatype::CDOUBLE: - return Action::template call >( + return Action::template call>( std::forward(args)...); // missing std::complex< long double > type in ADIOS2 v2.6.0 // case Datatype::CLONG_DOUBLE: diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index aed11094e3..18a4aad192 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -575,7 +575,7 @@ namespace detail detail::BufferedAttributeWrite ¶ms, T value); - static void readAttribute( + static Datatype readAttribute( detail::PreloadAdiosAttributes const &, std::string name, std::shared_ptr resource); @@ -614,7 +614,7 @@ namespace detail "attribute types"); } - static void readAttribute( + static Datatype readAttribute( detail::PreloadAdiosAttributes const &, std::string, std::shared_ptr) @@ -647,7 +647,7 @@ namespace detail "vector attribute types"); } - static void readAttribute( + static Datatype readAttribute( detail::PreloadAdiosAttributes const &, std::string, std::shared_ptr) @@ -675,7 +675,7 @@ namespace detail detail::BufferedAttributeWrite ¶ms, const std::vector &value); - static void readAttribute( + static Datatype readAttribute( detail::PreloadAdiosAttributes const &, std::string name, std::shared_ptr resource); @@ -713,7 +713,7 @@ namespace detail detail::BufferedAttributeWrite ¶ms, const std::vector &vec); - static void readAttribute( + static Datatype readAttribute( detail::PreloadAdiosAttributes const &, std::string name, std::shared_ptr resource); @@ -751,7 +751,7 @@ namespace detail detail::BufferedAttributeWrite ¶ms, const std::array &value); - static void readAttribute( + static Datatype readAttribute( detail::PreloadAdiosAttributes const &, std::string name, std::shared_ptr resource); @@ -816,7 +816,7 @@ namespace detail detail::BufferedAttributeWrite ¶ms, bool value); - static void readAttribute( + static Datatype readAttribute( detail::PreloadAdiosAttributes const &, std::string name, std::shared_ptr resource); diff --git a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp index 75c2613674..c2d9da0cbc 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp @@ -31,6 +31,7 @@ #include #include "openPMD/Datatype.hpp" +#include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" namespace openPMD { @@ -150,13 +151,6 @@ namespace detail } AttributeLocation const &location = it->second; Datatype determinedDatatype = determineDatatype(); - if (std::is_same::value) - { - // workaround: we use Datatype::CHAR to represent ADIOS2 signed char - // (ADIOS2 does not have chars with unspecified signed-ness - // anyway) - determinedDatatype = Datatype::CHAR; - } if (location.dt != determinedDatatype) { std::stringstream errorMsg; diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 844aaea12f..ae58e787e1 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -295,8 +295,6 @@ class Iteration : public Attributable * @brief End an IO step on the IO file (or file-like object) * containing this iteration. In case of group-based iteration * layout, this will be the complete Series. - * - * @return AdvanceStatus */ void endStep(); diff --git a/include/openPMD/auxiliary/Memory.hpp b/include/openPMD/auxiliary/Memory.hpp index 8752f0a4da..11a47d215f 100644 --- a/include/openPMD/auxiliary/Memory.hpp +++ b/include/openPMD/auxiliary/Memory.hpp @@ -133,6 +133,11 @@ namespace auxiliary data = new unsigned char[numPoints]; del = [](void *p) { delete[] static_cast(p); }; break; + case DT::VEC_SCHAR: + case DT::SCHAR: + data = new signed char[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; case DT::BOOL: data = new bool[numPoints]; del = [](void *p) { delete[] static_cast(p); }; diff --git a/include/openPMD/backend/Attribute.hpp b/include/openPMD/backend/Attribute.hpp index f83b99a2ae..0ac26d9dec 100644 --- a/include/openPMD/backend/Attribute.hpp +++ b/include/openPMD/backend/Attribute.hpp @@ -53,7 +53,8 @@ class Attribute : public auxiliary::Variant< Datatype, char, - unsigned char, // signed char, + unsigned char, + signed char, short, int, long, @@ -85,6 +86,7 @@ class Attribute std::vector >, std::vector >, std::vector >, + std::vector, std::vector, std::array, bool> diff --git a/include/openPMD/binding/python/Numpy.hpp b/include/openPMD/binding/python/Numpy.hpp index f0fc7c1b84..f28ac8dbc2 100644 --- a/include/openPMD/binding/python/Numpy.hpp +++ b/include/openPMD/binding/python/Numpy.hpp @@ -136,6 +136,8 @@ inline pybind11::dtype dtype_to_numpy(Datatype const dt) { case DT::CHAR: case DT::VEC_CHAR: + case DT::SCHAR: + case DT::VEC_SCHAR: case DT::STRING: case DT::VEC_STRING: return pybind11::dtype("b"); diff --git a/src/Datatype.cpp b/src/Datatype.cpp index 683cfbb06f..71565b3d52 100644 --- a/src/Datatype.cpp +++ b/src/Datatype.cpp @@ -20,6 +20,7 @@ */ #include "openPMD/Datatype.hpp" #include "openPMD/DatatypeHelpers.hpp" +#include "openPMD/Datatype_internal.hpp" #include #include @@ -46,6 +47,9 @@ std::ostream &operator<<(std::ostream &os, openPMD::Datatype const &d) case DT::UCHAR: os << "UCHAR"; break; + case DT::SCHAR: + os << "SCHAR"; + break; case DT::SHORT: os << "SHORT"; break; @@ -139,6 +143,9 @@ std::ostream &operator<<(std::ostream &os, openPMD::Datatype const &d) case DT::VEC_CLONG_DOUBLE: os << "VEC_CLONG_DOUBLE"; break; + case DT::VEC_SCHAR: + os << "VEC_SCHAR"; + break; case DT::VEC_STRING: os << "VEC_STRING"; break; @@ -161,6 +168,7 @@ Datatype stringToDatatype(std::string s) static std::unordered_map m{ {"CHAR", Datatype::CHAR}, {"UCHAR", Datatype::UCHAR}, + {"SCHAR", Datatype::SCHAR}, {"SHORT", Datatype::SHORT}, {"INT", Datatype::INT}, {"LONG", Datatype::LONG}, @@ -192,6 +200,7 @@ Datatype stringToDatatype(std::string s) {"VEC_CFLOAT", Datatype::VEC_CFLOAT}, {"VEC_CDOUBLE", Datatype::VEC_CDOUBLE}, {"VEC_CLONG_DOUBLE", Datatype::VEC_CLONG_DOUBLE}, + {"VEC_SCHAR", Datatype::VEC_SCHAR}, {"VEC_STRING", Datatype::VEC_STRING}, {"ARR_DBL_7", Datatype::ARR_DBL_7}, {"BOOL", Datatype::BOOL}, @@ -216,67 +225,71 @@ std::string datatypeToString(openPMD::Datatype dt) } std::vector openPMD_Datatypes{ - Datatype::CHAR, Datatype::UCHAR, Datatype::SHORT, - Datatype::INT, Datatype::LONG, Datatype::LONGLONG, - Datatype::USHORT, Datatype::UINT, Datatype::ULONG, - Datatype::ULONGLONG, Datatype::FLOAT, Datatype::DOUBLE, - Datatype::LONG_DOUBLE, Datatype::CFLOAT, Datatype::CDOUBLE, - Datatype::CLONG_DOUBLE, Datatype::STRING, Datatype::VEC_CHAR, - Datatype::VEC_SHORT, Datatype::VEC_INT, Datatype::VEC_LONG, - Datatype::VEC_LONGLONG, Datatype::VEC_UCHAR, Datatype::VEC_USHORT, - Datatype::VEC_UINT, Datatype::VEC_ULONG, Datatype::VEC_ULONGLONG, - Datatype::VEC_FLOAT, Datatype::VEC_DOUBLE, Datatype::VEC_LONG_DOUBLE, - Datatype::VEC_CFLOAT, Datatype::VEC_CDOUBLE, Datatype::VEC_CLONG_DOUBLE, - Datatype::VEC_STRING, Datatype::ARR_DBL_7, Datatype::BOOL, + Datatype::CHAR, + Datatype::UCHAR, + Datatype::SCHAR, + Datatype::SHORT, + Datatype::INT, + Datatype::LONG, + Datatype::LONGLONG, + Datatype::USHORT, + Datatype::UINT, + Datatype::ULONG, + Datatype::ULONGLONG, + Datatype::FLOAT, + Datatype::DOUBLE, + Datatype::LONG_DOUBLE, + Datatype::CFLOAT, + Datatype::CDOUBLE, + Datatype::CLONG_DOUBLE, + Datatype::STRING, + Datatype::VEC_CHAR, + Datatype::VEC_SHORT, + Datatype::VEC_INT, + Datatype::VEC_LONG, + Datatype::VEC_LONGLONG, + Datatype::VEC_UCHAR, + Datatype::VEC_USHORT, + Datatype::VEC_UINT, + Datatype::VEC_ULONG, + Datatype::VEC_ULONGLONG, + Datatype::VEC_FLOAT, + Datatype::VEC_DOUBLE, + Datatype::VEC_LONG_DOUBLE, + Datatype::VEC_CFLOAT, + Datatype::VEC_CDOUBLE, + Datatype::VEC_CLONG_DOUBLE, + Datatype::VEC_SCHAR, + Datatype::VEC_STRING, + Datatype::ARR_DBL_7, + Datatype::BOOL, Datatype::UNDEFINED}; -Datatype basicDatatype(Datatype dt) +namespace { - return switchType(dt); -} + struct DoDetermineDatatype + { + /* + * Suppress wrong compiler warnings. + * The typedef is needed in instantiation. + */ + using DT_enum [[maybe_unused]] = Datatype; -Datatype toVectorType(Datatype dt) -{ - auto initializer = []() { - std::map res; - for (Datatype d : openPMD_Datatypes) + template + static constexpr Datatype call() { - if (d == Datatype::ARR_DBL_7 || d == Datatype::UNDEFINED) - continue; - Datatype basic = basicDatatype(d); - if (basic == d) - continue; - res[basic] = d; + return determineDatatype(); } - return res; }; - static auto map(initializer()); - auto it = map.find(dt); - if (it != map.end()) - { - return it->second; - } - else - { - std::cerr << "Encountered non-basic type " << dt << ", aborting." - << std::endl; - throw std::runtime_error("toVectorType: passed non-basic type."); - } -} +} // namespace -namespace detail +Datatype basicDatatype(Datatype dt) { - template - Datatype BasicDatatype::call() - { - static auto res = BasicDatatypeHelper{}.m_dt; - return res; - } + return switchType>(dt); +} - template - Datatype BasicDatatype::call() - { - throw std::runtime_error("basicDatatype: received unknown datatype."); - } -} // namespace detail +Datatype toVectorType(Datatype dt) +{ + return switchType>(dt); +} } // namespace openPMD diff --git a/src/IO/ADIOS/ADIOS2Auxiliary.cpp b/src/IO/ADIOS/ADIOS2Auxiliary.cpp index 9a4b645c0f..d4c08408ce 100644 --- a/src/IO/ADIOS/ADIOS2Auxiliary.cpp +++ b/src/IO/ADIOS/ADIOS2Auxiliary.cpp @@ -22,6 +22,8 @@ #include "openPMD/config.hpp" #if openPMD_HAVE_ADIOS2 #include "openPMD/Datatype.hpp" +#include "openPMD/DatatypeHelpers.hpp" +#include "openPMD/Datatype_internal.hpp" #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include @@ -35,7 +37,7 @@ std::string ToDatatypeHelper::type() } template -std::string ToDatatypeHelper >::type() +std::string ToDatatypeHelper>::type() { return @@ -43,7 +45,7 @@ std::string ToDatatypeHelper >::type() } template -std::string ToDatatypeHelper >::type() +std::string ToDatatypeHelper>::type() { return @@ -72,7 +74,7 @@ Datatype fromADIOS2Type(std::string const &dt, bool verbose) static std::map map{ {"string", Datatype::STRING}, {"char", Datatype::CHAR}, - {"signed char", Datatype::CHAR}, + {"signed char", Datatype::SCHAR}, {"unsigned char", Datatype::UCHAR}, {"short", Datatype::SHORT}, {"unsigned short", Datatype::USHORT}, @@ -87,11 +89,11 @@ Datatype fromADIOS2Type(std::string const &dt, bool verbose) {"long double", Datatype::LONG_DOUBLE}, {"float complex", Datatype::CFLOAT}, {"double complex", Datatype::CDOUBLE}, - {"long double complex", - Datatype::CLONG_DOUBLE}, // does not exist as of 2.7.0 but might come - // later + // does not exist as of 2.7.0 but might come later + // {"long double complex", + // Datatype::CLONG_DOUBLE}, {"uint8_t", Datatype::UCHAR}, - {"int8_t", Datatype::CHAR}, + {"int8_t", Datatype::SCHAR}, {"uint16_t", determineDatatype()}, {"int16_t", determineDatatype()}, {"uint32_t", determineDatatype()}, @@ -217,7 +219,8 @@ Datatype attributeInfo( } else if ( shape.size() == 2 && - (basicType == Datatype::CHAR || basicType == Datatype::UCHAR)) + (basicType == Datatype::CHAR || basicType == Datatype::SCHAR || + basicType == Datatype::UCHAR)) { return Datatype::VEC_STRING; } diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 072c5cd285..2e53700bea 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -166,10 +166,10 @@ void ADIOS2IOHandlerImpl::init(json::TracingJSON cfg) m_schema = auxiliary::getEnvNum("OPENPMD2_ADIOS2_SCHEMA", m_schema); } -std::optional > +std::optional> ADIOS2IOHandlerImpl::getOperators(json::TracingJSON cfg) { - using ret_t = std::optional >; + using ret_t = std::optional>; std::vector res; if (!cfg.json().contains("dataset")) { @@ -222,7 +222,7 @@ ADIOS2IOHandlerImpl::getOperators(json::TracingJSON cfg) return std::make_optional(std::move(res)); } -std::optional > +std::optional> ADIOS2IOHandlerImpl::getOperators() { return getOperators(m_config); @@ -1436,14 +1436,13 @@ namespace detail auto attr = IO.InquireAttribute(metaAttr); if (attr.Data().size() == 1 && attr.Data()[0] == 1) { - AttributeTypes::readAttribute( + return AttributeTypes::readAttribute( preloadedAttributes, name, resource); - return determineDatatype(); } } } - AttributeTypes::readAttribute(preloadedAttributes, name, resource); - return determineDatatype(); + return AttributeTypes::readAttribute( + preloadedAttributes, name, resource); } template @@ -1756,7 +1755,7 @@ namespace detail } template - void AttributeTypes::readAttribute( + Datatype AttributeTypes::readAttribute( detail::PreloadAdiosAttributes const &preloadedAttributes, std::string name, std::shared_ptr resource) @@ -1771,10 +1770,11 @@ namespace detail std::to_string(attr.shape.size()) + "D: " + name); } *resource = *attr.data; + return determineDatatype(); } template - void AttributeTypes >::createAttribute( + void AttributeTypes>::createAttribute( adios2::IO &IO, adios2::Engine &engine, detail::BufferedAttributeWrite ¶ms, @@ -1797,7 +1797,7 @@ namespace detail } template - void AttributeTypes >::readAttribute( + Datatype AttributeTypes>::readAttribute( detail::PreloadAdiosAttributes const &preloadedAttributes, std::string name, std::shared_ptr resource) @@ -1812,9 +1812,10 @@ namespace detail std::vector res(attr.shape[0]); std::copy_n(attr.data, attr.shape[0], res.data()); *resource = std::move(res); + return determineDatatype>(); } - void AttributeTypes >::createAttribute( + void AttributeTypes>::createAttribute( adios2::IO &IO, adios2::Engine &engine, detail::BufferedAttributeWrite ¶ms, @@ -1859,7 +1860,7 @@ namespace detail attr, params.bufferForVecString.data(), adios2::Mode::Deferred); } - void AttributeTypes >::readAttribute( + Datatype AttributeTypes>::readAttribute( detail::PreloadAdiosAttributes const &preloadedAttributes, std::string name, std::shared_ptr resource) @@ -1935,9 +1936,10 @@ namespace detail *resource = res; }; /* - * If writing char variables in ADIOS2, they might become either int8_t - * or uint8_t on disk depending on the platform. - * So allow reading from both types. + * If writing char variables in ADIOS2, they might become either int8_t, + * uint8_t or char on disk depending on platform, ADIOS2 version and + * ADIOS2 engine. + * So allow reading from all these types. */ switch (preloadedAttributes.attributeType(name)) { @@ -1946,10 +1948,10 @@ namespace detail * ADIOS2 does not have an explicit char type, * we don't have an explicit schar type. * Until this is fixed, we use CHAR to represent ADIOS signed char. + * @todo revisit this workaround and its necessity */ case Datatype::CHAR: { - using schar_t = signed char; - loadFromDatatype(schar_t{}); + loadFromDatatype(char{}); break; } case Datatype::UCHAR: { @@ -1957,16 +1959,21 @@ namespace detail loadFromDatatype(uchar_t{}); break; } + case Datatype::SCHAR: { + using schar_t = signed char; + loadFromDatatype(schar_t{}); + break; + } default: { throw std::runtime_error( - "[ADIOS2] Expecting 2D ADIOS variable of " - "type signed or unsigned char."); + "[ADIOS2] Expecting 2D ADIOS variable of any char type."); } } + return Datatype::VEC_STRING; } template - void AttributeTypes >::createAttribute( + void AttributeTypes>::createAttribute( adios2::IO &IO, adios2::Engine &engine, detail::BufferedAttributeWrite ¶ms, @@ -1988,7 +1995,7 @@ namespace detail } template - void AttributeTypes >::readAttribute( + Datatype AttributeTypes>::readAttribute( detail::PreloadAdiosAttributes const &preloadedAttributes, std::string name, std::shared_ptr resource) @@ -2005,6 +2012,7 @@ namespace detail std::array res; std::copy_n(attr.data, n, res.data()); *resource = std::move(res); + return determineDatatype>(); } void AttributeTypes::createAttribute( @@ -2019,7 +2027,7 @@ namespace detail IO, engine, params, toRep(value)); } - void AttributeTypes::readAttribute( + Datatype AttributeTypes::readAttribute( detail::PreloadAdiosAttributes const &preloadedAttributes, std::string name, std::shared_ptr resource) @@ -2035,6 +2043,7 @@ namespace detail } *resource = fromRep(*attr.data); + return Datatype::BOOL; } void BufferedGet::run(BufferedActions &ba) diff --git a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp index 5e30ecabb3..7bed2f15c7 100644 --- a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp @@ -108,6 +108,9 @@ void CommonADIOS1IOHandlerImpl::flush_attribute( case DT::VEC_UCHAR: nelems = att.get >().size(); break; + case DT::VEC_SCHAR: + nelems = att.get >().size(); + break; case DT::VEC_USHORT: nelems = att.get >().size(); break; @@ -156,6 +159,11 @@ void CommonADIOS1IOHandlerImpl::flush_attribute( *ptr = att.get(); break; } + case DT::SCHAR: { + auto ptr = reinterpret_cast(values.get()); + *ptr = att.get(); + break; + } case DT::SHORT: { auto ptr = reinterpret_cast(values.get()); *ptr = att.get(); @@ -274,6 +282,13 @@ void CommonADIOS1IOHandlerImpl::flush_attribute( ptr[i] = vec[i]; break; } + case DT::VEC_SCHAR: { + auto ptr = reinterpret_cast(values.get()); + auto const &vec = att.get >(); + for (size_t i = 0; i < vec.size(); ++i) + ptr[i] = vec[i]; + break; + } case DT::VEC_USHORT: { auto ptr = reinterpret_cast(values.get()); auto const &vec = att.get >(); @@ -1149,6 +1164,7 @@ void CommonADIOS1IOHandlerImpl::readDataset( case DT::ULONGLONG: case DT::CHAR: case DT::UCHAR: + case DT::SCHAR: case DT::BOOL: break; case DT::UNDEFINED: diff --git a/src/IO/HDF5/HDF5Auxiliary.cpp b/src/IO/HDF5/HDF5Auxiliary.cpp index f77ddc68d2..33201d3ab5 100644 --- a/src/IO/HDF5/HDF5Auxiliary.cpp +++ b/src/IO/HDF5/HDF5Auxiliary.cpp @@ -62,6 +62,9 @@ hid_t openPMD::GetH5DataType::operator()(Attribute const &att) case DT::UCHAR: case DT::VEC_UCHAR: return H5Tcopy(H5T_NATIVE_UCHAR); + case DT::SCHAR: + case DT::VEC_SCHAR: + return H5Tcopy(H5T_NATIVE_SCHAR); case DT::SHORT: case DT::VEC_SHORT: return H5Tcopy(H5T_NATIVE_SHORT); @@ -145,6 +148,7 @@ hid_t openPMD::getH5DataSpace(Attribute const &att) { case DT::CHAR: case DT::UCHAR: + case DT::SCHAR: case DT::SHORT: case DT::INT: case DT::LONG: @@ -198,6 +202,12 @@ hid_t openPMD::getH5DataSpace(Attribute const &att) H5Sset_extent_simple(vec_t_id, 1, dims, nullptr); return vec_t_id; } + case DT::VEC_SCHAR: { + hid_t vec_t_id = H5Screate(H5S_SIMPLE); + hsize_t dims[1] = {att.get >().size()}; + H5Sset_extent_simple(vec_t_id, 1, dims, nullptr); + return vec_t_id; + } case DT::VEC_USHORT: { hid_t vec_t_id = H5Screate(H5S_SIMPLE); hsize_t dims[1] = {att.get >().size()}; diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 51a013fca8..ae4e6588aa 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -1247,6 +1247,7 @@ void HDF5IOHandlerImpl::writeDataset( case DT::ULONGLONG: case DT::CHAR: case DT::UCHAR: + case DT::SCHAR: case DT::BOOL: status = H5Dwrite( dataset_id, @@ -1376,6 +1377,11 @@ void HDF5IOHandlerImpl::writeAttribute( status = H5Awrite(attribute_id, dataType, &u); break; } + case DT::SCHAR: { + auto u = att.get(); + status = H5Awrite(attribute_id, dataType, &u); + break; + } case DT::SHORT: { auto i = att.get(); status = H5Awrite(attribute_id, dataType, &i); @@ -1476,6 +1482,12 @@ void HDF5IOHandlerImpl::writeAttribute( dataType, att.get >().data()); break; + case DT::VEC_SCHAR: + status = H5Awrite( + attribute_id, + dataType, + att.get >().data()); + break; case DT::VEC_USHORT: status = H5Awrite( attribute_id, diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 9e7a93e6ea..bfa024a786 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -287,6 +287,20 @@ void RecordComponent::read() readBase(); } +namespace +{ + struct MakeConstant + { + template + static void call(RecordComponent rc, Attribute const &attr) + { + rc.makeConstant(attr.get()); + } + + static constexpr char const *errorMsg = "Unexpected constant datatype"; + }; +} // namespace + void RecordComponent::readBase() { using DT = Datatype; @@ -302,62 +316,7 @@ void RecordComponent::readBase() Attribute a(*aRead.resource); DT dtype = *aRead.dtype; written() = false; - switch (dtype) - { - case DT::LONG_DOUBLE: - makeConstant(a.get()); - break; - case DT::DOUBLE: - makeConstant(a.get()); - break; - case DT::FLOAT: - makeConstant(a.get()); - break; - case DT::CLONG_DOUBLE: - makeConstant(a.get >()); - break; - case DT::CDOUBLE: - makeConstant(a.get >()); - break; - case DT::CFLOAT: - makeConstant(a.get >()); - break; - case DT::SHORT: - makeConstant(a.get()); - break; - case DT::INT: - makeConstant(a.get()); - break; - case DT::LONG: - makeConstant(a.get()); - break; - case DT::LONGLONG: - makeConstant(a.get()); - break; - case DT::USHORT: - makeConstant(a.get()); - break; - case DT::UINT: - makeConstant(a.get()); - break; - case DT::ULONG: - makeConstant(a.get()); - break; - case DT::ULONGLONG: - makeConstant(a.get()); - break; - case DT::CHAR: - makeConstant(a.get()); - break; - case DT::UCHAR: - makeConstant(a.get()); - break; - case DT::BOOL: - makeConstant(a.get()); - break; - default: - throw std::runtime_error("Unexpected constant datatype"); - } + switchNonVectorType(dtype, *this, a); written() = true; aRead.name = "shape"; diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index b73c850409..aa2f600da1 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -333,6 +333,12 @@ void Attributable::readAttributes(ReadMode mode) a.get(), internal::SetAttributeMode::WhileReadingAttributes); break; + case DT::SCHAR: + setAttributeImpl( + att, + a.get(), + internal::SetAttributeMode::WhileReadingAttributes); + break; case DT::SHORT: setAttributeImpl( att, @@ -510,6 +516,12 @@ void Attributable::readAttributes(ReadMode mode) a.get > >(), internal::SetAttributeMode::WhileReadingAttributes); break; + case DT::VEC_SCHAR: + setAttributeImpl( + att, + a.get >(), + internal::SetAttributeMode::WhileReadingAttributes); + break; case DT::VEC_STRING: setAttributeImpl( att, diff --git a/src/binding/python/PatchRecordComponent.cpp b/src/binding/python/PatchRecordComponent.cpp index cecd57b252..a734e1d6b9 100644 --- a/src/binding/python/PatchRecordComponent.cpp +++ b/src/binding/python/PatchRecordComponent.cpp @@ -21,6 +21,7 @@ #include #include +#include "openPMD/DatatypeHelpers.hpp" #include "openPMD/auxiliary/ShareRaw.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" #include "openPMD/backend/PatchRecordComponent.hpp" @@ -29,6 +30,20 @@ namespace py = pybind11; using namespace openPMD; +namespace +{ +struct Prc_Load +{ + template + static void call(PatchRecordComponent &prc, py::array &a) + { + prc.load(shareRaw((T *)a.mutable_data())); + } + + static constexpr char const *errorMsg = "Datatype not known in 'load'!"; +}; +} // namespace + void init_PatchRecordComponent(py::module &m) { py::class_( @@ -49,44 +64,7 @@ void init_PatchRecordComponent(py::module &m) auto const dtype = dtype_to_numpy(prc.getDatatype()); auto a = py::array(dtype, prc.getExtent()[0]); - if (prc.getDatatype() == Datatype::CHAR) - prc.load(shareRaw((char *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::UCHAR) - prc.load( - shareRaw((unsigned char *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::SHORT) - prc.load(shareRaw((short *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::INT) - prc.load(shareRaw((int *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::LONG) - prc.load(shareRaw((long *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::LONGLONG) - prc.load( - shareRaw((long long *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::USHORT) - prc.load( - shareRaw((unsigned short *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::UINT) - prc.load( - shareRaw((unsigned int *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::ULONG) - prc.load( - shareRaw((unsigned long *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::ULONGLONG) - prc.load( - shareRaw((unsigned long long *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::LONG_DOUBLE) - prc.load( - shareRaw((long double *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::DOUBLE) - prc.load(shareRaw((double *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::FLOAT) - prc.load(shareRaw((float *)a.mutable_data())); - else if (prc.getDatatype() == Datatype::BOOL) - prc.load(shareRaw((bool *)a.mutable_data())); - else - throw std::runtime_error( - std::string("Datatype not known in 'load'!")); + switchNonVectorType(prc.getDatatype(), prc, a); return a; }) diff --git a/src/binding/python/RecordComponent.cpp b/src/binding/python/RecordComponent.cpp index 272b6b27cd..c2c1acfebb 100644 --- a/src/binding/python/RecordComponent.cpp +++ b/src/binding/python/RecordComponent.cpp @@ -558,6 +558,8 @@ void load_chunk( load_data((char)0); else if (r.getDatatype() == Datatype::UCHAR) load_data((unsigned char)0); + else if (r.getDatatype() == Datatype::SCHAR) + load_data((signed char)0); else if (r.getDatatype() == Datatype::SHORT) load_data((short)0); else if (r.getDatatype() == Datatype::INT) @@ -664,6 +666,8 @@ inline void load_chunk( load_data(char()); else if (r.getDatatype() == Datatype::UCHAR) load_data((unsigned char)0); + else if (r.getDatatype() == Datatype::SCHAR) + load_data((signed char)0); else if (r.getDatatype() == Datatype::SHORT) load_data(short()); else if (r.getDatatype() == Datatype::INT) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 155d4fc58d..6846ee0f14 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1237,6 +1237,8 @@ inline void dtype_test(const std::string &backend) s.setAttribute("char", c); unsigned char uc = 'u'; s.setAttribute("uchar", uc); + signed char sc = 's'; + s.setAttribute("schar", sc); int16_t i16 = 16; s.setAttribute("int16", i16); int32_t i32 = 32; @@ -1268,7 +1270,9 @@ inline void dtype_test(const std::string &backend) "vecInt64", std::vector({9223372036854775806, 9223372036854775807})); s.setAttribute( - "vecUchar", std::vector({'u', 'c', 'h', 'a', 'r'})); + "vecUchar", std::vector({'u', 'c', 'h', 'a', 'r'})); + s.setAttribute( + "vecSchar", std::vector({'s', 'c', 'h', 'a', 'r'})); s.setAttribute("vecUint16", std::vector({65534u, 65535u})); s.setAttribute( "vecUint32", std::vector({4294967294u, 4294967295u})); @@ -1349,6 +1353,7 @@ inline void dtype_test(const std::string &backend) REQUIRE(s.getAttribute("char").get() == 'c'); REQUIRE(s.getAttribute("uchar").get() == 'u'); + REQUIRE(s.getAttribute("schar").get() == 's'); REQUIRE(s.getAttribute("int16").get() == 16); REQUIRE(s.getAttribute("int32").get() == 32); REQUIRE(s.getAttribute("int64").get() == 64); @@ -1375,8 +1380,11 @@ inline void dtype_test(const std::string &backend) s.getAttribute("vecInt64").get >() == std::vector({9223372036854775806, 9223372036854775807})); REQUIRE( - s.getAttribute("vecUchar").get >() == - std::vector({'u', 'c', 'h', 'a', 'r'})); + s.getAttribute("vecUchar").get >() == + std::vector({'u', 'c', 'h', 'a', 'r'})); + REQUIRE( + s.getAttribute("vecSchar").get >() == + std::vector({'s', 'c', 'h', 'a', 'r'})); REQUIRE( s.getAttribute("vecUint16").get >() == std::vector({65534u, 65535u})); @@ -2292,7 +2300,7 @@ TEST_CASE("deletion_test", "[serial]") { for (auto const &t : testedFileExtensions()) { - if (t == "bp") + if (t == "bp" || t == "bp4" || t == "bp5") { continue; // deletion not implemented in ADIOS1 backend } @@ -5794,7 +5802,7 @@ void append_mode( } { Series read(filename, Access::READ_ONLY); - if (variableBased) + if (variableBased || extension == "bp5") { // in variable-based encodings, iterations are not parsed ahead of // time but as they go @@ -5825,7 +5833,7 @@ TEST_CASE("append_mode", "[serial]") { for (auto const &t : testedFileExtensions()) { - if (t == "bp") + if (t == "bp" || t == "bp4" || t == "bp5") { std::string jsonConfigOld = R"END( { From d64365bbc2348b5622e8e65d1a4f4dbfdc8ae313 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 20 Jun 2022 13:52:47 -0700 Subject: [PATCH 51/70] [pre-commit.ci] pre-commit autoupdate (#1292) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.2.0 → v4.3.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.2.0...v4.3.0) - [github.com/pre-commit/mirrors-clang-format: v14.0.3 → v14.0.4-1](https://github.com/pre-commit/mirrors-clang-format/compare/v14.0.3...v14.0.4-1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eff6613ea5..25439d83af 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ exclude: '^share/openPMD/thirdParty' # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.2.0 + rev: v4.3.0 hooks: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] @@ -65,7 +65,7 @@ repos: # clang-format v13 # to run manually, use .github/workflows/clang-format/clang-format.sh - repo: https://github.com/pre-commit/mirrors-clang-format - rev: v14.0.3 + rev: v14.0.5 hooks: - id: clang-format From 456605e11c13c9e5392d253ac73f6e734101cde5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 15 Jul 2022 19:49:54 +0200 Subject: [PATCH 52/70] Update catch2 to v2.13.9 (#1299) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update catch2 to v2.13.9 * Add to changelog * Docs: Catch2 2.13.9+ Co-authored-by: Martin Hořeňovský Co-authored-by: Axel Huebl --- CHANGELOG.rst | 1 + CMakeLists.txt | 4 +- NEWS.rst | 1 + README.md | 4 +- docs/source/dev/buildoptions.rst | 2 +- docs/source/dev/dependencies.rst | 2 +- .../catch2/include/catch2/catch.hpp | 537 ++++++++++-------- 7 files changed, 321 insertions(+), 230 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ad38af5cfd..45c42b1c54 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -25,6 +25,7 @@ Bug Fixes Other """"" +- Catch2: updated to 2.13.9 #1299 0.14.3 diff --git a/CMakeLists.txt b/CMakeLists.txt index dcebf9e12e..8cca88b188 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -536,9 +536,9 @@ if(openPMD_BUILD_TESTING) target_include_directories(openPMD::thirdparty::Catch2 SYSTEM INTERFACE $ ) - message(STATUS "Catch2: Using INTERNAL version '2.13.4'") + message(STATUS "Catch2: Using INTERNAL version '2.13.9'") else() - find_package(Catch2 2.13.4 REQUIRED CONFIG) + find_package(Catch2 2.13.9 REQUIRED CONFIG) target_link_libraries(openPMD::thirdparty::Catch2 INTERFACE Catch2::Catch2) message(STATUS "Catch2: Found version '${Catch2_VERSION}'") diff --git a/NEWS.rst b/NEWS.rst index 264225f275..7456daecfe 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -12,6 +12,7 @@ Building openPMD-api now requires a compiler that supports C++17 or newer. Python 3.10 is now supported. openPMD-api now depends on `toml11 `__ 3.7.1+. pybind11 2.9.1 is now the minimally supported version for Python support. +Catch2 2.13.9 is now the minimally supported version for tests. The following backend-specific members of the ``Dataset`` class have been removed: ``Dataset::setChunkSize()``, ``Dataset::setCompression()``, ``Dataset::setCustomTransform()``, ``Dataset::chunkSize``, ``Dataset::compression``, ``Dataset::transform``. They are replaced by backend-specific options in the JSON-based backend configuration. diff --git a/README.md b/README.md index 91e1a7550f..94b0531178 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ Required: * C++17 capable compiler, e.g., g++ 7+, clang 7+, MSVC 19.15+, icpc 19+, icpx Shipped internally in `share/openPMD/thirdParty/`: -* [Catch2](https://github.com/catchorg/Catch2) 2.13.4+ ([BSL-1.0](https://github.com/catchorg/Catch2/blob/master/LICENSE.txt)) +* [Catch2](https://github.com/catchorg/Catch2) 2.13.9+ ([BSL-1.0](https://github.com/catchorg/Catch2/blob/master/LICENSE.txt)) * [pybind11](https://github.com/pybind/pybind11) 2.9.1+ ([new BSD](https://github.com/pybind/pybind11/blob/master/LICENSE)) * [NLohmann-JSON](https://github.com/nlohmann/json) 3.9.1+ ([MIT](https://github.com/nlohmann/json/blob/develop/LICENSE.MIT)) * [toml11](https://github.com/ToruNiina/toml11) 3.7.1+ ([MIT](https://github.com/ToruNiina/toml11/blob/master/LICENSE)) @@ -271,7 +271,7 @@ The following options allow to switch to external installs: | CMake Option | Values | Library | Version | |---------------------------------|------------|---------------|---------| -| `openPMD_USE_INTERNAL_CATCH` | **ON**/OFF | Catch2 | 2.13.4+ | +| `openPMD_USE_INTERNAL_CATCH` | **ON**/OFF | Catch2 | 2.13.9+ | | `openPMD_USE_INTERNAL_PYBIND11` | **ON**/OFF | pybind11 | 2.9.1+ | | `openPMD_USE_INTERNAL_JSON` | **ON**/OFF | NLohmann-JSON | 3.9.1+ | | `openPMD_USE_INTERNAL_TOML11` | **ON**/OFF | toml11 | 3.7.1+ | diff --git a/docs/source/dev/buildoptions.rst b/docs/source/dev/buildoptions.rst index a897e78e77..b9cbcc9e40 100644 --- a/docs/source/dev/buildoptions.rst +++ b/docs/source/dev/buildoptions.rst @@ -68,7 +68,7 @@ The following options allow to switch to external installs of dependencies: ================================= =========== ======== ============= ======== CMake Option Values Installs Library Version ================================= =========== ======== ============= ======== -``openPMD_USE_INTERNAL_CATCH`` **ON**/OFF No Catch2 2.13.4+ +``openPMD_USE_INTERNAL_CATCH`` **ON**/OFF No Catch2 2.13.9+ ``openPMD_USE_INTERNAL_PYBIND11`` **ON**/OFF No pybind11 2.9.1+ ``openPMD_USE_INTERNAL_JSON`` **ON**/OFF No NLohmann-JSON 3.9.1+ ``openPMD_USE_INTERNAL_TOML11`` **ON**/OFF No toml11 3.7.1+ diff --git a/docs/source/dev/dependencies.rst b/docs/source/dev/dependencies.rst index c5da7454bb..6ff4c49906 100644 --- a/docs/source/dev/dependencies.rst +++ b/docs/source/dev/dependencies.rst @@ -17,7 +17,7 @@ Shipped internally The following libraries are shipped internally in ``share/openPMD/thirdParty/`` for convenience: -* `Catch2 `_ 2.13.4+ (`BSL-1.0 `__) +* `Catch2 `_ 2.13.9+ (`BSL-1.0 `__) * `pybind11 `_ 2.9.1+ (`new BSD `_) * `NLohmann-JSON `_ 3.9.1+ (`MIT `_) * `toml11 `_ 3.7.1+ (`MIT `__) diff --git a/share/openPMD/thirdParty/catch2/include/catch2/catch.hpp b/share/openPMD/thirdParty/catch2/include/catch2/catch.hpp index 0384171ae4..d2a12427b2 100644 --- a/share/openPMD/thirdParty/catch2/include/catch2/catch.hpp +++ b/share/openPMD/thirdParty/catch2/include/catch2/catch.hpp @@ -1,9 +1,9 @@ /* - * Catch v2.13.4 - * Generated: 2020-12-29 14:48:00.116107 + * Catch v2.13.9 + * Generated: 2022-04-12 22:37:23.260201 * ---------------------------------------------------------- * This file has been merged from multiple headers. Please don't edit it directly - * Copyright (c) 2020 Two Blue Cubes Ltd. All rights reserved. + * Copyright (c) 2022 Two Blue Cubes Ltd. All rights reserved. * * Distributed under the Boost Software License, Version 1.0. (See accompanying * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -15,7 +15,7 @@ #define CATCH_VERSION_MAJOR 2 #define CATCH_VERSION_MINOR 13 -#define CATCH_VERSION_PATCH 4 +#define CATCH_VERSION_PATCH 9 #ifdef __clang__ # pragma clang system_header @@ -66,13 +66,16 @@ #if !defined(CATCH_CONFIG_IMPL_ONLY) // start catch_platform.h +// See e.g.: +// https://opensource.apple.com/source/CarbonHeaders/CarbonHeaders-18.1/TargetConditionals.h.auto.html #ifdef __APPLE__ -# include -# if TARGET_OS_OSX == 1 -# define CATCH_PLATFORM_MAC -# elif TARGET_OS_IPHONE == 1 -# define CATCH_PLATFORM_IPHONE -# endif +# include +# if (defined(TARGET_OS_OSX) && TARGET_OS_OSX == 1) || \ + (defined(TARGET_OS_MAC) && TARGET_OS_MAC == 1) +# define CATCH_PLATFORM_MAC +# elif (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE == 1) +# define CATCH_PLATFORM_IPHONE +# endif #elif defined(linux) || defined(__linux) || defined(__linux__) # define CATCH_PLATFORM_LINUX @@ -132,9 +135,9 @@ namespace Catch { #endif -// We have to avoid both ICC and Clang, because they try to mask themselves -// as gcc, and we want only GCC in this block -#if defined(__GNUC__) && !defined(__clang__) && !defined(__ICC) && !defined(__CUDACC__) +// Only GCC compiler should be used in this block, so other compilers trying to +// mask themselves as GCC should be ignored. +#if defined(__GNUC__) && !defined(__clang__) && !defined(__ICC) && !defined(__CUDACC__) && !defined(__LCC__) # define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION _Pragma( "GCC diagnostic push" ) # define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION _Pragma( "GCC diagnostic pop" ) @@ -237,9 +240,6 @@ namespace Catch { // Visual C++ #if defined(_MSC_VER) -# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION __pragma( warning(push) ) -# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION __pragma( warning(pop) ) - // Universal Windows platform does not support SEH // Or console colours (or console at all...) # if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP) @@ -248,13 +248,18 @@ namespace Catch { # define CATCH_INTERNAL_CONFIG_WINDOWS_SEH # endif +# if !defined(__clang__) // Handle Clang masquerading for msvc + // MSVC traditional preprocessor needs some workaround for __VA_ARGS__ // _MSVC_TRADITIONAL == 0 means new conformant preprocessor // _MSVC_TRADITIONAL == 1 means old traditional non-conformant preprocessor -# if !defined(__clang__) // Handle Clang masquerading for msvc # if !defined(_MSVC_TRADITIONAL) || (defined(_MSVC_TRADITIONAL) && _MSVC_TRADITIONAL) # define CATCH_INTERNAL_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR # endif // MSVC_TRADITIONAL + +// Only do this if we're not using clang on Windows, which uses `diagnostic push` & `diagnostic pop` +# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION __pragma( warning(push) ) +# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION __pragma( warning(pop) ) # endif // __clang__ #endif // _MSC_VER @@ -323,7 +328,7 @@ namespace Catch { // Check if byte is available and usable # if __has_include() && defined(CATCH_CPP17_OR_GREATER) # include - # if __cpp_lib_byte > 0 + # if defined(__cpp_lib_byte) && (__cpp_lib_byte > 0) # define CATCH_INTERNAL_CONFIG_CPP17_BYTE # endif # endif // __has_include() && defined(CATCH_CPP17_OR_GREATER) @@ -1007,34 +1012,34 @@ struct AutoReg : NonCopyable { #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION(Name, Tags, ...) \ - INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, typename TestType, __VA_ARGS__ ) + INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, typename TestType, __VA_ARGS__ ) #else #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION(Name, Tags, ...) \ - INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, typename TestType, __VA_ARGS__ ) ) + INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, typename TestType, __VA_ARGS__ ) ) #endif #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_SIG_NO_REGISTRATION(Name, Tags, Signature, ...) \ - INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, Signature, __VA_ARGS__ ) + INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, Signature, __VA_ARGS__ ) #else #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_SIG_NO_REGISTRATION(Name, Tags, Signature, ...) \ - INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, Signature, __VA_ARGS__ ) ) + INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, Signature, __VA_ARGS__ ) ) #endif #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION( ClassName, Name, Tags,... ) \ - INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____C_L_A_S_S____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ) , ClassName, Name, Tags, typename T, __VA_ARGS__ ) + INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_C_L_A_S_S_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ) , ClassName, Name, Tags, typename T, __VA_ARGS__ ) #else #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION( ClassName, Name, Tags,... ) \ - INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____C_L_A_S_S____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ) , ClassName, Name, Tags, typename T, __VA_ARGS__ ) ) + INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_C_L_A_S_S_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ) , ClassName, Name, Tags, typename T, __VA_ARGS__ ) ) #endif #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_SIG_NO_REGISTRATION( ClassName, Name, Tags, Signature, ... ) \ - INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____C_L_A_S_S____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ) , ClassName, Name, Tags, Signature, __VA_ARGS__ ) + INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_C_L_A_S_S_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ) , ClassName, Name, Tags, Signature, __VA_ARGS__ ) #else #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_SIG_NO_REGISTRATION( ClassName, Name, Tags, Signature, ... ) \ - INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____C_L_A_S_S____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ) , ClassName, Name, Tags, Signature, __VA_ARGS__ ) ) + INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_NO_REGISTRATION_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_C_L_A_S_S_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ) , ClassName, Name, Tags, Signature, __VA_ARGS__ ) ) #endif #endif @@ -1047,7 +1052,7 @@ struct AutoReg : NonCopyable { CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION \ static void TestName() #define INTERNAL_CATCH_TESTCASE( ... ) \ - INTERNAL_CATCH_TESTCASE2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ), __VA_ARGS__ ) + INTERNAL_CATCH_TESTCASE2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ ), __VA_ARGS__ ) /////////////////////////////////////////////////////////////////////////////// #define INTERNAL_CATCH_METHOD_AS_TEST_CASE( QualifiedMethod, ... ) \ @@ -1069,7 +1074,7 @@ struct AutoReg : NonCopyable { CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION \ void TestName::test() #define INTERNAL_CATCH_TEST_CASE_METHOD( ClassName, ... ) \ - INTERNAL_CATCH_TEST_CASE_METHOD2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ), ClassName, __VA_ARGS__ ) + INTERNAL_CATCH_TEST_CASE_METHOD2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ ), ClassName, __VA_ARGS__ ) /////////////////////////////////////////////////////////////////////////////// #define INTERNAL_CATCH_REGISTER_TESTCASE( Function, ... ) \ @@ -1110,18 +1115,18 @@ struct AutoReg : NonCopyable { #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR #define INTERNAL_CATCH_TEMPLATE_TEST_CASE(Name, Tags, ...) \ - INTERNAL_CATCH_TEMPLATE_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, typename TestType, __VA_ARGS__ ) + INTERNAL_CATCH_TEMPLATE_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, typename TestType, __VA_ARGS__ ) #else #define INTERNAL_CATCH_TEMPLATE_TEST_CASE(Name, Tags, ...) \ - INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, typename TestType, __VA_ARGS__ ) ) + INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, typename TestType, __VA_ARGS__ ) ) #endif #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_SIG(Name, Tags, Signature, ...) \ - INTERNAL_CATCH_TEMPLATE_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, Signature, __VA_ARGS__ ) + INTERNAL_CATCH_TEMPLATE_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, Signature, __VA_ARGS__ ) #else #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_SIG(Name, Tags, Signature, ...) \ - INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, Signature, __VA_ARGS__ ) ) + INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, Signature, __VA_ARGS__ ) ) #endif #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2(TestName, TestFuncName, Name, Tags, Signature, TmplTypes, TypesList) \ @@ -1159,18 +1164,18 @@ struct AutoReg : NonCopyable { #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE(Name, Tags, ...)\ - INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, typename T,__VA_ARGS__) + INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, typename T,__VA_ARGS__) #else #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE(Name, Tags, ...)\ - INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, typename T, __VA_ARGS__ ) ) + INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, typename T, __VA_ARGS__ ) ) #endif #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_SIG(Name, Tags, Signature, ...)\ - INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, Signature, __VA_ARGS__) + INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, Signature, __VA_ARGS__) #else #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_SIG(Name, Tags, Signature, ...)\ - INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, Signature, __VA_ARGS__ ) ) + INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, Signature, __VA_ARGS__ ) ) #endif #define INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_2(TestName, TestFunc, Name, Tags, TmplList)\ @@ -1201,7 +1206,7 @@ struct AutoReg : NonCopyable { static void TestFunc() #define INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE(Name, Tags, TmplList) \ - INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, TmplList ) + INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), Name, Tags, TmplList ) #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( TestNameClass, TestName, ClassName, Name, Tags, Signature, ... ) \ CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \ @@ -1234,18 +1239,18 @@ struct AutoReg : NonCopyable { #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD( ClassName, Name, Tags,... ) \ - INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____C_L_A_S_S____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ) , ClassName, Name, Tags, typename T, __VA_ARGS__ ) + INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_C_L_A_S_S_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ) , ClassName, Name, Tags, typename T, __VA_ARGS__ ) #else #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD( ClassName, Name, Tags,... ) \ - INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____C_L_A_S_S____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ) , ClassName, Name, Tags, typename T, __VA_ARGS__ ) ) + INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_C_L_A_S_S_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ) , ClassName, Name, Tags, typename T, __VA_ARGS__ ) ) #endif #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_SIG( ClassName, Name, Tags, Signature, ... ) \ - INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____C_L_A_S_S____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ) , ClassName, Name, Tags, Signature, __VA_ARGS__ ) + INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_C_L_A_S_S_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ) , ClassName, Name, Tags, Signature, __VA_ARGS__ ) #else #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_SIG( ClassName, Name, Tags, Signature, ... ) \ - INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____C_L_A_S_S____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ) , ClassName, Name, Tags, Signature, __VA_ARGS__ ) ) + INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_C_L_A_S_S_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ) , ClassName, Name, Tags, Signature, __VA_ARGS__ ) ) #endif #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2(TestNameClass, TestName, ClassName, Name, Tags, Signature, TmplTypes, TypesList)\ @@ -1286,18 +1291,18 @@ struct AutoReg : NonCopyable { #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD( ClassName, Name, Tags, ... )\ - INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), ClassName, Name, Tags, typename T, __VA_ARGS__ ) + INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), ClassName, Name, Tags, typename T, __VA_ARGS__ ) #else #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD( ClassName, Name, Tags, ... )\ - INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), ClassName, Name, Tags, typename T,__VA_ARGS__ ) ) + INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), ClassName, Name, Tags, typename T,__VA_ARGS__ ) ) #endif #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_SIG( ClassName, Name, Tags, Signature, ... )\ - INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), ClassName, Name, Tags, Signature, __VA_ARGS__ ) + INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), ClassName, Name, Tags, Signature, __VA_ARGS__ ) #else #define INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_SIG( ClassName, Name, Tags, Signature, ... )\ - INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), ClassName, Name, Tags, Signature,__VA_ARGS__ ) ) + INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), ClassName, Name, Tags, Signature,__VA_ARGS__ ) ) #endif #define INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_METHOD_2( TestNameClass, TestName, ClassName, Name, Tags, TmplList) \ @@ -1331,7 +1336,7 @@ struct AutoReg : NonCopyable { void TestName::test() #define INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_METHOD(ClassName, Name, Tags, TmplList) \ - INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), ClassName, Name, Tags, TmplList ) + INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_ ), INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_M_P_L_A_T_E_T_E_S_T_F_U_N_C_ ), ClassName, Name, Tags, TmplList ) // end catch_test_registry.h // start catch_capture.hpp @@ -3088,7 +3093,7 @@ namespace Detail { Approx operator-() const; template ::value>::type> - Approx operator()( T const& value ) { + Approx operator()( T const& value ) const { Approx approx( static_cast(value) ); approx.m_epsilon = m_epsilon; approx.m_margin = m_margin; @@ -4160,7 +4165,7 @@ namespace Generators { if (!m_predicate(m_generator.get())) { // It might happen that there are no values that pass the // filter. In that case we throw an exception. - auto has_initial_value = next(); + auto has_initial_value = nextImpl(); if (!has_initial_value) { Catch::throw_exception(GeneratorException("No valid value found in filtered generator")); } @@ -4172,6 +4177,11 @@ namespace Generators { } bool next() override { + return nextImpl(); + } + + private: + bool nextImpl() { bool success = m_generator.next(); if (!success) { return false; @@ -5455,6 +5465,8 @@ namespace Catch { } // namespace Catch // end catch_outlier_classification.hpp + +#include #endif // CATCH_CONFIG_ENABLE_BENCHMARKING #include @@ -6339,9 +6351,10 @@ namespace Catch { void writeTestCase(TestCaseNode const& testCaseNode); - void writeSection(std::string const& className, - std::string const& rootName, - SectionNode const& sectionNode); + void writeSection( std::string const& className, + std::string const& rootName, + SectionNode const& sectionNode, + bool testOkToFail ); void writeAssertions(SectionNode const& sectionNode); void writeAssertion(AssertionStats const& stats); @@ -6876,7 +6889,7 @@ namespace Catch { } iters *= 2; } - throw optimized_away_error{}; + Catch::throw_exception(optimized_away_error{}); } } // namespace Detail } // namespace Benchmark @@ -6884,6 +6897,7 @@ namespace Catch { // end catch_run_for_at_least.hpp #include +#include namespace Catch { namespace Benchmark { @@ -7054,8 +7068,8 @@ namespace Catch { double b2 = bias - z1; double a1 = a(b1); double a2 = a(b2); - auto lo = std::max(cumn(a1), 0); - auto hi = std::min(cumn(a2), n - 1); + auto lo = (std::max)(cumn(a1), 0); + auto hi = (std::min)(cumn(a2), n - 1); return { point, resample[lo], resample[hi], confidence_level }; } @@ -7124,7 +7138,9 @@ namespace Catch { } template EnvironmentEstimate> estimate_clock_cost(FloatDuration resolution) { - auto time_limit = std::min(resolution * clock_cost_estimation_tick_limit, FloatDuration(clock_cost_estimation_time_limit)); + auto time_limit = (std::min)( + resolution * clock_cost_estimation_tick_limit, + FloatDuration(clock_cost_estimation_time_limit)); auto time_clock = [](int k) { return Detail::measure([k] { for (int i = 0; i < k; ++i) { @@ -7771,7 +7787,7 @@ namespace Catch { double sb = stddev.point; double mn = mean.point / n; double mg_min = mn / 2.; - double sg = std::min(mg_min / 4., sb / std::sqrt(n)); + double sg = (std::min)(mg_min / 4., sb / std::sqrt(n)); double sg2 = sg * sg; double sb2 = sb * sb; @@ -7790,7 +7806,7 @@ namespace Catch { return (nc / n) * (sb2 - nc * sg2); }; - return std::min(var_out(1), var_out(std::min(c_max(0.), c_max(mg_min)))) / sb2; + return (std::min)(var_out(1), var_out((std::min)(c_max(0.), c_max(mg_min)))) / sb2; } bootstrap_analysis analyse_samples(double confidence_level, int n_resamples, std::vector::iterator first, std::vector::iterator last) { @@ -7980,86 +7996,58 @@ namespace Catch { // start catch_fatal_condition.h -// start catch_windows_h_proxy.h - - -#if defined(CATCH_PLATFORM_WINDOWS) - -#if !defined(NOMINMAX) && !defined(CATCH_CONFIG_NO_NOMINMAX) -# define CATCH_DEFINED_NOMINMAX -# define NOMINMAX -#endif -#if !defined(WIN32_LEAN_AND_MEAN) && !defined(CATCH_CONFIG_NO_WIN32_LEAN_AND_MEAN) -# define CATCH_DEFINED_WIN32_LEAN_AND_MEAN -# define WIN32_LEAN_AND_MEAN -#endif - -#ifdef __AFXDLL -#include -#else -#include -#endif - -#ifdef CATCH_DEFINED_NOMINMAX -# undef NOMINMAX -#endif -#ifdef CATCH_DEFINED_WIN32_LEAN_AND_MEAN -# undef WIN32_LEAN_AND_MEAN -#endif - -#endif // defined(CATCH_PLATFORM_WINDOWS) - -// end catch_windows_h_proxy.h -#if defined( CATCH_CONFIG_WINDOWS_SEH ) +#include namespace Catch { - struct FatalConditionHandler { - - static LONG CALLBACK handleVectoredException(PEXCEPTION_POINTERS ExceptionInfo); + // Wrapper for platform-specific fatal error (signals/SEH) handlers + // + // Tries to be cooperative with other handlers, and not step over + // other handlers. This means that unknown structured exceptions + // are passed on, previous signal handlers are called, and so on. + // + // Can only be instantiated once, and assumes that once a signal + // is caught, the binary will end up terminating. Thus, there + class FatalConditionHandler { + bool m_started = false; + + // Install/disengage implementation for specific platform. + // Should be if-defed to work on current platform, can assume + // engage-disengage 1:1 pairing. + void engage_platform(); + void disengage_platform(); + public: + // Should also have platform-specific implementations as needed FatalConditionHandler(); - static void reset(); ~FatalConditionHandler(); - private: - static bool isSet; - static ULONG guaranteeSize; - static PVOID exceptionHandlerHandle; - }; - -} // namespace Catch - -#elif defined ( CATCH_CONFIG_POSIX_SIGNALS ) - -#include - -namespace Catch { - - struct FatalConditionHandler { - - static bool isSet; - static struct sigaction oldSigActions[]; - static stack_t oldSigStack; - static char altStackMem[]; - - static void handleSignal( int sig ); + void engage() { + assert(!m_started && "Handler cannot be installed twice."); + m_started = true; + engage_platform(); + } - FatalConditionHandler(); - ~FatalConditionHandler(); - static void reset(); + void disengage() { + assert(m_started && "Handler cannot be uninstalled without being installed first"); + m_started = false; + disengage_platform(); + } }; -} // namespace Catch - -#else - -namespace Catch { - struct FatalConditionHandler { - void reset(); + //! Simple RAII guard for (dis)engaging the FatalConditionHandler + class FatalConditionHandlerGuard { + FatalConditionHandler* m_handler; + public: + FatalConditionHandlerGuard(FatalConditionHandler* handler): + m_handler(handler) { + m_handler->engage(); + } + ~FatalConditionHandlerGuard() { + m_handler->disengage(); + } }; -} -#endif +} // end namespace Catch // end catch_fatal_condition.h #include @@ -8185,6 +8173,7 @@ namespace Catch { std::vector m_unfinishedSections; std::vector m_activeSections; TrackerContext m_trackerContext; + FatalConditionHandler m_fatalConditionhandler; bool m_lastAssertionPassed = false; bool m_shouldReportUnexpected = true; bool m_includeSuccessfulResults; @@ -10057,6 +10046,36 @@ namespace Catch { } // end catch_errno_guard.h +// start catch_windows_h_proxy.h + + +#if defined(CATCH_PLATFORM_WINDOWS) + +#if !defined(NOMINMAX) && !defined(CATCH_CONFIG_NO_NOMINMAX) +# define CATCH_DEFINED_NOMINMAX +# define NOMINMAX +#endif +#if !defined(WIN32_LEAN_AND_MEAN) && !defined(CATCH_CONFIG_NO_WIN32_LEAN_AND_MEAN) +# define CATCH_DEFINED_WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif + +#ifdef __AFXDLL +#include +#else +#include +#endif + +#ifdef CATCH_DEFINED_NOMINMAX +# undef NOMINMAX +#endif +#ifdef CATCH_DEFINED_WIN32_LEAN_AND_MEAN +# undef WIN32_LEAN_AND_MEAN +#endif + +#endif // defined(CATCH_PLATFORM_WINDOWS) + +// end catch_windows_h_proxy.h #include namespace Catch { @@ -10573,7 +10592,7 @@ namespace Catch { // Extracts the actual name part of an enum instance // In other words, it returns the Blue part of Bikeshed::Colour::Blue StringRef extractInstanceName(StringRef enumInstance) { - // Find last occurence of ":" + // Find last occurrence of ":" size_t name_start = enumInstance.size(); while (name_start > 0 && enumInstance[name_start - 1] != ':') { --name_start; @@ -10735,25 +10754,47 @@ namespace Catch { // end catch_exception_translator_registry.cpp // start catch_fatal_condition.cpp -#if defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wmissing-field-initializers" -#endif +#include + +#if !defined( CATCH_CONFIG_WINDOWS_SEH ) && !defined( CATCH_CONFIG_POSIX_SIGNALS ) + +namespace Catch { + + // If neither SEH nor signal handling is required, the handler impls + // do not have to do anything, and can be empty. + void FatalConditionHandler::engage_platform() {} + void FatalConditionHandler::disengage_platform() {} + FatalConditionHandler::FatalConditionHandler() = default; + FatalConditionHandler::~FatalConditionHandler() = default; + +} // end namespace Catch + +#endif // !CATCH_CONFIG_WINDOWS_SEH && !CATCH_CONFIG_POSIX_SIGNALS + +#if defined( CATCH_CONFIG_WINDOWS_SEH ) && defined( CATCH_CONFIG_POSIX_SIGNALS ) +#error "Inconsistent configuration: Windows' SEH handling and POSIX signals cannot be enabled at the same time" +#endif // CATCH_CONFIG_WINDOWS_SEH && CATCH_CONFIG_POSIX_SIGNALS #if defined( CATCH_CONFIG_WINDOWS_SEH ) || defined( CATCH_CONFIG_POSIX_SIGNALS ) namespace { - // Report the error condition + //! Signals fatal error message to the run context void reportFatal( char const * const message ) { Catch::getCurrentContext().getResultCapture()->handleFatalErrorCondition( message ); } -} -#endif // signals/SEH handling + //! Minimal size Catch2 needs for its own fatal error handling. + //! Picked anecdotally, so it might not be sufficient on all + //! platforms, and for all configurations. + constexpr std::size_t minStackSizeForErrors = 32 * 1024; +} // end unnamed namespace + +#endif // CATCH_CONFIG_WINDOWS_SEH || CATCH_CONFIG_POSIX_SIGNALS #if defined( CATCH_CONFIG_WINDOWS_SEH ) namespace Catch { + struct SignalDefs { DWORD id; const char* name; }; // There is no 1-1 mapping between signals and windows exceptions. @@ -10766,7 +10807,7 @@ namespace Catch { { static_cast(EXCEPTION_INT_DIVIDE_BY_ZERO), "Divide by zero error" }, }; - LONG CALLBACK FatalConditionHandler::handleVectoredException(PEXCEPTION_POINTERS ExceptionInfo) { + static LONG CALLBACK handleVectoredException(PEXCEPTION_POINTERS ExceptionInfo) { for (auto const& def : signalDefs) { if (ExceptionInfo->ExceptionRecord->ExceptionCode == def.id) { reportFatal(def.name); @@ -10777,38 +10818,50 @@ namespace Catch { return EXCEPTION_CONTINUE_SEARCH; } + // Since we do not support multiple instantiations, we put these + // into global variables and rely on cleaning them up in outlined + // constructors/destructors + static PVOID exceptionHandlerHandle = nullptr; + + // For MSVC, we reserve part of the stack memory for handling + // memory overflow structured exception. FatalConditionHandler::FatalConditionHandler() { - isSet = true; - // 32k seems enough for Catch to handle stack overflow, - // but the value was found experimentally, so there is no strong guarantee - guaranteeSize = 32 * 1024; - exceptionHandlerHandle = nullptr; + ULONG guaranteeSize = static_cast(minStackSizeForErrors); + if (!SetThreadStackGuarantee(&guaranteeSize)) { + // We do not want to fully error out, because needing + // the stack reserve should be rare enough anyway. + Catch::cerr() + << "Failed to reserve piece of stack." + << " Stack overflows will not be reported successfully."; + } + } + + // We do not attempt to unset the stack guarantee, because + // Windows does not support lowering the stack size guarantee. + FatalConditionHandler::~FatalConditionHandler() = default; + + void FatalConditionHandler::engage_platform() { // Register as first handler in current chain exceptionHandlerHandle = AddVectoredExceptionHandler(1, handleVectoredException); - // Pass in guarantee size to be filled - SetThreadStackGuarantee(&guaranteeSize); + if (!exceptionHandlerHandle) { + CATCH_RUNTIME_ERROR("Could not register vectored exception handler"); + } } - void FatalConditionHandler::reset() { - if (isSet) { - RemoveVectoredExceptionHandler(exceptionHandlerHandle); - SetThreadStackGuarantee(&guaranteeSize); - exceptionHandlerHandle = nullptr; - isSet = false; + void FatalConditionHandler::disengage_platform() { + if (!RemoveVectoredExceptionHandler(exceptionHandlerHandle)) { + CATCH_RUNTIME_ERROR("Could not unregister vectored exception handler"); } + exceptionHandlerHandle = nullptr; } - FatalConditionHandler::~FatalConditionHandler() { - reset(); - } +} // end namespace Catch -bool FatalConditionHandler::isSet = false; -ULONG FatalConditionHandler::guaranteeSize = 0; -PVOID FatalConditionHandler::exceptionHandlerHandle = nullptr; +#endif // CATCH_CONFIG_WINDOWS_SEH -} // namespace Catch +#if defined( CATCH_CONFIG_POSIX_SIGNALS ) -#elif defined( CATCH_CONFIG_POSIX_SIGNALS ) +#include namespace Catch { @@ -10817,10 +10870,6 @@ namespace Catch { const char* name; }; - // 32kb for the alternate stack seems to be sufficient. However, this value - // is experimentally determined, so that's not guaranteed. - static constexpr std::size_t sigStackSize = 32768 >= MINSIGSTKSZ ? 32768 : MINSIGSTKSZ; - static SignalDefs signalDefs[] = { { SIGINT, "SIGINT - Terminal interrupt signal" }, { SIGILL, "SIGILL - Illegal instruction signal" }, @@ -10830,7 +10879,32 @@ namespace Catch { { SIGABRT, "SIGABRT - Abort (abnormal termination) signal" } }; - void FatalConditionHandler::handleSignal( int sig ) { +// Older GCCs trigger -Wmissing-field-initializers for T foo = {} +// which is zero initialization, but not explicit. We want to avoid +// that. +#if defined(__GNUC__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wmissing-field-initializers" +#endif + + static char* altStackMem = nullptr; + static std::size_t altStackSize = 0; + static stack_t oldSigStack{}; + static struct sigaction oldSigActions[sizeof(signalDefs) / sizeof(SignalDefs)]{}; + + static void restorePreviousSignalHandlers() { + // We set signal handlers back to the previous ones. Hopefully + // nobody overwrote them in the meantime, and doesn't expect + // their signal handlers to live past ours given that they + // installed them after ours.. + for (std::size_t i = 0; i < sizeof(signalDefs) / sizeof(SignalDefs); ++i) { + sigaction(signalDefs[i].id, &oldSigActions[i], nullptr); + } + // Return the old stack + sigaltstack(&oldSigStack, nullptr); + } + + static void handleSignal( int sig ) { char const * name = ""; for (auto const& def : signalDefs) { if (sig == def.id) { @@ -10838,16 +10912,33 @@ namespace Catch { break; } } - reset(); - reportFatal(name); + // We need to restore previous signal handlers and let them do + // their thing, so that the users can have the debugger break + // when a signal is raised, and so on. + restorePreviousSignalHandlers(); + reportFatal( name ); raise( sig ); } FatalConditionHandler::FatalConditionHandler() { - isSet = true; + assert(!altStackMem && "Cannot initialize POSIX signal handler when one already exists"); + if (altStackSize == 0) { + altStackSize = std::max(static_cast(SIGSTKSZ), minStackSizeForErrors); + } + altStackMem = new char[altStackSize](); + } + + FatalConditionHandler::~FatalConditionHandler() { + delete[] altStackMem; + // We signal that another instance can be constructed by zeroing + // out the pointer. + altStackMem = nullptr; + } + + void FatalConditionHandler::engage_platform() { stack_t sigStack; sigStack.ss_sp = altStackMem; - sigStack.ss_size = sigStackSize; + sigStack.ss_size = altStackSize; sigStack.ss_flags = 0; sigaltstack(&sigStack, &oldSigStack); struct sigaction sa = { }; @@ -10859,40 +10950,17 @@ namespace Catch { } } - FatalConditionHandler::~FatalConditionHandler() { - reset(); - } +#if defined(__GNUC__) +# pragma GCC diagnostic pop +#endif - void FatalConditionHandler::reset() { - if( isSet ) { - // Set signals back to previous values -- hopefully nobody overwrote them in the meantime - for( std::size_t i = 0; i < sizeof(signalDefs)/sizeof(SignalDefs); ++i ) { - sigaction(signalDefs[i].id, &oldSigActions[i], nullptr); - } - // Return the old stack - sigaltstack(&oldSigStack, nullptr); - isSet = false; - } + void FatalConditionHandler::disengage_platform() { + restorePreviousSignalHandlers(); } - bool FatalConditionHandler::isSet = false; - struct sigaction FatalConditionHandler::oldSigActions[sizeof(signalDefs)/sizeof(SignalDefs)] = {}; - stack_t FatalConditionHandler::oldSigStack = {}; - char FatalConditionHandler::altStackMem[sigStackSize] = {}; - -} // namespace Catch - -#else - -namespace Catch { - void FatalConditionHandler::reset() {} -} - -#endif // signals/SEH handling +} // end namespace Catch -#if defined(__GNUC__) -# pragma GCC diagnostic pop -#endif +#endif // CATCH_CONFIG_POSIX_SIGNALS // end catch_fatal_condition.cpp // start catch_generators.cpp @@ -11447,7 +11515,8 @@ namespace { return lhs == rhs; } - auto ulpDiff = std::abs(lc - rc); + // static cast as a workaround for IBM XLC + auto ulpDiff = std::abs(static_cast(lc - rc)); return static_cast(ulpDiff) <= maxUlpDiff; } @@ -11621,7 +11690,6 @@ Floating::WithinRelMatcher WithinRel(float target) { } // namespace Matchers } // namespace Catch - // end catch_matchers_floating.cpp // start catch_matchers_generic.cpp @@ -12955,9 +13023,8 @@ namespace Catch { } void RunContext::invokeActiveTestCase() { - FatalConditionHandler fatalConditionHandler; // Handle signals + FatalConditionHandlerGuard _(&m_fatalConditionhandler); m_activeTestCase->invoke(); - fatalConditionHandler.reset(); } void RunContext::handleUnfinishedSections() { @@ -13325,6 +13392,10 @@ namespace Catch { filename.erase(0, lastSlash); filename[0] = '#'; } + else + { + filename.insert(0, "#"); + } auto lastDot = filename.find_last_of('.'); if (lastDot != std::string::npos) { @@ -15320,7 +15391,7 @@ namespace Catch { } Version const& libraryVersion() { - static Version version( 2, 13, 4, "", 0 ); + static Version version( 2, 13, 9, "", 0 ); return version; } @@ -16733,6 +16804,7 @@ CATCH_REGISTER_REPORTER("console", ConsoleReporter) #include #include #include +#include namespace Catch { @@ -16760,7 +16832,7 @@ namespace Catch { #else std::strftime(timeStamp, timeStampSize, fmt, timeInfo); #endif - return std::string(timeStamp); + return std::string(timeStamp, timeStampSize-1); } std::string fileNameTag(const std::vector &tags) { @@ -16771,6 +16843,17 @@ namespace Catch { return it->substr(1); return std::string(); } + + // Formats the duration in seconds to 3 decimal places. + // This is done because some genius defined Maven Surefire schema + // in a way that only accepts 3 decimal places, and tools like + // Jenkins use that schema for validation JUnit reporter output. + std::string formatDuration( double seconds ) { + ReusableStringStream rss; + rss << std::fixed << std::setprecision( 3 ) << seconds; + return rss.str(); + } + } // anonymous namespace JunitReporter::JunitReporter( ReporterConfig const& _config ) @@ -16840,7 +16923,7 @@ namespace Catch { if( m_config->showDurations() == ShowDurations::Never ) xml.writeAttribute( "time", "" ); else - xml.writeAttribute( "time", suiteTime ); + xml.writeAttribute( "time", formatDuration( suiteTime ) ); xml.writeAttribute( "timestamp", getCurrentTimestamp() ); // Write properties if there are any @@ -16885,12 +16968,13 @@ namespace Catch { if ( !m_config->name().empty() ) className = m_config->name() + "." + className; - writeSection( className, "", rootSection ); + writeSection( className, "", rootSection, stats.testInfo.okToFail() ); } - void JunitReporter::writeSection( std::string const& className, - std::string const& rootName, - SectionNode const& sectionNode ) { + void JunitReporter::writeSection( std::string const& className, + std::string const& rootName, + SectionNode const& sectionNode, + bool testOkToFail) { std::string name = trim( sectionNode.stats.sectionInfo.name ); if( !rootName.empty() ) name = rootName + '/' + name; @@ -16907,13 +16991,18 @@ namespace Catch { xml.writeAttribute( "classname", className ); xml.writeAttribute( "name", name ); } - xml.writeAttribute( "time", ::Catch::Detail::stringify( sectionNode.stats.durationInSeconds ) ); + xml.writeAttribute( "time", formatDuration( sectionNode.stats.durationInSeconds ) ); // This is not ideal, but it should be enough to mimic gtest's // junit output. // Ideally the JUnit reporter would also handle `skipTest` // events and write those out appropriately. xml.writeAttribute( "status", "run" ); + if (sectionNode.stats.assertions.failedButOk) { + xml.scopedElement("skipped") + .writeAttribute("message", "TEST_CASE tagged with !mayfail"); + } + writeAssertions( sectionNode ); if( !sectionNode.stdOut.empty() ) @@ -16923,9 +17012,9 @@ namespace Catch { } for( auto const& childNode : sectionNode.childSections ) if( className.empty() ) - writeSection( name, "", *childNode ); + writeSection( name, "", *childNode, testOkToFail ); else - writeSection( className, name, *childNode ); + writeSection( className, name, *childNode, testOkToFail ); } void JunitReporter::writeAssertions( SectionNode const& sectionNode ) { @@ -17570,9 +17659,9 @@ int main (int argc, char * const argv[]) { #if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) #define CATCH_BENCHMARK(...) \ - INTERNAL_CATCH_BENCHMARK(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), INTERNAL_CATCH_GET_1_ARG(__VA_ARGS__,,), INTERNAL_CATCH_GET_2_ARG(__VA_ARGS__,,)) + INTERNAL_CATCH_BENCHMARK(INTERNAL_CATCH_UNIQUE_NAME(C_A_T_C_H_B_E_N_C_H_), INTERNAL_CATCH_GET_1_ARG(__VA_ARGS__,,), INTERNAL_CATCH_GET_2_ARG(__VA_ARGS__,,)) #define CATCH_BENCHMARK_ADVANCED(name) \ - INTERNAL_CATCH_BENCHMARK_ADVANCED(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), name) + INTERNAL_CATCH_BENCHMARK_ADVANCED(INTERNAL_CATCH_UNIQUE_NAME(C_A_T_C_H_B_E_N_C_H_), name) #endif // CATCH_CONFIG_ENABLE_BENCHMARKING // If CATCH_CONFIG_PREFIX_ALL is not defined then the CATCH_ prefix is not required @@ -17674,9 +17763,9 @@ int main (int argc, char * const argv[]) { #if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) #define BENCHMARK(...) \ - INTERNAL_CATCH_BENCHMARK(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), INTERNAL_CATCH_GET_1_ARG(__VA_ARGS__,,), INTERNAL_CATCH_GET_2_ARG(__VA_ARGS__,,)) + INTERNAL_CATCH_BENCHMARK(INTERNAL_CATCH_UNIQUE_NAME(C_A_T_C_H_B_E_N_C_H_), INTERNAL_CATCH_GET_1_ARG(__VA_ARGS__,,), INTERNAL_CATCH_GET_2_ARG(__VA_ARGS__,,)) #define BENCHMARK_ADVANCED(name) \ - INTERNAL_CATCH_BENCHMARK_ADVANCED(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), name) + INTERNAL_CATCH_BENCHMARK_ADVANCED(INTERNAL_CATCH_UNIQUE_NAME(C_A_T_C_H_B_E_N_C_H_), name) #endif // CATCH_CONFIG_ENABLE_BENCHMARKING using Catch::Detail::Approx; @@ -17723,8 +17812,8 @@ using Catch::Detail::Approx; #define CATCH_WARN( msg ) (void)(0) #define CATCH_CAPTURE( msg ) (void)(0) -#define CATCH_TEST_CASE( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ )) -#define CATCH_TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ )) +#define CATCH_TEST_CASE( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ )) +#define CATCH_TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ )) #define CATCH_METHOD_AS_TEST_CASE( method, ... ) #define CATCH_REGISTER_TEST_CASE( Function, ... ) (void)(0) #define CATCH_SECTION( ... ) @@ -17733,7 +17822,7 @@ using Catch::Detail::Approx; #define CATCH_FAIL_CHECK( ... ) (void)(0) #define CATCH_SUCCEED( ... ) (void)(0) -#define CATCH_ANON_TEST_CASE() INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ )) +#define CATCH_ANON_TEST_CASE() INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ )) #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR #define CATCH_TEMPLATE_TEST_CASE( ... ) INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION(__VA_ARGS__) @@ -17756,8 +17845,8 @@ using Catch::Detail::Approx; #endif // "BDD-style" convenience wrappers -#define CATCH_SCENARIO( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ )) -#define CATCH_SCENARIO_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_METHOD_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ), className ) +#define CATCH_SCENARIO( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ )) +#define CATCH_SCENARIO_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_METHOD_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ ), className ) #define CATCH_GIVEN( desc ) #define CATCH_AND_GIVEN( desc ) #define CATCH_WHEN( desc ) @@ -17805,10 +17894,10 @@ using Catch::Detail::Approx; #define INFO( msg ) (void)(0) #define UNSCOPED_INFO( msg ) (void)(0) #define WARN( msg ) (void)(0) -#define CAPTURE( msg ) (void)(0) +#define CAPTURE( ... ) (void)(0) -#define TEST_CASE( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ )) -#define TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ )) +#define TEST_CASE( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ )) +#define TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ )) #define METHOD_AS_TEST_CASE( method, ... ) #define REGISTER_TEST_CASE( Function, ... ) (void)(0) #define SECTION( ... ) @@ -17816,7 +17905,7 @@ using Catch::Detail::Approx; #define FAIL( ... ) (void)(0) #define FAIL_CHECK( ... ) (void)(0) #define SUCCEED( ... ) (void)(0) -#define ANON_TEST_CASE() INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ )) +#define ANON_TEST_CASE() INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ )) #ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR #define TEMPLATE_TEST_CASE( ... ) INTERNAL_CATCH_TEMPLATE_TEST_CASE_NO_REGISTRATION(__VA_ARGS__) @@ -17846,8 +17935,8 @@ using Catch::Detail::Approx; #define CATCH_TRANSLATE_EXCEPTION( signature ) INTERNAL_CATCH_TRANSLATE_EXCEPTION_NO_REG( INTERNAL_CATCH_UNIQUE_NAME( catch_internal_ExceptionTranslator ), signature ) // "BDD-style" convenience wrappers -#define SCENARIO( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ) ) -#define SCENARIO_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_METHOD_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_S_T____ ), className ) +#define SCENARIO( ... ) INTERNAL_CATCH_TESTCASE_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ ) ) +#define SCENARIO_METHOD( className, ... ) INTERNAL_CATCH_TESTCASE_METHOD_NO_REGISTRATION(INTERNAL_CATCH_UNIQUE_NAME( C_A_T_C_H_T_E_S_T_ ), className ) #define GIVEN( desc ) #define AND_GIVEN( desc ) From 43e2b32e18f045487d7e995540cd3c090658bde2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 19 Jul 2022 12:26:03 +0200 Subject: [PATCH 53/70] [pre-commit.ci] pre-commit autoupdate (#1295) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/Lucas-C/pre-commit-hooks: v1.2.0 → v1.3.0](https://github.com/Lucas-C/pre-commit-hooks/compare/v1.2.0...v1.3.0) - [github.com/pre-commit/mirrors-clang-format: v14.0.5 → v14.0.6](https://github.com/pre-commit/mirrors-clang-format/compare/v14.0.5...v14.0.6) - [github.com/hadialqattan/pycln: v1.3.3 → v2.0.3](https://github.com/hadialqattan/pycln/compare/v1.3.3...v2.0.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 25439d83af..790125b569 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,7 +48,7 @@ repos: # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.2.0 + rev: v1.3.0 hooks: - id: remove-tabs @@ -65,13 +65,13 @@ repos: # clang-format v13 # to run manually, use .github/workflows/clang-format/clang-format.sh - repo: https://github.com/pre-commit/mirrors-clang-format - rev: v14.0.5 + rev: v14.0.6 hooks: - id: clang-format # Autoremoves unused Python imports - repo: https://github.com/hadialqattan/pycln - rev: v1.3.3 + rev: v2.0.3 hooks: - id: pycln name: pycln (python) From ca451b58210ba8f91d7a4c2aa9b6449c3a5e37d3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Jul 2022 17:25:04 -0700 Subject: [PATCH 54/70] [pre-commit.ci] pre-commit autoupdate (#1304) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/hadialqattan/pycln: v2.0.3 → v2.0.4](https://github.com/hadialqattan/pycln/compare/v2.0.3...v2.0.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 790125b569..20779214cb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -71,7 +71,7 @@ repos: # Autoremoves unused Python imports - repo: https://github.com/hadialqattan/pycln - rev: v2.0.3 + rev: v2.0.4 hooks: - id: pycln name: pycln (python) From b05171d371fe9568149ef58ae60ee113f4d0c4b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 28 Jul 2022 21:10:36 +0200 Subject: [PATCH 55/70] ADIOS2: Flush to disk within a step (#1207) * Add backendConfig parameter to flush call Parse string JSON config into nlohmann representation * ADIOS2 implementation: preferred_flush_target parameter * Testing * Documentation * Add override parameters, update tests and docs Also, use fstat in tests rather than * Flush attributes before PerformDataWrites() * Don't write buffered attributes upon PerformDataWrites There is no advantage from doing this, readers will only see the data after EndStep anyway, but there's the disadvantage that attributes cannot be overwritten within this step anymore * Use images from PR thread --- CMakeLists.txt | 2 + docs/source/backends/adios2.rst | 97 +++++- docs/source/backends/memory_filebased.png | Bin 13419 -> 0 bytes .../backends/memory_groupbased_nosteps.png | Bin 28565 -> 0 bytes docs/source/backends/memory_variablebased.png | Bin 11595 -> 0 bytes .../memory_variablebased_initialization.png | Bin 13675 -> 0 bytes docs/source/details/backendconfig.rst | 15 +- docs/source/usage/workflow.rst | 9 + include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp | 4 +- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 43 ++- .../IO/ADIOS/ParallelADIOS1IOHandler.hpp | 2 +- include/openPMD/IO/AbstractIOHandler.hpp | 23 +- include/openPMD/IO/DummyIOHandler.hpp | 2 +- .../openPMD/IO/FlushParametersInternal.hpp | 40 +++ include/openPMD/IO/HDF5/HDF5IOHandler.hpp | 2 +- .../openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp | 2 +- include/openPMD/IO/JSON/JSONIOHandler.hpp | 2 +- include/openPMD/Series.hpp | 7 +- include/openPMD/backend/Attributable.hpp | 7 +- include/openPMD/backend/Writable.hpp | 2 +- src/IO/ADIOS/ADIOS1IOHandler.cpp | 4 +- src/IO/ADIOS/ADIOS2IOHandler.cpp | 182 +++++++++++- src/IO/ADIOS/ParallelADIOS1IOHandler.cpp | 4 +- src/IO/AbstractIOHandler.cpp | 35 +++ src/IO/DummyIOHandler.cpp | 2 +- src/IO/FlushParams.cpp | 31 ++ src/IO/HDF5/HDF5IOHandler.cpp | 4 +- src/IO/HDF5/ParallelHDF5IOHandler.cpp | 4 +- src/IO/JSON/JSONIOHandler.cpp | 2 +- src/Series.cpp | 6 +- src/backend/Attributable.cpp | 4 +- src/backend/Writable.cpp | 4 +- src/binding/python/Attributable.cpp | 5 +- src/binding/python/Series.cpp | 2 +- test/SerialIOTest.cpp | 276 ++++++++++++++++++ 35 files changed, 769 insertions(+), 55 deletions(-) delete mode 100644 docs/source/backends/memory_filebased.png delete mode 100644 docs/source/backends/memory_groupbased_nosteps.png delete mode 100644 docs/source/backends/memory_variablebased.png delete mode 100644 docs/source/backends/memory_variablebased_initialization.png create mode 100644 include/openPMD/IO/FlushParametersInternal.hpp create mode 100644 src/IO/AbstractIOHandler.cpp create mode 100644 src/IO/FlushParams.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8cca88b188..699b4b9f06 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -469,10 +469,12 @@ set(CORE_SOURCE src/benchmark/mpi/OneDimensionalBlockSlicer.cpp src/helper/list_series.cpp) set(IO_SOURCE + src/IO/AbstractIOHandler.cpp src/IO/AbstractIOHandlerImpl.cpp src/IO/AbstractIOHandlerHelper.cpp src/IO/DummyIOHandler.cpp src/IO/IOTask.cpp + src/IO/FlushParams.cpp src/IO/HDF5/HDF5IOHandler.cpp src/IO/HDF5/ParallelHDF5IOHandler.cpp src/IO/HDF5/HDF5Auxiliary.cpp diff --git a/docs/source/backends/adios2.rst b/docs/source/backends/adios2.rst index 8996466544..34450bd2c0 100644 --- a/docs/source/backends/adios2.rst +++ b/docs/source/backends/adios2.rst @@ -130,12 +130,18 @@ This buffer is drained to storage only at specific times: The usage pattern of openPMD, especially the choice of iteration encoding influences the memory use of ADIOS2. The following graphs are created from a real-world application using openPMD (PIConGPU) using KDE Heaptrack. -Ignore the 30GB initialization phases. + +BP4 file engine +*************** + +The internal data structure of BP4 is one large buffer that holds all data written by a process. +It is drained to the disk upon ending a step or closing the engine (in parallel applications, data will usually be aggregated at the node-level before this). +This approach enables a very high IO performance by requiring only very few, very large IO operations, at the cost of a high memory consumption and some common usage pitfalls as detailed below: * **file-based iteration encoding:** A new ADIOS2 engine is opened for each iteration and closed upon ``Iteration::close()``. Each iteration has its own buffer: -.. image:: ./memory_filebased.png +.. figure:: https://user-images.githubusercontent.com/14241876/181477396-746ee21d-6efe-450b-bb2f-f53d49945fb9.png :alt: Memory usage of file-based iteration encoding * **variable-based iteration encoding and group-based iteration encoding with steps**: @@ -147,19 +153,100 @@ Ignore the 30GB initialization phases. These memory spikes can easily lead to out-of-memory (OOM) situations, motivating that the ``InitialBufferSize`` should not be chosen too small. Both behaviors are depicted in the following two pictures: -.. image:: ./memory_variablebased.png +.. figure:: https://user-images.githubusercontent.com/14241876/181477405-0439b017-256b-48d6-a169-014b3fe3aeb3.png :alt: Memory usage of variable-based iteration encoding -.. image:: ./memory_variablebased_initialization.png +.. figure:: https://user-images.githubusercontent.com/14241876/181477406-f6e2a173-2ec1-48df-a417-0cb97a160c91.png :alt: Memory usage of variable-based iteration encoding with bad ``InitialBufferSize`` * **group-based iteration encoding without steps:** This encoding **should be avoided** in ADIOS2. No data will be written to disk before closing the ``Series``, leading to a continuous buildup of memory, and most likely to an OOM situation: -.. image:: ./memory_groupbased_nosteps.png +.. figure:: https://user-images.githubusercontent.com/14241876/181477397-4d923061-7051-48c4-ae3a-a9efa10dcac7.png :alt: Memory usage of group-based iteration without using steps +SST staging engine +****************** + +Like the BP4 engine, the SST engine uses one large buffer as an internal data structure. + +Unlike the BP4 engine, however, a new buffer is allocated for each IO step, leading to a memory profile with clearly distinct IO steps: + +.. figure:: https://user-images.githubusercontent.com/14241876/181477403-7ed7810b-dedf-48b8-b17b-8ce89fd3c34a.png + :alt: Ideal memory usage of the SST engine + +The SST engine performs all IO asynchronously in the background and releases memory only as soon as the reader is done interacting with an IO step. +With slow readers, this can lead to a buildup of past IO steps in memory and subsequently to an out-of-memory condition: + +.. figure:: https://user-images.githubusercontent.com/14241876/181477400-f342135f-612e-464f-b0e7-c1978ef47a94.png + :alt: Memory congestion in SST due to a slow reader + +This can be avoided by specifying the `ADIOS2 parameter `_ ``QueueLimit``: + +.. code:: cpp + + std::string const adios2Config = R"( + {"adios2": {"engine": {"parameters": {"QueueLimit": 1}}}} + )"; + Series series("simData.sst", Access::CREATE, adios2Config); + +By default, the openPMD-api configures a queue limit of 2. +Depending on the value of the ADIOS2 parameter ``QueueFullPolicy``, the SST engine will either ``"Discard"`` steps or ``"Block"`` the writer upon reaching the queue limit. + +BP5 file engine +*************** + +The BP5 file engine internally uses a linked list of equally-sized buffers. +The size of each buffer can be specified up to a maximum of 2GB with the `ADIOS2 parameter `_ ``BufferChunkSize``: + +.. code:: cpp + + std::string const adios2Config = R"( + {"adios2": {"engine": {"parameters": {"BufferChunkSize": 2147381248}}}} + )"; + Series series("simData.bp5", Access::CREATE, adios2Config); + +This approach implies a sligthly lower IO performance due to more frequent and smaller writes, but it lets users control memory usage better and avoids out-of-memory issues when configuring ADIOS2 incorrectly. + +The buffer is drained upon closing a step or the engine, but draining to the filesystem can also be triggered manually. +In the openPMD-api, this can be done by specifying backend-specific parameters to the ``Series::flush()`` or ``Attributable::seriesFlush()`` calls: + +.. code:: cpp + + series.flush(R"({"adios2": {"preferred_flush_target": "disk"}})") + +The memory consumption of this approach shows that the 2GB buffer is first drained and then recreated after each ``flush()``: + +.. figure:: https://user-images.githubusercontent.com/14241876/181477392-7eff2020-7bfb-4ddb-b31c-27b9937e088a.png + :alt: Memory usage of BP5 when flushing directly to disk + +.. note:: + + KDE Heaptrack tracks the **virtual memory** consumption. + While the BP4 engine uses ``std::vector`` for its internal buffer, BP5 uses plain ``malloc()`` (hence the 2GB limit), which does not initialize memory. + Memory pages will only be allocated to physical memory upon writing. + In applications with small IO sizes on systems with virtual memory, the physical memory usage will stay well below 2GB even if specifying the BufferChunkSize as 2GB. + + **=> Specifying the buffer chunk size as 2GB as shown above is a good idea in most cases.** + +Alternatively, data can be flushed to the buffer. +Note that this involves data copies that can be avoided by either flushing directly to disk or by entirely avoiding to flush until ``Iteration::close()``: + +.. code:: cpp + + series.flush(R"({"adios2": {"preferred_flush_target": "buffer"}})") + +With this strategy, the BP5 engine will slowly build up its buffer until ending the step. +Rather than by reallocation as in BP4, this is done by appending a new chunk, leading to a clearly more acceptable memory profile: + +.. figure:: https://user-images.githubusercontent.com/14241876/181477384-ce4ea8ab-3bde-4210-991b-2e627dfcc7c9.png + :alt: Memory usage of BP5 when flushing to the engine buffer + +The default is to flush to disk, but the default ``preferred_flush_target`` can also be specified via JSON/TOML at the ``Series`` level. + + + Known Issues ------------ diff --git a/docs/source/backends/memory_filebased.png b/docs/source/backends/memory_filebased.png deleted file mode 100644 index a7fbc02d0ddc109cabe3435bb9380afccc77d7c6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13419 zcmbVzc|26@|MytNzLhQOgi6SkE$iHsC6(OSvrA-`ePYbm6-gx&LJP8okbQv{fo9zwO4pY8ow&gVKauK#_`gquT@1A#zro1QYX zKp>a^1cK3ojfp;k&^9xoe~N?}*@apLdWJ^01$!Va2L$?iD2BKPdw2wdcm;;8Gj$ly zsStmkwKAemD3G%lbm;(eaX;jSf-ol_XVIadp~AvK$XfxrdIE9~fIMZPa5Q8u07aaj zHr2uR1gKN1im6#C)aK{Zrv;GzQOHFSaukAmjzGcMP_QoKr%J7SOsy%U7C)fm+@d_n zq7>YvHrG+JZcv+^Q6Al=7G_dg>LD)$$nPlRs{}djgWP2x7YQf~4P7;aJPtuuw4h4| zDft=H@`se1bZX``YGXCEuAEx+nDRJ>dMA_oS83|KG-^f?HSad$CiUH01+?Iz)>ly5>Y*Sl$bK*6A_iTSgB*mRaD6D^ zBot*#Ey|NQBrCy8quxq{d^iV9_CYE8_go)@{8*rCqSV&27Xr zq8Mlvg0?gdHYm(^L^hNhefSpKE@-@i8l*@WuB&9nqH;0;v;|rRhCSH}ztv3Jrd{{(; zP~2cz`VGkX1S+G&!O01-Gp0*RmG(P|&--&z`LnJ^)J+p*)++1Ig> zS_VHNIOil)&-IKXhX2jZZfmg|CH<1;v+W)4+kNb}ZMa55HiH|MIUhT-IgQ7EyX|~_ zCyRU5jIBoRgyG|Nai{b(#M18))6W2t*f}TXYxf=<_$U*tqCVc{2#{RL=Jqaye(>F5 z$!dS~P0Vi6AtZOx%mK%ACSq)Y`z0%XS8&kVr|GAr>MU#7 zy3Q@?s~SYS00WOp7v$w@;1o;t=@XbFfi27vUK_XQoYWjcvUWS{ORatGKqd*!=kK2rG46AuUc@ zp)WMw+2CvkM;xUe%|PdzUd;$fLnY&X};Ks}F(YkY8GhZMHfc%rGuiz{%yy zNdtXdZ?_-be|X;_y4`M2U}f1eAn&Q4Xno0ZV9YivSE~|1hapf0j9PD6G5PWB+4kJp ztSsv4r`YB-UUc;M_&Uy~tGK$dC2%3jY-K&F>MJ`*OA?Qp88gwMtbX-351egi3+OtT zPI&=sT*iNUsXzF>5+B<4YA4J=t5W&vH`f`abp5SMGx6N7lztRB<F*!Ex}42D)D{%h9%V(m zv%&`nYt(rq>P3Z`ELS@vOzfv;h=%H%nBoI~^*(SGbLh8W)jZ(HSS-4Ift>uWt^W%D zp9t*Ew(=7S>IEpMKj^g15DI*vVN-vz`!8CO5~#Whp%yCz1;hGZYvV`2lYdBXnS_X4 z1o^-an>h5wng71?Um|=iSsA{GwUwRBy6Yr1Y~OVfJf+1?@%Rr3n2DtAfk7;ZLBlvh z1M_7^B{(G&3nLe$hR7R9RNMtEX2zVJ@|~k@ig_Eo_G&(p-%E4e-Q~3IK(GF=(o}07 zR`R(jZ1zJq;=Ed_&FeZQMO$>(o{RBpd(y^UCAheDs?FU82ChD76r%Y4v0@fSMzg^2 zdkEZ8*ewRy{168}UBAnnxd=!aR>>q5zvp*URKtlHF`&DP+P|()0eN>VLsAaFcV`gs zo^oLJALuA$G5FaNKpHHvAL#-kN)MtHcwKGI41pJgS(lQ-e2E?0@R}h&YuHDTgrQlA zrSZZnmt$zU@c0`p%9}X!G{^%_u*Tw<{y4Fk5$u5VV{J7hli-|O-w&+sdWh|}OPAu- z;pyLeeEC|-n;pLbkU0K8NwQt!88)$jEtv5bV;ij>b4aq$+eZ%r-UNo{$0-$>y`-C99- zGh5B^glopWZk^`#zfRFy=vd~n)72L`L!@#3v22=;f}UlhYe|3fjKgT;vO$mP$t8dA zZRIAqF#N{~7z3A4r-B+G3gaKs8d2cQ#B+ZD3q%9i|62VIfVN@dOn<fcadjj*DW}sLdR$mjU<1h7d)H-)pJ~}}2!9chv5P3k$Ul6o}#n^7K z82>VIMAj>CO0UO9FX-T3_)^E?|{EHfUlpO5jl zp&_5>SG~og_Z+3Revg5!6Dy}~yd}z199|kG+<6;!_gseGA=8pQ2WPww7v1ukV6@ZgUne<+}hVBQn>x$OA{F z>|*hZtNO9~uW!P&t$-15Hb1Q$1h1r89ckSV|2fa&zysyN_LBZ;5Ti)Ikn~yU%D5D9 z(H&!efojL#6c}TX-nI|ITLb+>Fa_2csTXn9{Svi3Uzk#<}sWvtDZT%Je$}_Hx zaG00fEOrwJnXHq|-CjJr|K8;(1PFF~G--V<1c%7Q;_q!2nqgbF8^q!I@}O#N zr4BO{hleXO?sn}B>>M|J9()oL@y6{?JKQxHD2@YLiM+dygcYR0a8K}VLC#<{kZFNA zSpwp?ir^zMZAtVCSOZp>%nUatFRocwBN}DW2N++6T}4aBgu&o?Bi6ag9_@XNOCbOJ z$3qW7yPr|j-^{r0UL&!S1y#)(yHDyB-gdEx%sYzb-iXH2_+iy9l+}{*pGxC7cD)nv z+1H}~lfp0)EIX+u*xF;J{YtzxEayn;nopCo?9|g&@0u+Njm@5 zV=&)uDf{ug^7BVV?xkJS9Q?tCwkFdur7r!Fc{w{fMNWT2@{!--p!v(|3FdR4`&b%s zrgj?!0}-;;vr(%Lzxz^LKDVC+5;jAQTp$Z$S! zDm9rq`#S^s)^;PE4XolCpD@~qrgM-4Oc_Y>W_ZJGJGiD}EeN_JNE+*R<#z$)Z~{X{lUlQCtPi?6mm=OhRQ(JZi^ThU)|T;4&1Ep;)FC|5#CNe zWjDu;4d;fd#?u(!nMtP>`(vd)4+#%^g1xg+0~8;k&OVWN{Sa=doTGrw&o?f4UkMG8 zg$Va24_Kjwp{=>h=z@TLyb;KKi!FIgwIM?u~)G^zT7ytQKtBSvbEozN-Bc8 z5tNn-^>goc`PqPa zU)s8@o$~nlQmqYwv{5dpX_V9!eS3-9zgVc-VZQEjVeV%sV1<#V?4@`# z{)T~RHpFj@NMnEvxOjRUZV$~lK^7RX?=cbPKCtGNmUAYVwIaOT&%#;IAI?1X{GvyEVq30M#3AQdF)A{)8R#t;tYJwLeqp5t~)k7DoNSxnFiH9ML5WxOCSgCV!{Tt=M@(#gD9N9 zZ>UJ#$K>;0k$0O)9BxGE8dfG{g1FnPQ;w9x4oOmW8_~geq+lBnwfjb(cs~0PPI*`8 z%d?*!o!$bt8|@olg(8SP4Zb6;IPrW zyp(HHKJm7^L9k)8x79mQ*4ku5C8cR~GNYdoED)lR;vrgh{_Rqe9K(yl`?9^sJc5$yN=J(Qt9W8O=6*EYgC#alyM$g1r8< z{rhe+dE$Lb`TKe#ez!%rSL;~B0?XdyAXyG6)LCh;(02DVZu0JvsdPYnt9!Vh)Lox; z@hZu}$>*Hg^8J3p>ic>YS`h5Az6pNq_aYSY-5|i-epMrW%sw8k46AwYXZWFcU(gm# zRau7CW*j=eamsg^h)2*}*vj`K_HMfTP5uoVOZo#3W{t}+t9-s&(-Sy4?$>{}U0|U} z!7>I56j^C+mSH}DJQU?Wee^%|0P2sY!LO6(Ea*Utb5pv+sy|Z__bNJsv`Mb5Gx<{jxk+jZvK6_5+XrGPefrORj z@MA^UMPG7R@g8lC(QAv!trW5|22;rMGyj1%8WSG)gE+bMd|@@7_VvZFtGK(DX$#fp z){UQmj{n5VERyUXdjhn`8ofJP75?gTB~b8$9x`F&z+&u|FXkoRQ?q2c$_$i}EhB>j zcDpBrR{y@FHRQ~f7l+EYRRc@U-c-vE(W?Z$KHjH5sa{~lue=I`UmC51W;;7QN;g`X z=(v~~@2(n{NaMeo9xrQNlD>HQBroh4Ny{BI6i>GU3dXaSg4EfK!cqp5=@>K337(W= zq($w5P2vL!XmGFk+Z8J;5WMXL1!6nkxT1T5z97RuGx_&sVc1J0NXmk*GQ>e&r7RI> z2bVZt*BL01OaH@&9opS#YOz7p^t=@LAc($xILsDku&w_(y^-JG6p^+kyfiU^hZLZeJSf5) zi1ha--;3ShPL|xye(0}tKG%$A%yBV8hnpUOLj7-~1mC#DN!;s>k$!T@NYuzizs(hix<8T`)hAL54#jBWhH*c=-S$F<&^V|Cq;OE!?J!Y3f(z=on?i(-A4Mg z9sPYZ2H&u+yW{G`gwY*pTi4@F%v{)x%kqFe9w#3F7F_9CNb2Fnhqn)(LDJ|OXi0Kn zTn9IR-~}$TQmPNckJD!uptmu$@c8?H@VpSOIi7X*CM2cfV;mhjIsqe*=AeV2cMXzc zqga|Qfl2TGc%Ue`Q7IfS!6O2V-b?=B`(G#Kte|xaD~5g?C+|vj-Uk}>l1Y{FqA<_Y z<~dRyOZERQfbtnh+hsB+fMNO%W3(1Pe>?c^HAG{89{2%=U_J#>06Q!OEkmZe_g`&W zPjYm8`VlaS))Bwgh}vvSHnH`*Jo#&Sb2E1CJPlkr?dlPd*T!qNeqs3q+BK{;7Mh&Q zd^a~W(7$n70IzYHfOfbsPtsHE^r60p48=$)x1OV|d}vx8G$(S?jCZ(EUEJwnl3J}Q zz|Y@mb;r==yj6BFq@zMV3N%0o{Yn&avP*qZrgEA$L*sCBDKqccEmzuV@@U}%cX8F~ zcq`gqzHm75(aO_-gp1=utN1d-C7(l9*^?31tk3OawAyvO>rMK0jcFz*<;z^YW?<0i zA{*8MrN1NbfVb4zp75YA6$6s2Vf>!lEqZ}{vh&)-o?~a$CYEhX>+&)m<&-s5e_Bni zdU|{-nHtD|*&&~tych2E^n!+)m+F?vUZ1mSd+lvTxHQ6DpC$Wv*`F}8fOyS5x*l*;$@3o^-ii|Vb6vNeKKzA)Li_6Q_YGznw6t}tmdqoa@Rni zr-DY5?veCk21XruP=vb|VNwRnkeujj(UlYsA_rfB&h`*w|ck zcG%TtrM_N;?jIkmaB5apHnx?n?>Lt9=nhW?cg?(?-Ep1&DrmcMsd1WPyE8{`BGlW9 zFr(iq^zGR#eY3AC!aCVPLJ!I{_b9#!uCe2dJW~_k1ntN>Q0|P5b`5Vj%5^tShW3wp zD|USg))M}yrt)%2tyA;Z%zI}aSBDzW58?64b9K%Q7qVuR_rJ0pS;9r(nV|hHTkv z2bdwu1+mR6SY>vgZ@_vwJz}b-H|>?cckX0mOIFm|Q%KSv5`R#h3G2KZOW)K=k}s?T<`=73 z11@(E9m|dyYRZBXHsMwW4lwQn50;H=8b@0jaYRz2xq;g(S9I}@ztX<--pu~i|!8=P&`apb9b(~8vuXF!tmX0%fu_qQ{ z(?9Ix#_^K9X93y)R$K)Gc<&O>mp77i#*`uO3b6$lpN`Or2Bh|qGbESGyxE(`C-net zIdc6zVM>A*po?&6T#lL~zaBrn_b4Iv zyb>nc3pj}|=cn{=fC19vnE~t6Gb{z-3waF*#|0=cUVtUSTmp;c27;g{ zf+9JxSNPqm`^m7PzasvVwc?!S5L&n8sE*@wleY8q`D$Z!@N(*TO0KN&L}fXubJ1$7 zb7f>uvU1xQTw2O19v}Dg)Eq0xu3%Z&c;gd&!EXkmX(_SwX)-C$J=%ZZ=Wrytt=(;~ zj?qlHzkl2L*;tV`Y3gkqcgsqU>Jy26+xz6YllJ^s^CknMDdK|eliGUn-L*$IFVviN zI%4CoVKLmZc~P!Evn$W<@VTm))zH$|M0{S}0yghm{QY4AZ1?B%%FipxDI=8=E6V0C z8+6vr_Q_zC%TY1)UD^#h_TFM*-Fvzp**nwrr}S+3>a&Y{0BQ4XYu8My1%D%KwSnUo$LRF(Zy zX(8$7w*b(8P7C`ZS)_e3Y9OmaZnQW+ z=y}dU{`E1={Hk_|;{7ZJXr7T5YoQL#7uQK55PU!2-vY$(0P4h!@|ZynuNW=Sn1Wke zxYPrnq$H@iK^BZD;W02{!b&eW)+Z_i6E6qnjR^JjB1opZu<`^;BmtGhMOp8Sv8_EoD3|LgQ#{RDy-A2NjYFlRuv7~o+5-kr9`Sff zhU!;dG3mNC{u%YjRp0 z{PI&hri8=bj3lMO53sM;@OC)Bf#ajMl>7iPlNqD-nae<0q&3GMI~9IC?x&|={P<%y zpN*v$3_fw;%ymF|Ju&40D_gGsZZRe7kbk#r3LD7g1e`9W^Wu5OT)@vW%M9-5Cdj-rW4SJMt z_OYNAW73D1178`&SL4i?FlFQ}bwZ$a{5t)shXWi2Zy2g&4dVwH^v(m-<>YvEg7Pt& zwdM&#H3kVR0o~ z6OE8b!G=PM-KmMccm>3pBZ`1GZ6`mq-a?zCC3A3W8^5pFkEXqer4rMA@|e~MFnh`! zdHu8B;)-%b^53b1Unk{0R}3~sRw->7d!Bz+RI$@sclmtx!S*=xhk4(?9UtB5eC|-) zh>V+9Q+nh z_msU-x@t`f7mISncWLDnv_PH{AFz-4N;^!J`U%|R>St_~&a%$NCDqBwr9Xx*lCTP> z?1ky_`43VLuZI-yi>(AAmBp`pjGJgVwW=5p<6se-*m7RhDV|RdQmV3ayH_T^!_iS@ z3-YpEO~fbl%M1I)-O2^2(&VS@h>W@myXBxP%v#0L4YtJd?EK36p>}> zKP0oz>QR)zQw*e5;p}?1E5PvO!P^lIQF2_EN8SzcbD9X!TmTc#Y;ZN75AasB1A2$( zT6;t|X#mss(VB!6s5_TvG1x}p#L{=LD*AX=SX*B zB0q=o^jS9WjrtLA@FDR`fxbHdmBi3douCI43Fys{r`Z`ucar<;e|UP))t{VMI+etL zlI6wqKKw_M!(i#f6NGG`3{J^S2?lnoG#6O;F}At_4v_VONXjWiA*{GMc=0MAQP34b zlHrA1Gj_-h!-v6BicT+MP`xtDt&I}d^>-@Cq_I!4l!ej9DRK&-NlIgsPe)1L z%kc>jOyoMw^eE?y(kHgtCK3Lhi5vSm6)r{Xlh!bMioOxkypOE8IR&h@Wkj(E53tA+Et z<8DOs`^CE=Y2(3fLYo$M{Ms&*^a;s-pS+m3$C@qqTVcRr)6@B!j(MYo_OrE;*7tuX=_FYZkfcsGPuHT<7d1&!01d2tCZ} zVth~GO1DQWN87P#)sIt@leOEK@ydF-#|Xj~d0o0maY?x$Pv+dxUXh!kvm2!AI=TN! zo78!D;_?_en{i3Gm(;S_y$SS1Nrs+es!1tF3!hXo_A(s#_HglD(W#fERol|N+}xIL zS5k+{4ZAF9$5@!#F9+=?(biegUKuPnUk02+IKA-sR zsq$d&C})~+`$7;$ejHwc=;G_I7|VcC^Y?ky%|_Pl8~V^3cUeI`Zdm@K^hciSO+(1N zNFHaJXPj1t4Y_&`8#GwjPGT+A0S$zAlzDk)2UM!=Yp{~0rbb7nMyICi-`Q_Wv-X%v zn9vH*opTs6U$Ep;42Ig4a5o~HExA#%_+#MC#jfInyJ>-I;Q7#l)TCHjF20FD14_pqkHG0UBg8I$}Wj!$q44k-oH~7>& zj}aEq5fk)$vg|`i6V0#YppxjoxIwF=+?eIu@@?hMRv$M%`M6ZcU0mC+%~wd_XnM7B z+xB`mc;-WlHBV1M_?OXQjnxVaRc4}IIo zV)%Zl1F*wBx%|p^ED|cS)_OY1D2d=sODe`8;%zc|r}npZwybdr=+URPMB^{}6bt|A z?#LQ>`$MHH*0y{I;e}8~00^&#i9+`Ow-2j}A6C?BiUu`Z8Cs5{UcT!#VY+f`D4TUA zx}p1clIjopWV>f<6-qP!9Px+Zz=_sdY2j1P5krIlaD%ti za>3Ivq&T2IZe*kLb=E(L!G3j#wKOy&Cz6Q-h zYd)T9;bwhq455xf<9a{cb%nYRT~!F=i*h8sBVQEve@bV42%d1@c?(?st0@PQ&c--$jzg(DB z7wV9;=7T;T@;F+&$9|PMAR>HrFEl<&t<&V`auXk4YBgySRh+F2RXH)+YL~0}icOYT zIrbJYHC`%1ZOPv%_)bfwlhimPIvUxr8beSRevHJy6#?3JE>cEx5COTG3BHE^`n=fk_gMPGp*OH?>w)bcw|N; z-zl|ev3sdBGuY-N9KNTBqksF~@WG%$sS zZTjaDQbGlq)%73nS;5-}$t|<%!T(tDfBgeZE^M>9WTE=*q%e>Mer$zp)b3c2-M9Z1 zYIs*8vga*GoSy=55 z-5@VM9Cj|_6))-C=dt~@8DDc<7z@sHB$@ZP4>zt1fA0ET`^r0_=rtjj5v@&PELeAN z&%muQ6@)MDXO&6{t?*ZPX&f$A9)0p)qvu%zcI;GExw`)JaPM1}a{rk(Ir2=Y?3Cx; z5mc2T((6wK9xlBn4g~mE?d&u5qvb&Y#@ptMsQmY@^~8myXI!4lO&oQ7=nG;)qrc<^ zTB#4Rdn11K&U)yVz8EeGSKGaio7|%Rg{V~yQJpSvZ zBTrCaiY(LmWo9w&&Vxv679kOgnCHT%0BG%Xdh@gGecllK`JYW~mgX$&Y*7e=P<>K$ zyyN>}Pu(VE|D#A}9uHX)sRpMDT1PjV4BltfoVYtI5_0traPe-Bug2G>N1xpzpbhNH zMQ1od-c_ADNzRSOe`F0^*t^zwWPd33-|wDt?;rQv@44sPbDZONpV#tyKAw-q^Z9(g-Y?H@ni+E*6+X(s z!oq2C_3~{NmIDwA%YM_t2bfRHvi01Uzl8ja?)Y2yx%=asFm5a!zJ5M#XR$6AH#cAG zeLw$|19ke$Su8iNTN>@|?lS!K8SXL+v?jw-k>PTRaaZ8$*RPBK{a3GEG3@ync6^Kn zXBmMAhU;mDgD}HMg5fI7@YiG5@-hPS829BENFj!s3==v>8|(hSN!gizLHQ zoN-T#fz@O9X)wH$87`+8_Ph+F2*XE>;iXCkDj42s40{1apgzOx3FF_aVlurXk6uwke@mo){y@)sLC;K}zt5+?&Y+V%h#lAy z;Mx;9vd6;mScG+N8!1YEmrKu2r`hr`9$q{xBg=n$kAb_eX~7LUN%0BnZ8@GC&rCcn zy0_^fb4-%4;jYNQT{>l?ui~?JQwD$__HU^%VD>~2)e{bL#S(M zF?<~v+NZfZX7;4m`9k{{cZ?Vp6a*r>E?+Ue#c#uKx2965oc9(OjyLw4x#+6gwYSLLB7`F@pA3hX`Ydo1!!?>=+z}@fY=vcgaYTZj+@m1k5+%Wwqua}R{^78U* z2zKH{6oo?B-rS_^>@d(=dzSkg9PbSc4NXl=F&<-XTU!2dIKxQM*j-+t_jWLzJ!)!d znvNno4hu7Qn=(H?uU?SLl>p%a>yrN$-k^GPANgU!wp3s0;LAU}T&JEZ-Cqyo)z$k3{rd5j3pKO-Zt8^K=Y; z?a^{s3^wDZr#>%y_YNU_v9IIE-P%_RVbX-6r=3o>pA&8WSQ^r^1vXSg16tp`8x>sg zxYHUFQu%QzK9lfgWUBSo&?-hE;Ri5MUR;>&{bq#H-!eXRN!7Ttu{__@;j`CTsPB*$ z`$ZO(lPo5eFIqkvU7~VdPl3hSR&xh~D;GZY>g=5#@FWB07e9L=&k5vSc>DRY-B0nQ z;r3%K27+CkqxYmvs>)6tJf`$|B8=@Cn`kFdw@l{q;i%9jB1bRXxFRBwtZw^KUCI9C zThjq)>PisgD!-*ySct)WsRMRN)4eh&ee<&`7c=K0?g>UZ{XhMrr;8<287qYc`eki? zC>Vgezd_opbPbzu!4G!J2(DUl4qmnauT(H9JIp%EeKeKl7tV8(Zm93*l_iMM**|vi z-w2w;!ph1q@hotZV)H#pqZ)OGur_wn{Li5SWNTUCoI^l|feQ)uD>~O<@c{Of zKV2sp1; zNWDCetS5+ewuo>_5Ylx(c+nkOUD3Kfz>;xH8Co zw35B}sPFZ`vl2(;|9fTt;#`%lmfF3byt<}j)gVpVWBF2a$3u`@%#jF8wd%fkw z^}3BFx#V2>LUnEqooZr`wf6mWQI$eBTbm|stS}ASsu?6D5>p2q+ImK>xdIZ_P2fQ)ZkjrZu)_LlvPd4Tk{~w-O%1G z>qOY3(bWGvtY|=zPg$)#Q@HJES~mM3m|rJGA5QpZTHB?_+y#u^(HL4(XHL^Ut>)`D zDW@2;cUb%7E8b}E>bA@D2SN`rEkvA9+r68y5!X_clMT1$ug~qytO~Ku>%rY%#z3K2 zCK=UmE@m#}X$zqi!v~?F@P9qFxZOsbd;a9J7fcHAo+*heRL<0)&tqi>q3QSxCD^vp z#`vEz#)Qbw!CCQ|*3x}&MrYlB{;|VHk&wB<(E2 zGMBTu##X-RfBOwP9?>71_&7on6)ze5z5G}gY*U46PU5h{eQ=EWtsEy~`&WGmcHUz8 zLoWgaL*25Fx{~ckw}wV~CuiYgwp0evs%;?2k<7 z-_t{%`KzH8U;pBLfjNeo1M3&yY0$*(RBeZ6=3;VxXc(19MdY0G{!XPDrln{Mi7Y|L zS=ASL@AGVOy0bne2hGb|;wX|?PyO+D=YtI*Lm-^^pT&2)eoT+OI|vL3qgLWEP)bI3@U`9ecX!(fv za@6K}NT}BzzbESsS-7h-qwF0~W@y@kek@|RkuKJ%Q)73;O@$upoenja=K}Zsj@@6+ zy=6cXXAW9O+_U$8b#)B-U|Fi;R!>pTgXTt)?sR^r+Q>sWCfVfJB-`kJN%?-iYvU^- zs6 ze75aNLf3zY)X_KAy&N7cv{;GQ)NXfnnf_3+{~XxSRjz0cxDLiBQ$EaXORX?9aZFsh z;2nUd`#|}iWAG4&-keA(KfBxsU=Bcerq>jZsL-$A_*144vmqBSeRa#n)iIvRioIzd zxzp&W5**WDdQAjV&ID>we-1P?zpI0{9-F~TEydwI4D->|ztI0c(Po3m^R*`+RA@U` z6>x=4&(MdxoG|6#|E6t%bsDS2=G`fg0@u?Sh@L;#DuLdIySmARdaLc1H((F98gNg1 zD&*TUyf`Q|9U=&!64LQy`tWh4iC*3E5`w7m$Fz3aBRG=5cPV>+pk(jW*qbhdf1K@X zraq3HnS0=h`7u5rI7kwb5J4%2)7rO9IhpY=0vAm$mnp&9oi@TS_hbkRjx%K=p#-ZL zLUO@i=&>)YrE7`%;J4KOGC~#}nX?Z*!b}G8eU0>sSc9WX&2~~QrQwZoWCv#%2bP7a zxC|ixC86EL+*5YS_eeN7_*+Nc1fw2UVfreCs04bS(ZW7j;17-!r zymmbR&Hkpf3!x1@2tZPE1m=2cQiU?}u5;c!-5p>i)}@0(Y2js(eJH`JY;aa7bY_gn zI}ym~(XIU6inRfe1f)J;XRRV`6mwUpEupJV%phcG?N8RPnfEsN__+SZ*{@LAs}F!9XNER;TrEI(`CA5!OU_;SW~G&MUV zGczUqlcc)BjoQCB%{qTmyQF>s)wQPBRr*YU@6~G@6wBIsF`LCA$~WXjMTq0_8_beS z9ih4e2#*P%~_#FA|;sT@IPN((Sn6%>gk6ojQq2hvjGZZ zfP+%F;f^`8EVH*p;_zWiolNwE2# z!5N3k|Ct_+KRT6D8%5JTgWG5qL#+ZH02ofRXeR`KQ7?2cL$l)Hr1_Q_8&2&yJLu4i zZw5^eV+{4onEeI}dS}En>M8a-Nz8qvU|KxBD4uqBqrQ@K{MlorKt3@B3|B(QMzb!lZGcafhbAa6T$KYWU5LnLXhyf~Y_oQm1J3DjE23C@P3mQJH` zr_c$KxRMcq0dve1Qe9CWrGC2BJBK{Z0`E^FE7wMVtIpzp{S-rnE}POdl6tV2&LQ-> zeC*fpEV9B^?XGDv@>fZC&I2CZD@?>MqUtVoaw?4VzdQ^3mp=v^OQ%EfR6#>M$oF43 z$)AXR$pXda_jcLLEwe)uY$tOfNV!s!IZ~SkU0ee<`cnQh7QjitcmQU`pqILRX5=7x zsB^^V1uZBUB&^yhXOUfF$rAtg;OjO&W%UgI8Khk5z691?32rN{mPZ51t0X3YR1bg# z7ygCJet7$MI9@rK#=2(23F~E%e;)o{)WWZaEU@#y*R;v$;UvFPxmmN&>Kah^5Ja5q zMTWj@@vymmJrTehnlJe1e)lBVA(W!`-mOcbxUFvH^0JZS+~;2*2gG(TJ|BY%Kax__ zjaruz#@8NZSh&8q>S1qxZSH#F>O->MBwxnPua>3w5d!}R<#w;q+q@-))LO^G4KvR~ zB9;Ci5U#4yUA`nR>yA@)=$>xT!&HC79+`5G^AK29foFvwZ4F#7C**q&S6(=M7~_Zx zTrq~;cObYi%0t!1S>cvj=RnF`$WEZdz#b6NOq|$2p(^VaP;-Y0NE4Nubd{-_c_SK=V;}Nq_#;*BAsZ zGg4U6{9h(v@MN!LX{~xDSY|Ipnu!kKSkl02zg59h?PtPnLA@jKN1|$$0da(72gU&)Ihr2 z-mlCgLVl+iJo=^Vx~J8S-1nbnbxx4KC|L-SVJdTtL|*Df3TiK-wA9)n-nuhs$_7aj zAdz-~lkmaj{|ptH6?ewMMIiAalktVEhv7n>ZsZ{Yy^b9<=6Hv>+E;x9sqIJK&%<%E zH2jQ9I{pVAP9Tl^YcnW`gnF-iB$?*Una1XL0Cl5b&BFwmH>0S0;~G;HUxyK3lSqxR*XE!-);s z@VQH9)c*f)a(hNoYeyVJI~te84fLjRKu40n+&S#tg~KE*FuXTRbE%8)+GED@3J+H{ zQ)a{_I!7}B%`&5O0PLDw((GW(2Q3~!$1(FQBNxLkx!S|mq55#)ncB}J#|V=0W6(tf zXV)9@$%(mEE%A0l2h}Bo=1arm23{`)Wkj3)V?%B@DXkP$IJ=Z~2V_?V^Y21_e@OT^ z6t^y$#?LKlYFI{hw`bdz$IP@MY%!F^4XhZn>|=6y3VzLrw1_IlfwxLDidTxLk}c}_lvDcpE3eF9FGgOXGs#)
>uR<5u5JV)GcooHpSA!7cR@87Wqi}J3C9rC+%vGPI%4N1`aaR9lo`! zq)Len>sB2b}%$WhFo8993$^QvAFCUpZe@7Av86=yW)5VTv$E2>)U(hxsvC^ z%BGltYTLtuQ6rQ?RrsZ%_v1H{$Cbr{CmM{024>;$kWFprksEE4izA4i`+Nk#s&px* zug1hKnX!Eq?uCl`#W2xCs%NfnT~soa{N~%xQWjN#>dh>d%2@6NjJf|nm*%d$jsO^kR7eFcVsGu1&DHjOt4_MH+0L>AMZlynl2yS>fMc+)nDIZ)^Z|7 z`2x7k0Z~ulWSWuW)9q>iE^|^Tq4sptFtZlRC!{qYq!a2MI6_VF+eb$ILMk&rZb?|Z z36a-@Xr6Ik0oZIB5YCbKYvP2bO-Qf1PLFBBF@d#KHXr>1HQNX*Ksi5Tf`BSJ5oe!( zhSsD3-V3)qPr^2-S5OHZE)pYYv}`MgnaZA$+U|gErXaf>G${=26G+Bi7Z1H9hhnOd zPcK9tHW*E#x%QoBW>=hH_iv1O%fhbAGd#zPI7$_e5d?|uKxp$k!>hsPF-?OhAn~QV zl{~TYKYQFcfO1<1xOZBRgRRMF13TCs@b`2#CAkCP8^(;!eNp&@UhNGN6~K$|fCL%_ z&N*bayQ;1KZWG1$2|<-JfjU&P>aK-6(^)?y=}{B5h6RjGUCL9SP!Tm`#&YNce9vP; zd!TKyOmLE+Z~#mV5%IulT}F`G=jkl`N4(1pxL{cEh9A;T$H(>&IyvA9 zpKD2NgzAaSRY2m?>Ij{Jxqb?MIjOWCTo|lMX9W~a!tbmgNnVh89XjN;BpZbv)~?Cu zquld?3Ppd6xrc*8D)5qS+XBnbbeq>4P#iV;`obCye6StqjKYP37Aa&7ER!YW5xITH z?lsch71Sz8#=D2GbuexqT&kTCK$DkI>ZJ_Nu%X-OwNp8fq`srB7Zb?AL<9aOfboA2%ALSDp2cw| z&?HscrfUxXUIxQXn-@VXj3XIbuK_1Qp$ygQBor0xhsX$lDXzWg#)U4UOrt z0t<|70$G8_(HXU8fQKapS_9Zvb~pRj>#mZuYdZ9vSVyZ+c+KvkJ$u2{X(+!xdVyMg6}mH;nM@C zC#_h4p(06)x!Mhx^Su<@g?WY=TA~fLryPy{*bO2NLe7VbTfpfTc&rG zex*Jmx|c3Jr7zuwd<^=%CgIUw-@7^AQy^~hu?D=%v8p~J&!DVBC_6=%9X*WK}H%hi8f9roV9=jTrC z7Gjvf9k}oLUQv{hRpLINj(1gFaer|%$Aqs6>5f0YP}Zl-qSMDeM_-R;47mi0+8ed@ zeG0jEj~XGbrt%`Nq&;?Z05Nc~fqQWTX+J{kHw1Mgh(r2fx18K4p3U%&St{Ct|5)xzvRMKVSbdgEXSq}uf2=t z>hESJ0h`fi{iEA`9}2VrnJ>1Q6=2I#g*(b&G^-ofzO{i7)vwMvI3F~w3yT}P zS$z}i8RVs|flqG(S=m>i{m;p^obXToH_tpPe}G zYkLU#R7}dsz5{9gYIoW3y8}tO44llM8P=4UkijU&!>}EZ78AdeG!K=U%=3U^bH8s0 zLR{yfb-z5Z5yt68lJE52P=(8X)!6Q+>*2IMrN$g5jek84-;TMm01oNkERKK*$!lPP zRxocoUOkD--h#~j6FDz#*+5W0;<8>)d80|=Z_MwfC=7GeSrP7Nr)Ly zDHbcx_BbqKaTz*RPl>K4NYS6Z1#o7Y*F6HD#c0~XT-TmfkdEV`+&IyF>NdnxrHG@g znyY}-5wWt6@mFwJ8e?EO6>|Cb^LqL^dOu9M1oeIFrte=_JhD!B01TB(hM3Pg^tK08v~Orpm0?iwEawE(To%OdXT{#*BhzfQ%YQ0hZHi&U3}o{ zNZNPVFfs}CG_jj*k`)LftPXg}W5jxXWT6Ja z9VqKNHtO6xZ!O5XB!V1OPsnZ;GYe6D`wTP~Qo)fP@*BBX_Tg+=2noHEQZVj9?~22| z&>vaj;LQ&mr(dG0Pm&xtgc3nGACTZHm4)Ju1LpSt^2iTn(~&TD7{ne0%stcbx#qjC zYwoA?(z9YjU(AZ%ZWMD^<~m8j4^QNl%mCpB7U>$I@U>>s*)R8@dk61*asnjo9)k9r zg3+8njKPyBsO=xBx`C$pT8BY$0MuFUfk&H5k#d=At49;@LQ*vCL!i%9$m%{s ze!T`>d`sw(e??LzyS|MjKMd)6G6`9=5uYcM%r8P58ThgQrsT6%;2W*)y04QALdo6z z*52;zLjy-5=f0E^W-Ue#=_6Mnu_@u|TDu(#}>!DK~@7{^xdX)*u?=^y;LEeyquzXoSlsZa# z;s3k~kd`_Nx!JGe0Zr_i#X$J63({^C`G7I-`M?8`j0w2XSoHoV30Lj|edvjGow)uZ zD@={}T%{ngU*Y&s;pnUL_Gj~6FG>}PULqdlg)a`12W?36$1Y9C-! zd>kAwc5|)})Jvb=&fwoywi+`(P5AZt7JsEr2-Ld-E{S<+lD|@If6bk!(N*UUwxGUd z>O`g?pJb}mmuvHIr0gZJaE!`Dw$N=UD~O4?HtrhpwLjpnUIn5qhJ>|=p{f~(%$kWk z@C@q8;;sdcaQm9QO; zyx8uijK6}H^*IgppQu(dYFL>4>b;yP%dL<`rSxl#oN9N+VK*WK5~&ug@mSyOC5UZ=8ru!sbFC9er5<>< z1U)HS?@3>R=10d~ipua*;!n9Yd;BZuXUw{xjtSvDsA)26SMrg$^7S7hC~goLl?+dDw^{?H!Im7Be!PSjp@ANKmAnHQi_!{H-1vM7#pAvG;^Uc4Ud zr)v`ky|Ul2N706s`*c%T7BuE?h)$3+b{R14~Zg*qALG z_SbPp@-*(DsoqO+#YIk>i|mC720<&CxNrtzUg`sRqVu-y>|7%iCcbVYes4VSSN}Ly zwMcUXzOGwz$c$m}P-|Sb$I)((9nDm`^yR=RgRKscKP%gDb-s@_s>grsR^`^!pi|1c zZnjbtyTry;qGx*jIcl!2fpG^RHcPC4-*aYrQ*wb%LWYOL)(+;X-~6~A+VEm@@AkKR zYB}I9y{s?cn+vV3CT9%JI^<)OxnXoW)h%fna%iQ7O)&79Pw-DX4bkR8KUr^>@xih~ z73{0Jsz*3rdP;}w6-e$BEb|y2-;VegPtxQnivFgh)=V4KMUzv)XcGUKoMDN^4Z602 zlhNw3kV!pNt`G5e7MjCn-9w>U4LpSmEJh;UN568L?Y8($eN&DMr=Zi`R!C~wsG9PZ3}d@9iM)Tt0bQ%J`; zTyVisdcV>B_m?gj`oFXE@Bc`_Nj7-Ytx$B6BYFVk{sZXwa%KI{H7F-{T=Q-lqI1IG z9=49C@#=UCd26p3i5GQIJYS9ty~eiu7L|L2;Q1x2k-D@jUPAad!im~@Dln&i7?U8& zJVP(;AbB`J0Hv#o+BD-Z2%HP=r5Llr6dl-YNB5`rK(4V>D(Uaz-Ub0m z%aKXp!I_@}3*=VamAjGhJGb$~`Uz26Br zjKh5=sAX`X42F(Q9$&I151vuiC>eOr#@UA>7GTC>^-Y9DSzyyooEzT_ z>a8}a@wE6vayg*r5AiorM-Rd^7uAJewPwWWH-7S%wCLLJ4l*A|Z5d#f$$?B44E#|A z<_tYAM%gUurORQe)iDX|z>(6LM4IL;_2;1%&-Wp()^efCUti^z$T;5bBO`Kpn&&5HHE&JZ@X$5cJ}%` zrlEB)@(27O>!>g7IxB_$5Y%FU24s(6`+@mymDse%b#;_+5G#D)0zaIg-b*ZfuwGjP zcRob7koIoRbG1s7msh`UwQR47r{j-ctG|(S2T}ipVg4TE_T@SE1@N3`H)TIY#$qa! zG`~nn%K(E|aZ2jvBI@|Xw7qB3-Rx#wC$FJZ&{S6~)lAYsTa$oHpt2yIETcGGam zd%arEIhY4WI}gq7`*M*JW~nC>HeH(!IUs%9OeB@LsQ@2I<^p8(jC|hM`fxp1;5Ipk zou`Lbfl)s+V4UBL%-~fjZFpg_?l)+&7a{D^dj&d_j@Q3B?^^}8N0+i&LgJFu14mG~ zE#NXUMtm_x(9iJHSDCT2zm;fZgf^@=#}G}%e|ZYjc{FN539WMpo|BBoGmsHcmnYo% zDEvWC==|^!AN1yHNl#fMKD!gyO?*nKD{V(eMx2D?4ngZYI8QCdXH8&%JkO`889+B=@z=#`(Nbrh7j1G-e+dOSP95X#aW?*t9b6Z?Vk zA4iCZ!JURKsa;{sCSdcqRX*XI642J@OP>L-rPe*_hrvGJlT$ z6i@rhixtRkAj^|z_UW|e?!Zbv;MJ2osdf6~v9lWcFFEV`pw%%{4tpKgN+(aB zk-x*kenlAOYbWvr7hrJTvC3ab$^2gTApU1GiJOjgiWflU@fb57*)}3AC-luL>)@BO?&IryKe>`h0o3=!?u-U~K%%bXDq z@{!ieiqLr2lQyoi{ll$4`^zIi%_wE|oMTmFpLhA#y_`I;W&CFKir{Ef9oATg%4|xf zMse@*DV&;yK5X29908(w!#|Wx?5su_(q4*Jh4wYNh`tQH`<|et8Y?kul4Rn!+Pf?k zwOK#{5hYb(g?8=Ng)0qZOd>iDoG`x8`IOVp)xcj$E^<-)8xSh#=aDLURvmiXLKz*! zO@uo&YRbfC zJYEEk^CWo2#QY%RQ7h2vzmSsYO>0+VY=1D!Cm{6k2?ek3nFm9C`9HN)k8VKNHS$vC zmLTEKZ)$;drL!PJ4YeC0`#EnwzV0gycNQtSKz3;8R_E4H?8*>e3JxKv7zOVQw!W=9 z22YlZ({>cLh_cObdQWw0_2LD)eUIWA1`;&C0(+&K9VCu8ytvU$Dc2!L=`mitEe!N) zU#Yah$Y}0%f_uC{*$bye-*!`{n4hTpQ>{?&XEoR$D>y14(`*s6VMNp@E^7Gg4q7k4 z1uGNPzq5wRc~sVFuRvCn6N*Iq;R&crq0(&L!FsB<;2xBEIxURU_ZTc^<_PnZ=6A>8 zsKn7*kKZdY3u-udV&PT;73>6+Gs|LyqN{_=J!i(~SRzdgHBY}XHr`H%SJd*_>?gIU z;*^Qx(srG+0s zvDPw%wFBzjVuvzE@-k^0lf4!I+s-QMQCzvLgDcVW8_DM8DR;Db8-g7jns_(nWduGq6k`3JkUudKRXdw7Q1^TX!#ZX> zX(n&5rZ0>Oxu=7? zYbV|62B`c=5uP$woN^46+n+(?gB)W9XvUX~-4J{R>px zLbv#J`~vwcvZ$sC?bmul9bCcpG!^)qF;}j3(A{dz{heB~bsqQgk^iUcKd5pi3h&R!NZ5zepFnuy! zY;-};hnf7XFm>N+KxK#Y(~gPT7ZhNM9SQ!Z2rs*@I?T2bZcVlsr)G5aSu{{b@8?j@+_>EOh$6T?~X$8C+0%D z`_Dhx*QQOxTNL@N)y6qK%s@%{G0rZ2_K|cW{>pgI2XW;KzD4)3NkVjsOV1h+mVJbr zRswHLE-S#Ele1TegY$f{aiaBurR56eksWs~rF{&IsQh+|)n(81&({j4jEcHI9C$Tx z_W^2jq_4e&bT)@SA<+EssLKkcDT*ed5&l)1JqzEM4u3YbX1ic&F*9npbUQ zZGSXqY-T&R!D}=`p~VuWg07{fI=Oo}q`R$-Ja5(~Fs&r-+1~`0+~;!yCsiP4_DQ8npmala*VVFmkj$%v`*iZ1AiN)l$7l{Z@!}F2=vagg zu?2~3LI?{*;9D~Am!CzRn&eqN^;YQ%Xk_?}v9F7`D!bh-)Q8FC5*242ps~Wb@9VqV zA?{x#-hbL9b>YLaV~Xn#npYeJvZ-1=z?#TbBD zXUpCngu73Idm^T-Nga2Nt~k@VbWdJ9_Y24&x*6LtP*@Yp=Q4K%jRx8Eg6{!%`!)qd%U};ePwg zrhT64WSL`NlN6f;53WCgWYe!-7KUcB;HBV?zQ`FRCvdNK%D1$s^-Kk+S`KD*)#w(W z5R-HcrmRxlT2mKae+w7*(oIjM8@604wSXoGe}{J0lwrf{Ie|C$$(+LpFSyz%EZBsb z(A@(QV-pVQSAe@$NCREoc@gNomqxMp1>qR78VAfZtKY@EXCpkcMFeE5BWUa~a$g1? z)cp*Ct9}zhBWuWPUo77{XzCj+10rY`i%4cg4YAHCy#~)|S3E)II$7d) zue&HzkTm7sl@Fxq{#K(p{Ji+Jm!>6sgt0BYPxHRbV#5aA`s<5Hw5M17=QjlwdmRhok-YZb27WaX~Er_S34p z>p&-(`>9LLg#MxvWRbkFPEV*30mXjaKV31s?WhM&d^S?z4zc&%Hfrgl+^#3h`+&(Q zwEc}le-W4V;%)GEQbT$o~mxSbJ$0A8lF#=cFCdO3TKK#&1*QIoy zP!NPI;|+moTd=^1B!6{RtMDeK3@4xEj*|YHP}=Jq>;T zlrOyk)t#cb*5%0FuWe?3&B?a=!Lb6hwT&lV_&V1za~{8LYkc!0^b~fJFJtJSHhg$t zznt3hKXD0HMXn}#X64^~04m8Exu@L$B;~J0$M#b86}JBw(Fi(Xni!PW7*Xe9d%?mr z5Iq{#HDmKIx?5@BUD6gdkU5YlX*3sn>g9c%x9xY_-4^gm5&{>kY1(wgf2*H0|rn`&*IR|EP21qYF`{1ZaSL01C&eg&15{`M8MI-XsE z@6W5T8K=S_j|NL<-0e43tl=iM-aP=y(oq2#s38J!rw;~=t~P|&C`Tq)L&dpXc0gLE zAe%<2PsfA9ghEAPmu+eI7o2tm1yV)>F7;=5X(c_o@jTPM;8Fx7x;8S+8e+r z`d(ALOdD+t7VJoi^OJ|+mvQ)Nis6`@s!aEj4yjuDU@n(g{^A2{u*iUpNU` zFtd~%2OKT_wSA&j#Mp{R^2i31#cO_p__0|?H@GrnExbNxoE2w^&*+2V+7^!7%hniO z9r<>Tbz)2;8{g83oYPf-Euj-ZtTXxyUNtt<-uz>5P^}lyFfn!yH6JxKc1?2xTrVT7 z=S+@msK7287f$|ZnB;_hY-;2t_>I;CSnNK35Mc3$b}ogPW>DPjyRHqLh!~8`5_l*U zzpc@Z;1%7I3lf}QSYdj%RBYI%mC{U-8z`JuZs`ng{V?1ejZS0Qjc6uxcO{I7X6Y~O zsESmB%3vtCSKpOj@TYm`^KLxjV{YiK!Z@cp3nXWd@d?v+57gGhS>0z|o9S5dnCN&E zDN%O>6Po@=^)SxvnF-sZcpgX=#69gmM4`CVNTv(IZF4#2#`>{yIje`^&xHW@x+PU- z!hz579Q<82myP6Dff`aKAA>kW;G~<=JW$$2$TnnGzL8$M8d%OO3TY-H)8aRh#TyZC zpK(C85%}l^W?tX|-d4}M2&QrWBkbUyOq(c?j?>XlSe0l>V z#u7Jh^wyE-oSsCcxxyyN%p;%l9z>Nqc&?1pR^l)@_V<23IotlwjM{-Q+ri%<5~%s7 zJbh7H1@T^W(srb_D~AR<5rL_X##6fWQ@RCO4>9Mgb! zhBl96isw>z2lf1e&5$XFzrQ}w{1G=L-0?A=Y^dfj(X~cBMFj{p6KwUc8k*Yzl4gfd zrC#&Chs7#UE5jt4%SM+;nodyTu~@tZ5A>bK_5w+!(MT-{&AiUXIi##aN)rRjfeFT3 zR&H=O84atIpB+SC&Z8KwXazMfUU9T)Qlw|ht+LPdrRjjol|vn!2y{m)A{OiSlceB$ zXvR!A-%GR^$sg~}1HHvw&1fLxG!S&V5tTvNjLPl?gW27<~TnI8yP z0y|3nNU8gd9`(Idff2Aj0VQ5;>qXqEC*gcRk-O~EWnrLcW8#@^YWDJr`D0L#XoexF zt#Rlkbj#!(#(=ZeJGOyX2(~l;Y(&ytE``P&_fv10CFe{;z|G&BC)yJ-VtW*vT%2CG zhw|+-wtmah@TQ*5`!4M~z_b5(&%x8Dq}8I8PYp+VhuymS=c#TXDlIlF`I4mRLcElF z_(ZI#Vfu~A-A{_NZ}GnV5eOa@PdQ9@=OLh}%SR^7C|!3W$lf8>cT}LY@{Sl4+zq4{PCACx;SGH^ z{}c=kU83Oy67KHaT>%MQCbtU!Hp9D;Er{!A<%?kXDKPCe*wxGiv=x5KJB%)$0At>x z_Y^v*DFPM0%ZhGk;8YHxViH@Z?>%Z`;hUFJ z4$dTIB{(q65Lz%tmnR*bDRTi0SG{GQfJT|*gIJrtMk!=zt6NZX+cY?G_3sao?#A^r z4rpNLIQxroPCrq|$paJmq^yRskIGx%ZW^cftjgeYXg)wQbj_J6>k7XcHmeln98F$( z=a!!NH*_Dg-W(2eB{RDroW=!`MHa4%+CvEAh2PFFm1eJhrb$S!*Zd&Ve_yucC)kq1+eCa!}0aK11+E zP(LdmfQC{n-lH_<&XI;xpkNK$fypvAEPM_~xa;d61L!>ps%fKVh@ON?d@y^rqA-3D zTVwB%^8J#1&g1-^fhOqk0%j`@&Z5(4$j7v04rqTfrTK;x4;1JHmdN6a<@3SWHdNi1 zK5kJJgjWnU2a%4?q9}MrtpJ|?eXFdDb00EMm$edY$N^=7DT-gKlY+c8Rn~*On z&MF*h`1zgP8CW(Mk9Bt}8F!LG8+-}_!tvm^!Ju_%b6efYhlppin(SXiwx{aup7bwf zSqZt!=#Q0qCnNlw_vDFmmM7`t`hz@=hEfuZ?WLkDDvqx-gQ5GIuQh{Gll&*GUw6}~ zC&u`6jleK z&f&H8y6-%9wLW-UVIOrvj=7$P6;VWC)w`J1r&C(9cfrP*K?kia^cq2b%0nCBWNs$cps?fq89sIDZgO!!`XUccX8-|ze} zbLY9wbMAT0@+{{$=UheSFaQ~%$?FAJkNtnX0GK=(m!mCo#Fd(g7T|DAYBMzoTQ{1? z`u&s`$Ago?<#%rNuPu}tbP??ZdKmW4SfZG^rb@u?4T;%x3tl|ZR^@RSdDSrTx5!I+o9%g;WMJZHoKA6cBbl`5Rprfya{9*L zbMt~kvi-d|r+ps-dNbMkFpcH|@FeqbJ1OE2b}a44yeQ}1o*b=tjRk$H<(x(I)R9kH zT0R(J0G&*eA@#ahuvTd`Ep(xnPg2~D(ENVEdW@fv1A3SKy z@mQ6N>f;>VsenO_w1tL>PHH!O;x{m5j|1_U73Xq@l>^KZj*TBv)EJkH@fByrxuX%8t@8H#zx z$7r})(}Gk{1~q%AlAEV+aQu)d`MNtG%00heTa#?wm9vZ{vRM&ESXM|k zta6U$^wS@Zggl#2N|-k%pd-_cM<-_CeMCSKYph$hC1>*DxhIszXn57ib@?DEQ`d3inz zPT}s}?HtWv4+KB!{Ir^iWwAScp%5nA=DoVFcx)gS-Z95y->|A5j- z6)IshSj>f(+7=W>78bHE?b~{taTYAdKK|h}PQRW0OgDn1TMVxpq6(@2vr{oZ+ z3rH<+2GV4vbD4Un_~_nOfU(Ut^wfJ`q6{>mg|YrDNUn>CJ$k_-p=bpt$jV$g0ez6F z1XgwX38A0adZ~v{N~i~&O;->;Xt7pO2KRG{=V{w~XW(0Gurhcc8*hF(BoQAA0Xq zv1foZZ1$W6db4BEjenR#3y4(Lfm|{KQ+}X{TZ^b2F{P+yB-K<=1?NO^uUl22`hO&Z zmBsrMKss_B{H54RD9+u}jB+|%(u}Ea1{9JgdoUHVfRT|gDY34J+7Q7zZ-D|^rS!lu zY~D9CQ4k=gaI{P}H#>-F@>gp8TN58nX1rzW5p7pL;`PPwT(OUQ#D~F#qK%gvd&LN7 zZ>>sVTaak(t!@sf)~0I{)yhxQ&@CUk?}D}X@gtBVmHOS0tQuDR*G>+~eKD2tJ(!4nfjkwc?K@7L!8X!gZ0S2~o4NUf;vYx4 zJ4}N*nO%BqArUN)v{$l~uy;`#)bpQh69?NF^XC+gpRzYUMUriOo2gI! zI-m^hpq5i8Z+|pQH(Ub)B*Q;vSe3<56o01&u|Y}biWs0w7iTlK?f&8PqX?G9YX>{9 z+I{~5=)Q%`GcIJG_vHdsb~=%1sGW~}eUF;A%ICM&_L7&K;K=h(qwgxXET*G`E)<2X zSOGLThmFCbXf=P7N&ZAs?6vYx|_%;I*5wH({A@lrWGQQ2$6~K3; zp&!uhS~gStK!fr7K;GmLfG9BQU->ZJtVR`#81%N{CyjL{4WA%61V-R<&fBL`@)#k! z!x*5VhvXoRJA`q%P^iI)qrJ;$&FLZ2_yF55%EOQD)+i#{2xV|&n$8PSabsJFL}-$= zanq}hK1ViTgAOq!XUX5R!8BpWBq@v0z~y0)&N{5FdJsEy5!~ChC0H*$W1%au;Jf(f zt8k<;biPWCp&yBACP#k=L_FZ&GlGVK0p5=b~i+yX27v#QD8z0j~M^_b8S?Yp@ z@b_S3=DwP)M=3Y`g}jq1-`^;w9*jiAUQ0t~)O3BguC*iOk!<(q;4t5d_QI59AG^O^ zW)Alo6!iNRJd*DLH18_D7kdj?bJcQMbdd}szZO3ad1;vFE%B+R05L@oCccW#1ATPD z>;L*|dAFV@#An$D2i7}yhSZn(2G%bd6Z7`(5GMxx9{~A_+Z>yP3r!=`a))Vu+p-5Df%B>m11^AXPBMH2 zue|)Cel;N-14C26#WTTDT}Y6j-;32cOy?ty75U7OytdHG=4&*h?FG&%j~T66&4U zI{2KTzuNE`Y>4vY@5V`X4jq`228?gOZ(2CsL^8Mi1S_#A%Kev3n+aJ{BZlx2e%JzGQ1P;*`8&E z03qPFs{rKDv2~C}AJ#CLo7_YC7EA;R6GIU<>-K^Xv)v?iW#~e{MX*7qyFD(N9qT7v zGu_Yw{GjBr%;ee$SF1%gy9oI8cgU~q(j%tiRh7Eu2>rmvJ26}|YV3A|J`Ib5kNXsd zf52I3j?I(g;xCe>8gD!MCLVp$g3Kbl#sYty!xpmS=&iq3lyg~ZOaHMy1RF{=B_ZwC z^Q2NYC&0tJ1@?p=y9=62&Wm=DGW=*;(JxXCKsF1$zB(YSLK=5W0nOX{*^l-K%p0SX?-X#8i~H%aj`|OO}hUI%q*=Ay3}zaRgiMSXioI5a12l(@u?rx z`FQ|q`y~kUc}`gh1G2<%_EIMWpd$+w9c2Eu>i57{NiJCY55rF3hUFzr9rs$O7W=_R zeF_T}g%L1%l2CypiWdt&>%G)WCMdF)M|one44qrK3#NzTpDxuGXI=g<6!69dQ=wskXD=T^`rIVy(1={EYBpaL?z)pphNJ1Pb zrgIG<#UTllG)6+2eF7pG4P68eKgVaa00FU=#38v*SMBe?*yBi9QiTu%I}psHD$W2J zh1ZriafxWi8b&rAT`|DE5`b&L>Y&TtgNfMWMLcYs1F)0di`_or zB!ZDBvFCWH8DC}aQ#jsQcINE-q%&Y(O16}9shbv!0wz*xv2e&OE}o5 z-gA7;=w0y4$_3ErynvS5;T^wf+=FK7bb}R;cgGAYqO{TFN9__nq%J=D6Ws2jh3y*0Ic=t!)JAgc@5Z2|(9txe*VSWhCB_TfJ z$WDrM=b1}z1ty~W{}C);3@Zkcj4TZX+j*P@5Ji_hq~h6;OfoHh<@jGAk7SAz#_!x! zwO%;L^P74Xdh9yvyq^IlfC1p3@BCplve@>wZ>tBO_Kju>iFo51s_>4_ZECXBju8I2QY#BH1DpeTXn@skiZMWX=B~M@z2jSC_|J3h- zk&&==Wl?JD0h;iG#*G1?P9nBCT9F#C@NtOI7~<1g{DhtOcbklO&N|MCi$pJ<2AHQH z&z>9_RT;ha&+4J)h--{5aJKR?!=+FoXDcJNM(n5bU$OCZnA2;4CxQ#=LMj)Ar} zt`xrk{7_{6jxCAblc!**n)8hTC>-Vw$Z~Mt;@M85iP#>%=>T~|vui6>9QVltnZ)|r z_j4eOb$J1Ns|Z#5sep~=Pg?%j0Ef$}O{S24|B$US>az8=2VyY9z{BJnPQ7dtBmj}a zStdhpPN4Y)p_&s3tXE$*Pw!l|pYx3Z5F7z&K(tjaN-IsLX={l%PTBdaS+28Z`c|u; z{bdJpkuQ85uQ_c@uem6jUpxB9V>HPB@%}ILMK-&)9mR|up8KYOm9{3g;y)c^e6(4p z^`L+8I8O1bu1z@mLcQu^pYkA^99Nm?dyV1KnqNHJcf_~$6#Hl3iEYVzL$fy}h}9AArB$(*>GgEsaz5G4-4|#e=Wi;or{Hni z4l+;z4|{u-@`D^eL-&K%unu>%q4UGk@9|*HdwYp9x65MlDVHP5k{ZofNn59{a-9^> z)bz(36nFX_pyHWv8nzi+m+mRaome_5+5S-BBWLVN`*qc8Pgz-Rt4Xh}3S$or$7-;7 z5|=_f`JWB|;tGSN3LgfOT-Gt2sATWf1n-Z^A=O34kmDw&X&~cl^UTU|<4E zxG5|baoQiC-|g7)-G_^GpiGfpC~Z|d#!r=w^TFv;;_Ve3zv^OVfaa+?+H35+X=>>$ zqTQ><`G9V5a7&W3lt}L<@7#Z<3Kn`zi>7IS0@{c3UIHn~G)&>spXYG( zhNts8&~>pq39Mi>a6|{xk4o#LzoGB3(}Myg8aCuHPHDitHO6*N>{@XNzDgA|szK4< zLmF*~>mY_3bS0@h{TWUXb|o3rTY;KgLL`~qSrD_`l8TtSMk@4?CQa2HC0E#kQ+6)K z@7LzQ40TXNq8&qPa!aRVFfDAOPJi?Va)LofkVIP|v5UJ-gDT8zyfrv-2B>(D%(7=F zif&3^DuUV2J$llo?ZBfCAJ1EG&2ZwnX&4$75U~;frbf}chA-toQ4>xuXl{}Ncrwd5 z=eV`MB^7TN)ssp&v9OCJ+}ZoWg#&$IBs%S3dZVMqL0ndSPMdJ`Wgzi*U7$;PqKi9BM+TkGE>V=BW z=dv5LB7buV;|k;F6^Er6Cp=e$aT#|GLXrMJt1?m)*ZpmsOpQ1=sX4R#oteZEWQf{h z0i|0#g#RKq_j?OMc2TC>IHgXc|4(4?89JTuE|~!)({;eGT*|)DMN+4P6CdPY@ynf^ z-6PxkxI4UXY1x|alBYD@F#+{^2t2d$I4v4|09)u~YPWs(TL3^!!zMPp4W&$kXw_v_Yo!g8w^v7!T6Rfk2%M+{T~A3S^JLg$>Oq1+{kKamEn~5atYTU+fI5J zQ!%5N&Rgfn2;s;0?Fa9n@Asn(tPDWkbbQi0I7^;*2TGKktODgQejMO%7nF=&17LHnSSuZzBdNW?gj49HWfHF40wnhISx|jud##G3 zwU(SVX`U&QgqK1knn@vqvRD@s0KHTe-_I{_*z2MT*(M?UlQqaMf?Gw8aqgdFnwL0xaLVOp-72Hff;YdWgJFYFy zjyweSN)*8-=&gjTP&|cTv@I?sVCG;z4uacl(3-=d2vmIo8zoJkF*aXVia zA!I|)NaI>lGb%%br-v^63vkV)d<*J9E|4R|8&2SC?SKj~VrqWj~IWMmIKt zA@Z+2|522HKP8!l&Ety)$YR{^(>ZYh9P(7`*l3-9`Ywn6Ai1 zkb~gtJJ1`_gSix#5+yQsS5*KX#Ry6p`X8R_-H$?|nW7iLF2X#Cx7-=*cbkY@wlx2t zPYm!}TKYrxR4Z3!FnXrq6Z0BNG$@5#lOH!Cm$hsg$Ju-)@RW&(+T}%08xpRjHsJSf z|MxZP52f*5W4Q^Hqu}HZpC5GJ=CMf&!?ZopAANSFJu73QUbm^`heQSP8(yiP1(;$g zZKbdJ-1#q}b_5IEhl-fstRV$T;z=6>!S+EO4Nps3)~P>MUa89o$)w75wEvoaoGkTy zLh{c_9u3pQ!c(t}`6B4n9DZuTMh8t=5HTYBf@@8a>+OGrll2P4h#8vO?Zv7{#SoWk zLZM*PWJ?^ruC!Gpu)N{R)1r3%`VFe+!c(=w(~je_8&@tU;bTO%kv=1n6_dxTH6m*H zCM%v*98&9aE;`oEQ=LKFi+}a2>(9dcRF{jipadoMVkX^q$f%lPSzA4xBbz2QPWKw~ zMbxzgjO!h$?5ReOHAJcrZqS=5yuZx@92PVRm=kp-Q)HpX-|v!Hu(nKa{m&_D> zGG_&(IO^fn=$BzuaDpm9KQT|R23A~ncZc&C#dcX79R7a#yd8M07!DZ`9!h8NERhPu^BIZ~u>4&E4$L`n!H$!@ z8IMx&w^5aV(GfQ?d;3wJi-V67@NC1+>B+1fl%*tOPwEYcpa}m5Fv23|MGIgJ#hdCD zD+nx9ufPK$gB%<2eS3OW`jhcnWD{9jr0{D%stKJqVx`@pCx2KU8N*Va3j!a|mdM9t zVNtdn(35^{Aa$6W(W%#g;x|mi9|Yua1%VN;YCrk%QqmDYBOG*l2oeZc|9Z4T1Pz~I zaEI>o=pAxQr--S_;~M=hQGR?bV>ImH3I=x({e`j=hnnq4GG}oaM+yNKU5L9ZM*>j$ zb@YeDfx0J5efCxO$x@J|IL_1>j>j=1%QSal9(X?BfQocqpVSKS2gI-%D}c?uU0vG; zsbealS?crPL4*0+$9sBnieX?b@;Dh9RTeJ>Q(3M}B!T;_3b=1yz?^<72A=b@IjVpc zUj$n8WVzk*B1!&$LzMyytAZO6=9Ylxuc?;+^~furP@)ZEIkH>tz}Ub};X7@lE<~xy zgLRzXO0txKGD$RRP~X%FRD-#}yqh_{oxETCV*bi1P`3`VEdj8NV$BPZcDQPptl()-WLts@|Jsc>_X)>4z>Cxlx=-JYXdIvnXm(Iu~EPmeJ z$4X2tX#AJG%~1)IDa~u5pxEa{!-VIzm1$dz%AHt)h=2T~iNm-v?;f8QjnJJBk=d{V z6Jru90f)`1@dodF@-EYJNVFE%Ivvc$nU3G@CJV6#f#&09(kYw1!~m7waOOjwYYt-q z>_6o+zZ)J&I$HgohHK_9;j3h%vE#Z&x`98z;ZseF-l#Eu+07H2wU@24fAT#254gxd zF0Vc&qjAM9<+I58JH6QQE;L2CdgyV>?KVE+SV8lgTQ*th#l!mM2&Uu29ifzkS+l6+ zKY9PLrwEJYt#15%RNmT~JjPFY!Xiv>Yi+@1_ieNR@j^4THP|bSRi{RlYk1BBPB*v$ zJ0AU8y;)6Lj$cU6SkhkO^#JbLk3Bfl6b2;jg>*@ICAKKB?&-aK_!o>|}Y{_h8fDBIENUT^=44?>B( zmm^!sl5Gw4gVWp7$i5vDKY$2_UYK?s%sLJV_F|4Mw%-H1+79q zXqJ3GDo zA9mUIt4_$DhqkJ}Ek}M<`pa#(c*Q8wErZQ_x0*!&t~k~wuD4h}zYV&^FBz#vXXy2{ z7N%>P8l)%qd+5d%T}UcE5*#hSgx|#P19lt4;Jp~@N%mhzmW%WRAq4PdyJWK%ugopr;#;IdRT}jq(42D=WWs%Yxc}edQ&?G^WUcowbhQ>j zb^vGu!}^XAbIzeqh5pqoY@+hQ7l*;Mw^(98mp%sw*Ej%cm6rglc8Om;CY`?8{k)HhXKIC-3g(ttwHmw_JPUx!_z*lS` z&Un^*Mw$uYY+@Se+l{^FZ?Sb=_3npV8Zr_Vzu)Gpq2M2E$$kev(*?%qh*OQvi6yH| zLI3(A=90_OhOab9h}+EVJB4cm_|Ih$@3(lC-6x;|X6qA#9Iyim1rK-)MXVnQ=K5cc zDovto_w3wwnnFLxXj1)BaIY?0ya_625wis;eH@>XsWt{;zmY z2)dC5DA~utEq1BGLmOlw%yF+ z-hVZGN6Ea1CJ{hjX7WD-Xk=M5yZzk*Mm~bXKE9xT_kTQOdF{ zJt2@S+1eqi8qNxDut;8fWd8}={!q!9_ff!BQNHtAs?X$ls;TQLulL zu`Oi-5Wfr_-faSoKtr9 z3DB(5M2zH1s|_~#-Ux3N`iI&MWn^d@JcG%LyPUDeQl@b^LT)HO?{`{XewYXTay%5j zv$V83Ht1YS88pxvHMrcmvR(NJ9vpP8SrYyydwP2Gb@Msjxcb(YH)g>VrEtwqybG4k z>FIAgWBUNiI5(10jjts1S@@5*1e!@gIL4E;5LqIH_+Q@EXN&@J%D&!s(tYb@WeQMo zw&rt#gSGwGtvi(d;o7>Yo;zzO*DTiwDN0G-glDWz!EmOYW^8^zms@#xZPGToLaiRl z%v`g$@ng-iz%p6O0O>#dpAh}SIl>ARD$h$|#EMnTkQ1i2%w&ITp88>5_T5viqOu#g zxM!)0P-k})%9vQUItfkbhh|Kj|<#N;dk#;p@r6_amO45NdN{_$H zRY`J|&V`zJ&3D>y-ObXj*@Mw7Q5rgMJ;QNNcY9{G6lZaO=Sxo7APdy~$oTqM7aS5u zaNz0YU)61VtpcUrVs7+j)ph69%q;Y06LX}51Iav)Xy97G6FHHuk&)KTY4vZuPWSwM z9amnyR$tz|R9{&B4Z_GCFQBZKXDB1?uRG*CP<=b)v1T++r;LVJ+_I@E9Ldsi8k?FUR!uOsKbjXSI?Wc_k)eZdbRXrn(ZsK@W3F&aj)N0!`8ljp-zIhn3NIO za`U|6sfB&HocewF`T2`ALCYxM>XLIVSHD-*);=n(gIlH5FmopSYhdim94LCUWjf(a zn7|S)qI&4-{a;+@NAAxi48eL@+{IVI!9wbS|F@6NN_2mZcbt39keUGSJNVkBmR$Av ztFeX_sBm`s`B!?ls)@66>f42PmUlir`gqYVEL>XU^%}$%L161HzGwWGGw*ko!sYhx z$Sm>2w3ZBN(1SCV4a3Hk-%Qz2-)&Yze+wFWsCha`qA&}wG~{WH>NX^~ynKkPJkNH+ zWxM#wy|%Bv0_+ou(qH9Cx!yWNYiuc%#by}A##Hz;Jll9_$$HopLE+tqsH3VGuLIm#^Ui)@vBz1g#uzuFg zQswtElkA_0K`ndZ$(B)}#%yO>@ap}d90CI)=KnOtrP>%;kqHLiySc8|Hj@!61jCcz_G3SgwN46ociJ5(>$_~Sa04-415&w zP(L`|s!?Rptidmm>9Xz7&X=it^x_E?!P+N#4Sna%%?HnhywULmUw;7Yj7r7v@(Vwle^Pa#%F$={nH@gB!s(njP7))Xl-`Q1B}cH zGdg-xN1ke17aMd#UZ;A?*Jk<0$w7=aJ(@9eR4EO4eCeEDa9OZa!PW82klF3G^0Rif zffy&?&5=hu9kaKMfb-$D(t0n)TZL8+J5Zde>5&ve2q*T_!Rw0s*_-uJ% zr5)YL8(nx{C%GYY>Tg|->3;Z4SpToj&qUvyh2gYh;OnWM{r;Wvzg;&?(1~-g?>%Jd zIkR=a-i7A|F|2ldOj|~J6js0EJabe(`WK0 zK6fC?CgdMTxCPO~*0G9Xe+%6Bm)ji=+PS5{_j=--IDJjQS zKVQE8^UmAIs-rp+btliIO^Em=WmE+S-5za>rckVv;wLWP2z-lj_1xJj03E2lHLfIb zrOfA<%%}2;e>R^vxXk_R#X^k?{VR3tZ>ZYByhHv1=kEbR0p?jh%z+bwE=-RKJ zFIIef3hf^|Wn?WdOwHfabyOEnvxO(`@#vLN8tpzCzsLoV(Mz}_RcJ#8ZY dKULM|R7u7+YK_H@qA6e*mgJ)64(> diff --git a/docs/source/backends/memory_variablebased.png b/docs/source/backends/memory_variablebased.png deleted file mode 100644 index 3da406d97851c1165f6f3a619b73e7571a6de0f0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11595 zcmbt)2{@GP+qb8l79=}mNl^)*tO=v+5wh&1@NZo>({RzK70s98Gz1*L3c?yuAJV} zO)D=1{ggllKG5|j{QuS!`r}esNf!NnJiV?8^pc@H%%Nk;=s6gAE{6WR6Lb>?J*7a$ zL$tzlFj5!1Z3?IYU zE%eSNI^ijJLkheh4SJjeZ>oTv(qNDpy{;TY3WI^_Ao2(pYXsi00R2_KaBUDN2Hv?s z?|nwkNusyZ(z6ojBm%v)j^5oy&%H+{66k++u&~_PvE$jX^YG&z59s+R^sbg+gF_N0 zB?UPdpwC|J34558LGO41X2^jlN5OlNQ+es2j}v&)m4dlNe^dl! zs~-K0!F#VC^sol=(>AY4&8I2TMhyzMCTH8E-`D+Ro21{PeT z&Ck8i6`73*_~Cz^_b>x2G6aL&!AO)jzwIkMq0NnTu*4Mf+RuFwB$KB$9S(CU&?u>Q zi9{mA1cn-r$n4@%GkMTWkcVE3d7q>*Vax9vj}#(J}2R|H|Eg5aPo)#r-ud@_^0q z*O{5>nwo`$g#(vY-;a(UCr#?>>)k%-gX$-MzzeIZtMLhmT>j*(UPy z1&zE>?^P;Gva+*RQOeCvo`8vwtJZuX>CXxaix$34|79oi?c29a%Y)0y%U3KcL3_5; zv{Vpvj5awoI5_yIqQbl?9yH+Hxx=?~&61jM>@RCQh7Mm)xgVH^GHWYdzb7|)TiU

4*sq(Xy2kDMY zAe%*po%MqW&FS{!W2y1Dqh;r+vqnv=-^v|CJ;_iyQLw>D(K|yvzC2`ltz5;S$SE4X z1DRYW+vaCgPE;mn-!8is>Unng5PQX%W-J1M2`9_emhBlmquR-pf(=rvexz82 zzEx^(cjz?jOn!2&MgIhH?(4A|yMQi7JDq-98ECw1a*)D3EL0f6A-RuxxN{G%;i%J$ z(K62^^Del1e=2WYIMSU}|1mY3C_z31NQe%K`hLf?>kN9!W`}qic=?hIePwX*f)QVM zDBS7cZp5&4ClN6*!sAbCozL%TcZc^!cCt>liHRa_XgB6Q&{-Y1!Id6P6h00H?*K#1 zX2m{QkzZfF*~pSF`L02ZrT&#!UR+gjzSAb99POpme!j+b-3zr|A0Js0G?T)A@>|R5 zVv2)iv{g*Y^WVD#TDz0SV@-k= zp{M>2sS`!2?;UkaP$qHTkx zg@N?vW*NKH2 zDvNLR!?ccT0y7Eei(Q09zfDbb1o&5L{c^Q`S6JJ|6W^)FI5g#!3o2}6R29#N5u-kK zDE(;jI2Ad*`Ur~;=)F6+;@gEDuAA8BjR;S-aU`@Zec?o{;|115o^*8jO~wB0t-m?Y z3&Z*^@1d`@?Nl8m690aTb)vQ0;Q5b-NIaD(XM-Ef#e`1gG+y{8hX0xwZJk26_qTA| zDLe=CK6)$J$K~LL!4IPkGns&oB|6I@>J5a05_6(;arf_s+~AL~`r)3!t8pJO zR&VWZAChAeHvZVkLV{Fiy*BYj?R!HPeyy3K#I*p-)eOy8%MB{;A^?B>!$fZ63Ci$M zD6pzVv_1dy@gMH~_pd4f3`cliC=bH`zP?BjJ6lTS{=dyd_m}%J^&u|-`~JzGe|b;y zC6+>y+5gih_&pv&WU&1o=KgDD*F13(wo6_;bI^El#?XbelZNP<2DWL-4Or6Lx~JQ6 z!~OKf))r06wSS}=;e7(`7`&a^Zuu^Yt$X`jPB_QN`;lEU#>Fu^N2m`=VJ!TFs$qMB z+W!+Mm5!Q!@P0}fu!UJLd5IparRAIw^$~ML@M1JPBm_j*qQ~Vz<(0tIKuCtW%0&`j$Q?(&2>eqD>z6wi# z=b3qFYeSp-^+NDJP_}KO8kg$9$t3}#GsCP;vsT0~uNh{36&7dA+T3_El4THzD zWm3nE!Pu%n*!GV6@34hx)>iUeu6MmscP?&FQ&#{@%{=_GBg4?=XDo&J8MTK=gkArF zgy?8N6aM9n>fi2kogfi}{yAh4el=-GYlE*46hFyZUrQB(L4;hegcwFp$2) zFy@0C>KF{asy?k0Rw99Kq9E`7oq)1Z;3qgVOX8ps3{ek}!OqkwBLm{#7pdF~&YxgG zntlnI8axG_{T1+&@byI!XOTHg_@`y~p?egqct`_TpQ6Pq_4@&}pQB4vs;FbIxe(GO z{WlX;@7q3?BbfaN??MpN7rlW9&53;V+OHDcIoY-9*l*?csQ? z2Rw9778aG7Lal8GYrCvlL!@5 zMc+B}EohB-BVNy>47rYvpVlm?$Y2Sf!uHFz8|DIZ&#xiWll`{&-5FLhugB zAEMedV(0qk!cfAe&>n8a=@N<-K{{ zs;g7W8g@B8Nwu(}F4B}0&LI}Sr5RBp)26Hs0VaQe_CcECyOHuqc-)?(6(#GVv7<13 zd{{1+^MZj#K=7W7hk#1!r+EDASw1jdh> zau+!miPDE_u~`~*{ZcEn6H<8;*eSQ~A7=Tb-ct2@th8Wk2CBbja%-Z@-b9dn+({03 z%A;l;r;QL4>9-o}DMWe_! zn$Sb4CrM0X#deCrfHobDVH^UI1jX^S2Gdukwwd0r%7o<#&uSpjD5d@&RKoMb@jS~8 zF&dN&h76? zgp^$tnw$V6#HK)^nM2L>!!;w!NSF0G`b~w7_&~WU;c|;A5%W41QiwqV70v+zTBfr6*5Xo$6QBh(`g?F6=Rf(%?NdCe_DWhEnX?v27hu!l6Amr@?wC3l0HfC)ASD)0>GuF^gH%_a97p zlmhpPg2!S5eqh7v;R1%3r*yYP*~r)4zYYXJTJBKe5$z8)*Wjfo`Z>w0wyi>TdxxqD ziToYlj|r4TzJ+6)C5NYNBsaN>zFi*Kbdv&&;i=(mYDnv}85jw(r{T(@pFX>E&O0Hg zp7t=elDX3%`6cC!7246U+?~z;Ek!CAqj)OcvBu$qiSc#?YBHjj7-bPf z3yRdxbTLX_sdat~LB?Gw0%-Oy?XT)xkfM|_pb81`G{f<_jm047xhqCkO>p|_&W%<>WZERakH&0tCbE0AkloJce z+BFJg57_Pw9H04c?aJ(M&U~}`?8i5Y1x>BeiL*@~70iqnxENrX?^pmc%;&?)s~2kK zvafGar`}A@0}7E{W3}nuU-_oOlw;@8YBjpsVUhX^YjqR;q|y*3Gr6YPD)H1t72Onc z9K>LIY?y~>l;23sb$}C^2^G~4-13@4?Q3E! z&CNl#sokv>vbcSqbA)d7jxgFmaC-|GA}rx+X1VP8rZ7VNw3NG#u2W zKzj&yjYTIzf?YDGEASiwD-iZmlQ+QJL<$}U)!{jNk{QejTf_yIp}+0X0JMG;?R0n= zI}d|d*sl?%;2~&5O6gBa-ctxLk-@#hspKkZIsx7bf@^o39G5kQ#|{p9ci6q04cyWO z7>ggB4IRCNo&n?*!3mj=(R7z^^T(sZq05ntwk;DO#6y{6RsKh#6YRVdEmHTnI8Gd7 z<=d?7SI}J#zbFM2<;Gm;Usq8{q z+=70-3F?d8!~xQKN*eO~aL@Gm-1GJk>t*V@_GyFmY50THqcOWECWLqk(iof@_2PgD zj94)jW2lI-!rbjK(*?tagctCi=^|qhtq$jqG!FyrZhr>c_7Ro2O*Tzp^c7L$;i-@` z=I$J;Ps?MTu#6nXbN@YoDU5bl5DBw|%nE2lSw_plV!gQ;%0D>-%di5N8+phgf2Mw0 zSx^doD@kPZ7g3nURJNH^0r(xHx=NMAJg$LM$H10h4WhfmAeCLblswo|Xy{agW%O=X z4@KK_DMBQzR_xJOLY^CV^Bj-k296`0B<@2aVBXa#` zuCXWL%7$~-9iN9t3{PkT6lA zGH&oPU=pT>lPJh>62Y({Q1}u(OX;qa5vMjK%lM>@)Zrv#>NB2LiJog06|4?qU8}{} z4DztBpC{&ew7U&1M)C4W-jxC_?M=Oqa8V*pX0QSi z(yq{c?UVT_tMjc~vx~d}-j)ve>WBfdwC%|oqbKwhvR=GmO=k-^(J7!fhe>Hzf6pFR zA^9kMY)I9X_DZTtY*LuKKg8xfl@V@9i!+J#mM=-)r}!r|+DSu!t1MhM|6U?dvYxL0 z`^AET-fB4AxPsT2+>c>0pFgex~ecL(Z9mnOEAY7u676au&lg{a-O58;$yQFGX|u;@7M(Zz7B( zCc|Y1>=V739Uc0zIo+iJ2`94Nghv~51;i{vPClh;e*@Wo%~ zTBUHxrd;-zxDlBs6zMQ2H+nA=iK_G6(wQEq>~Iuq;iy+n5K8!dW1^@|#bsI8^T-NV zJA19@CC3w&6%kQ>S6`iB)#+kqOddxb~+qWo3 zbYIjHshK)mQ9laj{1ZI03ZQJY%+Grl2DMe$3Q@c2_?|6YxYaceZRn~g@z-wom3erJzV!MW2LYt6YfP%Zrm_dYy# zMSW{_r{~`Mw7=O)R>ZidEPs1)-f5`md_0`JC}Y0(*vzZ;r6*1_9k1DF6g2A`lhfOp zc`lT1_Kdx-G8uy}-_swinAb3ac=CgH;dF}K=^X@un%iIe`clXi?NYSd14m~~w=i0| z1eq|6^ic`u4eFKYnx2#S))@(dF2*eey>bQQtFm$#!I5bYlKiC?yE-Xh%y73aTh%3uGVs z{0QKOF&l!!CDmUCHVqESGOpf)Qq5L6Rt%`aaPEZ?q=JH!Ae#WIuw`bKX#I*u5l5i? zjx}bgGdcVQNvko%48p^Rgw*aTqA0?Qub?gk%ut3)M^^*Si`Z)!)^o`B(!Sh0-n#hz z)2Eo}lSTN)!n2DW6pwRsv7R0nFqM{LF=@zD zKwGx+kn`)kb#{^2E1jr&$ih^C7yUkeaD8u4^pJ_!qu-+Wq`vHRd4u6$p+TG4K>_1J zFJWnq!KHlMtKy4&cpg&PcJ>R*z0nEa+axbm=>~bkHo`|Z+0DT9U{;IyH13Hnp^BvT) zLgaxqHC3cddy*<#LA6c*ykx#9AAT*Ucf<0DeY^n0PIUnycn(yzi0ZOZ92&uB=|UAz{WLT0S}2N1Vdk_bXf_V2km_=HXy>z;2Blu76hu2nDV`HJn8KJ3 zh%u)@Cl4;T@P38?VwHk~+a*R4)dd;|&O&FB^;Xdm6j8w`XN_rN2n+{+bAxN;4=Ugb zO@gruv!e}5NbpL$lc4m*DW57;j8zY~hT|^WExQ%iBAHftsyKYx)(Wlh0OaUFD60m% z%`VN|i7P*sy0P(*SpV_Md}+qv==EMHP7Hd>v1CQxCBmGxGW7{9j+l^d{fu5J9!5=i z3iTo#&Z{iBMlDtkxPtNS8(e#5V3_c6 zYJ9>~%TawtT`fpP*>Y<1`Q*nBj4#z07<9lDGA`R_rhB4UT2)5Af8R*Ygv)pJ+Ox>& zCP(#JFi(^7mAp)av(0CF-+Ct_`tbdx>WKT1Rs2D|R{5LOHY#I&6MXfAu)uRD<%O#@ zw{kRPmxibmKN-}CY-#P7m+c3bTzeDPHDqOfD5(Q@Zx;Qi=vQc(W_^> zYOmwFoesVu7py3u&My48+W1jk5vjJY{E+4$-E4K-eXg2jhM#b)+fsUE8-(<$y#I3i zrP&XBAZ6NQ_`b5N>Z*z4?E=XH`-m9C{R72vX@Nq1qnloctmcB!Ty>M)?)6PVOjIjH zHtap6UeqW=Cw3FVh+UX-sCJUdGal*wPQK(lnr|DE+JLM8pkFXT=U==T5&nO z7ooa}-?#c@fZiXJp=d84F!8wFCoC$0zQk`bGF(mVut#c|57d^9+r^BB&*3ZF?9PH? zh1Cp<>*;|%3yfEAG3z-q0(b422U`pf%J``W+DbFuigetQkHNNPUbH&CQjJ55NUAbg zW4n-kT2ah&@Pt3Z+ECz*lS>CmErxDhk*ORm#ufP zP&spS z8%V-1(8oNi_OthGKYP9l`m?rw|9j@H;Bzju)$?rp-eZFwLgTlC%LLE+rE`r0Q!3Ay z_i^-Y>=_U^rhm%C&ICNWo_jOJML9UPQU>$t^ug^)JL3}#&Xv^UFZPi~uh_}>L@{ac%|LXCwa}GL{(wMw) zYGlWv^;#cM=(u?TrchJ6E?{Xe#6e2>l?Ypr2UV_R`{IP8sdV=)~86SEy1PIQXV?l zn%(1O+7f$;_UAOb||%=7o#??YAA?55q8J!UQg>s z6t?`7BZNoAhnnwvb<#DhUz8Fac}~Uz_*Hu^u01r0>CH~|`JQk{H703z*g@}YK=C!p z2XUgp9iqm%Pkp%Y*x~0EPTj{|fbDMIBuX3l@FTodapDrTka++B95*q7cff(I6`4)=U^9mP|k z{M)heS7ah`GA^?qj*@n>kX1?hoGsgQ(q$aH&ectQoq9jx9eU`DyKu+CDsQ{@x$?g_ zNKx6&y{N1VbfRhh`@u)!Nr&0)AGmb^V|LS$)Rr3iby6Z($|c>=?fl8>wI-(tW=VQ( z{bL2w`6_=TcWVO<1nc)oW9gS`c;1F}*xZw4VG)-?vQq_$g-u$Kp%2UFPx$5+05RY}ct z=^f^us#t8g)gYmpZ4;2;&O)^6s!i+-l&K02UFNK9LP?Yr~-IOBjUgsVqe%Nw#4^7^c!w zA$xM{GC7ilnKQpLp6B~~KELIU-+6hR8Rx$5_j0|j_jR9{*O`ml%qA&Kpt|? z4Q0qf7IK%NHNAlDSVDKKpjaCy_5u{D1KqhmYk5I?_K=pHK+8&?AU1*M8Yk)}|R z0CZ25?;eS+EHsl5FT_oUd^w>^g^4F65_{>u-LUT|FV!%X0PjU48rZZLjfBsQltZ0SA5|I*%d^Dl|DH0iDt~ zGcq!A8yx{zYC)RP*S)-&V$dWK3G%RoOw}!{tmxM`INcZeZlFkKkF>V7LP743p1kqd zvo~+uf}G7ad{1&|Q#Nj>J32YFcXTKytMG=5-MJeJ-S$aNNuggqhT6OW3{)?}}x>(Gj`mR#gkJstko zvV+=)U&+V5HsIDT>O;Pg?o-B2m6*k3uSL|c9VNlgir~V0Czn?_zvTJ8UNoG?+zNVI z0S0Op4mTIpBoY$Nyh*yyw>U)2O(88TP#>gi-jI6h?;+sd^l{G~i9Ke<26nf{Rwo1f zM8bvNte1Ux8C)2E_bIq?%6&O4_N40@=)}WZ{aB&bcIEqn49^umJjUk#?8Zy}yKXjt zUVvc5hT{oGTaw@Bq68QDFLLQW9=x+ceW^KLF8q{uSxq1vX`_)IaL&&guRl?e^VKi3 zRqK{=$BtgY+S8Ry1^Yp-1wTHP&J!Hn|Bt_S;JbxcP5b2dtmn(W*%L4kc*xeKF0anJ z^$~yiVZud@*^Crfsa*WW8}-Tc-rCkRDYYK{vlKhyrLSBA7x!Nv1`B>fxo+2DQ@j`sd(7Uj*NPJbG z=Q0yoy!*t5%;d9=U!PNYd6`{rGw| zdLDDG>PeIqz+Xw z+`j!b$JENzr?z`gOivYq?|{%~S_|d8L8$~FISBjI0W^hTv8Zq`A4U8^@O1UYwEZY_ zTxtE4Bv0gv$HxKc8sSCLYEi-)+PN`)Ek2vw!mr!_D2TstegWM6h7IkySU00iF=l$kJJNIfcz1(G-D>|5bck1Bl2fp;- zz8?;edar$*rf0V><>75DdaW^Uwm<*Ynb;s^L^+YiqsYU$+~n2KdTTwm=b3A(l-}P5 zxmRh<)32?`t7yNSOZ0%?$<2>6rw&>Pc`X6<8<|cWJOgKxq_bei1BxP$`_ZxcCAt4S zO$$;`|3~uV2rv$dU_tj0%i?qlVzFcyiV@>~Vz4q}<%a|bz{LO(VwBCOG+2@at&uIx zNHH>v8Yp4S17a88ia_0&U*CynJ;gl^f_@A67&q2(Li<)3C%xNC5M3$}%Q{r#J{0;A<6~xJ435?tvjDZ-) zz$mpcP{1ssU^$(?d_;*1_|I(R>ZZVM^xf~0*`j|DELt_jr22b3|o;cvff zS=7i*m49oCSLol7gsbB!Mb~A-7z_lS(TCC)kf~%X$ipM0QiI4R(ky7}EIM?NQDt3Y z2*xiSpHHNVfcquM{1god7*v3P?BAzx$r$#(N1Xl#lhZIpe+Lk_htl+)`6KC61~z}g z6|O+>Vz4qinHssf5I%jZ0BDSt^T3w2J_Jk1|KL}Q7o1>%Gi*14knn^T3jODQyl?9? z8%56NBlbOhyf>0_)R%5ZYFuguJyt@cJnGnKzmIDSW0Aiy%v}G6SUd9cfScmh3_Ba0 zQ^Nm67P8gfd`}|M;ba(Y;ZwA3KeFfl;K#~2$lsJW_%EdYXoWTSzr*6Tl|$fSG}yEc zDE%)b|85+~!Kgzp1})aH*y&n;`(IO-zWzbOQV9k{Bn+Ce#p8Dy?*wI6bef1O3?fA) zQhEM^N-6aytg^5z_(W~Zrt>lS+xN)R_5zXO`!x4xwA2Z$BQHjKQb*?G2W?9kt@0nP?}ka;&SLa6EC44dFy&d$fm#0r z3Hb!Y=mR!}87}|-mD+!|6J<^w26k5`$XEggOnkWLa324Jz8n94PMA2V>faqi$1ZN?gN6s{Gh` zF&ivj4;5msnAmUV)jzn$|GjXtYoKyY$i5NFlk!Ng4uDD41cTjlutxTxy7+2u^JZMG z|E*T2A>|QIevq#O**EY5XS?Q45QP%E9sF&q(bsatx90iC5wd>^*O!rzfF`lFKZ6TV z9J9T=e|d*2HMFbpUFtHlaCk3%^7Lbg5X;ZC+`m(8V9fbqA3DWupxe17n+&JMKB=nK z9W&XG$N9?3?qvk29b4~bym%`o96Lw+f@-OaqTAtM64h+7`;{f6N35gtjR;p zGM7Hb^E*z(Uzp98$_k>%i>DwY;GHbQ=z_`>)HB}*S>I|y58G=#>IfTiubk}7MvN+o zM@%wiFSuWq|I7VG>P9#*xNLu$Vs&N0LO+rg3Of&v?OFk>VJ`l?DN>oSNM;73clQlK zDS-S}I>it;!NRkld=wm`9Q00k1xxkMte}vn6y2SnA;>c`@tX2IUeU<1;Ynj?ce8JL z@Zg$zW|aio3%~YJ)|+WbexoM9V00{8af)Eh3wpk~Opm2CrJ?LQtY#fUGQMfE9nIxN zr8_cx$9;d2V4|lx1qXjritfk=-YH5R z#Zd&K8FZX25-XXOy8)bk81@FF!q^<@2DO#g|q#|U!Re)$BNji5T5i0s8SYc zX8R7O|7^$X8HjW?tE@sgl4mjQV%|$RKl#Y)q@s}*m!>YwpQy3!B^z{R9s=LE^=dIM z`>=dEsU__h!Q)oD@McC;z zgvfY)PV^QO{|)tRaiM=}sB8_>Dk8S#t1SKAlinIcc~@Vqxfyi&n##yd5$WT++M#PR@p!eN;owzYte2krFDA+hMNg5zhRkJS=oDjBjqZICJ_&0{ zAlIxiJ{sWOHId~#5R^%+HU(n3nlG@RO=_KwkDNOIzC$E*{BbJ_kbTNWA*;i_q8;w) zY?zhH0qHjtE~XQ$#oU~a#p9&}mcBnfYgc7 zexSgbEvlX*WU1hpN=elNRP;+>BnIa^PuU7ySA{c+y2qY?b?~$lRJv;B(4nKvjAoNx zJqb$P@YQI#t?H7WML)oZK0H@hny$rom&oeiBI{ic83LxAw#504SRCCVTd(MHW>pGCSSJ;b`W@;aa{%n49jca#1wH;FMA z`g#nIfAXS%*tQ*eoM?OQYE{;>Yik;29|-#|@zbSdiUr?jxQgNykVMnCGVP6B@z5Bg#b zC8PzL?a9Zm8Qr|2>^#sPA=xr?b=OQBUesNr4^|F)M(E_@jEnGQ57Pw}VVDBmU}YG9 zGi0#E)(m0ENMbt?KF2!zE*h}%jl7_fKp`H z;1^HjnuC;1R-!En+Vfhtlh?+)VP&P>?6jj5s(}5#AhS_JRCjmOd(S5=t0)KMGR@N` zt6V>vU7d@F2^Jo6J{y~ba5yNg$HMdIYQ$S2v93qOY{wW{Iu+@T3#WV*`vHFbM;aP8 z=XPOUHqyIMpqx!-0r_pdUTwpl+%2~_Sj7E$IDsn5h$HkK<;ZSRydo`l)Mc?T-qZB; z3U=H4%r4ahb}MG2=k&o&X*ch9YGJJx7{sK8mnW z%HUk?Q@ah*=-o0#OH4;B)XFl|VF}B7Dcp><9Di~jr}XQIn%wSMgQ;qB`Oi)h-qW4{ zg3;L1%Yw_-0XcOQrxR5xhZ>hYtr@de&cF0 zI5*mPe%D+Z#vx>VNT3uOb8xqTJj{2Qj|3jev;Sjy&Sp)wPmJO4miX!)eZf)1+MxC#ATv24*EoCg` zq*6ugk-s_n{LDpP(!tr?o)ibiSs3$ANr+nooUI{+&oYED{G3TZWhEy?1Y5-een_dP zaBjGga>F=U0QNJbdC8!15@XEFK%iTB`hefBt4&01xMSJ0@8M~vL88r9QE-(36&grz z%=OFPkqF2GHVv)P5=h89m%AgGkg-bRjX7Mybd^7+gLkSG0 zLP0MEySUMePB(%DhzE4Bf;P$p7~Y7m`rNR6@P6F%asf4HWwXAh#;?4z=Ex%n&$@X8 zm@h2E7OGtPj&$e5a{uM|fZ4L5nx_;$#S6TTE~t0)$N=#8=CON)J$EjfYlQ9eA~i1k z6;T_iUoQ0sO=1T%bR-zfcmw5~gS(WF+~4c#b^6lX;`TX_#L+))DufwA!>?6ECXJ3W z>r2ymy6ZhDDI148o&#Pg2z$gf59 zHia&wFiXh%Z zXH_w)ad*MIa-ibQ^?dpvkbgP4XG+iBI!#}0+CBB@z1yufs026poI=7lRo11QfAbC% zYYM2SDf4evyr@!~AX@cL$HK_AUb%ebr!EC2XsU;@m8CXQ(CuAj5shCP_X`DPq(=S_=M{4;KQNMQ8t~Y+LbEf+Ri0(ab_d3~@;P zcV=%{f$@f^%05X#A8=4&S~!6`nKm7XZ$1+h7$MuC)~?mwtQDo*5g~RL>yNVLBU~0T0vO|K)mFdGlZy=rikl{ zf!L8?=e0B0@a;P`312!3-i@V-?l*L02kAfMDZYH4(~;a&2)XaSul)syWLhgu@s4~j zVyp^uzAiV*xyb!J>PoGFug>k1JWk;0sPxHrU9Yhn6;vmWZLT95wrI681s}9(hy?3!m7@XeUA(SxJbUy;}4xY=*jjigX5oR8XLV_5HE9b{fS_DI_P z&+U#c<8Fse`ar|K&6?Z0N9Dk+fWq}di*d=R$dAeSEhgjD{&EJV-#&<&SNpZFlnpdNI<=K?L?kmhU3qZzgQBU>g5dCccz)Qzh~T^+H@C>+9hue_&5NdG?{U~; zE#MMlD1B6XXg%d86>Uid=}CmI4jzU*KK{Z-iv0?GymxGA?=w=yLmzkZo#XkaJEk0X z*s#QSa;}2Kuk{5fKbK0sgE5i2f2aaTI``q&d08@xu3QO5aQ{BE1f5q$`A?KQrBB9d z!a^EX9eW-@hCdGM-$zl)rmJ9aZ{twl#92F zcy@dM+t=8KIS#h4F*v}OJJclWucvs}I2E~mOx-!R_6#M-`zqTODeo()oS1eyJ>SlG6;+RIzwY^h=g~t|qA#2s z6Gfq>wBolV*N4W;(6ID-!=WQFN1l<-avCNUQQh*bWTzDa9th^*4*+*{u!Pp-@(_87 zoMc5%c9V9iBhVzvjOJq$G{%0&_hY0vk@Va!HXk42`>rylZzU|&T<>a}z69zLNZ$ui z`Q&PCz->!|TW9)A+PJDt9#xULx3LFgp`)hk8kO|p#pwIq&aBevB zO_{VbyryC`8v=dmGf4uMc}S!wdI*wPtUvdm?`s{qh|Il;Y;qH*f%)J>3CB)RkM1;A{)3ASWl6`bvGlmxTA z`+5T1TAerADA-VAq5U_u9)e22Kwxa9yHUF)f6nj zmlHX_prbG?dfAji@ln4hD@$!L?b@{E4IJUaDF@nooapt2#j&N4)%hCxPZQ{G8dcVX zHQ^31le+3T}nqw9DiR^ka@k*Cz$6|`hj}XrJ%?u z1;QM48XphjU)ZRxa?Ll1IUF>(F9q$ACj=TH$8)&1QWITC+`3JNb#-*MLr2=wMs`N2 zsY06(;=2Cvv+i1YyroFok_+_X)Z|=|r{$KF*@&f!_RAvWHWI?mU=^5{`~0%-AsI}4 ztR9AL-+S8S>l~AT7)l9U{Lc7LmIY3P3(-(e9q9wgM;R3ZGG)eBQmv+J{F%{h+`A&z zIh6KzJe2dTlP~-FaLB@F$$3~a#wP$5kL!y7U*TH`mC)U)Ae$quK6eIkyhS&%x!vW> z5M^qX%;;bqxR2eiD8j(<(@QPHw%ch%)X2ZWmH_qaUI|>nt`>+p>()0o1KnpbZubwS zYQBeUzb6$O#WefZ$6(qlpt-dncwvPk8D?hvr9PK{W@3l|Vp z$ercvX>3+}i13db4c5`>DnEdnY^#Mtha_3)DXRB-?JBI0^0}H~TmDV_Wab}cl1WEV znNDwL2*R19!VxJ{-34Em3A2zSltK6siY}|fK3m8+>@}Lc1jTJdzO3ty4gb{)%9As3TZC_NyhLO zY6lptO7J$&8XCwNQKtg_f|O`Ew8GwfcQF#g$2YPaY;vUootBew0tJ+qDe;WytDv*+ zm6vC&%;Z#P*XM}mQ@br)uVoQb+*HZjk0{vdKxxYvVQN|2+FdiUqC{ya5VIAzF!6QYO zR0F-GJNgph9AG`0L!c4nvM5++V+PgzR>r56Xk%wIx;Kw!g9n4p zU^6m8=p_9^pLkzxOtVYW(ogc=5>JqislIIeWD#T})R)o5?c_qwUht^ZG;PHf2s| zd}K-KbwJ3{?fP~Nd{1@)g|&9>nr#*6yGF0aq9o6@*4`7E3wM{%u<#oBw5Y2>2y7+m zDgq;)PuLPQo0jJ%o(W|Jf9M$lYoPLD3x;yDZC(NE7uEy^^A)egD@q<=i<;Fgc#PPy zhj)|lEI?1IR^M0>v-VB|-ZR0FeR8WqH$`;%_o$FpKxNFhX_%Q%%)BzrI}74cyNw#` z7Dm!kUrr>oN9DNxfCfaa<2?v%!e~P|QPHi^V|2mQY*}qKEbbV3y`26IJ(%4b?V4Cr zZ4PYmYoG;)2h;FZ_n@u7m=+F7%r`#pOqgq;t-TbED9qU8sT5#VNRIv}S!YZ~r5h0x zN_0Br$P7;9umeafFqH|0P)s0;tfkT&-L3`j5f6YPd3ZJkuzHdznI-vG+gwCJ`J>F> z2xF60Fo%+)z_1;(@b!vito5u|0BFX(mSWxdyGRDZtW3YscT*9 zZe5A9|7=%N6tOgRvgi*ngT4M-$cUXchk|kUsz-A}9H}DX#0wVVfDb&LEy;17`a%Z} z^FC*4)H@c+VkyDi`;aeL0sN9Qd~&HAbT)-R2L-nS$v4dBOPGdSd^ve6rF0eycpyPdK4-tkn&CoPiSe90%n8I zUk=TmQBx@cs%1?mtHWvM-CkD)`Q+gCeR+4FhwW0ZddoFR*P+t8tqn&_~^76gbl(~q3NJA&YZ)t?bPM8z||oc zde{4$@MfvdMbO&_lHNHeL9pp3f7p*61+??@r#f@C>`a$mj8$<`XcB~=e#9O8KCmzP z5E^CrgiBlQmE_=LrUYSM|KRE6OzMc16ul$~L043_Nour};0BN9AX&iW?6PnDnWxS! zp9nq*7`S?ZJF617;}rQ){9j^EyU$7!jU}xazP-8u9b089V(Y)i2wj^chD+}OjAO;K}=$u579G$xYFf{?%zITN` z7}*QE(4$;~UFcnsr(EVmk7WE56||4jA=X&wZKz`%GBFeElIR7$KV1GUaFWY8;s;=~ zrX9@n&Kghoa!g;%W8Zb#*cW07MSmg2JiJe3=0$JW{o#6Ug*}KK>JNV#i;n=flZla- z8Ewqmf!*L)vH9mawjIp-Gow^w$ieaC7auuyvz?+zs1#9ID*%KsP&ENBSla_HD8X!2 zN#MlLnaR{<^?hgtTHAla=wjaAW&&(AcZHupGLq4v)I~97;*zO}%Pc^ZJ|pj#{eWk7 z;6i9#WDv9E0?%f#+d$A4{P|Eo1PffYS{K1*>DIG7vgOr?1cHN9II%|JS_RobT zqHbGo(vRdn>>+n3`{w8ZFiKJ2tb|HFM#yw(awV);*HxS{Cz%z8FW^q((p&k7f!)(v zv&5Mg_wz>&KAkKJm$7h-=yL&k$CN@fup%sg0Te1tF8L%%xm%r; z=}x*C0jC;zK6)SEnL|%j>YiglV@DLwP27M$4lbG*4%`V8qh$qVuu)#;)lJuXhTOZW zLtM$lckcyX_7mRcUG#-{HKK$L=Vz$MVQ;!O*;N+DJh1$Da`NIwUIXyD-a^dFk(}@~_vQFK`i5t~mV!vHu@= z$v7%Hp^@p@HQ2B|j(uMc?B|oy`MiLsz*(#@UNb{E_INq~jy@O4f8%F4_el2a`+v%0(sQi2Z;)@;v?+4GtV+QBORJPm#r3N2lpX7 z{`(5-Dth?9?l@~ftMv4tQ=mwJ+C4^tO=g!dIc?)zyr5bPKC7`VC3V2?$I7$A@xuFIf~Dg_~-l7=~7 z6wG9McJuIV*c!uk@Xny=12wICu%~U=GD`XyvB?B}kf}gTjqh&@hk?25GS30;35I&P zgJpN|hT!kRyOH2T51>7Pl^;&97;T>Km_1A+$r4hPA6N3i81?~??gp-MXceQ5GYlo` z18S@p-dIC~A2-6@wB)86#ev+c%pb6VjHpwr4P3=2*U92bYTsLTJ(og(XJRoPAPP}< zs9`BzUiI*&Kueplcs9!~RKFDKF0#}q+ZVZRrTsE3luyAk=9kZ?J6)_lfBf2LWZts% z4d0xI{L5=S{FCPnmDj4+B!4C5NzgW1AR{DVl#ZNbsr?3~PCq6dPLo~hBxmHkb zS5ek)iBog9Qb@${2Af5!75usslT(q+TJz5CvPpvMyzqHGbwYfhb=Rehf{6*uUvxH3-Q+BpSdYqf0pakpGzH9pFf~^!}NS^nTu$#8LpJ?oyCI4&$KY$$j`L2xg!r| zf-E1&MjOQVWhfTpED!1!a(V3dviVpjsE>oz2%!{nd42>f|u8uZ^r9k*=f(;X+HN8_DjMvbnOHs9@kvA3p@ul9c&A{ z5aVxc)-r*5riT7}r{%4Ur@ZPy;pgC%6O+ZhQvDO9I9apSnf6}dU}&I*Vc&_OB9s>pb3w1Wttl~mS=r9ieJW(5k~Na>MeO436i!>p z&$7RSZF(-?O6^{|Uc4=7d~8wWi_O!Yu_1L{x2!H-U!0Jg@tkj3x_HyMSt_sntL!EM z=QU?vrCc6g>e;;d=!NMOiznKttLw8gWJ*#e{Di#z-Bmdt&=we(daANw;iKW^=7#T| z9bk%^k>5&hpIOc)#TdrLt_E53-aW%}FwrdhcWz!-Q-d|7mGna*lj+vCqNr~UlW|R> z(@)yEjSI03x4!8gO=fx=Bf}2<#6fAa!=}ZXWshkR!KmZ zD`E-LeGGR>{m169RDM(8$$nYIBL?Vf12yyf`$qRqk5@W$Itr|=r5j$jq6D@C*zmtf zU$)AA^i3KBc+skCL6<9+bY!ho$`o*wwLKKn{nEVB>o}u3OXnrocwTyFkLbw+a07m^wHiOO=<;EH3^O z&l)tsXa4N-Qk78!?{WOjBL>ggIcs`WtfcaU_im}5jpic#CguBy?o`~=aiFu9(Y@J{D&&|xr!)+5=TD*r2^0O{t*=)pxxr0h%o!@dcRai>*&zhS_7JV zCf~$9b^e^%ITyBJm9nJU8}ZClk?-Y6f2JmM3@eSq8}vpjQ)4Tx){@p#j?Goejwio6 zmOs;Mxt2i{NQ|quPoB)yR9LRF;6?v++X0JT$>Sf1?0X?S6tu)IZDCLQ;ZGK9dQ

$GMh2F{||H42)(|3uyi{2ooc}zdKHoBO;ef<^1en+a+b14poTDE zl6a|v-RbslWaL|w?Pnr^6Wm!sCyPlA3m)&f#;0mlJRRojQ_PreR}oRG(x3^Uwn$aS zxMOT2!7AQjR@0ANl4&yOeUkP`|KZP68F2Q~yt`+n{a=OH3hhKu%RO(L;AeI*4$(JO z&le^T`*1Gr<)uqn*Y2zS@Zii;T^|=-DKpm7BlZYg3yBG%f4G*ncWTqwwRpg8rq1-r z3U(g-DpC#^eY7ui#qAAzM_Q1m9(Ox9N~fY*kqPeu6oeNxmPv_+#pqc~u^0#mnh8Fp ziDYrRbn1_+Xg=_~qmfSK{rn6^Tv)GL|7&GL@pi^5r;ml{`lrCn*E%)T`d09jA{d|1b#rkszUG7b>VO6QuJ-^rcacF z+SfKl0@7r=W`Wb2Eo6+iEvfjaolfS+3~FpGT*`W2D%>*Bk@+F$$ fe=+sn2eu2P|I0$7?j*Yy`_ for the available engine parameters. The openPMD-api does not interpret these values and instead simply forwards them to ADIOS2. * ``adios2.engine.usesteps``: Described more closely in the documentation for the :ref:`ADIOS2 backend` (usesteps). +* ``adios2.engine.preferred_flush_target`` Only relevant for BP5 engine, possible values are ``"disk"`` and ``"buffer"`` (default: ``"disk"``). + + * If ``"disk"``, data will be moved to disk on every flush. + * If ``"buffer"``, then only upon ending an IO step or closing an engine. + + This behavior can be overridden on a per-flush basis by specifying this JSON/TOML key as an optional parameter to the ``Series::flush()`` or ``Attributable::seriesFlush()`` methods. + + Additionally, specifying ``"disk_override"`` or ``"buffer_override"`` will take precedence over options specified without the ``_override`` suffix, allowing to invert the normal precedence order. + This way, a data producing code can hardcode the preferred flush target per ``flush()`` call, but users can e.g. still entirely deactivate flushing to disk in the ``Series`` constructor by specifying ``preferred_flush_target = buffer_override``. + This is useful when applying the asynchronous IO capabilities of the BP5 engine. * ``adios2.dataset.operators``: This key contains a list of ADIOS2 `operators `_, used to enable compression or dataset transformations. Each object in the list has two keys: diff --git a/docs/source/usage/workflow.rst b/docs/source/usage/workflow.rst index 7cf0e232d7..ac5398b4cf 100644 --- a/docs/source/usage/workflow.rst +++ b/docs/source/usage/workflow.rst @@ -63,3 +63,12 @@ Attributes are (currently) unaffected by this: * In writing, attributes are stored internally by value and can afterwards not be accessed by the user. * In reading, attributes are parsed upon opening the Series / an iteration and are available to read right-away. + +.. attention:: + + Note that the concrete implementation of ``Series::flush()`` and ``Attributable::seriesFlush()`` is backend-specific. + Using these calls does neither guarantee that data is moved to storage/transport nor that it can be accessed by independent readers at this point. + + Some backends (e.g. the BP5 engine of ADIOS2) have multiple implementations for the openPMD-api-level guarantees of flush points. + For user-guided selection of such implementations, ``Series::flush`` and ``Attributable::seriesFlush()`` take an optional JSON/TOML string as a parameter. + See the section on :ref:`backend-specific configuration ` for details. diff --git a/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp index a967433cf4..bef874c607 100644 --- a/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp @@ -50,7 +50,7 @@ class OPENPMDAPI_EXPORT ADIOS1IOHandler : public AbstractIOHandler return "ADIOS1"; } - std::future flush(internal::FlushParams const &) override; + std::future flush(internal::ParsedFlushParams &) override; void enqueue(IOTask const &) override; @@ -72,7 +72,7 @@ class OPENPMDAPI_EXPORT ADIOS1IOHandler : public AbstractIOHandler return "DUMMY_ADIOS1"; } - std::future flush(internal::FlushParams const &) override; + std::future flush(internal::ParsedFlushParams &) override; private: std::unique_ptr m_impl; diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 18a4aad192..af79d72d3d 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -26,6 +26,7 @@ #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" #include "openPMD/IO/AbstractIOHandlerImplCommon.hpp" +#include "openPMD/IO/FlushParametersInternal.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/IO/InvalidatableFile.hpp" #include "openPMD/IterationEncoding.hpp" @@ -140,7 +141,7 @@ class ADIOS2IOHandlerImpl ~ADIOS2IOHandlerImpl() override; - std::future flush(internal::FlushParams const &); + std::future flush(internal::ParsedFlushParams &); void createFile(Writable *, Parameter const &) override; @@ -209,6 +210,16 @@ class ADIOS2IOHandlerImpl */ adios2::Mode adios2AccessMode(std::string const &fullPath); + enum class FlushTarget : unsigned char + { + Buffer, + Buffer_Override, + Disk, + Disk_Override + }; + + FlushTarget m_flushTarget = FlushTarget::Disk; + private: adios2::ADIOS m_ADIOS; /* @@ -412,6 +423,7 @@ namespace ADIOS2Defaults constexpr const_str str_type = "type"; constexpr const_str str_params = "parameters"; constexpr const_str str_usesteps = "usesteps"; + constexpr const_str str_flushtarget = "preferred_flush_target"; constexpr const_str str_usesstepsAttribute = "__openPMD_internal/useSteps"; constexpr const_str str_adios2Schema = "__openPMD_internal/openPMD2_adios2_schema"; @@ -927,6 +939,8 @@ namespace detail friend struct BufferedGet; friend struct BufferedPut; + using FlushTarget = ADIOS2IOHandlerImpl::FlushTarget; + BufferedActions(BufferedActions const &) = delete; /** @@ -1039,10 +1053,26 @@ namespace detail template void enqueue(BA &&ba, decltype(m_buffer) &); + struct ADIOS2FlushParams + { + /* + * Only execute performPutsGets if UserFlush. + */ + FlushLevel level; + FlushTarget flushTarget = FlushTarget::Disk; + + ADIOS2FlushParams(FlushLevel level_in) : level(level_in) + {} + + ADIOS2FlushParams(FlushLevel level_in, FlushTarget flushTarget_in) + : level(level_in), flushTarget(flushTarget_in) + {} + }; + /** * Flush deferred IO actions. * - * @param level Flush Level. Only execute performPutsGets if UserFlush. + * @param flushParams Flush level and target. * @param performPutsGets A functor that takes as parameters (1) *this * and (2) the ADIOS2 engine. * Its task is to ensure that ADIOS2 performs Put/Get operations. @@ -1057,7 +1087,7 @@ namespace detail */ template void flush( - FlushLevel level, + ADIOS2FlushParams flushParams, F &&performPutsGets, bool writeAttributes, bool flushUnconditionally); @@ -1067,7 +1097,7 @@ namespace detail * and does not flush unconditionally. * */ - void flush(FlushLevel, bool writeAttributes = false); + void flush(ADIOS2FlushParams, bool writeAttributes = false); /** * @brief Begin or end an ADIOS step. @@ -1265,7 +1295,8 @@ class ADIOS2IOHandler : public AbstractIOHandler // we must not throw in a destructor try { - this->flush(internal::defaultFlushParams); + auto params = internal::defaultParsedFlushParams; + this->flush(params); } catch (std::exception const &ex) { @@ -1304,6 +1335,6 @@ class ADIOS2IOHandler : public AbstractIOHandler return "ADIOS2"; } - std::future flush(internal::FlushParams const &) override; + std::future flush(internal::ParsedFlushParams &) override; }; // ADIOS2IOHandler } // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp index eefac95854..e28122582b 100644 --- a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp @@ -54,7 +54,7 @@ class OPENPMDAPI_EXPORT ParallelADIOS1IOHandler : public AbstractIOHandler return "MPI_ADIOS1"; } - std::future flush(internal::FlushParams const &) override; + std::future flush(internal::ParsedFlushParams &) override; #if openPMD_HAVE_ADIOS1 void enqueue(IOTask const &) override; #endif diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 80616e62bc..4f6916ae55 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -103,12 +103,24 @@ namespace internal struct FlushParams { FlushLevel flushLevel = FlushLevel::InternalFlush; + std::string backendConfig = "{}"; + + explicit FlushParams() + {} + FlushParams(FlushLevel flushLevel_in) : flushLevel(flushLevel_in) + {} + FlushParams(FlushLevel flushLevel_in, std::string backendConfig_in) + : flushLevel(flushLevel_in) + , backendConfig{std::move(backendConfig_in)} + {} }; /* * To be used for reading */ - constexpr FlushParams defaultFlushParams{}; + FlushParams const defaultFlushParams{}; + + struct ParsedFlushParams; } // namespace internal /** Interface for communicating between logical and physically persistent data. @@ -164,7 +176,14 @@ class AbstractIOHandler * @return Future indicating the completion state of the operation for * backends that decide to implement this operation asynchronously. */ - virtual std::future flush(internal::FlushParams const &) = 0; + std::future flush(internal::FlushParams const &); + + /** Process operations in queue according to FIFO. + * + * @return Future indicating the completion state of the operation for + * backends that decide to implement this operation asynchronously. + */ + virtual std::future flush(internal::ParsedFlushParams &) = 0; /** The currently used backend */ virtual std::string backendName() const = 0; diff --git a/include/openPMD/IO/DummyIOHandler.hpp b/include/openPMD/IO/DummyIOHandler.hpp index 9a4f3c3852..7cd4123699 100644 --- a/include/openPMD/IO/DummyIOHandler.hpp +++ b/include/openPMD/IO/DummyIOHandler.hpp @@ -44,6 +44,6 @@ class DummyIOHandler : public AbstractIOHandler /** No-op consistent with the IOHandler interface to enable library use * without IO. */ - std::future flush(internal::FlushParams const &) override; + std::future flush(internal::ParsedFlushParams &) override; }; // DummyIOHandler } // namespace openPMD diff --git a/include/openPMD/IO/FlushParametersInternal.hpp b/include/openPMD/IO/FlushParametersInternal.hpp new file mode 100644 index 0000000000..2233ec002d --- /dev/null +++ b/include/openPMD/IO/FlushParametersInternal.hpp @@ -0,0 +1,40 @@ +/* Copyright 2022 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#pragma once + +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" + +#include + +namespace openPMD::internal +{ +struct ParsedFlushParams +{ + ParsedFlushParams(FlushParams const &); + + FlushLevel flushLevel = FlushLevel::InternalFlush; + json::TracingJSON backendConfig; +}; + +ParsedFlushParams const defaultParsedFlushParams{defaultFlushParams}; +} // namespace openPMD::internal diff --git a/include/openPMD/IO/HDF5/HDF5IOHandler.hpp b/include/openPMD/IO/HDF5/HDF5IOHandler.hpp index 85d6ab9d40..e81996b389 100644 --- a/include/openPMD/IO/HDF5/HDF5IOHandler.hpp +++ b/include/openPMD/IO/HDF5/HDF5IOHandler.hpp @@ -42,7 +42,7 @@ class HDF5IOHandler : public AbstractIOHandler return "HDF5"; } - std::future flush(internal::FlushParams const &) override; + std::future flush(internal::ParsedFlushParams &) override; private: std::unique_ptr m_impl; diff --git a/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp b/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp index e1c8d52257..18d43c93ab 100644 --- a/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp +++ b/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp @@ -48,7 +48,7 @@ class ParallelHDF5IOHandler : public AbstractIOHandler return "MPI_HDF5"; } - std::future flush(internal::FlushParams const &) override; + std::future flush(internal::ParsedFlushParams &) override; private: std::unique_ptr m_impl; diff --git a/include/openPMD/IO/JSON/JSONIOHandler.hpp b/include/openPMD/IO/JSON/JSONIOHandler.hpp index 37b00fa165..0cdc6f3c36 100644 --- a/include/openPMD/IO/JSON/JSONIOHandler.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandler.hpp @@ -38,7 +38,7 @@ class JSONIOHandler : public AbstractIOHandler return "JSON"; } - std::future flush(internal::FlushParams const &) override; + std::future flush(internal::ParsedFlushParams &) override; private: JSONIOHandlerImpl m_impl; diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index a5af2be601..39da51057b 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -442,8 +442,13 @@ class Series : public Attributable std::string backend() const; /** Execute all required remaining IO operations to write or read data. + * + * @param backendConfig Further backend-specific instructions on how to + * implement this flush call. + * Must be provided in-line, configuration is not read + * from files. */ - void flush(); + void flush(std::string backendConfig = "{}"); /** * @brief Entry point to the reading end of the streaming API. diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index bba3eb409c..1c9dda8429 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -238,8 +238,13 @@ class Attributable * of parents. This method will walk up the parent list until it reaches * an object that has no parent, which is the Series object, and flush()-es * it. + * + * @param backendConfig Further backend-specific instructions on how to + * implement this flush call. + * Must be provided in-line, configuration is not read + * from files. */ - void seriesFlush(); + void seriesFlush(std::string backendConfig = "{}"); /** String serialization to describe an Attributable * diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index 1ed7054a6f..81d83955f9 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -109,7 +109,7 @@ class Writable final * an object that has no parent, which is the Series object, and flush()-es * it. */ - void seriesFlush(); + void seriesFlush(std::string backendConfig = "{}"); // clang-format off OPENPMD_private diff --git a/src/IO/ADIOS/ADIOS1IOHandler.cpp b/src/IO/ADIOS/ADIOS1IOHandler.cpp index 14ea1aeaaa..92166ca259 100644 --- a/src/IO/ADIOS/ADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS1IOHandler.cpp @@ -336,7 +336,7 @@ ADIOS1IOHandler::ADIOS1IOHandler( ADIOS1IOHandler::~ADIOS1IOHandler() = default; -std::future ADIOS1IOHandler::flush(internal::FlushParams const &) +std::future ADIOS1IOHandler::flush(internal::ParsedFlushParams &) { return m_impl->flush(); } @@ -438,7 +438,7 @@ ADIOS1IOHandler::ADIOS1IOHandler(std::string path, Access at, json::TracingJSON) ADIOS1IOHandler::~ADIOS1IOHandler() = default; -std::future ADIOS1IOHandler::flush(internal::FlushParams const &) +std::future ADIOS1IOHandler::flush(internal::ParsedFlushParams &) { return std::future(); } diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 2e53700bea..f0e7976f16 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -253,16 +253,121 @@ std::string ADIOS2IOHandlerImpl::fileSuffix() const } } +using FlushTarget = ADIOS2IOHandlerImpl::FlushTarget; +static FlushTarget flushTargetFromString(std::string const &str) +{ + if (str == "buffer") + { + return FlushTarget::Buffer; + } + else if (str == "disk") + { + return FlushTarget::Disk; + } + else if (str == "buffer_override") + { + return FlushTarget::Buffer_Override; + } + else if (str == "disk_override") + { + return FlushTarget::Disk_Override; + } + else + { + throw error::BackendConfigSchema( + {"adios2", "engine", ADIOS2Defaults::str_flushtarget}, + "Flush target must be either 'disk' or 'buffer', but " + "was " + + str + "."); + } +} + +static FlushTarget & +overrideFlushTarget(FlushTarget &inplace, FlushTarget new_val) +{ + auto allowsOverride = [](FlushTarget ft) { + switch (ft) + { + case FlushTarget::Buffer: + case FlushTarget::Disk: + return true; + case FlushTarget::Buffer_Override: + case FlushTarget::Disk_Override: + return false; + } + return true; + }; + + if (allowsOverride(inplace)) + { + inplace = new_val; + } + else + { + if (!allowsOverride(new_val)) + { + inplace = new_val; + } + // else { keep the old value, no-op } + } + return inplace; +} + std::future -ADIOS2IOHandlerImpl::flush(internal::FlushParams const &flushParams) +ADIOS2IOHandlerImpl::flush(internal::ParsedFlushParams &flushParams) { auto res = AbstractIOHandlerImpl::flush(); + + detail::BufferedActions::ADIOS2FlushParams adios2FlushParams{ + flushParams.flushLevel, m_flushTarget}; + if (flushParams.backendConfig.json().contains("adios2")) + { + auto adios2Config = flushParams.backendConfig["adios2"]; + if (adios2Config.json().contains("engine")) + { + auto engineConfig = adios2Config["engine"]; + if (engineConfig.json().contains(ADIOS2Defaults::str_flushtarget)) + { + auto target = json::asLowerCaseStringDynamic( + engineConfig[ADIOS2Defaults::str_flushtarget].json()); + if (!target.has_value()) + { + throw error::BackendConfigSchema( + {"adios2", "engine", ADIOS2Defaults::str_flushtarget}, + "Flush target must be either 'disk' or 'buffer', but " + "was non-literal type."); + } + overrideFlushTarget( + adios2FlushParams.flushTarget, + flushTargetFromString(target.value())); + } + } + + if (auto shadow = adios2Config.invertShadow(); shadow.size() > 0) + { + switch (adios2Config.originallySpecifiedAs) + { + case json::SupportedLanguages::JSON: + std::cerr << "Warning: parts of the backend configuration for " + "ADIOS2 remain unused:\n" + << shadow << std::endl; + break; + case json::SupportedLanguages::TOML: { + auto asToml = json::jsonToToml(shadow); + std::cerr << "Warning: parts of the backend configuration for " + "ADIOS2 remain unused:\n" + << asToml << std::endl; + break; + } + } + } + } + for (auto &p : m_fileData) { if (m_dirty.find(p.first) != m_dirty.end()) { - p.second->flush( - flushParams.flushLevel, /* writeAttributes = */ false); + p.second->flush(adios2FlushParams, /* writeAttributes = */ false); } else { @@ -2310,6 +2415,22 @@ namespace detail streamStatus = bool(tmp) ? StreamStatus::OutsideOfStep : StreamStatus::NoStream; } + + if (engineConfig.json().contains(ADIOS2Defaults::str_flushtarget)) + { + auto target = json::asLowerCaseStringDynamic( + engineConfig[ADIOS2Defaults::str_flushtarget].json()); + if (!target.has_value()) + { + throw error::BackendConfigSchema( + {"adios2", "engine", ADIOS2Defaults::str_flushtarget}, + "Flush target must be either 'disk' or 'buffer', but " + "was non-literal type."); + } + overrideFlushTarget( + m_impl->m_flushTarget, + flushTargetFromString(target.value())); + } } auto shadow = impl.m_config.invertShadow(); @@ -2581,11 +2702,12 @@ namespace detail template void BufferedActions::flush( - FlushLevel level, + ADIOS2FlushParams flushParams, F &&performPutGets, bool writeAttributes, bool flushUnconditionally) { + auto level = flushParams.level; if (streamStatus == StreamStatus::StreamOver) { if (flushUnconditionally) @@ -2681,16 +2803,52 @@ namespace detail } } - void BufferedActions::flush(FlushLevel level, bool writeAttributes) + void + BufferedActions::flush(ADIOS2FlushParams flushParams, bool writeAttributes) { + auto decideFlushAPICall = [this, flushTarget = flushParams.flushTarget]( + adios2::Engine &engine) { +#if ADIOS2_VERSION_MAJOR * 1000000000 + ADIOS2_VERSION_MINOR * 100000000 + \ + ADIOS2_VERSION_PATCH * 1000000 + ADIOS2_VERSION_TWEAK >= \ + 2701001223 + bool performDataWrite{}; + switch (flushTarget) + { + case FlushTarget::Disk: + case FlushTarget::Disk_Override: + performDataWrite = true; + break; + case FlushTarget::Buffer: + case FlushTarget::Buffer_Override: + performDataWrite = false; + break; + } + performDataWrite = performDataWrite && m_engineType == "bp5"; + + if (performDataWrite) + { + engine.PerformDataWrite(); + } + else + { + engine.PerformPuts(); + } +#else + (void)this; + (void)flushTarget; + engine.PerformPuts(); +#endif + }; + flush( - level, - [](BufferedActions &ba, adios2::Engine &eng) { + flushParams, + [decideFlushAPICall = std::move(decideFlushAPICall)]( + BufferedActions &ba, adios2::Engine &eng) { switch (ba.m_mode) { case adios2::Mode::Write: case adios2::Mode::Append: - eng.PerformPuts(); + decideFlushAPICall(eng); break; case adios2::Mode::Read: eng.PerformGets(); @@ -2717,7 +2875,7 @@ namespace detail { m_IO.DefineAttribute( ADIOS2Defaults::str_usesstepsAttribute, 0); - flush(FlushLevel::UserFlush, /* writeAttributes = */ false); + flush({FlushLevel::UserFlush}, /* writeAttributes = */ false); return AdvanceStatus::OK; } @@ -2757,7 +2915,7 @@ namespace detail } } flush( - FlushLevel::UserFlush, + {FlushLevel::UserFlush}, [](BufferedActions &, adios2::Engine &eng) { eng.EndStep(); }, /* writeAttributes = */ true, /* flushUnconditionally = */ true); @@ -2917,7 +3075,7 @@ ADIOS2IOHandler::ADIOS2IOHandler( {} std::future -ADIOS2IOHandler::flush(internal::FlushParams const &flushParams) +ADIOS2IOHandler::flush(internal::ParsedFlushParams &flushParams) { return m_impl.flush(flushParams); } @@ -2937,7 +3095,7 @@ ADIOS2IOHandler::ADIOS2IOHandler( : AbstractIOHandler(std::move(path), at) {} -std::future ADIOS2IOHandler::flush(internal::FlushParams const &) +std::future ADIOS2IOHandler::flush(internal::ParsedFlushParams &) { return std::future(); } diff --git a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp index 3fbc9aa395..a451dd2c15 100644 --- a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp @@ -355,7 +355,7 @@ ParallelADIOS1IOHandler::ParallelADIOS1IOHandler( ParallelADIOS1IOHandler::~ParallelADIOS1IOHandler() = default; -std::future ParallelADIOS1IOHandler::flush(internal::FlushParams const &) +std::future ParallelADIOS1IOHandler::flush(internal::ParsedFlushParams &) { return m_impl->flush(); } @@ -478,7 +478,7 @@ ParallelADIOS1IOHandler::ParallelADIOS1IOHandler( ParallelADIOS1IOHandler::~ParallelADIOS1IOHandler() = default; -std::future ParallelADIOS1IOHandler::flush(internal::FlushParams const &) +std::future ParallelADIOS1IOHandler::flush(internal::ParsedFlushParams &) { return std::future(); } diff --git a/src/IO/AbstractIOHandler.cpp b/src/IO/AbstractIOHandler.cpp new file mode 100644 index 0000000000..25adae581e --- /dev/null +++ b/src/IO/AbstractIOHandler.cpp @@ -0,0 +1,35 @@ +/* Copyright 2022 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/IO/AbstractIOHandler.hpp" + +#include "openPMD/IO/FlushParametersInternal.hpp" + +namespace openPMD +{ +std::future AbstractIOHandler::flush(internal::FlushParams const ¶ms) +{ + internal::ParsedFlushParams parsedParams{params}; + auto future = this->flush(parsedParams); + json::warnGlobalUnusedOptions(parsedParams.backendConfig); + return future; +} +} // namespace openPMD diff --git a/src/IO/DummyIOHandler.cpp b/src/IO/DummyIOHandler.cpp index 308f584ce4..6bc6ec4d64 100644 --- a/src/IO/DummyIOHandler.cpp +++ b/src/IO/DummyIOHandler.cpp @@ -32,7 +32,7 @@ DummyIOHandler::DummyIOHandler(std::string path, Access at) void DummyIOHandler::enqueue(IOTask const &) {} -std::future DummyIOHandler::flush(internal::FlushParams const &) +std::future DummyIOHandler::flush(internal::ParsedFlushParams &) { return std::future(); } diff --git a/src/IO/FlushParams.cpp b/src/IO/FlushParams.cpp new file mode 100644 index 0000000000..d09018ef4f --- /dev/null +++ b/src/IO/FlushParams.cpp @@ -0,0 +1,31 @@ +/* Copyright 2022 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/IO/FlushParametersInternal.hpp" + +namespace openPMD::internal +{ +ParsedFlushParams::ParsedFlushParams(FlushParams const &flushParams) + : flushLevel(flushParams.flushLevel) + , backendConfig(json::parseOptions( + flushParams.backendConfig, /* considerFiles = */ false)) +{} +} // namespace openPMD::internal diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index ae4e6588aa..b663740077 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -2371,7 +2371,7 @@ HDF5IOHandler::HDF5IOHandler( HDF5IOHandler::~HDF5IOHandler() = default; -std::future HDF5IOHandler::flush(internal::FlushParams const &) +std::future HDF5IOHandler::flush(internal::ParsedFlushParams &) { return m_impl->flush(); } @@ -2385,7 +2385,7 @@ HDF5IOHandler::HDF5IOHandler( HDF5IOHandler::~HDF5IOHandler() = default; -std::future HDF5IOHandler::flush(internal::FlushParams const &) +std::future HDF5IOHandler::flush(internal::ParsedFlushParams &) { return std::future(); } diff --git a/src/IO/HDF5/ParallelHDF5IOHandler.cpp b/src/IO/HDF5/ParallelHDF5IOHandler.cpp index e0a17ed980..369ee2e317 100644 --- a/src/IO/HDF5/ParallelHDF5IOHandler.cpp +++ b/src/IO/HDF5/ParallelHDF5IOHandler.cpp @@ -54,7 +54,7 @@ ParallelHDF5IOHandler::ParallelHDF5IOHandler( ParallelHDF5IOHandler::~ParallelHDF5IOHandler() = default; -std::future ParallelHDF5IOHandler::flush(internal::FlushParams const &) +std::future ParallelHDF5IOHandler::flush(internal::ParsedFlushParams &) { return m_impl->flush(); } @@ -196,7 +196,7 @@ ParallelHDF5IOHandler::ParallelHDF5IOHandler( ParallelHDF5IOHandler::~ParallelHDF5IOHandler() = default; -std::future ParallelHDF5IOHandler::flush(internal::FlushParams const &) +std::future ParallelHDF5IOHandler::flush(internal::ParsedFlushParams &) { return std::future(); } diff --git a/src/IO/JSON/JSONIOHandler.cpp b/src/IO/JSON/JSONIOHandler.cpp index 15d18194c7..10ffce9927 100644 --- a/src/IO/JSON/JSONIOHandler.cpp +++ b/src/IO/JSON/JSONIOHandler.cpp @@ -29,7 +29,7 @@ JSONIOHandler::JSONIOHandler(std::string path, Access at) : AbstractIOHandler{path, at}, m_impl{JSONIOHandlerImpl{this}} {} -std::future JSONIOHandler::flush(internal::FlushParams const &) +std::future JSONIOHandler::flush(internal::ParsedFlushParams &) { return m_impl.flush(); } diff --git a/src/Series.cpp b/src/Series.cpp index 36a7bd90d6..a68d1270fa 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -372,13 +372,13 @@ std::string Series::backend() const return IOHandler()->backendName(); } -void Series::flush() +void Series::flush(std::string backendConfig) { auto &series = get(); flush_impl( series.iterations.begin(), series.iterations.end(), - {FlushLevel::UserFlush}); + {FlushLevel::UserFlush, std::move(backendConfig)}); } std::unique_ptr Series::parseInput(std::string filepath) @@ -1354,7 +1354,7 @@ AdvanceStatus Series::advance( iterations_iterator begin, Iteration &iteration) { - constexpr internal::FlushParams flushParams = {FlushLevel::UserFlush}; + internal::FlushParams const flushParams = {FlushLevel::UserFlush}; auto &series = get(); auto end = begin; ++end; diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index aa2f600da1..0c0b715e28 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -106,9 +106,9 @@ Attributable &Attributable::setComment(std::string const &c) return *this; } -void Attributable::seriesFlush() +void Attributable::seriesFlush(std::string backendConfig) { - writable().seriesFlush(); + writable().seriesFlush(std::move(backendConfig)); } Series Attributable::retrieveSeries() const diff --git a/src/backend/Writable.cpp b/src/backend/Writable.cpp index fae7630795..32d4cd9816 100644 --- a/src/backend/Writable.cpp +++ b/src/backend/Writable.cpp @@ -27,9 +27,9 @@ namespace openPMD Writable::Writable(internal::AttributableData *a) : attributable{a} {} -void Writable::seriesFlush() +void Writable::seriesFlush(std::string backendConfig) { - seriesFlush({FlushLevel::UserFlush}); + seriesFlush({FlushLevel::UserFlush, std::move(backendConfig)}); } void Writable::seriesFlush(internal::FlushParams flushParams) diff --git a/src/binding/python/Attributable.cpp b/src/binding/python/Attributable.cpp index 69708fc948..b60bd8cb48 100644 --- a/src/binding/python/Attributable.cpp +++ b/src/binding/python/Attributable.cpp @@ -371,7 +371,10 @@ void init_Attributable(py::module &m) return ""; }) - .def("series_flush", py::overload_cast<>(&Attributable::seriesFlush)) + .def( + "series_flush", + py::overload_cast(&Attributable::seriesFlush), + py::arg("backend_config") = "{}") .def_property_readonly( "attributes", diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index e95ed1bfbb..1c8ddd3df7 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -197,7 +197,7 @@ void init_Series(py::module &m) &Series::iterationFormat, &Series::setIterationFormat) .def_property("name", &Series::name, &Series::setName) - .def("flush", &Series::flush) + .def("flush", &Series::flush, py::arg("backend_config") = "{}") .def_property_readonly("backend", &Series::backend) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 6846ee0f14..c3543fcb7b 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -30,6 +30,14 @@ #include #include +#ifdef __unix__ +#include +#include +#include +#include +#include +#endif + using namespace openPMD; struct BackendSelection @@ -3945,6 +3953,274 @@ TEST_CASE("git_adios2_early_chunk_query", "[serial][adios2]") R"({"backend": "adios2"})"); } +/* + * Require __unix__ since we need all that filestat stuff for this test. + */ +#if defined(__unix__) && defined(ADIOS2_HAVE_BP5) + +enum class FlushDuringStep +{ + Never, + Default_No, + Default_Yes, + Always +}; + +void adios2_bp5_flush(std::string const &cfg, FlushDuringStep flushDuringStep) +{ + constexpr size_t size = 1024 * 1024; + + auto getsize = []() { + off_t res = 0; + int dirfd = open("../samples/bp5_flush.bp", O_RDONLY); + if (dirfd < 0) + { + throw std::system_error( + std::error_code(errno, std::system_category())); + } + DIR *directory = fdopendir(dirfd); + if (!directory) + { + close(dirfd); + throw std::system_error( + std::error_code(errno, std::system_category())); + } + dirent *entry; + struct stat statbuf; + while ((entry = readdir(directory)) != nullptr) + { + if (strcmp(entry->d_name, ".") == 0 || + strcmp(entry->d_name, "..") == 0) + { + continue; + } + int err = fstatat(dirfd, entry->d_name, &statbuf, 0); + if (err != 0) + { + closedir(directory); + close(dirfd); + throw std::system_error( + std::error_code(errno, std::system_category())); + } + res += statbuf.st_size; + } + closedir(directory); + close(dirfd); + return res; + }; + std::vector data(size, 10); + { + Series write("../samples/bp5_flush.bp", Access::CREATE, cfg); + + { + auto component = + write.writeIterations()[0] + .meshes["e_chargeDensity"][RecordComponent::SCALAR]; + component.resetDataset({Datatype::INT, {size}}); + component.storeChunk(data, {0}, {size}); + // component.seriesFlush(FlushMode::NonCollective); + component.seriesFlush(); + } + + auto currentSize = getsize(); + if (flushDuringStep == FlushDuringStep::Default_Yes || + flushDuringStep == FlushDuringStep::Always) + { + // should be roughly within 1% of 4Mb + REQUIRE(std::abs(1 - double(currentSize) / (4 * size)) <= 0.01); + } + else + { + // should be roughly zero + REQUIRE(currentSize <= 4096); + } + + { + auto component = + write.writeIterations()[0] + .meshes["i_chargeDensity"][RecordComponent::SCALAR]; + component.resetDataset({Datatype::INT, {size}}); + component.storeChunk(data, {0}, {size}); + } + + currentSize = getsize(); + if (flushDuringStep == FlushDuringStep::Default_Yes || + flushDuringStep == FlushDuringStep::Always) + { + // should still be roughly within 1% of 4Mb + REQUIRE(std::abs(1 - double(currentSize) / (4 * size)) <= 0.01); + } + else + { + // should be roughly zero + REQUIRE(currentSize <= 4096); + } + + write.flush(); + currentSize = getsize(); + if (flushDuringStep == FlushDuringStep::Default_Yes || + flushDuringStep == FlushDuringStep::Always) + { + // should now be roughly within 1% of 8Mb + REQUIRE(std::abs(1 - double(currentSize) / (8 * size)) <= 0.01); + } + else + { + // should be roughly zero + REQUIRE(currentSize <= 4096); + } + + { + auto component = + write.writeIterations()[0] + .meshes["temperature"][RecordComponent::SCALAR]; + component.resetDataset({Datatype::INT, {size}}); + component.storeChunk(data, {0}, {size}); + // component.seriesFlush(FlushMode::NonCollective); + component.seriesFlush( + "adios2.engine.preferred_flush_target = \"buffer\""); + } + currentSize = getsize(); + if (flushDuringStep == FlushDuringStep::Always) + { + // should now be roughly within 1% of 12Mb + REQUIRE(std::abs(1 - double(currentSize) / (12 * size)) <= 0.01); + } + else if (flushDuringStep == FlushDuringStep::Default_Yes) + { + // should now be roughly within 1% of 8Mb + REQUIRE(std::abs(1 - double(currentSize) / (8 * size)) <= 0.01); + } + else + { + // should be roughly zero + REQUIRE(currentSize <= 4096); + } + + { + auto component = + write.writeIterations()[0] + .meshes["temperature"][RecordComponent::SCALAR]; + component.resetDataset({Datatype::INT, {size}}); + component.storeChunk(data, {0}, {size}); + // component.seriesFlush(FlushMode::NonCollective); + component.seriesFlush( + "adios2.engine.preferred_flush_target = \"disk\""); + } + currentSize = getsize(); + if (flushDuringStep != FlushDuringStep::Never) + { + // should now be roughly within 1% of 16Mb + REQUIRE(std::abs(1 - double(currentSize) / (16 * size)) <= 0.01); + } + else + { + // should be roughly zero + REQUIRE(currentSize <= 4096); + } + } + auto currentSize = getsize(); + // should now be indiscriminately be roughly within 1% of 8Mb after + // closing the Series + REQUIRE(std::abs(1 - double(currentSize) / (16 * size)) <= 0.01); +} + +TEST_CASE("adios2_bp5_flush", "[serial][adios2]") +{ + std::string cfg1 = R"( +[adios2] + +[adios2.engine] +usesteps = true +type = "bp5" +preferred_flush_target = "disk" + +[adios2.engine.parameters] +AggregationType = "TwoLevelShm" +MaxShmSize = 3221225472 +NumSubFiles = 1 +NumAggregators = 1 +BufferChunkSize = 2147483646 # 2^31 - 2 +)"; + + adios2_bp5_flush( + cfg1, /* flushDuringStep = */ FlushDuringStep::Default_Yes); + + std::string cfg2 = R"( +[adios2] + +[adios2.engine] +usesteps = true +type = "bp5" +preferred_flush_target = "buffer" + +[adios2.engine.parameters] +AggregationType = "TwoLevelShm" +MaxShmSize = 3221225472 +NumSubFiles = 1 +NumAggregators = 1 +BufferChunkSize = 2147483646 # 2^31 - 2 +)"; + + adios2_bp5_flush(cfg2, /* flushDuringStep = */ FlushDuringStep::Default_No); + + std::string cfg3 = R"( +[adios2] + +[adios2.engine] +usesteps = true +type = "bp5" +# preferred_flush_target = + +[adios2.engine.parameters] +AggregationType = "TwoLevelShm" +MaxShmSize = 3221225472 +NumSubFiles = 1 +NumAggregators = 1 +BufferChunkSize = 2147483646 # 2^31 - 2 +)"; + + adios2_bp5_flush( + cfg3, /* flushDuringStep = */ FlushDuringStep::Default_Yes); + + std::string cfg4 = R"( +[adios2] + +[adios2.engine] +usesteps = true +type = "bp5" +preferred_flush_target = "buffer_override" + +[adios2.engine.parameters] +AggregationType = "TwoLevelShm" +MaxShmSize = 3221225472 +NumSubFiles = 1 +NumAggregators = 1 +BufferChunkSize = 2147483646 # 2^31 - 2 +)"; + + adios2_bp5_flush(cfg4, /* flushDuringStep = */ FlushDuringStep::Never); + + std::string cfg5 = R"( +[adios2] + +[adios2.engine] +usesteps = true +type = "bp5" +preferred_flush_target = "disk_override" + +[adios2.engine.parameters] +AggregationType = "TwoLevelShm" +MaxShmSize = 3221225472 +NumSubFiles = 1 +NumAggregators = 1 +BufferChunkSize = 2147483646 # 2^31 - 2 +)"; + + adios2_bp5_flush(cfg5, /* flushDuringStep = */ FlushDuringStep::Always); +} +#endif + TEST_CASE("serial_adios2_backend_config", "[serial][adios2]") { if (auxiliary::getEnvString("OPENPMD_BP_BACKEND", "NOT_SET") == "ADIOS1") From e08cf64902c94f29c4bd6bf593b682d9a2d31a7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 9 Aug 2022 23:07:36 +0200 Subject: [PATCH 56/70] Fix file existence check in parallel tests (#1303) --- test/ParallelIOTest.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 4f9c2acb3c..c2a7e8d99c 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1222,10 +1222,13 @@ doshuffle = "BLOSC_BITSHUFFLE" write("../samples/jsonConfiguredBP4Parallel.bp", writeConfigBP4); write("../samples/jsonConfiguredBP3Parallel.bp", writeConfigBP3); + MPI_Barrier(MPI_COMM_WORLD); + // BP3 engine writes files, BP4 writes directories - REQUIRE(openPMD::auxiliary::file_exists("../samples/jsonConfiguredBP3.bp")); + REQUIRE(openPMD::auxiliary::file_exists( + "../samples/jsonConfiguredBP3Parallel.bp")); REQUIRE(openPMD::auxiliary::directory_exists( - "../samples/jsonConfiguredBP4.bp")); + "../samples/jsonConfiguredBP4Parallel.bp")); std::string readConfigBP3 = R"END( { From 27e58a0be809b00a424a23eb43d2bedbec423352 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 Aug 2022 21:26:23 +0000 Subject: [PATCH 57/70] [pre-commit.ci] pre-commit autoupdate (#1307) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/hadialqattan/pycln: v2.0.4 → v2.1.1](https://github.com/hadialqattan/pycln/compare/v2.0.4...v2.1.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 20779214cb..c55de0eed7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -71,7 +71,7 @@ repos: # Autoremoves unused Python imports - repo: https://github.com/hadialqattan/pycln - rev: v2.0.4 + rev: v2.1.1 hooks: - id: pycln name: pycln (python) From 4cf406468a95c16c528050b7404d363d7d0562d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 10 Aug 2022 00:26:08 +0200 Subject: [PATCH 58/70] ADIOS2: more fine-grained control for file endings (#1218) * Frontend implementation * Backend implementation * Fix config bug: JSON has higher priority than env vars * Testing Adapt tests to BP5 Don't use parentheses in attribute names Enable BP5 dtype tests Add test for AppendAfterSteps in BP5 * Documentation * CI debugging * Revert CI Debugging --- docs/source/backends/adios2.rst | 39 +- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 19 +- .../openPMD/IO/AbstractIOHandlerHelper.hpp | 18 +- include/openPMD/IO/Format.hpp | 4 +- include/openPMD/Series.hpp | 5 + src/Format.cpp | 12 +- src/IO/ADIOS/ADIOS2IOHandler.cpp | 161 +++++-- src/IO/AbstractIOHandlerHelper.cpp | 99 ++++- src/Series.cpp | 100 +++-- src/config.cpp | 7 + test/AuxiliaryTest.cpp | 11 +- test/CoreTest.cpp | 16 +- test/ParallelIOTest.cpp | 8 +- test/SerialIOTest.cpp | 392 +++++++++++++++++- test/python/unittest/API/APITest.py | 27 +- 15 files changed, 780 insertions(+), 138 deletions(-) diff --git a/docs/source/backends/adios2.rst b/docs/source/backends/adios2.rst index 34450bd2c0..d639f01897 100644 --- a/docs/source/backends/adios2.rst +++ b/docs/source/backends/adios2.rst @@ -10,12 +10,43 @@ For further information, check out the :ref:`installation guide `, :ref:`build dependencies ` and the :ref:`build options `. -I/O Method ----------- +I/O Method and Engine Selection +------------------------------- ADIOS2 has several engines for alternative file formats and other kinds of backends, yet natively writes to ``.bp`` files. -The openPMD API uses the BP4 engine as the default file engine and the SST engine for streaming support. -We currently leverage the default ADIOS2 transport parameters, i.e. ``POSIX`` on Unix systems and ``FStream`` on Windows. +The openPMD API uses the File meta engine as the default file engine and the SST engine for streaming support. + +The ADIOS2 engine can be selected in different ways: + +1. Automatic detection via the selected file ending +2. Explicit selection of an engine by specifying the environment variable ``OPENPMD_ADIOS2_ENGINE`` (case-independent). + This overrides the automatically detected engine. +3. Explicit selection of an engine by specifying the JSON/TOML key ``adios2.engine.type`` as a case-independent string. + This overrides both previous options. + +Automatic engine detection supports the following extensions: + +.. list-table:: + :header-rows: 1 + + * - Extension + - Selected ADIOS2 Engine + * - ``.bp`` + - ``"file"`` + * - ``.bp4`` + - ``"bp4"`` + * - ``.bp5`` + - ``"bp5"`` + * - ``.sst`` + - ``"sst"`` + * - ``.ssc`` + - ``"ssc"`` + +Specifying any of these extensions will automatically activate the ADIOS2 backend. +The ADIOS2 backend will create file system paths exactly as they were specified and not change file extensions. +Exceptions to this are the BP3 and SST engines which require their endings ``.bp`` and ``.sst`` respectively. + +For file engines, we currently leverage the default ADIOS2 transport parameters, i.e. ``POSIX`` on Unix systems and ``FStream`` on Windows. Steps ----- diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index af79d72d3d..d4a199be53 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -132,12 +132,16 @@ class ADIOS2IOHandlerImpl AbstractIOHandler *, MPI_Comm, json::TracingJSON config, - std::string engineType); + std::string engineType, + std::string specifiedExtension); #endif // openPMD_HAVE_MPI explicit ADIOS2IOHandlerImpl( - AbstractIOHandler *, json::TracingJSON config, std::string engineType); + AbstractIOHandler *, + json::TracingJSON config, + std::string engineType, + std::string specifiedExtension); ~ADIOS2IOHandlerImpl() override; @@ -231,6 +235,11 @@ class ADIOS2IOHandlerImpl * The ADIOS2 engine type, to be passed to adios2::IO::SetEngine */ std::string m_engineType; + /* + * The filename extension specified by the user. + */ + std::string m_userSpecifiedExtension; + ADIOS2Schema::schema_t m_schema = ADIOS2Schema::schema_0000_00_00; enum class UseSpan : char @@ -1320,7 +1329,8 @@ class ADIOS2IOHandler : public AbstractIOHandler Access, MPI_Comm, json::TracingJSON options, - std::string engineType); + std::string engineType, + std::string specifiedExtension); #endif @@ -1328,7 +1338,8 @@ class ADIOS2IOHandler : public AbstractIOHandler std::string path, Access, json::TracingJSON options, - std::string engineType); + std::string engineType, + std::string specifiedExtension); std::string backendName() const override { diff --git a/include/openPMD/IO/AbstractIOHandlerHelper.hpp b/include/openPMD/IO/AbstractIOHandlerHelper.hpp index e6775a4757..0d1fe4c8da 100644 --- a/include/openPMD/IO/AbstractIOHandlerHelper.hpp +++ b/include/openPMD/IO/AbstractIOHandlerHelper.hpp @@ -35,6 +35,8 @@ namespace openPMD * @param access Access mode describing desired operations and permissions of the desired handler. * @param format Format describing the IO backend of the desired handler. + * @param originalExtension The filename extension as it was originally + * specified by the user. * @param comm MPI communicator used for IO. * @param options JSON-formatted option string, to be interpreted by * the backend. @@ -47,6 +49,7 @@ std::shared_ptr createIOHandler( std::string path, Access access, Format format, + std::string originalExtension, MPI_Comm comm, JSON options); #endif @@ -58,6 +61,8 @@ std::shared_ptr createIOHandler( * @param access Access describing desired operations and permissions * of the desired handler. * @param format Format describing the IO backend of the desired handler. + * @param originalExtension The filename extension as it was originally + * specified by the user. * @param options JSON-formatted option string, to be interpreted by * the backend. * @tparam JSON Substitute for nlohmann::json. Templated to avoid @@ -66,9 +71,16 @@ std::shared_ptr createIOHandler( */ template std::shared_ptr createIOHandler( - std::string path, Access access, Format format, JSON options = JSON()); + std::string path, + Access access, + Format format, + std::string originalExtension, + JSON options = JSON()); // version without configuration to use in AuxiliaryTest -std::shared_ptr -createIOHandler(std::string path, Access access, Format format); +std::shared_ptr createIOHandler( + std::string path, + Access access, + Format format, + std::string originalExtension); } // namespace openPMD diff --git a/include/openPMD/IO/Format.hpp b/include/openPMD/IO/Format.hpp index 993f7bae90..697e73f788 100644 --- a/include/openPMD/IO/Format.hpp +++ b/include/openPMD/IO/Format.hpp @@ -30,7 +30,9 @@ enum class Format { HDF5, ADIOS1, - ADIOS2, + ADIOS2_BP, + ADIOS2_BP4, + ADIOS2_BP5, ADIOS2_SST, ADIOS2_SSC, JSON, diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 39da51057b..e828bcd7f7 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -108,6 +108,11 @@ namespace internal * Filename after the expansion pattern without filename extension. */ std::string m_filenamePostfix; + /** + * Filename extension as specified by the user. + * (Not necessarily the backend's default suffix) + */ + std::string m_filenameExtension; /** * The padding in file-based iteration encoding. * 0 if no padding is given (%T pattern). diff --git a/src/Format.cpp b/src/Format.cpp index b92c0c8cf5..7f30e1a793 100644 --- a/src/Format.cpp +++ b/src/Format.cpp @@ -45,7 +45,7 @@ Format determineFormat(std::string const &filename) ); if (bp_backend == "ADIOS2") - return Format::ADIOS2; + return Format::ADIOS2_BP; else if (bp_backend == "ADIOS1") return Format::ADIOS1; else @@ -54,6 +54,10 @@ Format determineFormat(std::string const &filename) "neither ADIOS1 nor ADIOS2: " + bp_backend); } + if (auxiliary::ends_with(filename, ".bp4")) + return Format::ADIOS2_BP4; + if (auxiliary::ends_with(filename, ".bp5")) + return Format::ADIOS2_BP5; if (auxiliary::ends_with(filename, ".sst")) return Format::ADIOS2_SST; if (auxiliary::ends_with(filename, ".ssc")) @@ -72,8 +76,12 @@ std::string suffix(Format f) case Format::HDF5: return ".h5"; case Format::ADIOS1: - case Format::ADIOS2: + case Format::ADIOS2_BP: return ".bp"; + case Format::ADIOS2_BP4: + return ".bp4"; + case Format::ADIOS2_BP5: + return ".bp5"; case Format::ADIOS2_SST: return ".sst"; case Format::ADIOS2_SSC: diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index f0e7976f16..047373cdff 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -69,10 +69,12 @@ ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( AbstractIOHandler *handler, MPI_Comm communicator, json::TracingJSON cfg, - std::string engineType) + std::string engineType, + std::string specifiedExtension) : AbstractIOHandlerImplCommon(handler) , m_ADIOS{communicator} , m_engineType(std::move(engineType)) + , m_userSpecifiedExtension{std::move(specifiedExtension)} { init(std::move(cfg)); } @@ -80,10 +82,14 @@ ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( #endif // openPMD_HAVE_MPI ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( - AbstractIOHandler *handler, json::TracingJSON cfg, std::string engineType) + AbstractIOHandler *handler, + json::TracingJSON cfg, + std::string engineType, + std::string specifiedExtension) : AbstractIOHandlerImplCommon(handler) , m_ADIOS{} , m_engineType(std::move(engineType)) + , m_userSpecifiedExtension(std::move(specifiedExtension)) { init(std::move(cfg)); } @@ -118,6 +124,14 @@ ADIOS2IOHandlerImpl::~ADIOS2IOHandlerImpl() void ADIOS2IOHandlerImpl::init(json::TracingJSON cfg) { + // allow overriding through environment variable + m_engineType = + auxiliary::getEnvString("OPENPMD_ADIOS2_ENGINE", m_engineType); + std::transform( + m_engineType.begin(), + m_engineType.end(), + m_engineType.begin(), + [](unsigned char c) { return std::tolower(c); }); if (cfg.json().contains("adios2")) { m_config = cfg["adios2"]; @@ -146,6 +160,7 @@ void ADIOS2IOHandlerImpl::init(json::TracingJSON cfg) json::asLowerCaseStringDynamic(engineTypeConfig); if (maybeEngine.has_value()) { + // override engine type by JSON/TOML configuration m_engineType = std::move(maybeEngine.value()); } else @@ -228,28 +243,85 @@ ADIOS2IOHandlerImpl::getOperators() return getOperators(m_config); } +using AcceptedEndingsForEngine = std::map; + std::string ADIOS2IOHandlerImpl::fileSuffix() const { // SST engine adds its suffix unconditionally // so we don't add it - static std::map endings{ - {"sst", ""}, - {"staging", ""}, - {"bp4", ".bp"}, - {"bp5", ".bp"}, - {"bp3", ".bp"}, - {"file", ".bp"}, - {"hdf5", ".h5"}, - {"nullcore", ".nullcore"}, - {"ssc", ".ssc"}}; - auto it = endings.find(m_engineType); - if (it != endings.end()) - { - return it->second; + static std::map const endings{ + {"sst", {{"", ""}, {".sst", ""}}}, + {"staging", {{"", ""}, {".sst", ""}}}, + {"filestream", {{".bp", ".bp"}, {".bp4", ".bp4"}, {".bp5", ".bp5"}}}, + {"bp4", {{".bp4", ".bp4"}, {".bp", ".bp"}}}, + {"bp5", {{".bp5", ".bp5"}, {".bp", ".bp"}}}, + {"bp3", {{".bp", ".bp"}}}, + {"file", {{".bp", ".bp"}, {".bp4", ".bp4"}, {".bp5", ".bp5"}}}, + {"hdf5", {{".h5", ".h5"}}}, + {"nullcore", {{".nullcore", ".nullcore"}, {".bp", ".bp"}}}, + {"ssc", {{".ssc", ".ssc"}}}}; + + if (auto engine = endings.find(m_engineType); engine != endings.end()) + { + auto const &acceptedEndings = engine->second; + if (auto ending = acceptedEndings.find(m_userSpecifiedExtension); + ending != acceptedEndings.end()) + { + if ((m_engineType == "file" || m_engineType == "filestream") && + (m_userSpecifiedExtension == ".bp3" || + m_userSpecifiedExtension == ".bp4" || + m_userSpecifiedExtension == ".bp5")) + { + std::cerr + << "[ADIOS2] Explicit ending '" << m_userSpecifiedExtension + << "' was specified in combination with generic file " + "engine '" + << m_engineType + << "'. ADIOS2 will pick a default file ending " + "independent of specified suffix. (E.g. 'simData.bp5' " + "might actually be written as a BP4 dataset.)" + << std::endl; + } + return ending->second; + } + else if (m_userSpecifiedExtension.empty()) + { + std::cerr << "[ADIOS2] No file ending specified. Will not add one." + << std::endl; + if (m_engineType == "bp3") + { + std::cerr + << "Note that the ADIOS2 BP3 engine will add its " + "ending '.bp' if not specified (e.g. 'simData.bp3' " + "will appear on disk as 'simData.bp3.bp')." + << std::endl; + } + return ""; + } + else + { + std::cerr << "[ADIOS2] Specified ending '" + << m_userSpecifiedExtension + << "' does not match the selected engine '" + << m_engineType + << "'. Will use the specified ending anyway." + << std::endl; + if (m_engineType == "bp3") + { + std::cerr + << "Note that the ADIOS2 BP3 engine will add its " + "ending '.bp' if not specified (e.g. 'simData.bp3' " + "will appear on disk as 'simData.bp3.bp')." + << std::endl; + } + return m_userSpecifiedExtension; + } } else { - return ".adios2"; + throw error::WrongAPIUsage( + "[ADIOS2] Specified engine '" + m_engineType + + "' is not supported by ADIOS2 backend."); } } @@ -386,12 +458,7 @@ void ADIOS2IOHandlerImpl::createFile( if (!writable->written) { - std::string name = parameters.name; - std::string suffix(fileSuffix()); - if (!auxiliary::ends_with(name, suffix)) - { - name += suffix; - } + std::string name = parameters.name + fileSuffix(); auto res_pair = getPossiblyExisting(name); InvalidatableFile shared_name = InvalidatableFile(name); @@ -561,12 +628,7 @@ void ADIOS2IOHandlerImpl::openFile( m_handler->directory); } - std::string name = parameters.name; - std::string suffix(fileSuffix()); - if (!auxiliary::ends_with(name, suffix)) - { - name += suffix; - } + std::string name = parameters.name + fileSuffix(); auto file = std::get(getPossiblyExisting(name)); @@ -2299,15 +2361,6 @@ namespace detail // set engine type bool isStreaming = false; { - // allow overriding through environment variable - m_engineType = - auxiliary::getEnvString("OPENPMD_ADIOS2_ENGINE", m_engineType); - std::transform( - m_engineType.begin(), - m_engineType.end(), - m_engineType.begin(), - [](unsigned char c) { return std::tolower(c); }); - impl.m_engineType = this->m_engineType; m_IO.SetEngine(m_engineType); auto it = streamingEngines.find(m_engineType); if (it != streamingEngines.end()) @@ -2366,8 +2419,8 @@ namespace detail { throw std::runtime_error( "[ADIOS2IOHandler] Unknown engine type. Please choose " - "one out of " - "[sst, staging, bp4, bp3, hdf5, file, null]"); + "one out of [sst, staging, bp4, bp3, hdf5, file, " + "filestream, null]"); // not listing unsupported engines } } @@ -3058,9 +3111,15 @@ ADIOS2IOHandler::ADIOS2IOHandler( openPMD::Access at, MPI_Comm comm, json::TracingJSON options, - std::string engineType) + std::string engineType, + std::string specifiedExtension) : AbstractIOHandler(std::move(path), at, comm) - , m_impl{this, comm, std::move(options), std::move(engineType)} + , m_impl{ + this, + comm, + std::move(options), + std::move(engineType), + std::move(specifiedExtension)} {} #endif @@ -3069,9 +3128,14 @@ ADIOS2IOHandler::ADIOS2IOHandler( std::string path, Access at, json::TracingJSON options, - std::string engineType) + std::string engineType, + std::string specifiedExtension) : AbstractIOHandler(std::move(path), at) - , m_impl{this, std::move(options), std::move(engineType)} + , m_impl{ + this, + std::move(options), + std::move(engineType), + std::move(specifiedExtension)} {} std::future @@ -3084,14 +3148,19 @@ ADIOS2IOHandler::flush(internal::ParsedFlushParams &flushParams) #if openPMD_HAVE_MPI ADIOS2IOHandler::ADIOS2IOHandler( - std::string path, Access at, MPI_Comm comm, json::TracingJSON, std::string) + std::string path, + Access at, + MPI_Comm comm, + json::TracingJSON, + std::string, + std::string) : AbstractIOHandler(std::move(path), at, comm) {} #endif // openPMD_HAVE_MPI ADIOS2IOHandler::ADIOS2IOHandler( - std::string path, Access at, json::TracingJSON, std::string) + std::string path, Access at, json::TracingJSON, std::string, std::string) : AbstractIOHandler(std::move(path), at) {} diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index 22fa6af60f..0a0a32f53f 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -63,6 +63,8 @@ std::shared_ptr createIOHandler( std::string path, Access access, Format format, + std::string originalExtension, + MPI_Comm comm, json::TracingJSON options) { @@ -75,15 +77,51 @@ std::shared_ptr createIOHandler( case Format::ADIOS1: return constructIOHandler( "ADIOS1", path, access, std::move(options), comm); - case Format::ADIOS2: + case Format::ADIOS2_BP: + return constructIOHandler( + "ADIOS2", + path, + access, + comm, + std::move(options), + "file", + std::move(originalExtension)); + case Format::ADIOS2_BP4: + return constructIOHandler( + "ADIOS2", + path, + access, + comm, + std::move(options), + "bp4", + std::move(originalExtension)); + case Format::ADIOS2_BP5: return constructIOHandler( - "ADIOS2", path, access, comm, std::move(options), "bp4"); + "ADIOS2", + path, + access, + comm, + std::move(options), + "bp5", + std::move(originalExtension)); case Format::ADIOS2_SST: return constructIOHandler( - "ADIOS2", path, access, comm, std::move(options), "sst"); + "ADIOS2", + path, + access, + comm, + std::move(options), + "sst", + std::move(originalExtension)); case Format::ADIOS2_SSC: return constructIOHandler( - "ADIOS2", path, access, comm, std::move(options), "ssc"); + "ADIOS2", + path, + access, + comm, + std::move(options), + "ssc", + std::move(originalExtension)); default: throw std::runtime_error( "Unknown file format! Did you specify a file ending?"); @@ -93,7 +131,11 @@ std::shared_ptr createIOHandler( template <> std::shared_ptr createIOHandler( - std::string path, Access access, Format format, json::TracingJSON options) + std::string path, + Access access, + Format format, + std::string originalExtension, + json::TracingJSON options) { (void)options; switch (format) @@ -104,15 +146,46 @@ std::shared_ptr createIOHandler( case Format::ADIOS1: return constructIOHandler( "ADIOS1", path, access, std::move(options)); - case Format::ADIOS2: + case Format::ADIOS2_BP: + return constructIOHandler( + "ADIOS2", + path, + access, + std::move(options), + "file", + std::move(originalExtension)); + case Format::ADIOS2_BP4: + return constructIOHandler( + "ADIOS2", + path, + access, + std::move(options), + "bp4", + std::move(originalExtension)); + case Format::ADIOS2_BP5: return constructIOHandler( - "ADIOS2", path, access, std::move(options), "bp4"); + "ADIOS2", + path, + access, + std::move(options), + "bp5", + std::move(originalExtension)); case Format::ADIOS2_SST: return constructIOHandler( - "ADIOS2", path, access, std::move(options), "sst"); + "ADIOS2", + path, + access, + std::move(options), + "sst", + std::move(originalExtension)); case Format::ADIOS2_SSC: return constructIOHandler( - "ADIOS2", path, access, std::move(options), "ssc"); + "ADIOS2", + path, + access, + std::move(options), + "ssc", + std::move(originalExtension)); case Format::JSON: return constructIOHandler( "JSON", path, access); @@ -122,13 +195,17 @@ std::shared_ptr createIOHandler( } } -std::shared_ptr -createIOHandler(std::string path, Access access, Format format) +std::shared_ptr createIOHandler( + std::string path, + Access access, + Format format, + std::string originalExtension) { return createIOHandler( std::move(path), access, format, + std::move(originalExtension), json::TracingJSON(json::ParsedConfig{})); } } // namespace openPMD diff --git a/src/Series.cpp b/src/Series.cpp index a68d1270fa..814b0e2489 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -44,6 +44,18 @@ namespace openPMD { namespace { + struct CleanedFilename + { + std::string body; + std::string extension; + + std::tuple decompose() && + { + return std::tuple{ + std::move(body), std::move(extension)}; + } + }; + /** Remove the filename extension of a given storage format. * * @param filename String containing the filename, possibly with @@ -51,7 +63,8 @@ namespace * @param f File format to remove filename extension for. * @return String containing the filename without filename extension. */ - std::string cleanFilename(std::string const &filename, Format f); + CleanedFilename cleanFilename( + std::string const &filename, std::string const &filenameExtension); /** Compound return type for regex matching of filenames */ struct Match @@ -77,7 +90,7 @@ namespace * zero, any amount of padding is matched. * @param postfix String containing tail (i.e. after %T) of desired * filename without filename extension. - * @param f File format to check backend applicability for. + * @param filenameExtension Filename extension to match against * @return Functor returning tuple of bool and int. * bool is True if file could be of type f and matches the * iterationEncoding. False otherwise. int is the amount of padding present @@ -87,7 +100,7 @@ namespace std::string const &prefix, int padding, std::string const &postfix, - Format f); + std::string const &extension); } // namespace struct Series::ParsedInput @@ -98,6 +111,7 @@ struct Series::ParsedInput IterationEncoding iterationEncoding; std::string filenamePrefix; std::string filenamePostfix; + std::string filenameExtension; int filenamePadding = -1; }; // ParsedInput @@ -443,9 +457,10 @@ std::unique_ptr Series::parseInput(std::string filepath) "Can not determine iterationFormat from filename " + input->name); input->filenamePostfix = - cleanFilename(input->filenamePostfix, input->format); + cleanFilename(input->filenamePostfix, suffix(input->format)).body; - input->name = cleanFilename(input->name, input->format); + std::tie(input->name, input->filenameExtension) = + cleanFilename(input->name, suffix(input->format)).decompose(); return input; } @@ -536,6 +551,7 @@ void Series::init( series.m_filenamePrefix = input->filenamePrefix; series.m_filenamePostfix = input->filenamePostfix; series.m_filenamePadding = input->filenamePadding; + series.m_filenameExtension = input->filenameExtension; if (series.m_iterationEncoding == IterationEncoding::fileBased && !series.m_filenamePrefix.empty() && @@ -599,7 +615,7 @@ Given file pattern: ')END" series.m_filenamePrefix, series.m_filenamePadding, series.m_filenamePostfix, - series.m_format), + series.m_filenameExtension), IOHandler()->directory); switch (padding) { @@ -922,7 +938,7 @@ void Series::readFileBased() series.m_filenamePrefix, series.m_filenamePadding, series.m_filenamePostfix, - series.m_format); + series.m_filenameExtension); int padding = autoDetectPadding( std::move(isPartOfSeries), @@ -930,7 +946,11 @@ void Series::readFileBased() // foreach found file with `filename` and `index`: [&series](uint64_t index, std::string const &filename) { Iteration &i = series.iterations[index]; - i.deferParseAccess({std::to_string(index), index, true, filename}); + i.deferParseAccess( + {std::to_string(index), + index, + true, + cleanFilename(filename, series.m_filenameExtension).body}); }); if (series.iterations.empty()) @@ -1652,7 +1672,7 @@ void Series::parseJsonOptions(TracingJSON &options, ParsedInput &input) std::map const backendDescriptors{ {"hdf5", Format::HDF5}, {"adios1", Format::ADIOS1}, - {"adios2", Format::ADIOS2}, + {"adios2", Format::ADIOS2_BP}, {"json", Format::JSON}}; std::string backend; getJsonOptionLowerCase(options, "backend", backend); @@ -1661,7 +1681,20 @@ void Series::parseJsonOptions(TracingJSON &options, ParsedInput &input) auto it = backendDescriptors.find(backend); if (it != backendDescriptors.end()) { - if (input.format != Format::DUMMY && + if (backend == "adios2" && + (input.format == Format::ADIOS2_BP4 || + input.format == Format::ADIOS2_BP5 || + input.format == Format::ADIOS2_SST || + input.format == Format::ADIOS2_SSC || + input.format == Format::ADIOS2_BP)) + { + // backend = "adios2" should work as a catch-all for all + // different engines, using BP is just the default + // If the file ending was more explicit, keep it. + // -> Nothing to do + } + else if ( + input.format != Format::DUMMY && suffix(input.format) != suffix(it->second)) { std::cerr << "[Warning] Supplied filename extension '" @@ -1669,8 +1702,12 @@ void Series::parseJsonOptions(TracingJSON &options, ParsedInput &input) << "' contradicts the backend specified via the " "'backend' key. Will go on with backend " << it->first << "." << std::endl; + input.format = it->second; + } + else + { + input.format = it->second; } - input.format = it->second; } else { @@ -1768,8 +1805,13 @@ Series::Series( json::parseOptions(options, comm, /* considerFiles = */ true); auto input = parseInput(filepath); parseJsonOptions(optionsJson, *input); - auto handler = - createIOHandler(input->path, at, input->format, comm, optionsJson); + auto handler = createIOHandler( + input->path, + at, + input->format, + input->filenameExtension, + comm, + optionsJson); init(handler, std::move(input)); json::warnGlobalUnusedOptions(optionsJson); } @@ -1785,7 +1827,8 @@ Series::Series( json::parseOptions(options, /* considerFiles = */ true); auto input = parseInput(filepath); parseJsonOptions(optionsJson, *input); - auto handler = createIOHandler(input->path, at, input->format, optionsJson); + auto handler = createIOHandler( + input->path, at, input->format, input->filenameExtension, optionsJson); init(handler, std::move(input)); json::warnGlobalUnusedOptions(optionsJson); } @@ -1814,19 +1857,18 @@ WriteIterations Series::writeIterations() namespace { - std::string cleanFilename(std::string const &filename, Format f) + CleanedFilename cleanFilename( + std::string const &filename, std::string const &filenameExtension) { - switch (f) + std::string postfix = + auxiliary::replace_last(filename, filenameExtension, ""); + if (postfix == filename) { - case Format::HDF5: - case Format::ADIOS1: - case Format::ADIOS2: - case Format::ADIOS2_SST: - case Format::ADIOS2_SSC: - case Format::JSON: - return auxiliary::replace_last(filename, suffix(f), ""); - default: - return filename; + return {postfix, ""}; + } + else + { + return {postfix, filenameExtension}; } } @@ -1851,14 +1893,8 @@ namespace std::string const &prefix, int padding, std::string const &postfix, - Format f) + std::string const &filenameSuffix) { - std::string filenameSuffix = suffix(f); - if (filenameSuffix.empty()) - { - return [](std::string const &) -> Match { return {false, 0, 0}; }; - } - std::string nameReg = "^" + prefix; if (padding != 0) { diff --git a/src/config.cpp b/src/config.cpp index d37d9cc88e..b9e27be752 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -45,6 +45,13 @@ std::vector openPMD::getFileExtensions() #if openPMD_HAVE_ADIOS1 || openPMD_HAVE_ADIOS2 fext.emplace_back("bp"); #endif +#if openPMD_HAVE_ADIOS2 + // BP4 is always available in ADIOS2 + fext.emplace_back("bp4"); +#endif +#ifdef ADIOS2_HAVE_BP5 + fext.emplace_back("bp5"); +#endif #ifdef ADIOS2_HAVE_SST fext.emplace_back("sst"); #endif diff --git a/test/AuxiliaryTest.cpp b/test/AuxiliaryTest.cpp index 4a67507cbd..3af7b3741f 100644 --- a/test/AuxiliaryTest.cpp +++ b/test/AuxiliaryTest.cpp @@ -34,7 +34,7 @@ struct TestHelper : public Attributable TestHelper() { writable().IOHandler = - createIOHandler(".", Access::CREATE, Format::JSON); + createIOHandler(".", Access::CREATE, Format::JSON, ".json"); } }; } // namespace openPMD::test @@ -146,7 +146,8 @@ TEST_CASE("container_default_test", "[auxiliary]") { #if openPMD_USE_INVASIVE_TESTS Container c = Container(); - c.writable().IOHandler = createIOHandler(".", Access::CREATE, Format::JSON); + c.writable().IOHandler = + createIOHandler(".", Access::CREATE, Format::JSON, ".json"); REQUIRE(c.empty()); REQUIRE(c.erase("nonExistentKey") == false); @@ -183,7 +184,8 @@ TEST_CASE("container_retrieve_test", "[auxiliary]") #if openPMD_USE_INVASIVE_TESTS using structure = openPMD::test::structure; Container c = Container(); - c.writable().IOHandler = createIOHandler(".", Access::CREATE, Format::JSON); + c.writable().IOHandler = + createIOHandler(".", Access::CREATE, Format::JSON, ".json"); structure s; std::string text = @@ -255,7 +257,8 @@ TEST_CASE("container_access_test", "[auxiliary]") #if openPMD_USE_INVASIVE_TESTS using Widget = openPMD::test::Widget; Container c = Container(); - c.writable().IOHandler = createIOHandler(".", Access::CREATE, Format::JSON); + c.writable().IOHandler = + createIOHandler(".", Access::CREATE, Format::JSON, ".json"); c["1firstWidget"] = Widget(0); REQUIRE(c.size() == 1); diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index f737141bf4..fa2a1cad42 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -1074,12 +1074,24 @@ TEST_CASE("backend_via_json", "[core]") * BP4 engine should be selected even if ending .sst is given */ Series series( - "../samples/optionsViaJsonOverwritesAutomaticDetection.sst", + "../samples/optionsViaJsonOverwritesAutomaticDetectionFile.sst", + Access::CREATE, + R"({"adios2": {"engine": {"type": "file"}}})"); + } + REQUIRE(auxiliary::directory_exists( + "../samples/optionsViaJsonOverwritesAutomaticDetectionFile.sst")); + + { + /* + * BP4 engine should be selected even if ending .sst is given + */ + Series series( + "../samples/optionsViaJsonOverwritesAutomaticDetectionBp4.sst", Access::CREATE, R"({"adios2": {"engine": {"type": "bp4"}}})"); } REQUIRE(auxiliary::directory_exists( - "../samples/optionsViaJsonOverwritesAutomaticDetection.bp")); + "../samples/optionsViaJsonOverwritesAutomaticDetectionBp4.sst")); #if openPMD_HAVE_ADIOS1 setenv("OPENPMD_BP_BACKEND", "ADIOS1", 1); diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index c2a7e8d99c..1ca922c022 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1066,7 +1066,10 @@ void adios2_streaming(bool variableBasedLayout) if (rank == 0) { // write - Series writeSeries("../samples/adios2_stream.sst", Access::CREATE); + Series writeSeries( + "../samples/adios2_stream.sst", + Access::CREATE, + "adios2.engine.type = \"sst\""); if (variableBasedLayout) { writeSeries.setIterationEncoding(IterationEncoding::variableBased); @@ -1109,7 +1112,8 @@ void adios2_streaming(bool variableBasedLayout) Series readSeries( "../samples/adios2_stream.sst", Access::READ_ONLY, - "defer_iteration_parsing = true"); // inline TOML + // inline TOML + R"(defer_iteration_parsing = true)"); size_t last_iteration_index = 0; for (auto iteration : readSeries.readIterations()) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index c3543fcb7b..eac05b0214 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -55,7 +55,10 @@ std::vector testedBackends() { auto variants = getVariants(); std::map extensions{ - {"json", "json"}, {"adios1", "bp1"}, {"adios2", "bp"}, {"hdf5", "h5"}}; + {"json", "json"}, + {"adios1", "adios1.bp"}, + {"adios2", "bp"}, + {"hdf5", "h5"}}; std::vector res; for (auto const &pair : variants) { @@ -77,7 +80,9 @@ std::vector testedFileExtensions() auto allExtensions = getFileExtensions(); auto newEnd = std::remove_if( allExtensions.begin(), allExtensions.end(), [](std::string const &ext) { - return ext == "sst" || ext == "ssc"; + // sst and ssc need a receiver for testing + // bp4 is already tested via bp + return ext == "sst" || ext == "ssc" || ext == "bp4"; }); return {allExtensions.begin(), newEnd}; } @@ -279,6 +284,10 @@ TEST_CASE("write_and_read_many_iterations", "[serial]") auxiliary::remove_directory("../samples/many_iterations"); for (auto const &t : testedFileExtensions()) { + if (t == "bp4" || t == "bp5") + { + continue; + } write_and_read_many_iterations(t, intermittentFlushes); intermittentFlushes = !intermittentFlushes; } @@ -1484,7 +1493,9 @@ inline void dtype_test(const std::string &backend) TEST_CASE("dtype_test", "[serial]") { for (auto const &t : testedFileExtensions()) + { dtype_test(t); + } } inline void write_test(const std::string &backend) @@ -1604,7 +1615,8 @@ void test_complex(const std::string &backend) "../samples/serial_write_complex." + backend, Access::CREATE); o.setAttribute("lifeIsComplex", std::complex(4.56, 7.89)); o.setAttribute("butComplexFloats", std::complex(42.3, -99.3)); - if (backend != "bp") + if (o.backend() != "ADIOS2" && o.backend() != "ADIOS1" && + o.backend() != "MPI_ADIOS1") o.setAttribute( "longDoublesYouSay", std::complex(5.5, -4.55)); @@ -1625,7 +1637,8 @@ void test_complex(const std::string &backend) Cdbl.storeChunk(cdoubles, {0}); std::vector > cldoubles(3); - if (backend != "bp") + if (o.backend() != "ADIOS2" && o.backend() != "ADIOS1" && + o.backend() != "MPI_ADIOS1") { auto Cldbl = o.iterations[0].meshes["Cldbl"][RecordComponent::SCALAR]; @@ -1649,7 +1662,8 @@ void test_complex(const std::string &backend) REQUIRE( i.getAttribute("butComplexFloats").get >() == std::complex(42.3, -99.3)); - if (backend != "bp") + if (i.backend() != "ADIOS2" && i.backend() != "ADIOS1" && + i.backend() != "MPI_ADIOS1") { REQUIRE( i.getAttribute("longDoublesYouSay") @@ -1668,7 +1682,8 @@ void test_complex(const std::string &backend) REQUIRE(rcflt.get()[1] == std::complex(-3., 4.)); REQUIRE(rcdbl.get()[2] == std::complex(6, -5.)); - if (backend != "bp") + if (i.backend() != "ADIOS2" && i.backend() != "ADIOS1" && + i.backend() != "MPI_ADIOS1") { auto rcldbl = i.iterations[0] .meshes["Cldbl"][RecordComponent::SCALAR] @@ -2176,8 +2191,8 @@ inline void bool_test(const std::string &backend) Series o = Series("../samples/serial_bool." + backend, Access::CREATE); REQUIRE_THROWS_AS(o.setAuthor(""), std::runtime_error); - o.setAttribute("Bool attribute (true)", true); - o.setAttribute("Bool attribute (false)", false); + o.setAttribute("Bool attribute true", true); + o.setAttribute("Bool attribute false", false); } { Series o = @@ -2185,13 +2200,12 @@ inline void bool_test(const std::string &backend) auto attrs = o.attributes(); REQUIRE( - std::count(attrs.begin(), attrs.end(), "Bool attribute (true)") == - 1); + std::count(attrs.begin(), attrs.end(), "Bool attribute true") == 1); REQUIRE( - std::count(attrs.begin(), attrs.end(), "Bool attribute (false)") == + std::count(attrs.begin(), attrs.end(), "Bool attribute false") == 1); - REQUIRE(o.getAttribute("Bool attribute (true)").get() == true); - REQUIRE(o.getAttribute("Bool attribute (false)").get() == false); + REQUIRE(o.getAttribute("Bool attribute true").get() == true); + REQUIRE(o.getAttribute("Bool attribute false").get() == false); } { Series list{"../samples/serial_bool." + backend, Access::READ_ONLY}; @@ -4221,6 +4235,309 @@ BufferChunkSize = 2147483646 # 2^31 - 2 } #endif +TEST_CASE("adios2_engines_and_file_endings") +{ + size_t filenameCounter = 0; + auto groupbased_test_explicit_backend = + [&filenameCounter]( + std::string const &ext, + bool directory, + std::string const &requiredEngine, + std::string const &filesystemExt, + std::string const &jsonCfg = "{}") mutable { + // Env. var. OPENPMD_BP_BACKEND does not matter for this test as + // we always override it in the JSON config + auto basename = "../samples/file_endings/groupbased" + + std::to_string(filenameCounter++); + auto name = basename + ext; + std::cout << "Writing to file '" << name << "', should be engine " + << requiredEngine << " (explicit backend)." << std::endl; + auto filesystemname = + filesystemExt.empty() ? name : basename + filesystemExt; + { + Series write( + name, + Access::CREATE, + json::merge("backend = \"adios2\"", jsonCfg)); + } + if (directory) + { + REQUIRE(auxiliary::directory_exists(filesystemname)); + } + else + { + REQUIRE(auxiliary::file_exists(filesystemname)); + } + { + Series read( + name, + Access::READ_ONLY, + "backend = \"adios2\"\nadios2.engine.type = \"" + + requiredEngine + "\""); + } + }; + + groupbased_test_explicit_backend(".bp", true, "file", ""); + groupbased_test_explicit_backend(".bp4", true, "bp4", ""); + groupbased_test_explicit_backend( + ".bp", true, "bp4", "", "adios2.engine.type = \"bp4\""); + groupbased_test_explicit_backend( + ".bp4", true, "bp4", "", "adios2.engine.type = \"bp4\""); + groupbased_test_explicit_backend( + ".bp5", true, "bp4", "", "adios2.engine.type = \"bp4\""); + groupbased_test_explicit_backend( + ".bp", false, "bp3", "", "adios2.engine.type = \"bp3\""); + groupbased_test_explicit_backend( + ".sst", false, "bp3", ".sst.bp", "adios2.engine.type = \"bp3\""); + groupbased_test_explicit_backend( + "", false, "bp3", ".bp", "adios2.engine.type = \"bp3\""); + groupbased_test_explicit_backend( + "", true, "bp4", "", "adios2.engine.type = \"bp4\""); + +#ifdef ADIOS2_HAVE_BP5 + // BP5 tests + groupbased_test_explicit_backend(".bp5", true, "bp5", ""); + groupbased_test_explicit_backend( + ".bp", true, "bp5", "", "adios2.engine.type = \"bp5\""); + groupbased_test_explicit_backend( + ".bp4", true, "bp5", "", "adios2.engine.type = \"bp5\""); + groupbased_test_explicit_backend( + ".bp5", true, "bp5", "", "adios2.engine.type = \"bp5\""); + groupbased_test_explicit_backend( + "", true, "bp5", "", "adios2.engine.type = \"bp5\""); +#endif + + auto groupbased_test_no_explicit_backend = + [&filenameCounter]( + std::string const &ext, + bool directory, + std::string const &requiredEngine, + std::string const &filesystemExt, + std::string const &jsonCfg = "{}") mutable { + auto basename = "../samples/file_endings/groupbased" + + std::to_string(filenameCounter++); + auto name = basename + ext; + std::cout << "Writing to file '" << name << "', should be engine " + << requiredEngine << " (no explicit backend)." + << std::endl; + auto filesystemname = + filesystemExt.empty() ? name : basename + filesystemExt; + { + Series write(name, Access::CREATE, jsonCfg); + } + bool isThisADIOS1 = + auxiliary::getEnvString("OPENPMD_BP_BACKEND", "") == "ADIOS1" && + ext == ".bp"; + if (directory && !isThisADIOS1) + { + REQUIRE(auxiliary::directory_exists(filesystemname)); + } + else + { + REQUIRE(auxiliary::file_exists(filesystemname)); + } + { + Series read( + name, + Access::READ_ONLY, + isThisADIOS1 + ? "backend = \"adios1\"" + : "backend = \"adios2\"\nadios2.engine.type = \"" + + requiredEngine + "\""); + } + }; + + groupbased_test_no_explicit_backend(".bp", true, "file", ""); + groupbased_test_no_explicit_backend(".bp4", true, "bp4", ""); + groupbased_test_no_explicit_backend( + ".bp", true, "bp4", "", "adios2.engine.type = \"bp4\""); + groupbased_test_no_explicit_backend( + ".bp4", true, "bp4", "", "adios2.engine.type = \"bp4\""); + groupbased_test_no_explicit_backend( + ".bp5", true, "bp4", "", "adios2.engine.type = \"bp4\""); + groupbased_test_no_explicit_backend( + ".bp", false, "bp3", "", "adios2.engine.type = \"bp3\""); + groupbased_test_no_explicit_backend( + ".sst", false, "bp3", ".sst.bp", "adios2.engine.type = \"bp3\""); + REQUIRE_THROWS(groupbased_test_no_explicit_backend( + "", false, "bp3", ".bp", "adios2.engine.type = \"bp3\"")); + REQUIRE_THROWS(groupbased_test_no_explicit_backend( + "", true, "bp4", "", "adios2.engine.type = \"bp4\"")); + +#ifdef ADIOS2_HAVE_BP5 + // BP5 tests + groupbased_test_no_explicit_backend(".bp5", true, "bp5", ""); + groupbased_test_no_explicit_backend( + ".bp", true, "bp5", "", "adios2.engine.type = \"bp5\""); + groupbased_test_no_explicit_backend( + ".bp4", true, "bp5", "", "adios2.engine.type = \"bp5\""); + groupbased_test_no_explicit_backend( + ".bp5", true, "bp5", "", "adios2.engine.type = \"bp5\""); + REQUIRE_THROWS(groupbased_test_no_explicit_backend( + "", true, "bp5", "", "adios2.engine.type = \"bp5\"")); +#endif + + filenameCounter = 0; + auto filebased_test_explicit_backend = + [&filenameCounter]( + std::string const &ext, + bool directory, + std::string const &requiredEngine, + std::string const &filesystemExt, + std::string const &jsonCfg = "{}") mutable { + auto basename = "../samples/file_endings/filebased" + + std::to_string(filenameCounter++); + auto name = basename + "_%T" + ext; + std::cout << "Writing to file '" << name << "', should be engine " + << requiredEngine << " (explicit backend)." << std::endl; + auto filesystemname = + basename + "_0" + (filesystemExt.empty() ? ext : filesystemExt); + { + Series write( + name, + Access::CREATE, + json::merge("backend = \"adios2\"", jsonCfg)); + write.writeIterations()[0]; + } + if (directory) + { + REQUIRE(auxiliary::directory_exists(filesystemname)); + } + else + { + REQUIRE(auxiliary::file_exists(filesystemname)); + } + { + if (requiredEngine == "bp3" && + !auxiliary::ends_with(name, ".bp")) + { + /* + * File-based parsing procedures are not aware that BP3 + * engine adds its ending and won't find the iterations if + * we don't give it a little nudge. + */ + name += ".bp"; + } + Series read( + name, + Access::READ_ONLY, + "backend = \"adios2\"\nadios2.engine.type = \"" + + requiredEngine + "\""); + } + }; + + filebased_test_explicit_backend(".bp", true, "file", ""); + filebased_test_explicit_backend(".bp4", true, "bp4", ""); + filebased_test_explicit_backend( + ".bp", true, "bp4", "", "adios2.engine.type = \"bp4\""); + filebased_test_explicit_backend( + ".bp4", true, "bp4", "", "adios2.engine.type = \"bp4\""); + filebased_test_explicit_backend( + ".bp5", true, "bp4", "", "adios2.engine.type = \"bp4\""); + filebased_test_explicit_backend( + ".bp", false, "bp3", "", "adios2.engine.type = \"bp3\""); + filebased_test_explicit_backend( + ".sst", false, "bp3", ".sst.bp", "adios2.engine.type = \"bp3\""); + filebased_test_explicit_backend( + "", false, "bp3", ".bp", "adios2.engine.type = \"bp3\""); + filebased_test_explicit_backend( + "", true, "bp4", "", "adios2.engine.type = \"bp4\""); + +#ifdef ADIOS2_HAVE_BP5 + // BP5 tests + filebased_test_explicit_backend(".bp5", true, "bp5", ""); + filebased_test_explicit_backend( + ".bp", true, "bp5", "", "adios2.engine.type = \"bp5\""); + filebased_test_explicit_backend( + ".bp4", true, "bp5", "", "adios2.engine.type = \"bp5\""); + filebased_test_explicit_backend( + ".bp5", true, "bp5", "", "adios2.engine.type = \"bp5\""); + filebased_test_explicit_backend( + "", true, "bp5", "", "adios2.engine.type = \"bp5\""); +#endif + + auto filebased_test_no_explicit_backend = + [&filenameCounter]( + std::string const &ext, + bool directory, + std::string const &requiredEngine, + std::string const &filesystemExt, + std::string const &jsonCfg = "{}") mutable { + auto basename = "../samples/file_endings/filebased" + + std::to_string(filenameCounter++); + auto name = basename + "_%T" + ext; + std::cout << "Writing to file '" << name << "', should be engine " + << requiredEngine << " (no explicit backend)." + << std::endl; + auto filesystemname = + basename + "_0" + (filesystemExt.empty() ? ext : filesystemExt); + { + Series write(name, Access::CREATE, jsonCfg); + write.writeIterations()[0]; + } + bool isThisADIOS1 = + auxiliary::getEnvString("OPENPMD_BP_BACKEND", "") == "ADIOS1" && + ext == ".bp"; + if (directory && !isThisADIOS1) + { + REQUIRE(auxiliary::directory_exists(filesystemname)); + } + else + { + REQUIRE(auxiliary::file_exists(filesystemname)); + } + { + if (requiredEngine == "bp3" && + !auxiliary::ends_with(name, ".bp")) + { + /* + * File-based parsing procedures are not aware that BP3 + * engine adds its ending and won't find the iterations if + * we don't give it a little nudge. + */ + name += ".bp"; + } + Series read( + name, + Access::READ_ONLY, + isThisADIOS1 + ? "backend = \"adios1\"" + : "backend = \"adios2\"\nadios2.engine.type = \"" + + requiredEngine + "\""); + } + }; + + filebased_test_no_explicit_backend(".bp", true, "file", ""); + filebased_test_no_explicit_backend(".bp4", true, "bp4", ""); + filebased_test_no_explicit_backend( + ".bp", true, "bp4", "", "adios2.engine.type = \"bp4\""); + filebased_test_no_explicit_backend( + ".bp4", true, "bp4", "", "adios2.engine.type = \"bp4\""); + filebased_test_no_explicit_backend( + ".bp5", true, "bp4", "", "adios2.engine.type = \"bp4\""); + filebased_test_no_explicit_backend( + ".bp", false, "bp3", "", "adios2.engine.type = \"bp3\""); + filebased_test_no_explicit_backend( + ".sst", false, "bp3", ".sst.bp", "adios2.engine.type = \"bp3\""); + REQUIRE_THROWS(filebased_test_no_explicit_backend( + "", false, "bp3", ".bp", "adios2.engine.type = \"bp3\"")); + REQUIRE_THROWS(filebased_test_no_explicit_backend( + "", true, "bp4", "", "adios2.engine.type = \"bp4\"")); + +#ifdef ADIOS2_HAVE_BP5 + // BP5 tests + filebased_test_no_explicit_backend(".bp5", true, "bp5", ""); + filebased_test_no_explicit_backend( + ".bp", true, "bp5", "", "adios2.engine.type = \"bp5\""); + filebased_test_no_explicit_backend( + ".bp4", true, "bp5", "", "adios2.engine.type = \"bp5\""); + filebased_test_no_explicit_backend( + ".bp5", true, "bp5", "", "adios2.engine.type = \"bp5\""); + REQUIRE_THROWS(filebased_test_no_explicit_backend( + "", true, "bp5", "", "adios2.engine.type = \"bp5\"")); +#endif +} + TEST_CASE("serial_adios2_backend_config", "[serial][adios2]") { if (auxiliary::getEnvString("OPENPMD_BP_BACKEND", "NOT_SET") == "ADIOS1") @@ -5867,7 +6184,6 @@ void no_explicit_flush(std::string filename) "adios2": { "schema": 20210209, "engine": { - "type": "bp4", "usesteps": true } } @@ -6103,6 +6419,54 @@ void append_mode( */ helper::listSeries(read); } +#if 100000000 * ADIOS2_VERSION_MAJOR + 1000000 * ADIOS2_VERSION_MINOR + \ + 10000 * ADIOS2_VERSION_PATCH + 100 * ADIOS2_VERSION_TWEAK >= \ + 208002700 + // AppendAfterSteps has a bug before that version + if (extension == "bp5") + { + { + Series write( + filename, + Access::APPEND, + json::merge( + jsonConfig, + R"({"adios2":{"engine":{"parameters":{"AppendAfterSteps":-3}}}})")); + if (variableBased) + { + write.setIterationEncoding(IterationEncoding::variableBased); + } + if (write.backend() == "ADIOS1") + { + REQUIRE_THROWS_WITH( + write.flush(), + Catch::Equals( + "Operation unsupported in ADIOS1: Appending to " + "existing " + "file on disk (use Access::CREATE to overwrite)")); + // destructor will be noisy now + return; + } + + writeSomeIterations( + write.writeIterations(), std::vector{4, 5}); + write.flush(); + } + { + Series read(filename, Access::READ_ONLY); + // in variable-based encodings, iterations are not parsed ahead of + // time but as they go + unsigned counter = 0; + for (auto const &iteration : read.readIterations()) + { + REQUIRE(iteration.iterationIndex == counter); + ++counter; + } + REQUIRE(counter == 6); + helper::listSeries(read); + } + } +#endif } TEST_CASE("append_mode", "[serial]") diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index 0876e2c14a..2503e8c619 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -167,7 +167,7 @@ def attributeRoundTrip(self, file_ending): series.set_attribute("longdouble", np.longdouble(1.23456789)) series.set_attribute("csingle", np.complex64(1.+2.j)) series.set_attribute("cdouble", np.complex128(3.+4.j)) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: series.set_attribute("clongdouble", np.clongdouble(5.+6.j)) # array of ... series.set_attribute("arr_int16", (np.int16(23), np.int16(26), )) @@ -202,7 +202,7 @@ def attributeRoundTrip(self, file_ending): # series.set_attribute("l_cdouble", # [np.complex_(6.7+6.8j), # np.complex_(7.1+7.2j)]) - # if file_ending != "bp": + # if file_ending not in ["bp", "bp4", "bp5"]: # series.set_attribute("l_clongdouble", # [np.clongfloat(7.8e9-6.5e9j), # np.clongfloat(8.2e3-9.1e3j)]) @@ -231,7 +231,7 @@ def attributeRoundTrip(self, file_ending): series.set_attribute("nparr_cdouble", np.array([4.5 + 1.1j, 6.7 - 2.2j], dtype=np.complex128)) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: series.set_attribute("nparr_clongdouble", np.array([8.9 + 7.8j, 7.6 + 9.2j], dtype=np.clongdouble)) @@ -286,7 +286,7 @@ def attributeRoundTrip(self, file_ending): np.complex64(1.+2.j)) self.assertAlmostEqual(series.get_attribute("cdouble"), 3.+4.j) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: self.assertAlmostEqual(series.get_attribute("clongdouble"), 5.+6.j) # array of ... (will be returned as list) @@ -315,7 +315,7 @@ def attributeRoundTrip(self, file_ending): # self.assertListEqual(series.get_attribute("l_cdouble"), # [np.complex128(6.7 + 6.8j), # np.double(7.1 + 7.2j)]) - # if file_ending != "bp": + # if file_ending not in ["bp", "bp4", "bp5"]: # self.assertListEqual(series.get_attribute("l_clongdouble"), # [np.clongdouble(7.8e9 - 6.5e9j), # np.clongdouble(8.2e3 - 9.1e3j)]) @@ -342,7 +342,8 @@ def attributeRoundTrip(self, file_ending): np.testing.assert_almost_equal( series.get_attribute("nparr_cdouble"), [4.5 + 1.1j, 6.7 - 2.2j]) - if file_ending != "bp": # not in ADIOS 1.13.1 nor ADIOS 2.7.0 + # not in ADIOS 1.13.1 nor ADIOS 2.7.0 + if file_ending not in ["bp", "bp4", "bp5"]: np.testing.assert_almost_equal( series.get_attribute("nparr_clongdouble"), [8.9 + 7.8j, 7.6 + 9.2j]) @@ -453,7 +454,7 @@ def makeConstantRoundTrip(self, file_ending): DS(np.dtype("complex128"), extent)) ms["complex128"][SCALAR].make_constant( np.complex128(1.234567 + 2.345678j)) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: ms["clongdouble"][SCALAR].reset_dataset( DS(np.dtype("clongdouble"), extent)) ms["clongdouble"][SCALAR].make_constant( @@ -534,7 +535,7 @@ def makeConstantRoundTrip(self, file_ending): == np.dtype('complex64')) self.assertTrue(ms["complex128"][SCALAR].load_chunk(o, e).dtype == np.dtype('complex128')) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: self.assertTrue(ms["clongdouble"][SCALAR].load_chunk(o, e) .dtype == np.dtype('clongdouble')) @@ -561,7 +562,7 @@ def makeConstantRoundTrip(self, file_ending): np.complex64(1.234 + 2.345j)) self.assertEqual(ms["complex128"][SCALAR].load_chunk(o, e), np.complex128(1.234567 + 2.345678j)) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: self.assertEqual(ms["clongdouble"][SCALAR].load_chunk(o, e), np.clongdouble(1.23456789 + 2.34567890j)) @@ -600,7 +601,7 @@ def makeDataRoundTrip(self, file_ending): ms["complex128"][SCALAR].store_chunk( np.ones(extent, dtype=np.complex128) * np.complex128(1.234567 + 2.345678j)) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: ms["clongdouble"][SCALAR].reset_dataset( DS(np.dtype("clongdouble"), extent)) ms["clongdouble"][SCALAR].store_chunk( @@ -630,12 +631,12 @@ def makeDataRoundTrip(self, file_ending): dc64 = ms["complex64"][SCALAR].load_chunk(o, e) dc128 = ms["complex128"][SCALAR].load_chunk(o, e) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: dc256 = ms["clongdouble"][SCALAR].load_chunk(o, e) self.assertTrue(dc64.dtype == np.dtype('complex64')) self.assertTrue(dc128.dtype == np.dtype('complex128')) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: self.assertTrue( dc256.dtype == np.dtype('clongdouble')) @@ -645,7 +646,7 @@ def makeDataRoundTrip(self, file_ending): np.complex64(1.234 + 2.345j)) self.assertEqual(dc128, np.complex128(1.234567 + 2.345678j)) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: self.assertEqual(dc256, np.clongdouble(1.23456789 + 2.34567890j)) From 34e7df6398c8eb723ae5bac068f36edaaa109128 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 15 Aug 2022 02:54:31 -0700 Subject: [PATCH 59/70] Fix `operationAsString` Export (#1309) The `operationAsString` function is used in the ADIOS1 backend, which does symbol hiding to wrap ADIOS1's MPI mock library. Thus, we need to export this symbol once we wrap ADIOS1. --- include/openPMD/IO/IOTask.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index f8d08ab821..d4db58483b 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -67,7 +67,7 @@ namespace internal * The returned strings are compile-time constants, so no worries about * pointer validity. */ - std::string operationAsString(Operation); + OPENPMDAPI_EXPORT std::string operationAsString(Operation); } // namespace internal struct OPENPMDAPI_EXPORT AbstractParameter From 597ba19b814c35c017355635696264bb6dd0c19f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 16 Aug 2022 19:37:02 +0200 Subject: [PATCH 60/70] storeChunk: Add an overload for shared_ptr (#1296) * Add storeChunk(shared_ptr, ...) overload * Use overload in examples * Revert "Use overload in examples" This reverts commit 990b33ac2c824f9ae8b9725a57a937b8f3a20fb4. * Compatibility with clang-6 --- include/openPMD/RecordComponent.hpp | 3 +++ include/openPMD/RecordComponent.tpp | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index d30f7684f2..da5833794b 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -241,6 +241,9 @@ class RecordComponent : public BaseRecordComponent template void storeChunk(std::shared_ptr, Offset, Extent); + template + void storeChunk(std::shared_ptr, Offset, Extent); + template typename std::enable_if< traits::IsContiguousContainer::value>::type diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index 7e757fd7fd..a4731d5434 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -208,6 +208,16 @@ RecordComponent::storeChunk(std::shared_ptr data, Offset o, Extent e) rc.m_chunks.push(IOTask(this, dWrite)); } +template +inline void +RecordComponent::storeChunk(std::shared_ptr data, Offset o, Extent e) +{ + storeChunk( + std::static_pointer_cast(std::move(data)), + std::move(o), + std::move(e)); +} + template< typename T_ContiguousContainer > inline typename std::enable_if< traits::IsContiguousContainer< T_ContiguousContainer >::value @@ -310,6 +320,8 @@ RecordComponent::storeChunk( Offset o, Extent e, F && createBuffer ) auto &out = *getBufferView.out; if (!out.backendManagedBuffer) { + // note that data might have either + // type shared_ptr or shared_ptr auto data = std::forward(createBuffer)(size); out.ptr = static_cast(data.get()); storeChunk(std::move(data), std::move(o), std::move(e)); @@ -326,8 +338,12 @@ RecordComponent::storeChunk( Offset offset, Extent extent ) std::move( extent ), []( size_t size ) { +#if defined(__clang_major__) && __clang_major__ < 7 return std::shared_ptr< T >{ new T[ size ], []( auto * ptr ) { delete[] ptr; } }; +#else + return std::shared_ptr< T[] >{ new T[ size ] }; +#endif } ); } } From 0a6e360175a8673c5848034f201a8ce0ac89ec56 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 25 Aug 2022 15:11:57 +0200 Subject: [PATCH 61/70] [pre-commit.ci] pre-commit autoupdate (#1311) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/Lucas-C/pre-commit-hooks: v1.3.0 → v1.3.1](https://github.com/Lucas-C/pre-commit-hooks/compare/v1.3.0...v1.3.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c55de0eed7..e02ef0b083 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,7 +48,7 @@ repos: # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.3.0 + rev: v1.3.1 hooks: - id: remove-tabs From 5edf3efff6c9e25d88c223886b951fd58e05d1d4 Mon Sep 17 00:00:00 2001 From: Nils Schild <69214149+DerNils-git@users.noreply.github.com> Date: Mon, 19 Sep 2022 17:47:10 +0200 Subject: [PATCH 62/70] Remove caching cmake vars (#1313) * Remove Caching of global CMake variables. * Add comment to non standard cmake variables which previously given in CACHE string. * Need a commit to trigger workflows. * Need a commit to trigger workflows. * Replace set_property by set since no Cache value has to be changed. Co-authored-by: Nils Schild --- CMakeLists.txt | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 699b4b9f06..55f73954e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,26 +44,21 @@ endif() # # temporary build directories if(NOT CMAKE_ARCHIVE_OUTPUT_DIRECTORY) - set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" - CACHE PATH "Build directory for archives") + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") endif() if(NOT CMAKE_LIBRARY_OUTPUT_DIRECTORY) - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" - CACHE PATH "Build directory for libraries") + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") endif() if(NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY) - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" - CACHE PATH "Build directory for binaries") + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") endif() # install directories if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) include(GNUInstallDirs) - set(CMAKE_INSTALL_CMAKEDIR "${CMAKE_INSTALL_LIBDIR}/cmake/openPMD" - CACHE PATH "CMake config package location for installed targets") + set(CMAKE_INSTALL_CMAKEDIR "${CMAKE_INSTALL_LIBDIR}/cmake/openPMD") if(WIN32) - set(CMAKE_INSTALL_LIBDIR Lib - CACHE PATH "Object code libraries") - set_property(CACHE CMAKE_INSTALL_CMAKEDIR PROPERTY VALUE "cmake") + set(CMAKE_INSTALL_LIBDIR Lib) + set(CMAKE_INSTALL_CMAKEDIR "cmake") endif() endif() @@ -98,8 +93,7 @@ option(openPMD_USE_VERIFY "Enable internal VERIFY (assert) macro independent of set(CMAKE_CONFIGURATION_TYPES "Release;Debug;MinSizeRel;RelWithDebInfo") if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE "Release" CACHE STRING - "Choose the build type, e.g. Release or Debug." FORCE) + set(CMAKE_BUILD_TYPE "Release") endif() include(CMakeDependentOption) @@ -746,20 +740,16 @@ if(openPMD_HAVE_PYTHON) if(WIN32) set(CMAKE_INSTALL_PYTHONDIR_DEFAULT - "${CMAKE_INSTALL_LIBDIR}/site-packages" - ) + "${CMAKE_INSTALL_LIBDIR}/site-packages") else() set(CMAKE_INSTALL_PYTHONDIR_DEFAULT "${CMAKE_INSTALL_LIBDIR}/python${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}/site-packages" ) endif() - set(CMAKE_INSTALL_PYTHONDIR "${CMAKE_INSTALL_PYTHONDIR_DEFAULT}" - CACHE STRING "Location for installed python package" - ) - set(CMAKE_PYTHON_OUTPUT_DIRECTORY - "${openPMD_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR}" - CACHE PATH "Build directory for python modules" - ) + # Location for installed python package + set(CMAKE_INSTALL_PYTHONDIR "${CMAKE_INSTALL_PYTHONDIR_DEFAULT}") + # Build directory for python modules + set(CMAKE_PYTHON_OUTPUT_DIRECTORY "${openPMD_BINARY_DIR}/${CMAKE_INSTALL_PYTHONDIR}") set_target_properties(openPMD.py PROPERTIES ARCHIVE_OUTPUT_NAME openpmd_api_cxx LIBRARY_OUTPUT_NAME openpmd_api_cxx From 273466e194bfc3684517a7214ec570c99d546667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 7 Oct 2022 10:29:30 +0200 Subject: [PATCH 63/70] Constant scalars: Don't flush double (#1315) Paths were created in duplicate --- src/Mesh.cpp | 28 ++++++++++++++++------------ src/Record.cpp | 28 +++++++++++++++++----------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/src/Mesh.cpp b/src/Mesh.cpp index 6f9b891635..d570840e98 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -243,25 +243,29 @@ void Mesh::flush_impl( pCreate.path = name; IOHandler()->enqueue(IOTask(this, pCreate)); for (auto &comp : *this) + { comp.second.parent() = &this->writable(); + comp.second.flush(comp.first, flushParams); + } } } - - if (scalar()) + else { - for (auto &comp : *this) + if (scalar()) { - comp.second.flush(name, flushParams); - writable().abstractFilePosition = - comp.second.writable().abstractFilePosition; + for (auto &comp : *this) + { + comp.second.flush(name, flushParams); + writable().abstractFilePosition = + comp.second.writable().abstractFilePosition; + } + } + else + { + for (auto &comp : *this) + comp.second.flush(comp.first, flushParams); } } - else - { - for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); - } - flushAttributes(flushParams); break; } diff --git a/src/Record.cpp b/src/Record.cpp index bbd74ed0f5..717d709072 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -73,23 +73,29 @@ void Record::flush_impl( pCreate.path = name; IOHandler()->enqueue(IOTask(this, pCreate)); for (auto &comp : *this) + { comp.second.parent() = getWritable(this); + comp.second.flush(comp.first, flushParams); + } } } - - if (scalar()) + else { - for (auto &comp : *this) + + if (scalar()) { - comp.second.flush(name, flushParams); - writable().abstractFilePosition = - comp.second.writable().abstractFilePosition; + for (auto &comp : *this) + { + comp.second.flush(name, flushParams); + writable().abstractFilePosition = + comp.second.writable().abstractFilePosition; + } + } + else + { + for (auto &comp : *this) + comp.second.flush(comp.first, flushParams); } - } - else - { - for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); } flushAttributes(flushParams); From 230afddf72e537b957408ec5ebb4702711abdd09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 7 Oct 2022 10:33:02 +0200 Subject: [PATCH 64/70] Fix append mode double attributes (#1302) * Add failing test * Allow creating file-based Series with Append mode even if the specified directory does not exist * Add Mpi.hpp with MPI helpers Needed later for checking file presence in parallel situations * Add CHECK_FILE IO task and implement in the backends * Only write top-level attributes when really needed * Enable Series creation in Append mode * Add Windows CI workaround * Windows Workaround: Fallback to Create mode * Avoid MPI mocking, use ifdefs Also use parallel logical or for file existence check * Append mode: test in parallel * Don't alias sendbfr and recvbfr * Undo unused changes * Remove unused Mock_MPI_Comm * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Replace MPI specializations by if constexpr * Try making this constexpr I think that this won't work with every MPI implementation * Link ADIOS2 issue https://github.com/ornladios/ADIOS2/issues/3358 Co-authored-by: Axel Huebl Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 10 +- .../IO/ADIOS/CommonADIOS1IOHandler.hpp | 1 + include/openPMD/IO/AbstractIOHandlerImpl.hpp | 17 ++ include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp | 10 + include/openPMD/IO/IOTask.hpp | 29 ++- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 2 + include/openPMD/Series.hpp | 2 +- include/openPMD/auxiliary/Mpi.hpp | 77 ++++++ src/IO/ADIOS/ADIOS1IOHandler.cpp | 7 + src/IO/ADIOS/ADIOS2IOHandler.cpp | 112 +++++++-- src/IO/ADIOS/CommonADIOS1IOHandler.cpp | 8 + src/IO/ADIOS/ParallelADIOS1IOHandler.cpp | 7 + src/IO/HDF5/HDF5IOHandler.cpp | 35 +++ src/IO/HDF5/ParallelHDF5IOHandler.cpp | 3 + src/IO/JSON/JSONIOHandlerImpl.cpp | 26 ++- src/Iteration.cpp | 1 + src/Series.cpp | 55 ++++- src/auxiliary/Filesystem.cpp | 38 +-- test/ParallelIOTest.cpp | 221 ++++++++++++++++++ test/SerialIOTest.cpp | 24 +- 20 files changed, 608 insertions(+), 77 deletions(-) create mode 100644 include/openPMD/auxiliary/Mpi.hpp diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index d4a199be53..d3ccc93258 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -150,6 +150,11 @@ class ADIOS2IOHandlerImpl void createFile(Writable *, Parameter const &) override; + void checkFile(Writable *, Parameter &) override; + + // MPI Collective + bool checkFile(std::string fullFilePath) const; + void createPath(Writable *, Parameter const &) override; @@ -226,6 +231,9 @@ class ADIOS2IOHandlerImpl private: adios2::ADIOS m_ADIOS; +#if openPMD_HAVE_MPI + std::optional m_communicator; +#endif /* * If the iteration encoding is variableBased, we default to using the * 2021_02_09 schema since it allows mutable attributes. @@ -329,7 +337,7 @@ class ADIOS2IOHandlerImpl // use m_config std::optional > getOperators(); - std::string fileSuffix() const; + std::string fileSuffix(bool verbose = true) const; /* * We need to give names to IO objects. These names are irrelevant diff --git a/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp index ea89f033d6..9f43b9ae18 100644 --- a/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp @@ -52,6 +52,7 @@ class CommonADIOS1IOHandlerImpl : public AbstractIOHandlerImpl public: void createFile(Writable *, Parameter const &) override; + void checkFile(Writable *, Parameter &) override; void createPath(Writable *, Parameter const &) override; void createDataset( diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index 518b9dac0a..55ff021d06 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -58,6 +58,12 @@ class AbstractIOHandlerImpl deref_dynamic_cast >( i.parameter.get())); break; + case O::CHECK_FILE: + checkFile( + i.writable, + deref_dynamic_cast >( + i.parameter.get())); + break; case O::CREATE_PATH: createPath( i.writable, @@ -220,6 +226,17 @@ class AbstractIOHandlerImpl virtual void closeFile(Writable *, Parameter const &) = 0; + /** + * Check if the file specified by the parameter is already present on disk. + * The Writable is irrelevant for this method. + * A backend can choose to ignore this task and specify FileExists::DontKnow + * in the out parameter. + * The consequence will be that some top-level attributes might be defined + * a second time when appending to an existing file, because the frontend + * cannot be sure that the file already has these attributes. + */ + virtual void checkFile(Writable *, Parameter &) = 0; + /** Advance the file/stream that this writable belongs to. * * If the backend is based around usage of IO steps (especially streaming diff --git a/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp b/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp index adee717ff0..7502e36e3f 100644 --- a/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp +++ b/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp @@ -43,6 +43,7 @@ class HDF5IOHandlerImpl : public AbstractIOHandlerImpl void createFile(Writable *, Parameter const &) override; + void checkFile(Writable *, Parameter &) override; void createPath(Writable *, Parameter const &) override; void createDataset( @@ -92,6 +93,15 @@ class HDF5IOHandlerImpl : public AbstractIOHandlerImpl hid_t m_H5T_CDOUBLE; hid_t m_H5T_CLONG_DOUBLE; +protected: +#if openPMD_HAVE_MPI + /* + * Not defined in ParallelHDF5IOHandlerImpl, so we don't have to write + * some methods twice. + */ + std::optional m_communicator; +#endif + private: json::TracingJSON m_config; std::string m_chunks = "auto"; diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index d4db58483b..3ba7d09e24 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -44,7 +44,8 @@ Writable *getWritable(Attributable *); /** Type of IO operation between logical and persistent data. */ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){ - CREATE_FILE, OPEN_FILE, CLOSE_FILE, DELETE_FILE, + CREATE_FILE, CHECK_FILE, OPEN_FILE, CLOSE_FILE, + DELETE_FILE, CREATE_PATH, CLOSE_PATH, OPEN_PATH, DELETE_PATH, LIST_PATHS, @@ -118,6 +119,32 @@ struct OPENPMDAPI_EXPORT Parameter IterationEncoding encoding = IterationEncoding::groupBased; }; +template <> +struct OPENPMDAPI_EXPORT Parameter + : public AbstractParameter +{ + Parameter() = default; + Parameter(Parameter const &p) + : AbstractParameter(), name(p.name), fileExists(p.fileExists) + {} + + std::unique_ptr clone() const override + { + return std::unique_ptr( + new Parameter(*this)); + } + + std::string name = ""; + enum class FileExists + { + DontKnow, + Yes, + No + }; + std::shared_ptr fileExists = + std::make_shared(FileExists::DontKnow); +}; + template <> struct OPENPMDAPI_EXPORT Parameter : public AbstractParameter diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index d090d0b687..738891f33e 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -160,6 +160,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl void createFile(Writable *, Parameter const &) override; + void checkFile(Writable *, Parameter &) override; + void createPath(Writable *, Parameter const &) override; diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index e828bcd7f7..cda0956b52 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -534,7 +534,7 @@ OPENPMD_private bool hasExpansionPattern(std::string filenameWithExtension); bool reparseExpansionPattern(std::string filenameWithExtension); void init(std::shared_ptr, std::unique_ptr); - void initDefaults(IterationEncoding); + void initDefaults(IterationEncoding, bool initAll = false); /** * @brief Internal call for flushing a Series. * diff --git a/include/openPMD/auxiliary/Mpi.hpp b/include/openPMD/auxiliary/Mpi.hpp new file mode 100644 index 0000000000..5201f68bf2 --- /dev/null +++ b/include/openPMD/auxiliary/Mpi.hpp @@ -0,0 +1,77 @@ +/* Copyright 2022 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/config.hpp" + +#if openPMD_HAVE_MPI +#include +#endif + +#include + +namespace openPMD::auxiliary +{ +#if openPMD_HAVE_MPI + +namespace detail +{ + namespace + { + // see https://en.cppreference.com/w/cpp/language/if + template + inline constexpr bool dependent_false_v = false; + } // namespace +} // namespace detail + +namespace +{ + template + constexpr MPI_Datatype openPMD_MPI_type() + { + using T_decay = std::decay_t; + if constexpr (std::is_same_v) + { + return MPI_CHAR; + } + else if constexpr (std::is_same_v) + { + return MPI_UNSIGNED; + } + else if constexpr (std::is_same_v) + { + return MPI_UNSIGNED_LONG; + } + else if constexpr (std::is_same_v) + { + return MPI_UNSIGNED_LONG_LONG; + } + else + { + static_assert( + detail::dependent_false_v, + "openPMD_MPI_type: Unsupported type."); + } + } +} // namespace + +#endif +} // namespace openPMD::auxiliary diff --git a/src/IO/ADIOS/ADIOS1IOHandler.cpp b/src/IO/ADIOS/ADIOS1IOHandler.cpp index 92166ca259..019e5a8078 100644 --- a/src/IO/ADIOS/ADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS1IOHandler.cpp @@ -104,6 +104,12 @@ std::future ADIOS1IOHandlerImpl::flush() deref_dynamic_cast >( i.parameter.get())); break; + case O::CHECK_FILE: + checkFile( + i.writable, + deref_dynamic_cast >( + i.parameter.get())); + break; case O::CREATE_PATH: createPath( i.writable, @@ -346,6 +352,7 @@ void ADIOS1IOHandler::enqueue(IOTask const &i) switch (i.operation) { case Operation::CREATE_FILE: + case Operation::CHECK_FILE: case Operation::CREATE_PATH: case Operation::OPEN_PATH: case Operation::CREATE_DATASET: diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 047373cdff..fcdfffc1f8 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -27,6 +27,7 @@ #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/auxiliary/Mpi.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" @@ -73,6 +74,7 @@ ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( std::string specifiedExtension) : AbstractIOHandlerImplCommon(handler) , m_ADIOS{communicator} + , m_communicator{communicator} , m_engineType(std::move(engineType)) , m_userSpecifiedExtension{std::move(specifiedExtension)} { @@ -245,7 +247,7 @@ ADIOS2IOHandlerImpl::getOperators() using AcceptedEndingsForEngine = std::map; -std::string ADIOS2IOHandlerImpl::fileSuffix() const +std::string ADIOS2IOHandlerImpl::fileSuffix(bool verbose) const { // SST engine adds its suffix unconditionally // so we don't add it @@ -267,7 +269,8 @@ std::string ADIOS2IOHandlerImpl::fileSuffix() const if (auto ending = acceptedEndings.find(m_userSpecifiedExtension); ending != acceptedEndings.end()) { - if ((m_engineType == "file" || m_engineType == "filestream") && + if (verbose && + (m_engineType == "file" || m_engineType == "filestream") && (m_userSpecifiedExtension == ".bp3" || m_userSpecifiedExtension == ".bp4" || m_userSpecifiedExtension == ".bp5")) @@ -288,7 +291,7 @@ std::string ADIOS2IOHandlerImpl::fileSuffix() const { std::cerr << "[ADIOS2] No file ending specified. Will not add one." << std::endl; - if (m_engineType == "bp3") + if (verbose && m_engineType == "bp3") { std::cerr << "Note that the ADIOS2 BP3 engine will add its " @@ -300,19 +303,22 @@ std::string ADIOS2IOHandlerImpl::fileSuffix() const } else { - std::cerr << "[ADIOS2] Specified ending '" - << m_userSpecifiedExtension - << "' does not match the selected engine '" - << m_engineType - << "'. Will use the specified ending anyway." - << std::endl; - if (m_engineType == "bp3") + if (verbose) { - std::cerr - << "Note that the ADIOS2 BP3 engine will add its " - "ending '.bp' if not specified (e.g. 'simData.bp3' " - "will appear on disk as 'simData.bp3.bp')." - << std::endl; + std::cerr << "[ADIOS2] Specified ending '" + << m_userSpecifiedExtension + << "' does not match the selected engine '" + << m_engineType + << "'. Will use the specified ending anyway." + << std::endl; + if (m_engineType == "bp3") + { + std::cerr + << "Note that the ADIOS2 BP3 engine will add its " + "ending '.bp' if not specified (e.g. 'simData.bp3' " + "will appear on disk as 'simData.bp3.bp')." + << std::endl; + } } return m_userSpecifiedExtension; } @@ -496,6 +502,60 @@ void ADIOS2IOHandlerImpl::createFile( } } +void ADIOS2IOHandlerImpl::checkFile( + Writable *, Parameter ¶meters) +{ + std::string name = + fullPath(parameters.name + fileSuffix(/* verbose = */ false)); + + using FileExists = Parameter::FileExists; + *parameters.fileExists = checkFile(name) ? FileExists::Yes : FileExists::No; +} + +bool ADIOS2IOHandlerImpl::checkFile(std::string fullFilePath) const +{ + if (m_engineType == "bp3") + { + if (!auxiliary::ends_with(fullFilePath, ".bp")) + { + /* + * BP3 will add this ending if not specified + */ + fullFilePath += ".bp"; + } + } + else if (m_engineType == "sst") + { + /* + * SST will add this ending indiscriminately + */ + fullFilePath += ".sst"; + } + bool fileExists = auxiliary::directory_exists(fullFilePath) || + auxiliary::file_exists(fullFilePath); + +#if openPMD_HAVE_MPI + if (m_communicator.has_value()) + { + bool fileExistsRes = false; + int status = MPI_Allreduce( + &fileExists, + &fileExistsRes, + 1, + MPI_C_BOOL, + MPI_LOR, // logical or + m_communicator.value()); + if (status != 0) + { + throw std::runtime_error("MPI Reduction failed!"); + } + fileExists = fileExistsRes; + } +#endif + + return fileExists; +} + void ADIOS2IOHandlerImpl::createPath( Writable *writable, const Parameter ¶meters) { @@ -2271,11 +2331,14 @@ namespace detail : m_file(impl.fullPath(std::move(file))) , m_IOName(std::to_string(impl.nameCounter++)) , m_ADIOS(impl.m_ADIOS) - , m_IO(impl.m_ADIOS.DeclareIO(m_IOName)) - , m_mode(impl.adios2AccessMode(m_file)) , m_impl(&impl) , m_engineType(impl.m_engineType) { + // Declaring these members in the constructor body to avoid + // initialization order hazards. Need the IO_ prefix since in some + // situation there seems to be trouble with number-only IO names + m_IO = impl.m_ADIOS.DeclareIO("IO_" + m_IOName); + m_mode = impl.adios2AccessMode(m_file); if (!m_IO) { throw std::runtime_error( @@ -2616,9 +2679,22 @@ namespace detail { if (!m_engine) { + auto tempMode = m_mode; switch (m_mode) { case adios2::Mode::Append: +#ifdef _WIN32 + /* + * On Windows, ADIOS2 v2.8. Append mode only works with existing + * files. So, we first check for file existence and switch to + * create mode if it does not exist. + * + * See issue: https://github.com/ornladios/ADIOS2/issues/3358 + */ + tempMode = m_impl->checkFile(m_file) ? adios2::Mode::Append + : adios2::Mode::Write; + [[fallthrough]]; +#endif case adios2::Mode::Write: { // usesSteps attribute only written upon ::advance() // this makes sure that the attribute is only put in case @@ -2626,7 +2702,7 @@ namespace detail m_IO.DefineAttribute( ADIOS2Defaults::str_adios2Schema, m_impl->m_schema); m_engine = std::make_optional( - adios2::Engine(m_IO.Open(m_file, m_mode))); + adios2::Engine(m_IO.Open(m_file, tempMode))); break; } case adios2::Mode::Read: { diff --git a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp index 7bed2f15c7..3dbee3e2db 100644 --- a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp @@ -449,6 +449,14 @@ void CommonADIOS1IOHandlerImpl::createFile( } } +template +void CommonADIOS1IOHandlerImpl::checkFile( + Writable *, Parameter ¶meter) +{ + *parameter.fileExists = + Parameter::FileExists::DontKnow; +} + template void CommonADIOS1IOHandlerImpl::createPath( Writable *writable, Parameter const ¶meters) diff --git a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp index a451dd2c15..20f571e980 100644 --- a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp @@ -126,6 +126,12 @@ std::future ParallelADIOS1IOHandlerImpl::flush() deref_dynamic_cast >( i.parameter.get())); break; + case O::CHECK_FILE: + checkFile( + i.writable, + deref_dynamic_cast >( + i.parameter.get())); + break; case O::CREATE_PATH: createPath( i.writable, @@ -365,6 +371,7 @@ void ParallelADIOS1IOHandler::enqueue(IOTask const &i) switch (i.operation) { case Operation::CREATE_FILE: + case Operation::CHECK_FILE: case Operation::CREATE_PATH: case Operation::OPEN_PATH: case Operation::CREATE_DATASET: diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index b663740077..94dce55c1c 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -29,6 +29,7 @@ #include "openPMD/IO/HDF5/HDF5FilePosition.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/auxiliary/Mpi.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/backend/Attribute.hpp" @@ -284,6 +285,40 @@ void HDF5IOHandlerImpl::createFile( } } +void HDF5IOHandlerImpl::checkFile( + Writable *, Parameter ¶meters) +{ + std::string name = m_handler->directory + parameters.name; + if (!auxiliary::ends_with(name, ".h5")) + { + name += ".h5"; + } + bool fileExists = + auxiliary::file_exists(name) || auxiliary::directory_exists(name); + +#if openPMD_HAVE_MPI + if (m_communicator.has_value()) + { + bool fileExistsRes = false; + int status = MPI_Allreduce( + &fileExists, + &fileExistsRes, + 1, + MPI_C_BOOL, + MPI_LOR, // logical or + m_communicator.value()); + if (status != 0) + { + throw std::runtime_error("MPI Reduction failed!"); + } + fileExists = fileExistsRes; + } +#endif + + using FileExists = Parameter::FileExists; + *parameters.fileExists = fileExists ? FileExists::Yes : FileExists::No; +} + void HDF5IOHandlerImpl::createPath( Writable *writable, Parameter const ¶meters) { diff --git a/src/IO/HDF5/ParallelHDF5IOHandler.cpp b/src/IO/HDF5/ParallelHDF5IOHandler.cpp index 369ee2e317..f7a6dc1a1c 100644 --- a/src/IO/HDF5/ParallelHDF5IOHandler.cpp +++ b/src/IO/HDF5/ParallelHDF5IOHandler.cpp @@ -65,6 +65,9 @@ ParallelHDF5IOHandlerImpl::ParallelHDF5IOHandlerImpl( , m_mpiComm{comm} , m_mpiInfo{MPI_INFO_NULL} /* MPI 3.0+: MPI_INFO_ENV */ { + // Set this so the parent class can use the MPI communicator in functions + // that are written with special implemenations for MPI-enabled HDF5. + m_communicator = m_mpiComm; m_datasetTransferProperty = H5Pcreate(H5P_DATASET_XFER); m_fileAccessProperty = H5Pcreate(H5P_FILE_ACCESS); m_fileCreateProperty = H5Pcreate(H5P_FILE_CREATE); diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 6f8666fae5..062c736433 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -102,11 +102,12 @@ void JSONIOHandlerImpl::createFile( } auto res_pair = getPossiblyExisting(name); + auto fullPathToFile = fullPath(std::get<0>(res_pair)); File shared_name = File(name); VERIFY_ALWAYS( !(m_handler->m_backendAccess == Access::READ_WRITE && (!std::get<2>(res_pair) || - auxiliary::file_exists(fullPath(std::get<0>(res_pair))))), + auxiliary::file_exists(fullPathToFile))), "[JSON] Can only overwrite existing file in CREATE mode."); if (!std::get<2>(res_pair)) @@ -127,9 +128,12 @@ void JSONIOHandlerImpl::createFile( associateWithFile(writable, shared_name); this->m_dirty.emplace(shared_name); - if (m_handler->m_backendAccess != Access::APPEND) + if (m_handler->m_backendAccess != Access::APPEND || + !auxiliary::file_exists(fullPathToFile)) { - // make sure to overwrite! + // if in create mode: make sure to overwrite + // if in append mode and the file does not exist: create an empty + // dataset this->m_jsonVals[shared_name] = std::make_shared(); } // else: the JSON value is not available in m_jsonVals and will be @@ -140,6 +144,22 @@ void JSONIOHandlerImpl::createFile( } } +void JSONIOHandlerImpl::checkFile( + Writable *, Parameter ¶meters) +{ + std::string name = parameters.name; + if (!auxiliary::ends_with(name, ".json")) + { + name += ".json"; + } + name = fullPath(name); + using FileExists = Parameter::FileExists; + *parameters.fileExists = + (auxiliary::file_exists(name) || auxiliary::directory_exists(name)) + ? FileExists::Yes + : FileExists::No; +} + void JSONIOHandlerImpl::createPath( Writable *writable, Parameter const ¶meter) { diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 1cf77ec180..575610ea16 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -28,6 +28,7 @@ #include "openPMD/backend/Writable.hpp" #include +#include #include namespace openPMD diff --git a/src/Series.cpp b/src/Series.cpp index 814b0e2489..6387258cf6 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -502,15 +502,18 @@ namespace int padding; uint64_t iterationIndex; std::set paddings; - for (auto const &entry : auxiliary::list_directory(directory)) + if (auxiliary::directory_exists(directory)) { - std::tie(isContained, padding, iterationIndex) = - isPartOfSeries(entry); - if (isContained) + for (auto const &entry : auxiliary::list_directory(directory)) { - paddings.insert(padding); - // no std::forward as this is called repeatedly - mappingFunction(iterationIndex, entry); + std::tie(isContained, padding, iterationIndex) = + isPartOfSeries(entry); + if (isContained) + { + paddings.insert(padding); + // no std::forward as this is called repeatedly + mappingFunction(iterationIndex, entry); + } } } if (paddings.size() == 1u) @@ -637,12 +640,8 @@ Given file pattern: ')END" series.m_lastFlushSuccessful = true; } -void Series::initDefaults(IterationEncoding ie) +void Series::initDefaults(IterationEncoding ie, bool initAll) { - if (!containsAttribute("openPMD")) - setOpenPMD(getStandard()); - if (!containsAttribute("openPMDextension")) - setOpenPMDextension(0); if (!containsAttribute("basePath")) { if (ie == IterationEncoding::variableBased) @@ -655,6 +654,21 @@ void Series::initDefaults(IterationEncoding ie) setAttribute("basePath", std::string(BASEPATH)); } } + if (!containsAttribute("openPMD")) + setOpenPMD(getStandard()); + /* + * In Append mode, only init the rest of the defaults after checking that + * the file does not yet exist to avoid overriding more than needed. + * In file-based iteration encoding, files are always truncated in Append + * mode (Append mode works on a per-iteration basis). + */ + if (!initAll && IOHandler()->m_frontendAccess == Access::APPEND && + ie != IterationEncoding::fileBased) + { + return; + } + if (!containsAttribute("openPMDextension")) + setOpenPMDextension(0); if (!containsAttribute("date")) setDate(auxiliary::getDateString()); if (!containsAttribute("software")) @@ -845,6 +859,23 @@ void Series::flushGorVBased( case Access::APPEND: { if (!written()) { + if (IOHandler()->m_frontendAccess == Access::APPEND) + { + Parameter param; + param.name = series.m_name; + IOHandler()->enqueue(IOTask(this, param)); + IOHandler()->flush(internal::defaultFlushParams); + switch (*param.fileExists) + { + using FE = Parameter::FileExists; + case FE::DontKnow: + case FE::No: + initDefaults(iterationEncoding(), /* initAll = */ true); + break; + case FE::Yes: + break; + } + } Parameter fCreate; fCreate.name = series.m_name; fCreate.encoding = iterationEncoding(); diff --git a/src/auxiliary/Filesystem.cpp b/src/auxiliary/Filesystem.cpp index 3e2c65f3af..38d8e209f8 100644 --- a/src/auxiliary/Filesystem.cpp +++ b/src/auxiliary/Filesystem.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/auxiliary/Mpi.hpp" #include "openPMD/auxiliary/StringManip.hpp" #ifdef _WIN32 @@ -175,41 +176,6 @@ bool remove_file(std::string const &path) #if openPMD_HAVE_MPI -namespace -{ - template - struct MPI_Types; - - template <> - struct MPI_Types - { - static MPI_Datatype const value; - }; - - template <> - struct MPI_Types - { - static MPI_Datatype const value; - }; - - template <> - struct MPI_Types - { - static MPI_Datatype const value; - }; - - /* - * Only some of these are actually instanciated, - * so suppress warnings for the others. - */ - [[maybe_unused]] MPI_Datatype const MPI_Types::value = - MPI_UNSIGNED; - [[maybe_unused]] MPI_Datatype const MPI_Types::value = - MPI_UNSIGNED_LONG; - [[maybe_unused]] MPI_Datatype const MPI_Types::value = - MPI_UNSIGNED_LONG_LONG; -} // namespace - std::string collective_file_read(std::string const &path, MPI_Comm comm) { int rank, size; @@ -232,7 +198,7 @@ std::string collective_file_read(std::string const &path, MPI_Comm comm) } stringLength = res.size() + 1; } - MPI_Datatype datatype = MPI_Types::value; + MPI_Datatype datatype = openPMD_MPI_type(); int err = MPI_Bcast(&stringLength, 1, datatype, 0, comm); if (err) { diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 1ca922c022..aaf1fd4378 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -40,6 +40,18 @@ std::vector getBackends() auto const backends = getBackends(); +std::vector testedFileExtensions() +{ + auto allExtensions = getFileExtensions(); + auto newEnd = std::remove_if( + allExtensions.begin(), allExtensions.end(), [](std::string const &ext) { + // sst and ssc need a receiver for testing + // bp4 is already tested via bp + return ext == "sst" || ext == "ssc" || ext == "bp4" | ext == "json"; + }); + return {allExtensions.begin(), newEnd}; +} + #else TEST_CASE("none", "[parallel]") @@ -1368,4 +1380,213 @@ TEST_CASE("adios2_ssc", "[parallel][adios2]") { adios2_ssc(); } + +void append_mode( + std::string const &extension, + bool variableBased, + std::string jsonConfig = "{}") +{ + std::string filename = + (variableBased ? "../samples/append/append_variablebased." + : "../samples/append/append_groupbased.") + + extension; + MPI_Barrier(MPI_COMM_WORLD); + if (auxiliary::directory_exists("../samples/append")) + { + auxiliary::remove_directory("../samples/append"); + } + MPI_Barrier(MPI_COMM_WORLD); + std::vector data(10, 0); + auto writeSomeIterations = [&data]( + WriteIterations &&writeIterations, + std::vector indices) { + for (auto index : indices) + { + auto it = writeIterations[index]; + auto dataset = it.meshes["E"]["x"]; + dataset.resetDataset({Datatype::INT, {10}}); + dataset.storeChunk(data, {0}, {10}); + // test that it works without closing too + it.close(); + } + }; + { + Series write(filename, Access::APPEND, MPI_COMM_WORLD, jsonConfig); + if (variableBased) + { + if (write.backend() != "ADIOS2") + { + return; + } + write.setIterationEncoding(IterationEncoding::variableBased); + } + writeSomeIterations( + write.writeIterations(), std::vector{0, 1}); + } + { + Series write(filename, Access::APPEND, MPI_COMM_WORLD, jsonConfig); + if (variableBased) + { + write.setIterationEncoding(IterationEncoding::variableBased); + } + if (write.backend() == "MPI_ADIOS1") + { + REQUIRE_THROWS_WITH( + write.flush(), + Catch::Equals( + "Operation unsupported in ADIOS1: Appending to existing " + "file on disk (use Access::CREATE to overwrite)")); + // destructor will be noisy now + return; + } + + writeSomeIterations( + write.writeIterations(), std::vector{2, 3}); + write.flush(); + } + { + using namespace std::chrono_literals; + /* + * Put a little sleep here to trigger writing of a different /date + * attribute. ADIOS2 v2.7 does not like that so this test ensures that + * we deal with it. + */ + std::this_thread::sleep_for(1s); + Series write(filename, Access::APPEND, MPI_COMM_WORLD, jsonConfig); + if (variableBased) + { + write.setIterationEncoding(IterationEncoding::variableBased); + } + if (write.backend() == "MPI_ADIOS1") + { + REQUIRE_THROWS_WITH( + write.flush(), + Catch::Equals( + "Operation unsupported in ADIOS1: Appending to existing " + "file on disk (use Access::CREATE to overwrite)")); + // destructor will be noisy now + return; + } + + writeSomeIterations( + write.writeIterations(), std::vector{4, 3}); + write.flush(); + } + { + Series read(filename, Access::READ_ONLY, MPI_COMM_WORLD); + if (variableBased || extension == "bp5") + { + // in variable-based encodings, iterations are not parsed ahead of + // time but as they go + unsigned counter = 0; + for (auto const &iteration : read.readIterations()) + { + REQUIRE(iteration.iterationIndex == counter); + ++counter; + } + REQUIRE(counter == 5); + } + else + { + REQUIRE(read.iterations.size() == 5); + } + /* + * Roadmap: for now, reading this should work by ignoring the last + * duplicate iteration. + * After merging https://github.com/openPMD/openPMD-api/pull/949, we + * should see both instances when reading. + * Final goal: Read only the last instance. + */ + helper::listSeries(read); + } +#if 100000000 * ADIOS2_VERSION_MAJOR + 1000000 * ADIOS2_VERSION_MINOR + \ + 10000 * ADIOS2_VERSION_PATCH + 100 * ADIOS2_VERSION_TWEAK >= \ + 208002700 + // AppendAfterSteps has a bug before that version + if (extension == "bp5") + { + { + Series write( + filename, + Access::APPEND, + MPI_COMM_WORLD, + json::merge( + jsonConfig, + R"({"adios2":{"engine":{"parameters":{"AppendAfterSteps":-3}}}})")); + if (variableBased) + { + write.setIterationEncoding(IterationEncoding::variableBased); + } + if (write.backend() == "ADIOS1") + { + REQUIRE_THROWS_WITH( + write.flush(), + Catch::Equals( + "Operation unsupported in ADIOS1: Appending to " + "existing " + "file on disk (use Access::CREATE to overwrite)")); + // destructor will be noisy now + return; + } + + writeSomeIterations( + write.writeIterations(), std::vector{4, 5}); + write.flush(); + } + { + Series read(filename, Access::READ_ONLY, MPI_COMM_WORLD); + // in variable-based encodings, iterations are not parsed ahead of + // time but as they go + unsigned counter = 0; + for (auto const &iteration : read.readIterations()) + { + REQUIRE(iteration.iterationIndex == counter); + ++counter; + } + REQUIRE(counter == 6); + helper::listSeries(read); + } + } +#endif +} + +TEST_CASE("append_mode", "[parallel]") +{ + for (auto const &t : testedFileExtensions()) + { + if (t == "bp" || t == "bp4" || t == "bp5") + { + std::string jsonConfigOld = R"END( +{ + "adios2": + { + "schema": 0, + "engine": + { + "usesteps" : true + } + } +})END"; + std::string jsonConfigNew = R"END( +{ + "adios2": + { + "schema": 20210209, + "engine": + { + "usesteps" : true + } + } +})END"; + append_mode(t, false, jsonConfigOld); + append_mode(t, false, jsonConfigNew); + append_mode(t, true, jsonConfigOld); + append_mode(t, true, jsonConfigNew); + } + else + { + append_mode(t, false); + } + } +} #endif diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index eac05b0214..20d20d6a71 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -27,6 +28,7 @@ #include #include #include +#include #include #include @@ -6320,9 +6322,14 @@ void append_mode( std::string jsonConfig = "{}") { - std::string filename = (variableBased ? "../samples/append_variablebased." - : "../samples/append_groupbased.") + + std::string filename = + (variableBased ? "../samples/append/append_variablebased." + : "../samples/append/append_groupbased.") + extension; + if (auxiliary::directory_exists("../samples/append")) + { + auxiliary::remove_directory("../samples/append"); + } std::vector data(10, 0); auto writeSomeIterations = [&data]( WriteIterations &&writeIterations, @@ -6338,7 +6345,7 @@ void append_mode( } }; { - Series write(filename, Access::CREATE, jsonConfig); + Series write(filename, Access::APPEND, jsonConfig); if (variableBased) { if (write.backend() != "ADIOS2") @@ -6372,6 +6379,13 @@ void append_mode( write.flush(); } { + using namespace std::chrono_literals; + /* + * Put a little sleep here to trigger writing of a different /date + * attribute. ADIOS2 v2.7 does not like that so this test ensures that + * we deal with it. + */ + std::this_thread::sleep_for(1s); Series write(filename, Access::APPEND, jsonConfig); if (variableBased) { @@ -6540,8 +6554,8 @@ void append_mode_filebased(std::string const &extension) } { Series write( - "../samples/append/append_%T." + extension, - Access::CREATE, + "../samples/append/append_%06T." + extension, + Access::APPEND, jsonConfig); writeSomeIterations( write.writeIterations(), std::vector{0, 1}); From a8436ab5cd49d6ba5976a048b245e32509678e7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 25 Oct 2022 19:33:25 +0200 Subject: [PATCH 65/70] Deprecate shareRaw (#1229) * Deprecate shareRaw * Load/StoreChunk documentation * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Introduce and use auxiliary::shareRaw * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add determineDatatypeContiguous, rename determineDatatypeRaw * Update some further instances after rebasing * Fix docu after rebasing * Use enable_if_t * Add missing shared_ptr overloads * Throw static_assert errors for contiguous types at determineDatatype * Fix CI errors Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- examples/3_write_serial.cpp | 4 +- examples/3b_write_resizable_particles.cpp | 2 +- include/openPMD/Datatype.hpp | 207 +++++------------- include/openPMD/RecordComponent.hpp | 172 +++++++++++---- include/openPMD/RecordComponent.tpp | 51 ++++- include/openPMD/auxiliary/Mpi.hpp | 15 +- include/openPMD/auxiliary/ShareRaw.hpp | 18 +- .../openPMD/auxiliary/ShareRawInternal.hpp | 79 +++++++ include/openPMD/auxiliary/TypeTraits.hpp | 18 ++ .../openPMD/backend/PatchRecordComponent.hpp | 25 +++ src/binding/python/PatchRecordComponent.cpp | 3 +- src/binding/python/RecordComponent.cpp | 1 - test/CoreTest.cpp | 7 +- test/ParallelIOTest.cpp | 2 +- test/SerialIOTest.cpp | 19 +- 15 files changed, 386 insertions(+), 237 deletions(-) create mode 100644 include/openPMD/auxiliary/ShareRawInternal.hpp diff --git a/examples/3_write_serial.cpp b/examples/3_write_serial.cpp index 71628bc671..54f9c384b9 100644 --- a/examples/3_write_serial.cpp +++ b/examples/3_write_serial.cpp @@ -49,7 +49,7 @@ int main(int argc, char *argv[]) cout << "Created a scalar mesh Record with all required openPMD " "attributes\n"; - Datatype datatype = determineDatatype(shareRaw(global_data)); + Datatype datatype = determineDatatype(global_data.data()); Extent extent = {size, size}; Dataset dataset = Dataset(datatype, extent); cout << "Created a Dataset of size " << dataset.extent[0] << 'x' @@ -63,7 +63,7 @@ int main(int argc, char *argv[]) cout << "File structure and required attributes have been written\n"; Offset offset = {0, 0}; - rho.storeChunk(shareRaw(global_data), offset, extent); + rho.storeChunk(global_data, offset, extent); cout << "Stored the whole Dataset contents as a single chunk, " "ready to write content\n"; diff --git a/examples/3b_write_resizable_particles.cpp b/examples/3b_write_resizable_particles.cpp index d15dba92c6..78c752313e 100644 --- a/examples/3b_write_resizable_particles.cpp +++ b/examples/3b_write_resizable_particles.cpp @@ -39,7 +39,7 @@ int main() std::vector y{-2., -3., -4., -5., -6.}; // both x and y the same type, otherwise we use two distinct datasets - Datatype dtype = determineDatatype(shareRaw(x)); + Datatype dtype = determineDatatype(x.data()); Extent size = {x.size()}; auto dataset = Dataset(dtype, size, "{ \"resizable\": true }"); diff --git a/include/openPMD/Datatype.hpp b/include/openPMD/Datatype.hpp index 39be86ab43..9ca2cb0675 100644 --- a/include/openPMD/Datatype.hpp +++ b/include/openPMD/Datatype.hpp @@ -20,6 +20,8 @@ */ #pragma once +#include "openPMD/auxiliary/TypeTraits.hpp" + #include #include #include @@ -277,161 +279,62 @@ inline constexpr Datatype determineDatatype() template inline constexpr Datatype determineDatatype(std::shared_ptr) { - using DT = Datatype; - if (decay_equiv::value) - { - return DT::CHAR; - } - else if (decay_equiv::value) - { - return DT::UCHAR; - } - else if (decay_equiv::value) - { - return DT::SCHAR; - } - else if (decay_equiv::value) - { - return DT::SHORT; - } - else if (decay_equiv::value) - { - return DT::INT; - } - else if (decay_equiv::value) - { - return DT::LONG; - } - else if (decay_equiv::value) - { - return DT::LONGLONG; - } - else if (decay_equiv::value) - { - return DT::USHORT; - } - else if (decay_equiv::value) - { - return DT::UINT; - } - else if (decay_equiv::value) - { - return DT::ULONG; - } - else if (decay_equiv::value) - { - return DT::ULONGLONG; - } - else if (decay_equiv::value) - { - return DT::FLOAT; - } - else if (decay_equiv::value) - { - return DT::DOUBLE; - } - else if (decay_equiv::value) - { - return DT::LONG_DOUBLE; - } - else if (decay_equiv>::value) - { - return DT::CFLOAT; - } - else if (decay_equiv>::value) - { - return DT::CDOUBLE; - } - else if (decay_equiv>::value) - { - return DT::CLONG_DOUBLE; - } - else if (decay_equiv::value) - { - return DT::STRING; - } - else if (decay_equiv>::value) - { - return DT::VEC_CHAR; - } - else if (decay_equiv>::value) - { - return DT::VEC_SHORT; - } - else if (decay_equiv>::value) - { - return DT::VEC_INT; - } - else if (decay_equiv>::value) - { - return DT::VEC_LONG; - } - else if (decay_equiv>::value) - { - return DT::VEC_LONGLONG; - } - else if (decay_equiv>::value) - { - return DT::VEC_UCHAR; - } - else if (decay_equiv>::value) - { - return DT::VEC_SCHAR; - } - else if (decay_equiv>::value) - { - return DT::VEC_USHORT; - } - else if (decay_equiv>::value) - { - return DT::VEC_UINT; - } - else if (decay_equiv>::value) - { - return DT::VEC_ULONG; - } - else if (decay_equiv>::value) - { - return DT::VEC_ULONGLONG; - } - else if (decay_equiv>::value) - { - return DT::VEC_FLOAT; - } - else if (decay_equiv>::value) - { - return DT::VEC_DOUBLE; - } - else if (decay_equiv>::value) - { - return DT::VEC_LONG_DOUBLE; - } - else if (decay_equiv>>::value) - { - return DT::VEC_CFLOAT; - } - else if (decay_equiv>>::value) - { - return DT::VEC_CDOUBLE; - } - else if (decay_equiv>>::value) - { - return DT::VEC_CLONG_DOUBLE; - } - else if (decay_equiv>::value) - { - return DT::VEC_STRING; - } - else if (decay_equiv>::value) - { - return DT::ARR_DBL_7; + return determineDatatype(); +} + +template +inline constexpr Datatype determineDatatype(T *) +{ + return determineDatatype(); +} + +/* + * Catch-all overload for unsupported types, with static_assert errors + * triggered at compile-time. + */ +template +inline constexpr Datatype determineDatatype(T_ContiguousContainer &&) +{ + using T_ContiguousContainer_stripped = + std::remove_reference_t; + if constexpr (auxiliary::IsContiguousContainer_v< + T_ContiguousContainer_stripped>) + { + static_assert(auxiliary::dependent_false_v, R"( +Error: Passed a contiguous container type to determineDatatype<>(). +These types are not directly supported due to colliding semantics. +Assuming a vector object `std::vector vec;`, +use one of the following alternatives: + +1) If what you want is a direct openPMD::Datatype equivalent + of the container type, use: + `determineDatatype()` + OR + `determineDatatype>()`. + The result will be `Datatype::VECTOR_FLOAT`. +2) If what you want is the openPMD::Datatype equivalent of the *contained type*, + use the raw pointer overload by: + `determineDatatype(vec.data())` + The result will be `Datatype::FLOAT`. + This is the variant that you likely wish to use if intending to write data + from the vector via `storeChunk()`, e.g. `storeChunk(vec, {0}, {10})`. + )"); } - else if (decay_equiv::value) + else { - return DT::BOOL; + static_assert(auxiliary::dependent_false_v, R"( +Error: Unknown datatype passed to determineDatatype<>(). +For a direct translation from C++ type to the openPMD::Datatype enum, use: +`auto determineDatatype() -> Datatype`. + +For detecting the contained datatpye of a pointer type (shared or raw pointer), +use either of the following overloads: +`auto determineDatatype(std::shared_ptr) -> Datatype` or +`auto determineDatatype(T *) -> Datatype`. + )"); } - else - return DT::UNDEFINED; + // Unreachable, but C++ does not know it + return Datatype::UNDEFINED; } /** Return number of bytes representing a Datatype diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index da5833794b..5ef9585520 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -21,7 +21,7 @@ #pragma once #include "openPMD/Dataset.hpp" -#include "openPMD/auxiliary/ShareRaw.hpp" +#include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" #include @@ -42,34 +42,6 @@ namespace openPMD { -namespace traits -{ - /** Emulate in the C++17 concept ContiguousContainer - * - * Users can implement this trait for a type to signal it can be used as - * contiguous container. - * - * See: - * https://en.cppreference.com/w/cpp/named_req/ContiguousContainer - */ - template - struct IsContiguousContainer - { - static constexpr bool value = false; - }; - - template - struct IsContiguousContainer > - { - static constexpr bool value = true; - }; - - template - struct IsContiguousContainer > - { - static constexpr bool value = true; - }; -} // namespace traits template class DynamicMemoryView; @@ -225,29 +197,137 @@ class RecordComponent : public BaseRecordComponent template std::shared_ptr loadChunk(Offset = {0u}, Extent = {-1u}); - /** Load a chunk of data into pre-allocated memory + /** Load a chunk of data into pre-allocated memory. + * + * @param data Preallocated, contiguous buffer, large enough to load the + * the requested data into it. + * The shared pointer must either own and manage the buffer + * or have been created via shareRaw(). + * If using shareRaw(), it is in the user of this API call's + * responsibility to ensure that the lifetime of the buffer + * exceeds the next + * flush point. + * Optimizations might be implemented based on this + * assumption (e.g. skipping the operation if the backend + * is the unique owner). + * For raw pointers, use loadChunkRaw(). + * @param offset Offset within the dataset. Set to {0u} for full selection. + * @param extent Extent within the dataset, counted from the offset. + * Set to {-1u} for full selection. + * If offset is non-zero and extent is {-1u} the leftover + * extent in the record component will be selected. + */ + template + void loadChunk(std::shared_ptr data, Offset offset, Extent extent); + + /** Load a chunk of data into pre-allocated memory, array version. * - * shared_ptr for data must be pre-allocated, contiguous and large enough - * for extent + * @param data Preallocated, contiguous buffer, large enough to load the + * the requested data into it. + * The shared pointer must own and manage the buffer. + * Optimizations might be implemented based on this + * assumption (e.g. skipping the operation if the backend + * is the unique owner). + * The array-based overload helps avoid having to manually + * specify the delete[] destructor (C++17 feature). + * @param offset Offset within the dataset. Set to {0u} for full selection. + * @param extent Extent within the dataset, counted from the offset. + * Set to {-1u} for full selection. + * If offset is non-zero and extent is {-1u} the leftover + * extent in the record component will be selected. + */ + template + void loadChunk(std::shared_ptr data, Offset offset, Extent extent); + + /** Load a chunk of data into pre-allocated memory, raw pointer version. * - * Set offset to {0u} and extent to {-1u} for full selection. + * @param data Preallocated, contiguous buffer, large enough to load the + * the requested data into it. + * It is in the user of this API call's responsibility to + * ensure that the lifetime of the buffer exceeds the next + * + * flush point. + * @param offset Offset within the dataset. Set to {0u} for full selection. + * @param extent Extent within the dataset, counted from the offset. + * Set to {-1u} for full selection. + * If offset is non-zero and extent is {-1u} the leftover + * extent in the record component will be selected. + */ + template + void loadChunkRaw(T *data, Offset offset, Extent extent); + + /** Store a chunk of data from a chunk of memory. * - * If offset is non-zero and extent is {-1u} the leftover extent in the - * record component will be selected. + * @param data Preallocated, contiguous buffer, large enough to read the + * the specified data from it. + * The shared pointer must either own and manage the buffer + * or have been created via shareRaw(). + * If using shareRaw(), it is in the user of this API call's + * responsibility to ensure that the lifetime of the buffer + * exceeds the next + * flush point. + * Optimizations might be implemented based on this + * assumption (e.g. further deferring the operation if the + * backend is the unique owner). + * For raw pointers, use storeChunkRaw(). + * @param offset Offset within the dataset. + * @param extent Extent within the dataset, counted from the offset. */ template - void loadChunk(std::shared_ptr, Offset, Extent); + void storeChunk(std::shared_ptr data, Offset offset, Extent extent); + /** Store a chunk of data from a chunk of memory, array version. + * + * @param data Preallocated, contiguous buffer, large enough to read the + * the specified data from it. + * The array-based overload helps avoid having to manually + * specify the delete[] destructor (C++17 feature). + * @param offset Offset within the dataset. + * @param extent Extent within the dataset, counted from the offset. + */ template - void storeChunk(std::shared_ptr, Offset, Extent); + void storeChunk(std::shared_ptr data, Offset offset, Extent extent); + /** Store a chunk of data from a chunk of memory, raw pointer version. + * + * @param data Preallocated, contiguous buffer, large enough to read the + * the specified data from it. + * It is in the user of this API call's responsibility to + * ensure that the lifetime of the buffer exceeds the next + * + * flush point. + * @param offset Offset within the dataset. + * @param extent Extent within the dataset, counted from the offset. + */ template - void storeChunk(std::shared_ptr, Offset, Extent); + void storeChunkRaw(T *data, Offset offset, Extent extent); + /** Store a chunk of data from a contiguous container. + * + * @param data + * Contiguous container, large enough to read the the + * specified data from it. A contiguous container in here is + * either a std::vector or a std::array. + * It is in the user of this API call's responsibility to + * ensure that the lifetime of the container exceeds the next + * + * flush point. + * @param offset Offset within the dataset. + * @param extent Extent within the dataset, counted from the offset. + */ template - typename std::enable_if< - traits::IsContiguousContainer::value>::type - storeChunk(T_ContiguousContainer &, Offset = {0u}, Extent = {-1u}); + typename std::enable_if_t< + auxiliary::IsContiguousContainer_v> + storeChunk( + T_ContiguousContainer &data, + Offset offset = {0u}, + Extent extent = {-1u}); /** * @brief Overload of storeChunk() that lets the openPMD API allocate @@ -257,14 +337,17 @@ class RecordComponent : public BaseRecordComponent * users a view into its own buffers, avoiding the need to allocate * a new buffer. * - * Data can be written into the returned buffer until the next call to - * Series::flush() at which time the data will be read from. + * Data can be written into the returned buffer until the next + * flush point at which time the data will be read from. * * In order to provide a view into backend buffers, this call must possibly * create files and datasets in the backend, making it MPI-collective. * In order to avoid this, calling Series::flush() prior to this is * recommended to flush definitions. * + * @param offset Offset within the dataset. + * @param extent Extent within the dataset, counted from the offset. * @param createBuffer If the backend in use has no special support for this * operation, the openPMD API will fall back to creating a buffer, * queuing it for writing and returning a view into that buffer to @@ -280,7 +363,8 @@ class RecordComponent : public BaseRecordComponent * @return View into a buffer that can be filled with data. */ template - DynamicMemoryView storeChunk(Offset, Extent, F &&createBuffer); + DynamicMemoryView + storeChunk(Offset offset, Extent extent, F &&createBuffer); /** * Overload of span-based storeChunk() that uses operator new() to create diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index a4731d5434..671932e470 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -23,6 +23,8 @@ #include "openPMD/RecordComponent.hpp" #include "openPMD/Span.hpp" +#include "openPMD/auxiliary/ShareRawInternal.hpp" +#include "openPMD/auxiliary/TypeTraits.hpp" namespace openPMD { @@ -76,9 +78,16 @@ inline std::shared_ptr< T > RecordComponent::loadChunk( for( auto const& dimensionSize : extent ) numPoints *= dimensionSize; - auto newData = std::shared_ptr(new T[numPoints], []( T *p ){ delete [] p; }); +#if defined(__clang_major__) && __clang_major__ < 7 + auto newData = + std::shared_ptr(new T[numPoints], [](T *p) { delete[] p; }); loadChunk(newData, offset, extent); return newData; +#else + auto newData = std::shared_ptr(new T[numPoints]); + loadChunk(newData, offset, extent); + return std::static_pointer_cast(std::move(newData)); +#endif } template< typename T > @@ -157,6 +166,25 @@ inline void RecordComponent::loadChunk( } } +template +inline void RecordComponent::loadChunk( + std::shared_ptr ptr, Offset offset, Extent extent) +{ + loadChunk( + std::static_pointer_cast(std::move(ptr)), + std::move(offset), + std::move(extent)); +} + +template +inline void RecordComponent::loadChunkRaw(T *ptr, Offset offset, Extent extent) +{ + loadChunk( + auxiliary::shareRaw(ptr), + std::move(offset), + std::move(extent)); +} + template< typename T > inline void RecordComponent::storeChunk(std::shared_ptr data, Offset o, Extent e) @@ -218,11 +246,19 @@ RecordComponent::storeChunk(std::shared_ptr data, Offset o, Extent e) std::move(e)); } +template +void RecordComponent::storeChunkRaw(T *ptr, Offset offset, Extent extent) +{ + storeChunk( + auxiliary::shareRaw(ptr), + std::move(offset), + std::move(extent)); +} + template< typename T_ContiguousContainer > -inline typename std::enable_if< - traits::IsContiguousContainer< T_ContiguousContainer >::value ->::type -RecordComponent::storeChunk(T_ContiguousContainer & data, Offset o, Extent e) +inline typename std::enable_if_t< + auxiliary::IsContiguousContainer_v > +RecordComponent::storeChunk(T_ContiguousContainer &data, Offset o, Extent e) { uint8_t dim = getDimensionality(); @@ -241,7 +277,10 @@ RecordComponent::storeChunk(T_ContiguousContainer & data, Offset o, Extent e) else extent = e; - storeChunk(shareRaw(data), offset, extent); + storeChunk( + auxiliary::shareRaw(data.data()), + offset, + extent); } template< typename T, typename F > diff --git a/include/openPMD/auxiliary/Mpi.hpp b/include/openPMD/auxiliary/Mpi.hpp index 5201f68bf2..940ec026a3 100644 --- a/include/openPMD/auxiliary/Mpi.hpp +++ b/include/openPMD/auxiliary/Mpi.hpp @@ -22,6 +22,8 @@ #include "openPMD/config.hpp" +#include "openPMD/auxiliary/TypeTraits.hpp" + #if openPMD_HAVE_MPI #include #endif @@ -32,16 +34,6 @@ namespace openPMD::auxiliary { #if openPMD_HAVE_MPI -namespace detail -{ - namespace - { - // see https://en.cppreference.com/w/cpp/language/if - template - inline constexpr bool dependent_false_v = false; - } // namespace -} // namespace detail - namespace { template @@ -67,8 +59,7 @@ namespace else { static_assert( - detail::dependent_false_v, - "openPMD_MPI_type: Unsupported type."); + dependent_false_v, "openPMD_MPI_type: Unsupported type."); } } } // namespace diff --git a/include/openPMD/auxiliary/ShareRaw.hpp b/include/openPMD/auxiliary/ShareRaw.hpp index e3d2d1efb7..550560e43a 100644 --- a/include/openPMD/auxiliary/ShareRaw.hpp +++ b/include/openPMD/auxiliary/ShareRaw.hpp @@ -42,18 +42,29 @@ namespace openPMD * reference counting. */ template -std::shared_ptr shareRaw(T *x) +[[deprecated( + "For storing/loading data via raw pointers use " + "storeChunkRaw<>()/loadChunkRaw<>()")]] // +std::shared_ptr +shareRaw(T *x) { return std::shared_ptr(x, [](T *) {}); } template -std::shared_ptr shareRaw(T const *x) +[[deprecated( + "For storing/loading data via raw pointers use " + "storeChunkRaw<>()/loadChunkRaw<>()")]] // +std::shared_ptr +shareRaw(T const *x) { return std::shared_ptr(x, [](T const *) {}); } template +[[deprecated( + "For storing/loading data via raw pointers use " + "storeChunkRaw<>()/loadChunkRaw<>()")]] // auto shareRaw(T &c) -> std::shared_ptr::type> { @@ -62,6 +73,9 @@ auto shareRaw(T &c) } template +[[deprecated( + "For storing/loading data via raw pointers use " + "storeChunkRaw<>()/loadChunkRaw<>()")]] // auto shareRaw(T const &c) -> std::shared_ptr::type> { diff --git a/include/openPMD/auxiliary/ShareRawInternal.hpp b/include/openPMD/auxiliary/ShareRawInternal.hpp new file mode 100644 index 0000000000..b26f0b59f9 --- /dev/null +++ b/include/openPMD/auxiliary/ShareRawInternal.hpp @@ -0,0 +1,79 @@ +/* Copyright 2018-2021 Axel Huebl + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include +#include +#include +#include + +/* + * This header copies ShareRaw.hpp, but: + * 1. without deprecation markings + * 2. for internal usage only + * 3. inside auxiliary namespace + */ + +namespace openPMD::auxiliary +{ +//! @{ +/** Share ownership with a raw pointer + * + * Helper function to share load/store data ownership + * unprotected and without reference counting with a + * raw pointer or stdlib container (that implements a + * contiguous data storage). + * + * @warning this is a helper function to bypass the shared-pointer + * API for storing data behind raw pointers. Using it puts + * the responsibility of buffer-consistency between stores + * and flushes to the users side without an indication via + * reference counting. + */ +template +std::shared_ptr shareRaw(T *x) +{ + return std::shared_ptr(x, [](T *) {}); +} + +template +std::shared_ptr shareRaw(T const *x) +{ + return std::shared_ptr(x, [](T const *) {}); +} + +template +auto shareRaw(T &c) + -> std::shared_ptr::type> +{ + using value_type = typename std::remove_pointer::type; + return std::shared_ptr(c.data(), [](value_type *) {}); +} + +template +auto shareRaw(T const &c) + -> std::shared_ptr::type> +{ + using value_type = typename std::remove_pointer::type; + return std::shared_ptr(c.data(), [](value_type *) {}); +} +//! @} +} // namespace openPMD::auxiliary diff --git a/include/openPMD/auxiliary/TypeTraits.hpp b/include/openPMD/auxiliary/TypeTraits.hpp index c3ed8a1cd8..b7333cfdfe 100644 --- a/include/openPMD/auxiliary/TypeTraits.hpp +++ b/include/openPMD/auxiliary/TypeTraits.hpp @@ -59,4 +59,22 @@ inline constexpr bool IsVector_v = detail::IsVector::value; template inline constexpr bool IsArray_v = detail::IsArray::value; + +/** Emulate in the C++ concept ContiguousContainer + * + * Users can implement this trait for a type to signal it can be used as + * contiguous container. + * + * See: + * https://en.cppreference.com/w/cpp/named_req/ContiguousContainer + */ +template +inline constexpr bool IsContiguousContainer_v = IsVector_v || IsArray_v; + +namespace +{ + // see https://en.cppreference.com/w/cpp/language/if + template + inline constexpr bool dependent_false_v = false; +} // namespace } // namespace openPMD::auxiliary diff --git a/include/openPMD/backend/PatchRecordComponent.hpp b/include/openPMD/backend/PatchRecordComponent.hpp index 280b674ceb..718adee7d5 100644 --- a/include/openPMD/backend/PatchRecordComponent.hpp +++ b/include/openPMD/backend/PatchRecordComponent.hpp @@ -20,6 +20,7 @@ */ #pragma once +#include "openPMD/auxiliary/ShareRawInternal.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" #include @@ -84,6 +85,12 @@ class PatchRecordComponent : public BaseRecordComponent template void load(std::shared_ptr); + template + void load(std::shared_ptr); + + template + void loadRaw(T *); + template void store(uint64_t idx, T); @@ -137,10 +144,16 @@ template inline std::shared_ptr PatchRecordComponent::load() { uint64_t numPoints = getExtent()[0]; +#if defined(__clang_major__) && __clang_major__ < 7 auto newData = std::shared_ptr(new T[numPoints], [](T *p) { delete[] p; }); load(newData); return newData; +#else + auto newData = std::shared_ptr(new T[numPoints]); + load(newData); + return std::static_pointer_cast(std::move(newData)); +#endif } template @@ -169,6 +182,18 @@ inline void PatchRecordComponent::load(std::shared_ptr data) rc.m_chunks.push(IOTask(this, dRead)); } +template +inline void PatchRecordComponent::load(std::shared_ptr data) +{ + load(std::static_pointer_cast(std::move(data))); +} + +template +inline void PatchRecordComponent::loadRaw(T *data) +{ + load(auxiliary::shareRaw(data)); +} + template inline void PatchRecordComponent::store(uint64_t idx, T data) { diff --git a/src/binding/python/PatchRecordComponent.cpp b/src/binding/python/PatchRecordComponent.cpp index a734e1d6b9..f2fe0aa753 100644 --- a/src/binding/python/PatchRecordComponent.cpp +++ b/src/binding/python/PatchRecordComponent.cpp @@ -22,7 +22,6 @@ #include #include "openPMD/DatatypeHelpers.hpp" -#include "openPMD/auxiliary/ShareRaw.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" #include "openPMD/backend/PatchRecordComponent.hpp" #include "openPMD/binding/python/Numpy.hpp" @@ -37,7 +36,7 @@ struct Prc_Load template static void call(PatchRecordComponent &prc, py::array &a) { - prc.load(shareRaw((T *)a.mutable_data())); + prc.loadRaw((T *)a.mutable_data()); } static constexpr char const *errorMsg = "Datatype not known in 'load'!"; diff --git a/src/binding/python/RecordComponent.cpp b/src/binding/python/RecordComponent.cpp index c2c1acfebb..1b6c4ebee5 100644 --- a/src/binding/python/RecordComponent.cpp +++ b/src/binding/python/RecordComponent.cpp @@ -25,7 +25,6 @@ #include "openPMD/DatatypeHelpers.hpp" #include "openPMD/RecordComponent.hpp" #include "openPMD/Series.hpp" -#include "openPMD/auxiliary/ShareRaw.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" #include "openPMD/binding/python/Numpy.hpp" #include "openPMD/binding/python/Pickle.hpp" diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index fa2a1cad42..42045789a5 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -836,7 +836,7 @@ TEST_CASE("wrapper_test", "[core]") MeshRecordComponent mrc2 = o.iterations[4].meshes["E"]["y"]; REQUIRE(mrc2.constant()); double loadData; - mrc2.loadChunk(shareRaw(&loadData), {0}, {1}); + mrc2.loadChunkRaw(&loadData, {0}, {1}); o.flush(); REQUIRE(loadData == value); // TODO: do we want to be able to make data constant after already writing @@ -846,7 +846,7 @@ TEST_CASE("wrapper_test", "[core]") Catch::Equals("A recordComponent can not (yet) be made constant after " "it has been written.")); std::array moreData = {{112233.}}; - o.iterations[4].meshes["E"]["y"].loadChunk(shareRaw(moreData), {0}, {1}); + o.iterations[4].meshes["E"]["y"].loadChunkRaw(moreData.data(), {0}, {1}); o.flush(); REQUIRE(moreData[0] == value); auto all_data = o.iterations[4].meshes["E"]["y"].loadChunk(); @@ -862,8 +862,7 @@ TEST_CASE("wrapper_test", "[core]") Dataset(Datatype::DOUBLE, {1})); int wrongData = 42; REQUIRE_THROWS_WITH( - o.iterations[5].meshes["E"]["y"].storeChunk( - shareRaw(&wrongData), {0}, {1}), + o.iterations[5].meshes["E"]["y"].storeChunkRaw(&wrongData, {0}, {1}), Catch::Equals("Datatypes of chunk data (INT) and record component " "(DOUBLE) do not match.")); std::shared_ptr storeData = std::make_shared(44); diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index aaf1fd4378..99cd472166 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -858,7 +858,7 @@ void file_based_write_read(std::string file_ending) Offset chunk_offset = {0, local_Nz * mpi_rank}; Extent chunk_extent = {global_Nx, local_Nz}; - E_x.storeChunk(io::shareRaw(E_x_data), chunk_offset, chunk_extent); + E_x.storeChunk(E_x_data, chunk_offset, chunk_extent); series.flush(); } } diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 20d20d6a71..61172aa5f4 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1814,8 +1814,7 @@ inline void fileBased_write_test(const std::string &backend) for (uint64_t i = 0; i < 4; ++i) { double const position_local_2 = position_global.at(i); - e_2["position"]["x"].storeChunk( - shareRaw(&position_local_2), {i}, {1}); + e_2["position"]["x"].storeChunkRaw(&position_local_2, {i}, {1}); *positionOffset_local_2 = positionOffset_global[i]; e_2["positionOffset"]["x"].storeChunk( positionOffset_local_2, {i}, {1}); @@ -3572,7 +3571,7 @@ TEST_CASE("hzdr_hdf5_sample_content_test", "[serial][hdf5]") isSame(e_extent_z.getDatatype(), determineDatatype())); std::vector data(e_patches.size()); - e_extent_z.load(shareRaw(data.data())); + e_extent_z.loadRaw(data.data()); species_e.seriesFlush(); REQUIRE(data.at(0) == static_cast(80)); REQUIRE(data.at(1) == static_cast(80)); @@ -3594,7 +3593,7 @@ TEST_CASE("hzdr_hdf5_sample_content_test", "[serial][hdf5]") e_numParticles_scalar.getDatatype(), determineDatatype())); - e_numParticles_scalar.load(shareRaw(data.data())); + e_numParticles_scalar.loadRaw(data.data()); o.flush(); REQUIRE(data.at(0) == static_cast(512000)); REQUIRE(data.at(1) == static_cast(819200)); @@ -3640,7 +3639,7 @@ TEST_CASE("hzdr_hdf5_sample_content_test", "[serial][hdf5]") REQUIRE( isSame(e_offset_y.getDatatype(), determineDatatype())); - e_offset_y.load(shareRaw(data.data())); + e_offset_y.loadRaw(data.data()); o.flush(); REQUIRE(data.at(0) == static_cast(0)); REQUIRE(data.at(1) == static_cast(128)); @@ -6615,8 +6614,8 @@ void groupbased_read_write(std::string const &ext) auto E_y = write.iterations[0].meshes["E"]["y"]; E_x.resetDataset(ds); E_y.resetDataset(ds); - E_x.storeChunk(shareRaw(&data), {0}, {1}); - E_y.storeChunk(shareRaw(&data), {0}, {1}); + E_x.storeChunkRaw(&data, {0}, {1}); + E_y.storeChunkRaw(&data, {0}, {1}); E_x.setAttribute("updated_in_run", 0); E_y.setAttribute("updated_in_run", 0); @@ -6633,8 +6632,8 @@ void groupbased_read_write(std::string const &ext) data = 1; - E_x.storeChunk(shareRaw(&data), {0}, {1}); - E_y.storeChunk(shareRaw(&data), {0}, {1}); + E_x.storeChunkRaw(&data, {0}, {1}); + E_y.storeChunkRaw(&data, {0}, {1}); E_x.setAttribute("updated_in_run", 1); E_y.setAttribute("updated_in_run", 1); @@ -6670,7 +6669,7 @@ void groupbased_read_write(std::string const &ext) data = 2; - E_x.storeChunk(shareRaw(&data), {0}, {1}); + E_x.storeChunkRaw(&data, {0}, {1}); E_x.setAttribute("updated_in_run", 2); } From a83bc225b5c347d07e11b38cca75ff8970ca3f30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 25 Oct 2022 19:47:29 +0200 Subject: [PATCH 66/70] Mapping between ADIOS steps and openPMD iterations (#949) * Backend additions 1) New streaming status: RANDOM_ACCESS, for non-streaming situations 2) Variable attributes, to be written only if the backend has support for steps * Writing changes: Write current step(s) to snapshot attribute Only set snapshot attribute if Iteration is not yet written For v-based iteration encoding, the snapshot attribute is already being set before this PR. Just add a comment there. Also add missing includes Co-authored-by: Axel Huebl * Reading changes: Use snapshot attribute This means that the snapshot attribute, if present, is used for accessing iterations inside `series.readIterations()`. Fallback to the old behavior (linear progression through iterations) if the attribute is not found. Variable-b. encoding: Allow several (equivalent) iterations per step This means that a single step can be marked by /data/snapshot to represent iterations 0,10,20,30 at the same time. The underlying data is the same, but the API will treat it as 4 times a different iteration with equivalent content. Avoid const_cast by introducing a parsing state and use that when re-parsing. Skip repeated iterations that occur in Append mode Before the explicit iteration-step mapping, these were not seen by reading procedures at all. Now they are, so we skip the second instance. Better error message when calling readIterations() too late This commit includes some refactoring 1. Remove recursion of operator++(), this leads to constant memory usage rather than filling the stack at some point 2. Extract subroutines from operator++() 3. Steal some refactoring that solved some bugs on topic-read-leniently, so it stands to reason that we should apply it here already * Testing In the tests, don't try to read the series with listSeries after already having fully drained it Combined test: append mode and weird iteration order Deactivate troublesome Schema 2021 Append test * Add -wd1011 flag to Icc workflow * Fix priority of JSON/envvar config for ADIOS2 schema * Preview support for Linear read mode without snapshot attribute Currently only available for BP5 engine, will be generalized into Linear read mode in #1291. If the backend does not support the snapshot attribute, then iterate in ascending order, skipping duplicate and non-linear iteration indices. Not possible if the Series is parsed ahead of time. * Test edge cases of snapshot attribute Co-authored-by: Axel Huebl --- .github/workflows/intel.yml | 7 +- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 15 +- include/openPMD/IO/AbstractIOHandler.hpp | 23 ++ include/openPMD/IO/AbstractIOHandlerImpl.hpp | 15 +- include/openPMD/IO/IOTask.hpp | 8 + include/openPMD/Iteration.hpp | 50 ++- include/openPMD/ReadIterations.hpp | 38 ++- include/openPMD/Series.hpp | 29 +- include/openPMD/Streaming.hpp | 5 +- include/openPMD/backend/Attributable.hpp | 4 +- include/openPMD/backend/Container.hpp | 8 +- src/IO/ADIOS/ADIOS2IOHandler.cpp | 21 +- src/IO/ADIOS/CommonADIOS1IOHandler.cpp | 5 + src/IO/HDF5/HDF5IOHandler.cpp | 5 + src/IO/JSON/JSONIOHandlerImpl.cpp | 5 + src/Iteration.cpp | 97 ++++-- src/ReadIterations.cpp | 309 +++++++++++++++---- src/Series.cpp | 269 +++++++++++++--- test/JSONTest.cpp | 87 ++++++ test/ParallelIOTest.cpp | 29 +- test/SerialIOTest.cpp | 259 +++++++++++++--- test/python/unittest/API/APITest.py | 5 +- 22 files changed, 1093 insertions(+), 200 deletions(-) diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index 2b3cea6233..39a3d51e28 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -17,7 +17,12 @@ jobs: run: | sudo .github/workflows/dependencies/install_icc - name: Build - env: {CXXFLAGS: -Werror} + # Due to compiler bugs in Intel compiler, we need to disable warning 1011 + # (missing return value), otherwise `if constexpr` functions + # don't compile. + # See https://community.intel.com/t5/Intel-C-Compiler/quot-if-constexpr-quot-and-quot-missing-return-statement-quot-in/td-p/1154551 + # Using a local pragma does not work due to the reasons stated there. + env: {CXXFLAGS: -Werror -wd1011} run: | set +e; source /opt/intel/oneapi/setvars.sh; set -e share/openPMD/download_samples.sh build diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index d3ccc93258..bc8ea80ad5 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -1155,13 +1155,6 @@ namespace detail */ void invalidateVariablesMap(); - private: - ADIOS2IOHandlerImpl *m_impl; - std::optional m_engine; //! ADIOS engine - /** - * The ADIOS2 engine type, to be passed to adios2::IO::SetEngine - */ - std::string m_engineType; /* * streamStatus is NoStream for file-based ADIOS engines. * This is relevant for the method BufferedActions::requireActiveStep, @@ -1253,6 +1246,14 @@ namespace detail }; StreamStatus streamStatus = StreamStatus::OutsideOfStep; + private: + ADIOS2IOHandlerImpl *m_impl; + std::optional m_engine; //! ADIOS engine + /** + * The ADIOS2 engine type, to be passed to adios2::IO::SetEngine + */ + std::string m_engineType; + /** * See documentation for StreamStatus::Parsing. * Will be set true under the circumstance described there in order to diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 4f6916ae55..7627b66524 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -121,6 +121,28 @@ namespace internal FlushParams const defaultFlushParams{}; struct ParsedFlushParams; + + /** + * Some parts of the openPMD object model are read-only when accessing + * a Series in Access::READ_ONLY mode, notably Containers and Attributes. + * They are filled at parse time and not modified afterwards. + * Such state-changing operations are hence allowed under either of two + * conditions: + * 1) The Series is opened in an open mode that allows writing in any way. + * (Currently any but Access::READ_ONLY). + * 2) The Series is in Parsing state. This way, modifying the open mode + * during parsing can be avoided. + */ + enum class SeriesStatus : unsigned char + { + Default, ///< Mutability of objects in the openPMD object model is + ///< determined by the open mode (Access enum), normal state in + ///< which the user interacts with the Series. + Parsing ///< All objects in the openPMD object model are temporarily + ///< mutable to allow inserting newly-parsed data. + ///< Special state only active while internal routines are + ///< running. + }; } // namespace internal /** Interface for communicating between logical and physically persistent data. @@ -192,6 +214,7 @@ class AbstractIOHandler // why do these need to be separate? Access const m_backendAccess; Access const m_frontendAccess; + internal::SeriesStatus m_seriesStatus = internal::SeriesStatus::Default; std::queue m_work; }; // AbstractIOHandler diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index 55ff021d06..04820a28d4 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -252,8 +252,10 @@ class AbstractIOHandlerImpl * The advance mode is determined by parameters.mode. * The return status code shall be stored as parameters.status. */ - virtual void advance(Writable *, Parameter &) - {} + virtual void advance(Writable *, Parameter ¶meters) + { + *parameters.status = AdvanceStatus::RANDOMACCESS; + } /** Close an openPMD group. * @@ -488,8 +490,13 @@ class AbstractIOHandlerImpl * datatype parameters.dtype. Any existing attribute with the same name * should be overwritten. If possible, only the value should be changed if * the datatype stays the same. The attribute should be written to physical - * storage after the operation completes successfully. All datatypes of - * Datatype should be supported in a type-safe way. + * storage after the operation completes successfully. If the parameter + * changesOverSteps is true, then the attribute must be able to hold + * different values across IO steps. If the backend does not support IO + * steps in such a way, the attribute should not be written. (IO steps are + * an optional backend feature and the frontend must implement fallback + * measures in such a case) All datatypes of Datatype should be supported in + * a type-safe way. */ virtual void writeAttribute(Writable *, Parameter const &) = 0; diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 3ba7d09e24..88c7d0380b 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -537,6 +537,7 @@ struct OPENPMDAPI_EXPORT Parameter : AbstractParameter() , name(p.name) , dtype(p.dtype) + , changesOverSteps(p.changesOverSteps) , resource(p.resource) {} @@ -548,6 +549,13 @@ struct OPENPMDAPI_EXPORT Parameter std::string name = ""; Datatype dtype = Datatype::UNDEFINED; + /* + * If true, this attribute changes across IO steps. + * It should only be written in backends that support IO steps, + * otherwise writing should be skipped. + * The frontend is responsible for handling both situations. + */ + bool changesOverSteps = false; Attribute::resource resource; }; diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index ae58e787e1..2d14313cfa 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -28,7 +28,10 @@ #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" +#include +#include #include +#include namespace openPMD { @@ -282,14 +285,57 @@ class Iteration : public Attributable void readGorVBased(std::string const &groupPath, bool beginStep); void read_impl(std::string const &groupPath); + /** + * Status after beginning an IO step. Currently includes: + * * The advance status (OK, OVER, RANDOMACCESS) + * * The opened iterations, in case the snapshot attribute is found + */ + struct BeginStepStatus + { + using AvailableIterations_t = std::optional >; + + AdvanceStatus stepStatus{}; + /* + * If the iteration attribute `snapshot` is present, the value of that + * attribute. Otherwise empty. + */ + AvailableIterations_t iterationsInOpenedStep; + + /* + * Most of the time, the AdvanceStatus part of this struct is what we + * need, so let's make it easy to access. + */ + inline operator AdvanceStatus() const + { + return stepStatus; + } + + /* + * Support for std::tie() + */ + inline operator std::tuple() + { + return std::tuple{ + stepStatus, iterationsInOpenedStep}; + } + }; + /** * @brief Begin an IO step on the IO file (or file-like object) * containing this iteration. In case of group-based iteration * layout, this will be the complete Series. * - * @return AdvanceStatus + * @return BeginStepStatus + */ + BeginStepStatus beginStep(bool reread); + + /* + * Iteration-independent variant for beginStep(). + * Useful in group-based iteration encoding where the Iteration will only + * be known after opening the step. */ - AdvanceStatus beginStep(bool reread); + static BeginStepStatus + beginStep(std::optional thisObject, Series &series, bool reread); /** * @brief End an IO step on the IO file (or file-like object) diff --git a/include/openPMD/ReadIterations.hpp b/include/openPMD/ReadIterations.hpp index 473a4fae36..c5b2720dce 100644 --- a/include/openPMD/ReadIterations.hpp +++ b/include/openPMD/ReadIterations.hpp @@ -23,6 +23,8 @@ #include "openPMD/Iteration.hpp" #include "openPMD/Series.hpp" +#include +#include #include namespace openPMD @@ -54,7 +56,8 @@ class SeriesIterator using maybe_series_t = std::optional; maybe_series_t m_series; - iteration_index_t m_currentIteration = 0; + std::deque m_iterationsInCurrentStep; + uint64_t m_currentIteration{}; public: //! construct the end() iterator @@ -71,6 +74,39 @@ class SeriesIterator bool operator!=(SeriesIterator const &other) const; static SeriesIterator end(); + +private: + inline bool setCurrentIteration() + { + if (m_iterationsInCurrentStep.empty()) + { + std::cerr << "[ReadIterations] Encountered a step without " + "iterations. Closing the Series." + << std::endl; + *this = end(); + return false; + } + m_currentIteration = *m_iterationsInCurrentStep.begin(); + return true; + } + + inline std::optional peekCurrentIteration() + { + if (m_iterationsInCurrentStep.empty()) + { + return std::nullopt; + } + else + { + return {*m_iterationsInCurrentStep.begin()}; + } + } + + std::optional nextIterationInStep(); + + std::optional nextStep(); + + std::optional loopBody(); }; /** diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index cda0956b52..362001605e 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -37,8 +37,11 @@ #include #endif +#include +#include #include #include +#include #include // expose private and protected members for invasive testing @@ -82,6 +85,12 @@ namespace internal * the same instance. */ std::optional m_writeIterations; + /** + * For writing: Remember which iterations have been written in the + * currently active output step. Use this later when writing the + * snapshot attribute. + */ + std::set m_currentlyActiveIterations; /** * Needed if reading a single iteration of a file-based series. * Users may specify the concrete filename of one iteration instead of @@ -576,8 +585,10 @@ OPENPMD_private * Note on re-parsing of a Series: * If init == false, the parsing process will seek for new * Iterations/Records/Record Components etc. + * If series.iterations contains the attribute `snapshot`, returns its + * value. */ - void readGorVBased(bool init = true); + std::optional > readGorVBased(bool init = true); void readBase(); std::string iterationFilename(uint64_t i); @@ -627,6 +638,22 @@ OPENPMD_private internal::AttributableData &file, iterations_iterator it, Iteration &iteration); + + AdvanceStatus advance(AdvanceMode mode); + + /** + * @brief Called at the end of an IO step to store the iterations defined + * in the IO step to the snapshot attribute. + * + * @param doFlush If true, flush the IO handler. + */ + void flushStep(bool doFlush); + + /* + * Returns the current content of the /data/snapshot attribute. + * (We could also add this to the public API some time) + */ + std::optional > currentSnapshot() const; }; // Series } // namespace openPMD diff --git a/include/openPMD/Streaming.hpp b/include/openPMD/Streaming.hpp index 7bc84341aa..94854662fa 100644 --- a/include/openPMD/Streaming.hpp +++ b/include/openPMD/Streaming.hpp @@ -19,8 +19,9 @@ namespace openPMD */ enum class AdvanceStatus : unsigned char { - OK, /* stream goes on */ - OVER /* stream is over */ + OK, ///< stream goes on + OVER, ///< stream is over + RANDOMACCESS ///< there is no stream, it will never be over }; /** diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 1c9dda8429..8d34ee7935 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -463,7 +463,9 @@ inline bool Attributable::setAttributeImpl( internal::attr_value_check(key, value, setAttributeMode); auto &attri = get(); - if (IOHandler() && Access::READ_ONLY == IOHandler()->m_frontendAccess) + if (IOHandler() && + IOHandler()->m_seriesStatus == internal::SeriesStatus::Default && + Access::READ_ONLY == IOHandler()->m_frontendAccess) { auxiliary::OutOfRangeMsg const out_of_range_msg( "Attribute", "can not be set (read-only)."); diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 9dd163ecae..8db82c69f0 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -288,7 +288,9 @@ class Container : public Attributable return it->second; else { - if (Access::READ_ONLY == IOHandler()->m_frontendAccess) + if (IOHandler()->m_seriesStatus != + internal::SeriesStatus::Parsing && + Access::READ_ONLY == IOHandler()->m_frontendAccess) { auxiliary::OutOfRangeMsg const out_of_range_msg; throw std::out_of_range(out_of_range_msg(key)); @@ -321,7 +323,9 @@ class Container : public Attributable return it->second; else { - if (Access::READ_ONLY == IOHandler()->m_frontendAccess) + if (IOHandler()->m_seriesStatus != + internal::SeriesStatus::Parsing && + Access::READ_ONLY == IOHandler()->m_frontendAccess) { auxiliary::OutOfRangeMsg out_of_range_msg; throw std::out_of_range(out_of_range_msg(key)); diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index fcdfffc1f8..95362b2687 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -134,6 +134,10 @@ void ADIOS2IOHandlerImpl::init(json::TracingJSON cfg) m_engineType.end(), m_engineType.begin(), [](unsigned char c) { return std::tolower(c); }); + + // environment-variable based configuration + m_schema = auxiliary::getEnvNum("OPENPMD2_ADIOS2_SCHEMA", m_schema); + if (cfg.json().contains("adios2")) { m_config = cfg["adios2"]; @@ -179,8 +183,6 @@ void ADIOS2IOHandlerImpl::init(json::TracingJSON cfg) defaultOperators = std::move(operators.value()); } } - // environment-variable based configuration - m_schema = auxiliary::getEnvNum("OPENPMD2_ADIOS2_SCHEMA", m_schema); } std::optional> @@ -815,6 +817,11 @@ void ADIOS2IOHandlerImpl::writeAttribute( switch (attributeLayout()) { case AttributeLayout::ByAdiosAttributes: + if (parameters.changesOverSteps) + { + // cannot do this + return; + } switchType( parameters.dtype, this, writable, parameters); break; @@ -829,6 +836,13 @@ void ADIOS2IOHandlerImpl::writeAttribute( auto prefix = filePositionToString(pos); auto &filedata = getFileData(file, IfFileNotOpen::ThrowError); + if (parameters.changesOverSteps && + filedata.streamStatus == + detail::BufferedActions::StreamStatus::NoStream) + { + // cannot do this + return; + } filedata.requireActiveStep(); filedata.invalidateAttributesMap(); m_dirty.emplace(std::move(file)); @@ -2802,6 +2816,7 @@ namespace detail "[ADIOS2] Operation requires active step but no step is " "left."); case AdvanceStatus::OK: + case AdvanceStatus::RANDOMACCESS: // pass break; } @@ -3005,7 +3020,7 @@ namespace detail m_IO.DefineAttribute( ADIOS2Defaults::str_usesstepsAttribute, 0); flush({FlushLevel::UserFlush}, /* writeAttributes = */ false); - return AdvanceStatus::OK; + return AdvanceStatus::RANDOMACCESS; } /* diff --git a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp index 3dbee3e2db..3a13dc7fc8 100644 --- a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp @@ -1129,6 +1129,11 @@ template void CommonADIOS1IOHandlerImpl::writeAttribute( Writable *writable, Parameter const ¶meters) { + if (parameters.changesOverSteps) + { + // cannot do this + return; + } if (m_handler->m_backendAccess == Access::READ_ONLY) throw std::runtime_error( "[ADIOS1] Writing an attribute in a file opened as read only is " diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 94dce55c1c..01979d8071 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -1328,6 +1328,11 @@ void HDF5IOHandlerImpl::writeDataset( void HDF5IOHandlerImpl::writeAttribute( Writable *writable, Parameter const ¶meters) { + if (parameters.changesOverSteps) + { + // cannot do this + return; + } if (m_handler->m_backendAccess == Access::READ_ONLY) throw std::runtime_error( "[HDF5] Writing an attribute in a file opened as read only is not " diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 062c736433..272478789b 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -797,6 +797,11 @@ void JSONIOHandlerImpl::writeDataset( void JSONIOHandlerImpl::writeAttribute( Writable *writable, Parameter const ¶meter) { + if (parameter.changesOverSteps) + { + // cannot do this + return; + } if (m_handler->m_backendAccess == Access::READ_ONLY) { throw std::runtime_error( diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 575610ea16..5ef4ac0274 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -282,6 +282,13 @@ void Iteration::flushVariableBased( Parameter pOpen; pOpen.path = ""; IOHandler()->enqueue(IOTask(this, pOpen)); + /* + * In v-based encoding, the snapshot attribute must always be written, + * so don't set the `changesOverSteps` flag of the IOTask here. + * Reason: Even in backends that don't support changing attributes, + * variable-based iteration encoding can be used to write one single + * iteration. Then, this attribute determines which iteration it is. + */ this->setAttribute("snapshot", i); } @@ -566,49 +573,92 @@ void Iteration::read_impl(std::string const &groupPath) readAttributes(ReadMode::FullyReread); } -AdvanceStatus Iteration::beginStep(bool reread) +auto Iteration::beginStep(bool reread) -> BeginStepStatus { - using IE = IterationEncoding; + BeginStepStatus res; auto series = retrieveSeries(); + return beginStep({*this}, series, reread); +} + +auto Iteration::beginStep( + std::optional thisObject, Series &series, bool reread) + -> BeginStepStatus +{ + BeginStepStatus res; + using IE = IterationEncoding; // Initialize file with this to quiet warnings // The following switch is comprehensive internal::AttributableData *file = nullptr; switch (series.iterationEncoding()) { case IE::fileBased: - file = &Attributable::get(); + if (thisObject.has_value()) + { + file = &static_cast(*thisObject).get(); + } + else + { + throw error::Internal( + "Advancing a step in file-based iteration encoding is " + "iteration-specific."); + } break; case IE::groupBased: case IE::variableBased: file = &series.get(); break; } - AdvanceStatus status = series.advance( - AdvanceMode::BEGINSTEP, *file, series.indexOf(*this), *this); - if (status != AdvanceStatus::OK) + + AdvanceStatus status; + if (thisObject.has_value()) + { + status = series.advance( + AdvanceMode::BEGINSTEP, + *file, + series.indexOf(*thisObject), + *thisObject); + } + else + { + status = series.advance(AdvanceMode::BEGINSTEP); + } + + switch (status) { - return status; + case AdvanceStatus::OVER: + res.stepStatus = status; + return res; + case AdvanceStatus::OK: + case AdvanceStatus::RANDOMACCESS: + break; } // re-read -> new datasets might be available - if (reread && + auto IOHandl = series.IOHandler(); + if (reread && status != AdvanceStatus::RANDOMACCESS && (series.iterationEncoding() == IE::groupBased || series.iterationEncoding() == IE::variableBased) && - (this->IOHandler()->m_frontendAccess == Access::READ_ONLY || - this->IOHandler()->m_frontendAccess == Access::READ_WRITE)) + (IOHandl->m_frontendAccess == Access::READ_ONLY || + IOHandl->m_frontendAccess == Access::READ_WRITE)) { - switch (IOHandler()->m_frontendAccess) + switch (IOHandl->m_frontendAccess) { case Access::READ_ONLY: case Access::READ_WRITE: { bool previous = series.iterations.written(); series.iterations.written() = false; - auto oldType = this->IOHandler()->m_frontendAccess; - auto newType = - const_cast(&this->IOHandler()->m_frontendAccess); - *newType = Access::READ_WRITE; - series.readGorVBased(false); - *newType = oldType; + auto oldStatus = IOHandl->m_seriesStatus; + IOHandl->m_seriesStatus = internal::SeriesStatus::Parsing; + try + { + res.iterationsInOpenedStep = series.readGorVBased(false); + } + catch (...) + { + IOHandl->m_seriesStatus = oldStatus; + throw; + } + IOHandl->m_seriesStatus = oldStatus; series.iterations.written() = previous; break; } @@ -619,7 +669,8 @@ AdvanceStatus Iteration::beginStep(bool reread) } } - return status; + res.stepStatus = status; + return res; } void Iteration::endStep() @@ -641,6 +692,7 @@ void Iteration::endStep() } // @todo filebased check series.advance(AdvanceMode::ENDSTEP, *file, series.indexOf(*this), *this); + series.get().m_currentlyActiveIterations.clear(); } StepStatus Iteration::getStepStatus() @@ -724,9 +776,8 @@ void Iteration::runDeferredParseAccess() } auto const &deferred = it.m_deferredParseAccess.value(); - auto oldAccess = IOHandler()->m_frontendAccess; - auto newAccess = const_cast(&IOHandler()->m_frontendAccess); - *newAccess = Access::READ_WRITE; + auto oldStatus = IOHandler()->m_seriesStatus; + IOHandler()->m_seriesStatus = internal::SeriesStatus::Parsing; try { if (deferred.fileBased) @@ -743,12 +794,12 @@ void Iteration::runDeferredParseAccess() { // reset this thing it.m_deferredParseAccess = std::optional(); - *newAccess = oldAccess; + IOHandler()->m_seriesStatus = oldStatus; throw; } // reset this thing it.m_deferredParseAccess = std::optional(); - *newAccess = oldAccess; + IOHandler()->m_seriesStatus = oldStatus; break; } case Access::CREATE: diff --git a/src/ReadIterations.cpp b/src/ReadIterations.cpp index b677d97535..b567aa0ff1 100644 --- a/src/ReadIterations.cpp +++ b/src/ReadIterations.cpp @@ -37,19 +37,26 @@ SeriesIterator::SeriesIterator(Series series) : m_series(std::move(series)) *this = end(); return; } + else if ( + it->second.get().m_closed == internal::CloseStatus::ClosedInBackend) + { + throw error::WrongAPIUsage( + "Trying to call Series::readIterations() on a (partially) read " + "Series."); + } else { - auto openIteration = [&it]() { + auto openIteration = [](Iteration &iteration) { /* * @todo * Is that really clean? * Use case: See Python ApiTest testListSeries: * Call listSeries twice. */ - if (it->second.get().m_closed != + if (iteration.get().m_closed != internal::CloseStatus::ClosedInBackend) { - it->second.open(); + iteration.open(); } }; AdvanceStatus status{}; @@ -62,101 +69,289 @@ SeriesIterator::SeriesIterator(Series series) : m_series(std::move(series)) * so do that now. There is only one step per file, so beginning * the step after parsing the file is ok. */ - openIteration(); + + openIteration(series.iterations.begin()->second); status = it->second.beginStep(/* reread = */ true); + for (auto const &pair : m_series.value().iterations) + { + m_iterationsInCurrentStep.push_back(pair.first); + } break; case IterationEncoding::groupBased: - case IterationEncoding::variableBased: + case IterationEncoding::variableBased: { /* * In group-based iteration layout, we have definitely already had * access to the file until now. Better to begin a step right away, * otherwise we might get another step's data. */ - status = it->second.beginStep(/* reread = */ true); - openIteration(); + Iteration::BeginStepStatus::AvailableIterations_t + availableIterations; + std::tie(status, availableIterations) = + it->second.beginStep(/* reread = */ true); + /* + * In random-access mode, do not use the information read in the + * `snapshot` attribute, instead simply go through iterations + * one by one in ascending order (fallback implementation in the + * second if branch). + */ + if (availableIterations.has_value() && + status != AdvanceStatus::RANDOMACCESS) + { + m_iterationsInCurrentStep = availableIterations.value(); + if (!m_iterationsInCurrentStep.empty()) + { + openIteration( + series.iterations.at(m_iterationsInCurrentStep.at(0))); + } + } + else if (!series.iterations.empty()) + { + /* + * Fallback implementation: Assume that each step corresponds + * with an iteration in ascending order. + */ + m_iterationsInCurrentStep = {series.iterations.begin()->first}; + openIteration(series.iterations.begin()->second); + } + else + { + // this is a no-op, but let's keep it explicit + m_iterationsInCurrentStep = {}; + } + break; } + } + if (status == AdvanceStatus::OVER) { *this = end(); return; } + if (!setCurrentIteration()) + { + *this = end(); + return; + } it->second.setStepStatus(StepStatus::DuringStep); } - m_currentIteration = it->first; } -SeriesIterator &SeriesIterator::operator++() +std::optional SeriesIterator::nextIterationInStep() { - if (!m_series.has_value()) + using ret_t = std::optional; + + if (m_iterationsInCurrentStep.empty()) + { + return ret_t{}; + } + m_iterationsInCurrentStep.pop_front(); + if (m_iterationsInCurrentStep.empty()) + { + return ret_t{}; + } + auto oldIterationIndex = m_currentIteration; + m_currentIteration = *m_iterationsInCurrentStep.begin(); + auto &series = m_series.value(); + + switch (series.iterationEncoding()) + { + case IterationEncoding::groupBased: + case IterationEncoding::variableBased: { + auto begin = series.iterations.find(oldIterationIndex); + auto end = begin; + ++end; + series.flush_impl( + begin, + end, + {FlushLevel::UserFlush}, + /* flushIOHandler = */ true); + + series.iterations[m_currentIteration].open(); + return {this}; + } + case IterationEncoding::fileBased: + series.iterations[m_currentIteration].open(); + series.iterations[m_currentIteration].beginStep(/* reread = */ true); + return {this}; + } + throw std::runtime_error("Unreachable!"); +} + +std::optional SeriesIterator::nextStep() +{ + // since we are in group-based iteration layout, it does not + // matter which iteration we begin a step upon + AdvanceStatus status; + Iteration::BeginStepStatus::AvailableIterations_t availableIterations; + std::tie(status, availableIterations) = + Iteration::beginStep({}, *m_series, /* reread = */ true); + + if (availableIterations.has_value() && + status != AdvanceStatus::RANDOMACCESS) + { + m_iterationsInCurrentStep = availableIterations.value(); + } + else + { + /* + * Fallback implementation: Assume that each step corresponds + * with an iteration in ascending order. + */ + auto &series = m_series.value(); + auto it = series.iterations.find(m_currentIteration); + auto itEnd = series.iterations.end(); + if (it == itEnd) + { + if (status == AdvanceStatus::RANDOMACCESS || + status == AdvanceStatus::OVER) + { + *this = end(); + return {this}; + } + else + { + /* + * Stream still going but there was no iteration found in the + * current IO step? + * Might be a duplicate iteration resulting from appending, + * will skip such iterations and hope to find something in a + * later IO step. No need to finish right now. + */ + m_iterationsInCurrentStep = {}; + m_series->advance(AdvanceMode::ENDSTEP); + } + } + else + { + ++it; + + if (it == itEnd) + { + if (status == AdvanceStatus::RANDOMACCESS || + status == AdvanceStatus::OVER) + { + *this = end(); + return {this}; + } + else + { + /* + * Stream still going but there was no iteration found in + * the current IO step? Might be a duplicate iteration + * resulting from appending, will skip such iterations and + * hope to find something in a later IO step. No need to + * finish right now. + */ + m_iterationsInCurrentStep = {}; + m_series->advance(AdvanceMode::ENDSTEP); + } + } + else + { + m_iterationsInCurrentStep = {it->first}; + } + } + } + + if (status == AdvanceStatus::OVER) { *this = end(); - return *this; + return {this}; } + + return {this}; +} + +std::optional SeriesIterator::loopBody() +{ Series &series = m_series.value(); auto &iterations = series.iterations; - auto ¤tIteration = iterations[m_currentIteration]; - if (!currentIteration.closed()) + + /* + * Might not be present because parsing might have failed in previous step + */ + if (iterations.contains(m_currentIteration)) { - currentIteration.close(); + auto ¤tIteration = iterations[m_currentIteration]; + if (!currentIteration.closed()) + { + currentIteration.close(); + } } - switch (series.iterationEncoding()) + + auto guardReturn = + [&iterations]( + auto const &option) -> std::optional { + if (!option.has_value() || *option.value() == end()) + { + return option; + } + auto currentIterationIndex = option.value()->peekCurrentIteration(); + if (!currentIterationIndex.has_value()) + { + return std::nullopt; + } + auto iteration = iterations.at(currentIterationIndex.value()); + if (iteration.get().m_closed != internal::CloseStatus::ClosedInBackend) + { + iteration.open(); + option.value()->setCurrentIteration(); + return option; + } + else + { + // we had this iteration already, skip it + iteration.endStep(); + return std::nullopt; // empty, go into next iteration + } + }; + { - using IE = IterationEncoding; - case IE::groupBased: - case IE::variableBased: { - // since we are in group-based iteration layout, it does not - // matter which iteration we begin a step upon - AdvanceStatus status{}; - status = currentIteration.beginStep(/* reread = */ true); - if (status == AdvanceStatus::OVER) + auto optionallyAStep = nextIterationInStep(); + if (optionallyAStep.has_value()) { - *this = end(); - return *this; + return guardReturn(optionallyAStep); } - currentIteration.setStepStatus(StepStatus::DuringStep); - break; - } - default: - break; } - auto it = iterations.find(m_currentIteration); - auto itEnd = iterations.end(); - if (it == itEnd) + + // The currently active iterations have been exhausted. + // Now see if there are further iterations to be found. + + if (series.iterationEncoding() == IterationEncoding::fileBased) { + // this one is handled above, stream is over once it proceeds to here *this = end(); - return *this; + return {this}; } - ++it; - if (it == itEnd) + + auto option = nextStep(); + return guardReturn(option); +} + +SeriesIterator &SeriesIterator::operator++() +{ + if (!m_series.has_value()) { *this = end(); return *this; } - m_currentIteration = it->first; - if (it->second.get().m_closed != internal::CloseStatus::ClosedInBackend) + std::optional res; + /* + * loopBody() might return an empty option to indicate a skipped iteration. + * Loop until it returns something real for us. + */ + do { - it->second.open(); - } - switch (series.iterationEncoding()) + res = loopBody(); + } while (!res.has_value()); + + auto resvalue = res.value(); + if (*resvalue != end()) { - using IE = IterationEncoding; - case IE::fileBased: { - auto &iteration = series.iterations[m_currentIteration]; - AdvanceStatus status{}; - status = iteration.beginStep(/* reread = */ true); - if (status == AdvanceStatus::OVER) - { - *this = end(); - return *this; - } - iteration.setStepStatus(StepStatus::DuringStep); - break; - } - default: - break; + (**resvalue).setStepStatus(StepStatus::DuringStep); } - return *this; + return *resvalue; } IndexedIteration SeriesIterator::operator*() diff --git a/src/Series.cpp b/src/Series.cpp index 6387258cf6..7d87e88e13 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -577,28 +577,34 @@ Given file pattern: ')END" case Access::READ_WRITE: { /* Allow creation of values in Containers and setting of Attributes * Would throw for Access::READ_ONLY */ - auto oldType = IOHandler()->m_frontendAccess; - auto newType = const_cast(&IOHandler()->m_frontendAccess); - *newType = Access::READ_WRITE; + IOHandler()->m_seriesStatus = internal::SeriesStatus::Parsing; - if (input->iterationEncoding == IterationEncoding::fileBased) - readFileBased(); - else - readGorVBased(); - - if (series.iterations.empty()) + try { - /* Access::READ_WRITE can be used to create a new Series - * allow setting attributes in that case */ - written() = false; + if (input->iterationEncoding == IterationEncoding::fileBased) + readFileBased(); + else + readGorVBased(); - initDefaults(input->iterationEncoding); - setIterationEncoding(input->iterationEncoding); + if (series.iterations.empty()) + { + /* Access::READ_WRITE can be used to create a new Series + * allow setting attributes in that case */ + written() = false; - written() = true; + initDefaults(input->iterationEncoding); + setIterationEncoding(input->iterationEncoding); + + written() = true; + } + } + catch (...) + { + IOHandler()->m_seriesStatus = internal::SeriesStatus::Default; + throw; } - *newType = oldType; + IOHandler()->m_seriesStatus = internal::SeriesStatus::Default; break; } case Access::CREATE: { @@ -732,11 +738,16 @@ void Series::flushFileBased( switch (openIterationIfDirty(it->first, it->second)) { using IO = IterationOpened; + case IO::RemainsClosed: + // we might need to proceed further if the close status is + // ClosedInFrontend + // hence no continue here + // otherwise, we might forget to close files physically + break; case IO::HasBeenOpened: + // continue below it->second.flush(flushParams); break; - case IO::RemainsClosed: - break; } // Phase 2 @@ -769,13 +780,21 @@ void Series::flushFileBased( case IO::HasBeenOpened: { /* as there is only one series, * emulate the file belonging to each iteration as not yet - * written + * written, even if the iteration itself is already written + * (to ensure that the Series gets reassociated with the + * current iteration) */ written() = false; series.iterations.written() = false; dirty() |= it->second.dirty(); std::string filename = iterationFilename(it->first); + + if (!it->second.written()) + { + series.m_currentlyActiveIterations.emplace(it->first); + } + it->second.flushFileBased(filename, it->first, flushParams); series.iterations.flush( @@ -831,11 +850,15 @@ void Series::flushGorVBased( switch (openIterationIfDirty(it->first, it->second)) { using IO = IterationOpened; + case IO::RemainsClosed: + // we might need to proceed further if the close status is + // ClosedInFrontend + // hence no continue here + break; case IO::HasBeenOpened: + // continue below it->second.flush(flushParams); break; - case IO::RemainsClosed: - break; } // Phase 2 @@ -895,6 +918,7 @@ void Series::flushGorVBased( if (!it->second.written()) { it->second.parent() = getWritable(&series.iterations); + series.m_currentlyActiveIterations.emplace(it->first); } switch (iterationEncoding()) { @@ -1115,7 +1139,7 @@ void Series::readOneIterationFileBased(std::string const &filePath) series.iterations.readAttributes(ReadMode::OverrideExisting); } -void Series::readGorVBased(bool do_init) +std::optional> Series::readGorVBased(bool do_init) { auto &series = get(); Parameter fOpen; @@ -1187,16 +1211,35 @@ void Series::readGorVBased(bool do_init) IOHandler()->enqueue(IOTask(&series.iterations, pOpen)); readAttributes(ReadMode::IgnoreExisting); + + auto withRWAccess = [this](auto &&functor) { + auto oldStatus = IOHandler()->m_seriesStatus; + IOHandler()->m_seriesStatus = internal::SeriesStatus::Parsing; + try + { + std::forward(functor)(); + } + catch (...) + { + IOHandler()->m_seriesStatus = oldStatus; + throw; + } + IOHandler()->m_seriesStatus = oldStatus; + }; + /* * 'snapshot' changes over steps, so reread that. */ - series.iterations.readAttributes(ReadMode::OverrideExisting); + withRWAccess([&series]() { + series.iterations.readAttributes(ReadMode::OverrideExisting); + }); + /* obtain all paths inside the basepath (i.e. all iterations) */ Parameter pList; IOHandler()->enqueue(IOTask(&series.iterations, pList)); IOHandler()->flush(internal::defaultFlushParams); - auto readSingleIteration = [&series, &pOpen, this]( + auto readSingleIteration = [&series, &pOpen, this, withRWAccess]( uint64_t index, std::string path, bool guardAgainstRereading, @@ -1215,7 +1258,7 @@ void Series::readGorVBased(bool do_init) { pOpen.path = path; IOHandler()->enqueue(IOTask(&i, pOpen)); - i.reread(path); + withRWAccess([&i, &path]() { i.reread(path); }); } } else @@ -1235,13 +1278,18 @@ void Series::readGorVBased(bool do_init) } }; + /* + * @todo in BP5, a BeginStep() might be necessary before this + */ + auto currentSteps = currentSnapshot(); + switch (iterationEncoding()) { case IterationEncoding::groupBased: /* * Sic! This happens when a file-based Series is opened in group-based mode. */ - case IterationEncoding::fileBased: + case IterationEncoding::fileBased: { for (auto const &it : *pList.paths) { uint64_t index = std::stoull(it); @@ -1250,23 +1298,37 @@ void Series::readGorVBased(bool do_init) * (beginStep = false) * A streaming read mode might come in a future API addition. */ - readSingleIteration(index, it, true, false); + withRWAccess( + [&]() { readSingleIteration(index, it, true, false); }); } - break; + if (currentSteps.has_value()) + { + auto const &vec = currentSteps.value(); + return std::deque{vec.begin(), vec.end()}; + } + else + { + return std::optional>(); + } + } case IterationEncoding::variableBased: { - uint64_t index = 0; - if (series.iterations.containsAttribute("snapshot")) + std::deque res = {0}; + if (currentSteps.has_value() && !currentSteps.value().empty()) { - index = series.iterations.getAttribute("snapshot").get(); + res = {currentSteps.value().begin(), currentSteps.value().end()}; } - /* - * Variable-based iteration encoding relies on steps, so parsing must - * happen after opening the first step. - */ - readSingleIteration(index, "", false, true); - break; + for (auto it : res) + { + /* + * Variable-based iteration encoding relies on steps, so parsing + * must happen after opening the first step. + */ + withRWAccess([&]() { readSingleIteration(it, "", false, true); }); + } + return res; } } + throw std::runtime_error("Unreachable!"); } void Series::readBase() @@ -1458,9 +1520,15 @@ AdvanceStatus Series::advance( * opening an iteration's file by beginning a step on it. * So, return now. */ + iteration.get().m_closed = internal::CloseStatus::ClosedInBackend; return AdvanceStatus::OK; } + if (mode == AdvanceMode::ENDSTEP) + { + flushStep(/* doFlush = */ false); + } + Parameter param; if (itData.m_closed == internal::CloseStatus::ClosedTemporarily && series.m_iterationEncoding == IterationEncoding::fileBased) @@ -1517,6 +1585,92 @@ AdvanceStatus Series::advance( return *param.status; } +AdvanceStatus Series::advance(AdvanceMode mode) +{ + auto &series = get(); + if (series.m_iterationEncoding == IterationEncoding::fileBased) + { + throw error::Internal( + "Advancing a step in file-based iteration encoding is " + "iteration-specific."); + } + internal::FlushParams const flushParams = {FlushLevel::UserFlush}; + /* + * We call flush_impl() with flushIOHandler = false, meaning that tasks are + * not yet propagated to the backend. + * We will append ADVANCE and CLOSE_FILE tasks manually and then flush the + * IOHandler manually. + * In order to avoid having those tasks automatically appended by + * flush_impl(), set CloseStatus to Open for now. + */ + + auto begin = iterations.end(); + auto end = iterations.end(); + + switch (mode) + { + case AdvanceMode::ENDSTEP: + flush_impl(begin, end, flushParams, /* flushIOHandler = */ false); + break; + case AdvanceMode::BEGINSTEP: + /* + * When beginning a step, there is nothing to flush yet. + * Data is not written in between steps. + * So only make sure that files are accessed. + */ + flush_impl( + begin, + end, + {FlushLevel::CreateOrOpenFiles}, + /* flushIOHandler = */ false); + break; + } + + if (mode == AdvanceMode::ENDSTEP) + { + flushStep(/* doFlush = */ false); + } + + Parameter param; + param.mode = mode; + IOTask task(&series.m_writable, param); + IOHandler()->enqueue(task); + + // We cannot call Series::flush now, since the IO handler is still filled + // from calling flush(Group|File)based, but has not been emptied yet + // Do that manually + IOHandler()->flush(flushParams); + + return *param.status; +} + +void Series::flushStep(bool doFlush) +{ + auto &series = get(); + if (!series.m_currentlyActiveIterations.empty() && + IOHandler()->m_frontendAccess != Access::READ_ONLY) + { + /* + * Warning: changing attribute extents over time (probably) unsupported + * by this so far. + * Not (yet) needed as there is no way to pack several iterations within + * one IO step. + */ + Parameter wAttr; + wAttr.changesOverSteps = true; + wAttr.name = "snapshot"; + wAttr.resource = std::vector{ + series.m_currentlyActiveIterations.begin(), + series.m_currentlyActiveIterations.end()}; + wAttr.dtype = Datatype::VEC_ULONGLONG; + IOHandler()->enqueue(IOTask(&series.iterations, wAttr)); + if (doFlush) + { + IOHandler()->flush(internal::defaultFlushParams); + } + } +} + auto Series::openIterationIfDirty(uint64_t index, Iteration iteration) -> IterationOpened { @@ -1795,6 +1949,7 @@ namespace internal { Series impl{{this, [](auto const *) {}}}; impl.flush(); + impl.flushStep(/* doFlush = */ true); } if (m_writeIterations.has_value()) { @@ -1886,6 +2041,46 @@ WriteIterations Series::writeIterations() return series.m_writeIterations.value(); } +std::optional> Series::currentSnapshot() const +{ + using vec_t = std::vector; + auto &series = get(); + /* + * In variable-based iteration encoding, iterations have no distinct + * group within `series.iterations`, meaning that the `snapshot` + * attribute is not found at `/data/0/snapshot`, but at + * `/data/snapshot`. This makes it possible to retrieve it from + * `series.iterations`. + */ + if (series.iterations.containsAttribute("snapshot")) + { + auto const &attribute = series.iterations.getAttribute("snapshot"); + switch (attribute.dtype) + { + case Datatype::ULONGLONG: + case Datatype::VEC_ULONGLONG: { + auto const &vec = attribute.get>(); + return vec_t{vec.begin(), vec.end()}; + } + case Datatype::ULONG: + case Datatype::VEC_ULONG: { + auto const &vec = attribute.get>(); + return vec_t{vec.begin(), vec.end()}; + } + default: { + std::stringstream s; + s << "Unexpected datatype for '/data/snapshot': " << attribute.dtype + << std::endl; + throw std::runtime_error(s.str()); + } + } + } + else + { + return std::optional>{}; + } +} + namespace { CleanedFilename cleanFilename( diff --git a/test/JSONTest.cpp b/test/JSONTest.cpp index 686306700e..485689acb4 100644 --- a/test/JSONTest.cpp +++ b/test/JSONTest.cpp @@ -1,8 +1,10 @@ #include "openPMD/auxiliary/JSON.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" +#include "openPMD/openPMD.hpp" #include +#include #include using namespace openPMD; @@ -172,3 +174,88 @@ TEST_CASE("json_merging", "auxiliary") json::merge(defaultVal, overwrite) == json::parseOptions(expect, false).config.dump()); } + +/* + * This tests two things about the /data/snapshot attribute: + * + * 1) Reading a variable-based series without the snapshot attribute should be + * possible by assuming a default /data/snapshot = 0. + * 2) The snapshot attribute might be a vector of iterations. The Read API + * should then return the same iteration multiple times, with different + * indices. + * + * Such files are currently not created by the openPMD-api (the API currently + * supports creating a variable-based series with a scalar snapshot attribute). + * But the standard will allow both options above, so reading should at least + * be possible. + * This test creates a variable-based JSON series and then uses the nlohmann + * json library to modifiy the resulting series for testing purposes. + */ +TEST_CASE("variableBasedModifiedSnapshot", "[auxiliary]") +{ + constexpr auto file = "../samples/variableBasedModifiedSnapshot.json"; + { + Series writeSeries(file, Access::CREATE); + writeSeries.setIterationEncoding(IterationEncoding::variableBased); + REQUIRE( + writeSeries.iterationEncoding() == + IterationEncoding::variableBased); + auto iterations = writeSeries.writeIterations(); + auto iteration = iterations[10]; + auto E_z = iteration.meshes["E"]["x"]; + E_z.resetDataset({Datatype::INT, {1}}); + E_z.makeConstant(72); + + iteration.close(); + } + + { + nlohmann::json series; + { + std::fstream fstream; + fstream.open(file, std::ios_base::in); + fstream >> series; + } + series["data"]["attributes"].erase("snapshot"); + { + std::fstream fstream; + fstream.open(file, std::ios_base::out | std::ios_base::trunc); + fstream << series; + } + } + + /* + * Need generic capture here since the compilers are being + * annoying otherwise. + */ + auto testRead = [&](std::vector const &requiredIterations) { + Series readSeries(file, Access::READ_ONLY); + size_t counter = 0; + for (auto const &iteration : readSeries.readIterations()) + { + REQUIRE(iteration.iterationIndex == requiredIterations[counter++]); + } + REQUIRE(counter == requiredIterations.size()); + }; + testRead(std::vector{0}); + + { + nlohmann::json series; + { + std::fstream fstream; + fstream.open(file, std::ios_base::in); + fstream >> series; + } + series["data"]["attributes"].erase("snapshot"); + auto &snapshot = series["data"]["attributes"]["snapshot"]; + snapshot["datatype"] = "VEC_ULONG"; + snapshot["value"] = std::vector{1, 2, 3, 4, 5}; + { + std::fstream fstream; + fstream.open(file, std::ios_base::out | std::ios_base::trunc); + fstream << series; + } + } + + testRead(std::vector{1, 2, 3, 4, 5}); +} diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 99cd472166..b21bcc2e97 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1489,15 +1489,8 @@ void append_mode( else { REQUIRE(read.iterations.size() == 5); + helper::listSeries(read); } - /* - * Roadmap: for now, reading this should work by ignoring the last - * duplicate iteration. - * After merging https://github.com/openPMD/openPMD-api/pull/949, we - * should see both instances when reading. - * Final goal: Read only the last instance. - */ - helper::listSeries(read); } #if 100000000 * ADIOS2_VERSION_MAJOR + 1000000 * ADIOS2_VERSION_MINOR + \ 10000 * ADIOS2_VERSION_PATCH + 100 * ADIOS2_VERSION_TWEAK >= \ @@ -1578,10 +1571,24 @@ TEST_CASE("append_mode", "[parallel]") } } })END"; + /* + * Troublesome combination: + * 1) ADIOS2 v2.7 + * 2) Parallel writer + * 3) Append mode + * 4) Writing to a scalar variable + * + * 4) is done by schema 2021 which will be phased out, so the tests + * are just deactivated. + */ + if (auxiliary::getEnvNum("OPENPMD2_ADIOS2_SCHEMA", 0) != 0) + { + continue; + } append_mode(t, false, jsonConfigOld); - append_mode(t, false, jsonConfigNew); - append_mode(t, true, jsonConfigOld); - append_mode(t, true, jsonConfigNew); + // append_mode(t, true, jsonConfigOld); + // append_mode(t, false, jsonConfigNew); + // append_mode(t, true, jsonConfigNew); } else { diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 61172aa5f4..ac187a5d02 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -4900,8 +4900,10 @@ void serial_iterator(std::string const &file) Series readSeries(file, Access::READ_ONLY); size_t last_iteration_index = 0; + size_t numberOfIterations = 0; for (auto iteration : readSeries.readIterations()) { + ++numberOfIterations; auto E_x = iteration.meshes["E"]["x"]; REQUIRE(E_x.getDimensionality() == 1); REQUIRE(E_x.getExtent()[0] == extent); @@ -4914,6 +4916,7 @@ void serial_iterator(std::string const &file) last_iteration_index = iteration.iterationIndex; } REQUIRE(last_iteration_index == 9); + REQUIRE(numberOfIterations == 10); } TEST_CASE("serial_iterator", "[serial][adios2]") @@ -5665,7 +5668,8 @@ void iterate_nonstreaming_series( auto E_x = iteration.meshes["E"]["x"]; E_x.resetDataset( openPMD::Dataset(openPMD::Datatype::INT, {2, extent})); - std::vector data(extent, i); + int value = variableBasedLayout ? 0 : i; + std::vector data(extent, value); E_x.storeChunk(data, {0, 0}, {1, extent}); bool taskSupportedByBackend = true; DynamicMemoryView memoryView; @@ -5753,9 +5757,10 @@ void iterate_nonstreaming_series( iteration.close(); } + int value = variableBasedLayout ? 0 : iteration.iterationIndex; for (size_t i = 0; i < extent; ++i) { - REQUIRE(chunk.get()[i] == int(iteration.iterationIndex)); + REQUIRE(chunk.get()[i] == value); REQUIRE(chunk2.get()[i] == int(i)); } last_iteration_index = iteration.iterationIndex; @@ -6175,11 +6180,12 @@ TEST_CASE("deferred_parsing", "[serial]") } } -// @todo merge this back with the chaotic_stream test of PR #949 -// (bug noticed while working on that branch) -void no_explicit_flush(std::string filename) +void chaotic_stream(std::string filename, bool variableBased) { - std::vector sampleData{5, 9, 1, 3, 4, 6, 7, 8, 2, 0}; + /* + * We will write iterations in the following order. + */ + std::vector iterations{5, 9, 1, 3, 4, 6, 7, 8, 2, 0}; std::string jsonConfig = R"( { "adios2": { @@ -6190,16 +6196,31 @@ void no_explicit_flush(std::string filename) } })"; + bool weirdOrderWhenReading{}; + { Series series(filename, Access::CREATE, jsonConfig); - for (uint64_t currentIteration = 0; currentIteration < 10; - ++currentIteration) + /* + * When using ADIOS2 steps, iterations are read not by logical order + * (iteration index), but by order of writing. + */ + weirdOrderWhenReading = series.backend() == "ADIOS2" && + series.iterationEncoding() != IterationEncoding::fileBased; + if (variableBased) + { + if (series.backend() != "ADIOS2") + { + return; + } + series.setIterationEncoding(IterationEncoding::variableBased); + } + for (auto currentIteration : iterations) { auto dataset = series.writeIterations()[currentIteration] .meshes["iterationOrder"][MeshRecordComponent::SCALAR]; dataset.resetDataset({determineDatatype(), {10}}); - dataset.storeChunk(sampleData, {0}, {10}); + dataset.storeChunk(iterations, {0}, {10}); // series.writeIterations()[ currentIteration ].close(); } } @@ -6209,19 +6230,27 @@ void no_explicit_flush(std::string filename) size_t index = 0; for (const auto &iteration : series.readIterations()) { - REQUIRE(iteration.iterationIndex == index); + if (weirdOrderWhenReading) + { + REQUIRE(iteration.iterationIndex == iterations[index]); + } + else + { + REQUIRE(iteration.iterationIndex == index); + } ++index; } - REQUIRE(index == 10); + REQUIRE(index == iterations.size()); } } -TEST_CASE("no_explicit_flush", "[serial]") +TEST_CASE("chaotic_stream", "[serial]") { for (auto const &t : testedFileExtensions()) { - no_explicit_flush("../samples/no_explicit_flush_filebased_%T." + t); - no_explicit_flush("../samples/no_explicit_flush." + t); + chaotic_stream("../samples/chaotic_stream_filebased_%T." + t, false); + chaotic_stream("../samples/chaotic_stream." + t, false); + chaotic_stream("../samples/chaotic_stream_vbased." + t, true); } } @@ -6315,9 +6344,48 @@ TEST_CASE("varying_zero_pattern", "[serial]") } } +enum class ParseMode +{ + /* + * Conventional workflow. Just parse the whole thing and yield iterations + * in rising order. + */ + NoSteps, + /* + * NOTE: This mode is only temporary until the topic-linear-read PR, + * no longer necessary after that. + * The Series is parsed ahead of time upon opening, but it has steps. + * Parsing ahead of time is the conventional workflow to support + * random-access. + * Reading such a Series with the streaming API is only possible if all + * steps are in ascending order, otherwise the openPMD-api has no way of + * associating IO steps with interation indices. + * Reading such a Series with the Streaming API will become possible with + * the Linear read mode to be introduced by #1291. + */ + AheadOfTimeWithoutSnapshot, + /* + * A Series of the BP5 engine is not parsed ahead of time, but step-by-step, + * giving the openPMD-api a way to associate IO steps with iterations. + * No snapshot attribute exists, so the fallback mode is chosen: + * Iterations are returned in ascending order. + * If an IO step returns an iteration whose index is lower than the + * last one, it will be skipped. + * This mode of parsing will be generalized into the Linear read mode with + * PR #1291. + */ + LinearWithoutSnapshot, + /* + * Snapshot attribute exists and dictates the iteration index returned by + * an IO step. Duplicate iterations will be skipped. + */ + WithSnapshot +}; + void append_mode( std::string const &extension, bool variableBased, + ParseMode parseMode, std::string jsonConfig = "{}") { @@ -6402,35 +6470,94 @@ void append_mode( } writeSomeIterations( - write.writeIterations(), std::vector{4, 3}); + write.writeIterations(), std::vector{4, 3, 10}); + write.flush(); + } + { + Series write(filename, Access::APPEND, jsonConfig); + if (variableBased) + { + write.setIterationEncoding(IterationEncoding::variableBased); + } + if (write.backend() == "ADIOS1") + { + REQUIRE_THROWS_AS( + write.flush(), error::OperationUnsupportedInBackend); + // destructor will be noisy now + return; + } + + writeSomeIterations( + write.writeIterations(), std::vector{7, 1, 11}); write.flush(); } { Series read(filename, Access::READ_ONLY); - if (variableBased || extension == "bp5") + switch (parseMode) { + case ParseMode::NoSteps: { + unsigned counter = 0; + uint64_t iterationOrder[] = {0, 1, 2, 3, 4, 7, 10, 11}; + for (auto const &iteration : read.readIterations()) + { + REQUIRE(iteration.iterationIndex == iterationOrder[counter]); + ++counter; + } + REQUIRE(counter == 8); + } + break; + case ParseMode::LinearWithoutSnapshot: { + unsigned counter = 0; + uint64_t iterationOrder[] = {0, 1, 2, 3, 4, 10, 11}; + for (auto const &iteration : read.readIterations()) + { + REQUIRE(iteration.iterationIndex == iterationOrder[counter]); + ++counter; + } + REQUIRE(counter == 7); + } + break; + case ParseMode::WithSnapshot: { // in variable-based encodings, iterations are not parsed ahead of // time but as they go unsigned counter = 0; + uint64_t iterationOrder[] = {0, 1, 2, 3, 4, 10, 7, 11}; for (auto const &iteration : read.readIterations()) { - REQUIRE(iteration.iterationIndex == counter); + REQUIRE(iteration.iterationIndex == iterationOrder[counter]); ++counter; } - REQUIRE(counter == 5); + REQUIRE(counter == 8); + // Cannot do listSeries here because the Series is already drained + REQUIRE_THROWS_AS(helper::listSeries(read), error::WrongAPIUsage); } - else - { - REQUIRE(read.iterations.size() == 5); + break; + case ParseMode::AheadOfTimeWithoutSnapshot: { + REQUIRE(read.iterations.size() == 8); + unsigned counter = 0; + uint64_t iterationOrder[] = {0, 1, 2, 3, 4, 7, 10, 11}; + /* + * Use conventional read API since streaming API is not possible + * without Linear read mode. + * (See also comments inside ParseMode enum). + */ + for (auto const &iteration : read.iterations) + { + REQUIRE(iteration.first == iterationOrder[counter]); + ++counter; + } + REQUIRE(counter == 8); + /* + * Roadmap: for now, reading this should work by ignoring the last + * duplicate iteration. + * After merging https://github.com/openPMD/openPMD-api/pull/949, we + * should see both instances when reading. + * Final goal: Read only the last instance. + */ + helper::listSeries(read); + } + break; } - /* - * Roadmap: for now, reading this should work by ignoring the last - * duplicate iteration. - * After merging https://github.com/openPMD/openPMD-api/pull/949, we - * should see both instances when reading. - * Final goal: Read only the last instance. - */ - helper::listSeries(read); } #if 100000000 * ADIOS2_VERSION_MAJOR + 1000000 * ADIOS2_VERSION_MINOR + \ 10000 * ADIOS2_VERSION_PATCH + 100 * ADIOS2_VERSION_TWEAK >= \ @@ -6467,16 +6594,47 @@ void append_mode( } { Series read(filename, Access::READ_ONLY); - // in variable-based encodings, iterations are not parsed ahead of - // time but as they go - unsigned counter = 0; - for (auto const &iteration : read.readIterations()) + switch (parseMode) { - REQUIRE(iteration.iterationIndex == counter); - ++counter; + case ParseMode::LinearWithoutSnapshot: { + uint64_t iterationOrder[] = {0, 1, 2, 3, 4, 10}; + unsigned counter = 0; + for (auto const &iteration : read.readIterations()) + { + REQUIRE( + iteration.iterationIndex == iterationOrder[counter]); + ++counter; + } + REQUIRE(counter == 6); + // Cannot do listSeries here because the Series is already + // drained + REQUIRE_THROWS_AS( + helper::listSeries(read), error::WrongAPIUsage); + } + break; + case ParseMode::WithSnapshot: { + // in variable-based encodings, iterations are not parsed ahead + // of time but as they go + unsigned counter = 0; + uint64_t iterationOrder[] = {0, 1, 2, 3, 4, 10, 7, 5}; + for (auto const &iteration : read.readIterations()) + { + REQUIRE( + iteration.iterationIndex == iterationOrder[counter]); + ++counter; + } + REQUIRE(counter == 8); + // Cannot do listSeries here because the Series is already + // drained + REQUIRE_THROWS_AS( + helper::listSeries(read), error::WrongAPIUsage); + } + break; + case ParseMode::NoSteps: + case ParseMode::AheadOfTimeWithoutSnapshot: + throw std::runtime_error("Test configured wrong."); + break; } - REQUIRE(counter == 6); - helper::listSeries(read); } } #endif @@ -6486,9 +6644,7 @@ TEST_CASE("append_mode", "[serial]") { for (auto const &t : testedFileExtensions()) { - if (t == "bp" || t == "bp4" || t == "bp5") - { - std::string jsonConfigOld = R"END( + std::string jsonConfigOld = R"END( { "adios2": { @@ -6499,7 +6655,7 @@ TEST_CASE("append_mode", "[serial]") } } })END"; - std::string jsonConfigNew = R"END( + std::string jsonConfigNew = R"END( { "adios2": { @@ -6510,14 +6666,25 @@ TEST_CASE("append_mode", "[serial]") } } })END"; - append_mode(t, false, jsonConfigOld); - append_mode(t, false, jsonConfigNew); - append_mode(t, true, jsonConfigOld); - append_mode(t, true, jsonConfigNew); + if (t == "bp5") + { + append_mode( + t, false, ParseMode::LinearWithoutSnapshot, jsonConfigOld); + append_mode(t, false, ParseMode::WithSnapshot, jsonConfigNew); + append_mode(t, true, ParseMode::WithSnapshot, jsonConfigOld); + append_mode(t, true, ParseMode::WithSnapshot, jsonConfigNew); + } + else if (t == "bp" || t == "bp4" || t == "bp5") + { + append_mode( + t, false, ParseMode::AheadOfTimeWithoutSnapshot, jsonConfigOld); + append_mode(t, false, ParseMode::WithSnapshot, jsonConfigNew); + append_mode(t, true, ParseMode::WithSnapshot, jsonConfigOld); + append_mode(t, true, ParseMode::WithSnapshot, jsonConfigNew); } else { - append_mode(t, false); + append_mode(t, false, ParseMode::NoSteps); } } } diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index 2503e8c619..93093626a4 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -1067,8 +1067,9 @@ def testListSeries(self): series = self.__series self.assertRaises(TypeError, io.list_series) io.list_series(series) - io.list_series(series, False) - io.list_series(series, True) + # @todo make list_series callable repeatedly + # io.list_series(series, False) + # io.list_series(series, True) print(io.list_series.__doc__) From 8c8d14fe1d880c38e5bf036d63d47b0a13fe9519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 1 Nov 2022 18:05:21 +0100 Subject: [PATCH 67/70] Add Attribute::getOptional() and use to add some more dynamic datatype conversions at read time (#1278) * Add and use Attribute::getOptional() Useful for backends that might not report back the right numerical type * Unify code paths for both get() and getOptional() Note that this is slightly API breaking since the shared code was in a public header. All implementation details are now in the detail namespace. * CI fixes, clean up append_test --- include/openPMD/backend/Attribute.hpp | 222 +++++++++++++++---------- include/openPMD/backend/BaseRecord.hpp | 30 ++-- src/Iteration.cpp | 13 +- src/Mesh.cpp | 15 +- src/RecordComponent.cpp | 13 +- src/Series.cpp | 26 +-- src/backend/MeshRecordComponent.cpp | 3 + src/backend/PatchRecord.cpp | 9 +- src/backend/PatchRecordComponent.cpp | 5 +- test/SerialIOTest.cpp | 64 +++++-- 10 files changed, 240 insertions(+), 160 deletions(-) diff --git a/include/openPMD/backend/Attribute.hpp b/include/openPMD/backend/Attribute.hpp index 0ac26d9dec..13c267b19f 100644 --- a/include/openPMD/backend/Attribute.hpp +++ b/include/openPMD/backend/Attribute.hpp @@ -29,10 +29,12 @@ #include #include #include +#include #include #include #include #include +#include #include namespace openPMD @@ -119,100 +121,117 @@ class Attribute */ template U get() const; + + /** Retrieve a stored specific Attribute and cast if convertible. + * Like Attribute::get<>(), but returns an empty std::optional if no + * conversion is possible instead of throwing an exception. + * + * @note This performs a static_cast and might introduce precision loss if + * requested. Check dtype explicitly beforehand if needed. + * + * @tparam U Type of the object to be casted to. + * @return Copy of the retrieved object, casted to type U. + * An empty std::optional if no conversion is possible. + */ + template + std::optional getOptional() const; }; -template -auto doConvert(T *pv) -> U +namespace detail { - (void)pv; - if constexpr (std::is_convertible_v) - { - return static_cast(*pv); - } - else if constexpr (auxiliary::IsVector_v && auxiliary::IsVector_v) + template + auto doConvert(T *pv) -> std::variant { - if constexpr (std::is_convertible_v< - typename T::value_type, - typename U::value_type>) - { - U res{}; - res.reserve(pv->size()); - std::copy(pv->begin(), pv->end(), std::back_inserter(res)); - return res; - } - else + (void)pv; + if constexpr (std::is_convertible_v) { - throw std::runtime_error("getCast: no vector cast possible."); + return static_cast(*pv); } - } - // conversion cast: array to vector - // if a backend reports a std::array<> for something where - // the frontend expects a vector - else if constexpr (auxiliary::IsArray_v && auxiliary::IsVector_v) - { - if constexpr (std::is_convertible_v< - typename T::value_type, - typename U::value_type>) + else if constexpr (auxiliary::IsVector_v && auxiliary::IsVector_v) { - U res{}; - res.reserve(pv->size()); - std::copy(pv->begin(), pv->end(), std::back_inserter(res)); - return res; - } - else - { - throw std::runtime_error( - "getCast: no array to vector conversion possible."); + if constexpr (std::is_convertible_v< + typename T::value_type, + typename U::value_type>) + { + U res{}; + res.reserve(pv->size()); + std::copy(pv->begin(), pv->end(), std::back_inserter(res)); + return res; + } + else + { + return std::runtime_error("getCast: no vector cast possible."); + } } - } - // conversion cast: vector to array - // if a backend reports a std::vector<> for something where - // the frontend expects an array - else if constexpr (auxiliary::IsVector_v && auxiliary::IsArray_v) - { - if constexpr (std::is_convertible_v< - typename T::value_type, - typename U::value_type>) + // conversion cast: array to vector + // if a backend reports a std::array<> for something where + // the frontend expects a vector + else if constexpr (auxiliary::IsArray_v && auxiliary::IsVector_v) { - U res{}; - if (res.size() != pv->size()) + if constexpr (std::is_convertible_v< + typename T::value_type, + typename U::value_type>) { - throw std::runtime_error( - "getCast: no vector to array conversion possible (wrong " - "requested array size)."); + U res{}; + res.reserve(pv->size()); + std::copy(pv->begin(), pv->end(), std::back_inserter(res)); + return res; } - for (size_t i = 0; i < res.size(); ++i) + else { - res[i] = static_cast((*pv)[i]); + return std::runtime_error( + "getCast: no array to vector conversion possible."); } - return res; } - else + // conversion cast: vector to array + // if a backend reports a std::vector<> for something where + // the frontend expects an array + else if constexpr (auxiliary::IsVector_v && auxiliary::IsArray_v) { - throw std::runtime_error( - "getCast: no vector to array conversion possible."); + if constexpr (std::is_convertible_v< + typename T::value_type, + typename U::value_type>) + { + U res{}; + if (res.size() != pv->size()) + { + return std::runtime_error( + "getCast: no vector to array conversion possible " + "(wrong " + "requested array size)."); + } + for (size_t i = 0; i < res.size(); ++i) + { + res[i] = static_cast((*pv)[i]); + } + return res; + } + else + { + return std::runtime_error( + "getCast: no vector to array conversion possible."); + } } - } - // conversion cast: turn a single value into a 1-element vector - else if constexpr (auxiliary::IsVector_v) - { - if constexpr (std::is_convertible_v) + // conversion cast: turn a single value into a 1-element vector + else if constexpr (auxiliary::IsVector_v) { - U res{}; - res.reserve(1); - res.push_back(static_cast(*pv)); - return res; + if constexpr (std::is_convertible_v) + { + U res{}; + res.reserve(1); + res.push_back(static_cast(*pv)); + return res; + } + else + { + return std::runtime_error( + "getCast: no scalar to vector conversion possible."); + } } else { - throw std::runtime_error( - "getCast: no scalar to vector conversion possible."); + return std::runtime_error("getCast: no cast possible."); } - } - else - { - throw std::runtime_error("getCast: no cast possible."); - } #if defined(__INTEL_COMPILER) /* * ICPC has trouble with if constexpr, thinking that return statements are @@ -222,35 +241,58 @@ auto doConvert(T *pv) -> U * https://community.intel.com/t5/Intel-C-Compiler/quot-if-constexpr-quot-and-quot-missing-return-statement-quot-in/td-p/1154551 */ #pragma warning(disable : 1011) -} + } #pragma warning(default : 1011) #else -} + } #endif +} // namespace detail -/** Retrieve a stored specific Attribute and cast if convertible. - * - * @throw std::runtime_error if stored object is not static castable to U. - * @tparam U Type of the object to be casted to. - * @return Copy of the retrieved object, casted to type U. - */ template -inline U getCast(Attribute const &a) +U Attribute::get() const { - auto v = a.getResource(); - + auto eitherValueOrError = std::visit( + [](auto &&containedValue) -> std::variant { + using containedType = std::decay_t; + return detail::doConvert(&containedValue); + }, + Variant::getResource()); return std::visit( [](auto &&containedValue) -> U { - using containedType = std::decay_t; - return doConvert(&containedValue); + using T = std::decay_t; + if constexpr (std::is_same_v) + { + throw std::move(containedValue); + } + else + { + return std::move(containedValue); + } }, - v); + std::move(eitherValueOrError)); } template -U Attribute::get() const +std::optional Attribute::getOptional() const { - return getCast(Variant::getResource()); + auto eitherValueOrError = std::visit( + [](auto &&containedValue) -> std::variant { + using containedType = std::decay_t; + return detail::doConvert(&containedValue); + }, + Variant::getResource()); + return std::visit( + [](auto &&containedValue) -> std::optional { + using T = std::decay_t; + if constexpr (std::is_same_v) + { + return std::nullopt; + } + else + { + return {std::move(containedValue)}; + } + }, + std::move(eitherValueOrError)); } - } // namespace openPMD diff --git a/include/openPMD/backend/BaseRecord.hpp b/include/openPMD/backend/BaseRecord.hpp index 65ae298da9..b35709b240 100644 --- a/include/openPMD/backend/BaseRecord.hpp +++ b/include/openPMD/backend/BaseRecord.hpp @@ -317,24 +317,10 @@ inline void BaseRecord::readBase() aRead.name = "unitDimension"; this->IOHandler()->enqueue(IOTask(this, aRead)); this->IOHandler()->flush(internal::defaultFlushParams); - if (*aRead.dtype == DT::ARR_DBL_7) - this->setAttribute( - "unitDimension", - Attribute(*aRead.resource).template get >()); - else if (*aRead.dtype == DT::VEC_DOUBLE) - { - auto vec = - Attribute(*aRead.resource).template get >(); - if (vec.size() == 7) - { - std::array arr; - std::copy(vec.begin(), vec.end(), arr.begin()); - this->setAttribute("unitDimension", arr); - } - else - throw std::runtime_error( - "Unexpected Attribute datatype for 'unitDimension'"); - } + if (auto val = + Attribute(*aRead.resource).getOptional >(); + val.has_value()) + this->setAttribute("unitDimension", val.value()); else throw std::runtime_error( "Unexpected Attribute datatype for 'unitDimension'"); @@ -344,10 +330,14 @@ inline void BaseRecord::readBase() this->IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::FLOAT) this->setAttribute( - "timeOffset", Attribute(*aRead.resource).template get()); + "timeOffset", Attribute(*aRead.resource).get()); else if (*aRead.dtype == DT::DOUBLE) this->setAttribute( - "timeOffset", Attribute(*aRead.resource).template get()); + "timeOffset", Attribute(*aRead.resource).get()); + // conversion cast if a backend reports an integer type + else if (auto val = Attribute(*aRead.resource).getOptional(); + val.has_value()) + this->setAttribute("timeOffset", val.value()); else throw std::runtime_error( "Unexpected Attribute datatype for 'timeOffset'"); diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 5ef4ac0274..0a7de3e6bb 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -425,6 +425,10 @@ void Iteration::read_impl(std::string const &groupPath) setDt(Attribute(*aRead.resource).get()); else if (*aRead.dtype == DT::LONG_DOUBLE) setDt(Attribute(*aRead.resource).get()); + // conversion cast if a backend reports an integer type + else if (auto val = Attribute(*aRead.resource).getOptional(); + val.has_value()) + setDt(val.value()); else throw std::runtime_error("Unexpected Attribute datatype for 'dt'"); @@ -437,14 +441,19 @@ void Iteration::read_impl(std::string const &groupPath) setTime(Attribute(*aRead.resource).get()); else if (*aRead.dtype == DT::LONG_DOUBLE) setTime(Attribute(*aRead.resource).get()); + // conversion cast if a backend reports an integer type + else if (auto val = Attribute(*aRead.resource).getOptional(); + val.has_value()) + setTime(val.value()); else throw std::runtime_error("Unexpected Attribute datatype for 'time'"); aRead.name = "timeUnitSI"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (*aRead.dtype == DT::DOUBLE) - setTimeUnitSI(Attribute(*aRead.resource).get()); + if (auto val = Attribute(*aRead.resource).getOptional(); + val.has_value()) + setTimeUnitSI(val.value()); else throw std::runtime_error( "Unexpected Attribute datatype for 'timeUnitSI'"); diff --git a/src/Mesh.cpp b/src/Mesh.cpp index d570840e98..037e2ca1de 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -341,6 +341,9 @@ void Mesh::read() else if ( *aRead.dtype == DT::VEC_LONG_DOUBLE || *aRead.dtype == DT::LONG_DOUBLE) setGridSpacing(a.get >()); + // conversion cast if a backend reports an integer type + else if (auto val = a.getOptional >(); val.has_value()) + setGridSpacing(val.value()); else throw std::runtime_error( "Unexpected Attribute datatype for 'gridSpacing'"); @@ -348,9 +351,10 @@ void Mesh::read() aRead.name = "gridGlobalOffset"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (*aRead.dtype == DT::VEC_DOUBLE || *aRead.dtype == DT::DOUBLE) - setGridGlobalOffset( - Attribute(*aRead.resource).get >()); + if (auto val = + Attribute(*aRead.resource).getOptional >(); + val.has_value()) + setGridGlobalOffset(val.value()); else throw std::runtime_error( "Unexpected Attribute datatype for 'gridGlobalOffset'"); @@ -358,8 +362,9 @@ void Mesh::read() aRead.name = "gridUnitSI"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (*aRead.dtype == DT::DOUBLE) - setGridUnitSI(Attribute(*aRead.resource).get()); + if (auto val = Attribute(*aRead.resource).getOptional(); + val.has_value()) + setGridUnitSI(val.value()); else throw std::runtime_error( "Unexpected Attribute datatype for 'gridUnitSI'"); diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index bfa024a786..83d5ac688e 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -326,11 +326,9 @@ void RecordComponent::readBase() Extent e; // uint64_t check - Datatype const attrDtype = *aRead.dtype; - if (isSame(attrDtype, determineDatatype >()) || - isSame(attrDtype, determineDatatype())) - for (auto const &val : a.get >()) - e.push_back(val); + if (auto val = a.getOptional >(); val.has_value()) + for (auto const &shape : val.value()) + e.push_back(shape); else { std::ostringstream oss; @@ -348,8 +346,9 @@ void RecordComponent::readBase() aRead.name = "unitSI"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (*aRead.dtype == DT::DOUBLE) - setUnitSI(Attribute(*aRead.resource).get()); + if (auto val = Attribute(*aRead.resource).getOptional(); + val.has_value()) + setUnitSI(val.value()); else throw std::runtime_error("Unexpected Attribute datatype for 'unitSI'"); diff --git a/src/Series.cpp b/src/Series.cpp index 7d87e88e13..478aea5d1c 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -1334,22 +1334,23 @@ std::optional> Series::readGorVBased(bool do_init) void Series::readBase() { auto &series = get(); - using DT = Datatype; Parameter aRead; aRead.name = "openPMD"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (*aRead.dtype == DT::STRING) - setOpenPMD(Attribute(*aRead.resource).get()); + if (auto val = Attribute(*aRead.resource).getOptional(); + val.has_value()) + setOpenPMD(val.value()); else throw std::runtime_error("Unexpected Attribute datatype for 'openPMD'"); aRead.name = "openPMDextension"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (*aRead.dtype == determineDatatype()) - setOpenPMDextension(Attribute(*aRead.resource).get()); + if (auto val = Attribute(*aRead.resource).getOptional(); + val.has_value()) + setOpenPMDextension(val.value()); else throw std::runtime_error( "Unexpected Attribute datatype for 'openPMDextension'"); @@ -1357,8 +1358,9 @@ void Series::readBase() aRead.name = "basePath"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (*aRead.dtype == DT::STRING) - setAttribute("basePath", Attribute(*aRead.resource).get()); + if (auto val = Attribute(*aRead.resource).getOptional(); + val.has_value()) + setAttribute("basePath", val.value()); else throw std::runtime_error( "Unexpected Attribute datatype for 'basePath'"); @@ -1373,13 +1375,14 @@ void Series::readBase() aRead.name = "meshesPath"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (*aRead.dtype == DT::STRING) + if (auto val = Attribute(*aRead.resource).getOptional(); + val.has_value()) { /* allow setting the meshes path after completed IO */ for (auto &it : series.iterations) it.second.meshes.written() = false; - setMeshesPath(Attribute(*aRead.resource).get()); + setMeshesPath(val.value()); for (auto &it : series.iterations) it.second.meshes.written() = true; @@ -1397,13 +1400,14 @@ void Series::readBase() aRead.name = "particlesPath"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (*aRead.dtype == DT::STRING) + if (auto val = Attribute(*aRead.resource).getOptional(); + val.has_value()) { /* allow setting the meshes path after completed IO */ for (auto &it : series.iterations) it.second.particles.written() = false; - setParticlesPath(Attribute(*aRead.resource).get()); + setParticlesPath(val.value()); for (auto &it : series.iterations) it.second.particles.written() = true; diff --git a/src/backend/MeshRecordComponent.cpp b/src/backend/MeshRecordComponent.cpp index 9602868a07..49f2a99d64 100644 --- a/src/backend/MeshRecordComponent.cpp +++ b/src/backend/MeshRecordComponent.cpp @@ -43,6 +43,9 @@ void MeshRecordComponent::read() else if ( *aRead.dtype == DT::VEC_LONG_DOUBLE || *aRead.dtype == DT::LONG_DOUBLE) setPosition(a.get >()); + // conversion cast if a backend reports an integer type + else if (auto val = a.getOptional >(); val.has_value()) + setPosition(val.value()); else throw std::runtime_error( "Unexpected Attribute datatype for 'position'"); diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index f31b135b62..5b9b708311 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -63,11 +63,10 @@ void PatchRecord::read() IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (*aRead.dtype == Datatype::ARR_DBL_7 || - *aRead.dtype == Datatype::VEC_DOUBLE) - this->setAttribute( - "unitDimension", - Attribute(*aRead.resource).template get >()); + if (auto val = + Attribute(*aRead.resource).getOptional >(); + val.has_value()) + this->setAttribute("unitDimension", val.value()); else throw std::runtime_error( "Unexpected Attribute datatype for 'unitDimension'"); diff --git a/src/backend/PatchRecordComponent.cpp b/src/backend/PatchRecordComponent.cpp index a5178a9911..4cac01424b 100644 --- a/src/backend/PatchRecordComponent.cpp +++ b/src/backend/PatchRecordComponent.cpp @@ -126,8 +126,9 @@ void PatchRecordComponent::read() aRead.name = "unitSI"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (*aRead.dtype == Datatype::DOUBLE) - setUnitSI(Attribute(*aRead.resource).get()); + if (auto val = Attribute(*aRead.resource).getOptional(); + val.has_value()) + setUnitSI(val.value()); else throw std::runtime_error("Unexpected Attribute datatype for 'unitSI'"); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index ac187a5d02..2f445c5a50 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -6383,16 +6383,11 @@ enum class ParseMode }; void append_mode( - std::string const &extension, + std::string const &filename, bool variableBased, ParseMode parseMode, std::string jsonConfig = "{}") { - - std::string filename = - (variableBased ? "../samples/append/append_variablebased." - : "../samples/append/append_groupbased.") + - extension; if (auxiliary::directory_exists("../samples/append")) { auxiliary::remove_directory("../samples/append"); @@ -6559,11 +6554,11 @@ void append_mode( break; } } + // AppendAfterSteps has a bug before that version #if 100000000 * ADIOS2_VERSION_MAJOR + 1000000 * ADIOS2_VERSION_MINOR + \ 10000 * ADIOS2_VERSION_PATCH + 100 * ADIOS2_VERSION_TWEAK >= \ 208002700 - // AppendAfterSteps has a bug before that version - if (extension == "bp5") + if (auxiliary::ends_with(filename, ".bp5")) { { Series write( @@ -6669,22 +6664,55 @@ TEST_CASE("append_mode", "[serial]") if (t == "bp5") { append_mode( - t, false, ParseMode::LinearWithoutSnapshot, jsonConfigOld); - append_mode(t, false, ParseMode::WithSnapshot, jsonConfigNew); - append_mode(t, true, ParseMode::WithSnapshot, jsonConfigOld); - append_mode(t, true, ParseMode::WithSnapshot, jsonConfigNew); + "../samples/append/groupbased." + t, + false, + ParseMode::LinearWithoutSnapshot, + jsonConfigOld); + append_mode( + "../samples/append/groupbased_newschema." + t, + false, + ParseMode::WithSnapshot, + jsonConfigNew); + append_mode( + "../samples/append/variablebased." + t, + true, + ParseMode::WithSnapshot, + jsonConfigOld); + append_mode( + "../samples/append/variablebased_newschema." + t, + true, + ParseMode::WithSnapshot, + jsonConfigNew); } - else if (t == "bp" || t == "bp4" || t == "bp5") + else if (t == "bp" || t == "bp4") { append_mode( - t, false, ParseMode::AheadOfTimeWithoutSnapshot, jsonConfigOld); - append_mode(t, false, ParseMode::WithSnapshot, jsonConfigNew); - append_mode(t, true, ParseMode::WithSnapshot, jsonConfigOld); - append_mode(t, true, ParseMode::WithSnapshot, jsonConfigNew); + "../samples/append/append_groupbased." + t, + false, + ParseMode::AheadOfTimeWithoutSnapshot, + jsonConfigOld); + append_mode( + "../samples/append/append_groupbased." + t, + false, + ParseMode::WithSnapshot, + jsonConfigNew); + append_mode( + "../samples/append/append_variablebased." + t, + true, + ParseMode::WithSnapshot, + jsonConfigOld); + append_mode( + "../samples/append/append_variablebased." + t, + true, + ParseMode::WithSnapshot, + jsonConfigNew); } else { - append_mode(t, false, ParseMode::NoSteps); + append_mode( + "../samples/append/append_groupbased." + t, + false, + ParseMode::NoSteps); } } } From 62aaf6c20b1f1c02a9f901e9e7e5255644f81247 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 2 Nov 2022 16:23:54 -0500 Subject: [PATCH 68/70] pybind11: v2.10.1+ (#1322) * Docs: pybind11 v2.10.1+ * Internal: pybind11 v2.10.1 * CI: macOS Verbose Test * macOS CI: Install mpi4py, numpy, pandas Co-authored-by: Henry Schreiner --- .github/workflows/macos.yml | 3 +- CHANGELOG.rst | 2 +- CMakeLists.txt | 8 +- NEWS.rst | 2 +- README.md | 6 +- docs/source/dev/buildoptions.rst | 2 +- docs/source/dev/dependencies.rst | 4 +- pyproject.toml | 2 +- .../thirdParty/pybind11/CMakeLists.txt | 19 + share/openPMD/thirdParty/pybind11/MANIFEST.in | 1 - share/openPMD/thirdParty/pybind11/README.rst | 16 +- .../pybind11/include/pybind11/attr.h | 263 ++- .../pybind11/include/pybind11/buffer_info.h | 123 +- .../pybind11/include/pybind11/cast.h | 1033 ++++++---- .../pybind11/include/pybind11/chrono.h | 140 +- .../pybind11/include/pybind11/complex.h | 29 +- .../pybind11/include/pybind11/detail/class.h | 310 +-- .../pybind11/include/pybind11/detail/common.h | 919 +++++---- .../pybind11/include/pybind11/detail/descr.h | 81 +- .../pybind11/include/pybind11/detail/init.h | 270 ++- .../include/pybind11/detail/internals.h | 182 +- .../pybind11/detail/type_caster_base.h | 579 +++--- .../pybind11/include/pybind11/detail/typeid.h | 28 +- .../pybind11/include/pybind11/eigen.h | 497 +++-- .../pybind11/include/pybind11/embed.h | 171 +- .../pybind11/include/pybind11/eval.h | 103 +- .../pybind11/include/pybind11/functional.h | 37 +- .../pybind11/include/pybind11/gil.h | 118 +- .../pybind11/include/pybind11/iostream.h | 68 +- .../pybind11/include/pybind11/numpy.h | 1288 +++++++----- .../pybind11/include/pybind11/operators.h | 237 ++- .../pybind11/include/pybind11/options.h | 41 +- .../pybind11/include/pybind11/pybind11.h | 1773 ++++++++++------- .../pybind11/include/pybind11/pytypes.h | 1569 ++++++++++----- .../pybind11/include/pybind11/stl.h | 235 ++- .../include/pybind11/stl/filesystem.h | 73 +- .../pybind11/include/pybind11/stl_bind.h | 568 +++--- .../thirdParty/pybind11/pybind11/__init__.py | 10 +- .../thirdParty/pybind11/pybind11/__main__.py | 18 +- .../thirdParty/pybind11/pybind11/_version.py | 6 +- .../thirdParty/pybind11/pybind11/commands.py | 32 +- .../pybind11/pybind11/setup_helpers.py | 226 ++- .../thirdParty/pybind11/tools/FindCatch.cmake | 2 + .../pybind11/tools/FindPythonLibsNew.cmake | 60 +- .../thirdParty/pybind11/tools/JoinPaths.cmake | 23 + .../codespell_ignore_lines_from_errors.py | 35 + .../thirdParty/pybind11/tools/libsize.py | 7 +- .../pybind11/tools/make_changelog.py | 1 - .../thirdParty/pybind11/tools/pybind11.pc.in | 7 + .../pybind11/tools/pybind11Common.cmake | 28 +- .../pybind11/tools/pybind11Config.cmake.in | 6 +- .../pybind11/tools/pybind11NewTools.cmake | 36 +- .../pybind11/tools/pybind11Tools.cmake | 46 +- .../pybind11/tools/setup_global.py.in | 10 +- .../pybind11/tools/setup_main.py.in | 7 +- 55 files changed, 6772 insertions(+), 4588 deletions(-) create mode 100644 share/openPMD/thirdParty/pybind11/tools/JoinPaths.cmake create mode 100644 share/openPMD/thirdParty/pybind11/tools/codespell_ignore_lines_from_errors.py create mode 100644 share/openPMD/thirdParty/pybind11/tools/pybind11.pc.in diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 40528ae36e..209dd4e2ab 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -25,6 +25,7 @@ jobs: brew install adios2 || true brew install hdf5-mpi || true brew install python || true + python3 -m pip install -U mpi4py numpy pandas set -e - name: Build env: {CXXFLAGS: -Werror -DTOML11_DISABLE_STD_FILESYSTEM, MACOSX_DEPLOYMENT_TARGET: 10.13} @@ -41,7 +42,7 @@ jobs: -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 2 - ctest --test-dir build --output-on-failure + ctest --test-dir build --verbose # TODO: apple_conda_ompi_all (similar to conda_ompi_all on Linux) # both OpenMPI and MPICH cause startup (MPI_Init) issues on GitHub Actions diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 45c42b1c54..8668f57ffd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,7 +18,7 @@ Features """""""" - include internally shipped toml11 v3.7.1 #1148 #1227 -- pybind11: require version 2.9.1+ #1220 +- pybind11: require version 2.10.1+ #1220 #1322 Bug Fixes """"""""" diff --git a/CMakeLists.txt b/CMakeLists.txt index 55f73954e5..d9ca34333c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -401,9 +401,9 @@ if(openPMD_USE_PYTHON STREQUAL AUTO) if(openPMD_USE_INTERNAL_PYBIND11) add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/pybind11") set(openPMD_HAVE_PYTHON TRUE) - message(STATUS "pybind11: Using INTERNAL version 2.9.1") + message(STATUS "pybind11: Using INTERNAL version 2.10.1") else() - find_package(pybind11 2.9.1 CONFIG) + find_package(pybind11 2.10.1 CONFIG) if(pybind11_FOUND) set(openPMD_HAVE_PYTHON TRUE) message(STATUS "pybind11: Found version '${pybind11_VERSION}'") @@ -419,9 +419,9 @@ elseif(openPMD_USE_PYTHON) if(openPMD_USE_INTERNAL_PYBIND11) add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/pybind11") set(openPMD_HAVE_PYTHON TRUE) - message(STATUS "pybind11: Using INTERNAL version 2.9.1") + message(STATUS "pybind11: Using INTERNAL version 2.10.1") else() - find_package(pybind11 2.9.1 REQUIRED CONFIG) + find_package(pybind11 2.10.1 REQUIRED CONFIG) set(openPMD_HAVE_PYTHON TRUE) message(STATUS "pybind11: Found version '${pybind11_VERSION}'") endif() diff --git a/NEWS.rst b/NEWS.rst index 7456daecfe..c97f218534 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -11,7 +11,7 @@ Building openPMD-api now requires a compiler that supports C++17 or newer. Python 3.10 is now supported. openPMD-api now depends on `toml11 `__ 3.7.1+. -pybind11 2.9.1 is now the minimally supported version for Python support. +pybind11 2.10.1 is now the minimally supported version for Python support. Catch2 2.13.9 is now the minimally supported version for tests. The following backend-specific members of the ``Dataset`` class have been removed: ``Dataset::setChunkSize()``, ``Dataset::setCompression()``, ``Dataset::setCustomTransform()``, ``Dataset::chunkSize``, ``Dataset::compression``, ``Dataset::transform``. diff --git a/README.md b/README.md index 94b0531178..e39bd05ad0 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ Required: Shipped internally in `share/openPMD/thirdParty/`: * [Catch2](https://github.com/catchorg/Catch2) 2.13.9+ ([BSL-1.0](https://github.com/catchorg/Catch2/blob/master/LICENSE.txt)) -* [pybind11](https://github.com/pybind/pybind11) 2.9.1+ ([new BSD](https://github.com/pybind/pybind11/blob/master/LICENSE)) +* [pybind11](https://github.com/pybind/pybind11) 2.10.1+ ([new BSD](https://github.com/pybind/pybind11/blob/master/LICENSE)) * [NLohmann-JSON](https://github.com/nlohmann/json) 3.9.1+ ([MIT](https://github.com/nlohmann/json/blob/develop/LICENSE.MIT)) * [toml11](https://github.com/ToruNiina/toml11) 3.7.1+ ([MIT](https://github.com/ToruNiina/toml11/blob/master/LICENSE)) @@ -120,7 +120,7 @@ while those can be built either with or without: Optional language bindings: * Python: * Python 3.6 - 3.10 - * pybind11 2.9.1+ + * pybind11 2.10.1+ * numpy 1.15+ * mpi4py 2.1+ (optional, for MPI) * pandas 1.0+ (optional, for dataframes) @@ -272,7 +272,7 @@ The following options allow to switch to external installs: | CMake Option | Values | Library | Version | |---------------------------------|------------|---------------|---------| | `openPMD_USE_INTERNAL_CATCH` | **ON**/OFF | Catch2 | 2.13.9+ | -| `openPMD_USE_INTERNAL_PYBIND11` | **ON**/OFF | pybind11 | 2.9.1+ | +| `openPMD_USE_INTERNAL_PYBIND11` | **ON**/OFF | pybind11 | 2.10.1+ | | `openPMD_USE_INTERNAL_JSON` | **ON**/OFF | NLohmann-JSON | 3.9.1+ | | `openPMD_USE_INTERNAL_TOML11` | **ON**/OFF | toml11 | 3.7.1+ | diff --git a/docs/source/dev/buildoptions.rst b/docs/source/dev/buildoptions.rst index b9cbcc9e40..e63c1c8e9d 100644 --- a/docs/source/dev/buildoptions.rst +++ b/docs/source/dev/buildoptions.rst @@ -69,7 +69,7 @@ The following options allow to switch to external installs of dependencies: CMake Option Values Installs Library Version ================================= =========== ======== ============= ======== ``openPMD_USE_INTERNAL_CATCH`` **ON**/OFF No Catch2 2.13.9+ -``openPMD_USE_INTERNAL_PYBIND11`` **ON**/OFF No pybind11 2.9.1+ +``openPMD_USE_INTERNAL_PYBIND11`` **ON**/OFF No pybind11 2.10.1+ ``openPMD_USE_INTERNAL_JSON`` **ON**/OFF No NLohmann-JSON 3.9.1+ ``openPMD_USE_INTERNAL_TOML11`` **ON**/OFF No toml11 3.7.1+ ================================= =========== ======== ============= ======== diff --git a/docs/source/dev/dependencies.rst b/docs/source/dev/dependencies.rst index 6ff4c49906..f86a854252 100644 --- a/docs/source/dev/dependencies.rst +++ b/docs/source/dev/dependencies.rst @@ -18,7 +18,7 @@ Shipped internally The following libraries are shipped internally in ``share/openPMD/thirdParty/`` for convenience: * `Catch2 `_ 2.13.9+ (`BSL-1.0 `__) -* `pybind11 `_ 2.9.1+ (`new BSD `_) +* `pybind11 `_ 2.10.1+ (`new BSD `_) * `NLohmann-JSON `_ 3.9.1+ (`MIT `_) * `toml11 `_ 3.7.1+ (`MIT `__) @@ -40,7 +40,7 @@ Optional: language bindings * Python: * Python 3.6 - 3.10 - * pybind11 2.9.1+ + * pybind11 2.10.1+ * numpy 1.15+ * mpi4py 2.1+ (optional, for MPI) * pandas 1.0+ (optional, for dataframes) diff --git a/pyproject.toml b/pyproject.toml index 3de6ddfbd4..215901d25f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,6 +3,6 @@ requires = [ "setuptools>=42", "wheel", "cmake>=3.15.0,<4.0.0", - "pybind11>=2.9.1,<3.0.0" + "pybind11>=2.10.1,<3.0.0" ] build-backend = "setuptools.build_meta" diff --git a/share/openPMD/thirdParty/pybind11/CMakeLists.txt b/share/openPMD/thirdParty/pybind11/CMakeLists.txt index 3787982cbd..3be6f7a49b 100644 --- a/share/openPMD/thirdParty/pybind11/CMakeLists.txt +++ b/share/openPMD/thirdParty/pybind11/CMakeLists.txt @@ -91,10 +91,16 @@ endif() option(PYBIND11_INSTALL "Install pybind11 header files?" ${PYBIND11_MASTER_PROJECT}) option(PYBIND11_TEST "Build pybind11 test suite?" ${PYBIND11_MASTER_PROJECT}) option(PYBIND11_NOPYTHON "Disable search for Python" OFF) +option(PYBIND11_SIMPLE_GIL_MANAGEMENT + "Use simpler GIL management logic that does not support disassociation" OFF) set(PYBIND11_INTERNALS_VERSION "" CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.") +if(PYBIND11_SIMPLE_GIL_MANAGEMENT) + add_compile_definitions(PYBIND11_SIMPLE_GIL_MANAGEMENT) +endif() + cmake_dependent_option( USE_PYTHON_INCLUDE_DIR "Install pybind11 headers in Python include directory instead of default installation prefix" @@ -198,6 +204,9 @@ else() endif() include("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11Common.cmake") +# https://github.com/jtojnar/cmake-snips/#concatenating-paths-when-building-pkg-config-files +# TODO: cmake 3.20 adds the cmake_path() function, which obsoletes this snippet +include("${CMAKE_CURRENT_SOURCE_DIR}/tools/JoinPaths.cmake") # Relative directory setting if(USE_PYTHON_INCLUDE_DIR AND DEFINED Python_INCLUDE_DIRS) @@ -262,6 +271,16 @@ if(PYBIND11_INSTALL) NAMESPACE "pybind11::" DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) + # pkg-config support + if(NOT prefix_for_pc_file) + set(prefix_for_pc_file "${CMAKE_INSTALL_PREFIX}") + endif() + join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}") + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11.pc.in" + "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" @ONLY) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" + DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig/") + # Uninstall target if(PYBIND11_MASTER_PROJECT) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake_uninstall.cmake.in" diff --git a/share/openPMD/thirdParty/pybind11/MANIFEST.in b/share/openPMD/thirdParty/pybind11/MANIFEST.in index aed183e874..033303a74a 100644 --- a/share/openPMD/thirdParty/pybind11/MANIFEST.in +++ b/share/openPMD/thirdParty/pybind11/MANIFEST.in @@ -1,6 +1,5 @@ recursive-include pybind11/include/pybind11 *.h recursive-include pybind11 *.py recursive-include pybind11 py.typed -recursive-include pybind11 *.pyi include pybind11/share/cmake/pybind11/*.cmake include LICENSE README.rst pyproject.toml setup.py setup.cfg diff --git a/share/openPMD/thirdParty/pybind11/README.rst b/share/openPMD/thirdParty/pybind11/README.rst index 45c4af5a60..3c75edb575 100644 --- a/share/openPMD/thirdParty/pybind11/README.rst +++ b/share/openPMD/thirdParty/pybind11/README.rst @@ -32,9 +32,9 @@ this heavy machinery has become an excessively large and unnecessary dependency. Think of this library as a tiny self-contained version of Boost.Python -with everything stripped away that isn’t relevant for binding +with everything stripped away that isn't relevant for binding generation. Without comments, the core header files only require ~4K -lines of code and depend on Python (2.7 or 3.5+, or PyPy) and the C++ +lines of code and depend on Python (3.6+, or PyPy) and the C++ standard library. This compact implementation was possible thanks to some of the new C++11 language features (specifically: tuples, lambda functions and variadic templates). Since its creation, this library has @@ -78,8 +78,8 @@ Goodies In addition to the core functionality, pybind11 provides some extra goodies: -- Python 2.7, 3.5+, and PyPy/PyPy3 7.3 are supported with an - implementation-agnostic interface. +- Python 3.6+, and PyPy3 7.3 are supported with an implementation-agnostic + interface (pybind11 2.9 was the last version to support Python 2 and 3.5). - It is possible to bind C++11 lambda functions with captured variables. The lambda capture data is stored inside the resulting @@ -88,8 +88,8 @@ goodies: - pybind11 uses C++11 move constructors and move assignment operators whenever possible to efficiently transfer custom data types. -- It’s easy to expose the internal storage of custom data types through - Pythons’ buffer protocols. This is handy e.g. for fast conversion +- It's easy to expose the internal storage of custom data types through + Pythons' buffer protocols. This is handy e.g. for fast conversion between C++ matrix classes like Eigen and NumPy without expensive copy operations. @@ -119,10 +119,10 @@ goodies: Supported compilers ------------------- -1. Clang/LLVM 3.3 or newer (for Apple Xcode’s clang, this is 5.0.0 or +1. Clang/LLVM 3.3 or newer (for Apple Xcode's clang, this is 5.0.0 or newer) 2. GCC 4.8 or newer -3. Microsoft Visual Studio 2015 Update 3 or newer +3. Microsoft Visual Studio 2017 or newer 4. Intel classic C++ compiler 18 or newer (ICC 20.2 tested in CI) 5. Cygwin/GCC (previously tested on 2.5.1) 6. NVCC (CUDA 11.0 tested in CI) diff --git a/share/openPMD/thirdParty/pybind11/include/pybind11/attr.h b/share/openPMD/thirdParty/pybind11/include/pybind11/attr.h index f1b66fb80c..db7cd8efff 100644 --- a/share/openPMD/thirdParty/pybind11/include/pybind11/attr.h +++ b/share/openPMD/thirdParty/pybind11/include/pybind11/attr.h @@ -10,6 +10,7 @@ #pragma once +#include "detail/common.h" #include "cast.h" #include @@ -20,65 +21,72 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) /// @{ /// Annotation for methods -struct is_method { handle class_; +struct is_method { + handle class_; explicit is_method(const handle &c) : class_(c) {} }; /// Annotation for operators -struct is_operator { }; +struct is_operator {}; /// Annotation for classes that cannot be subclassed -struct is_final { }; +struct is_final {}; /// Annotation for parent scope -struct scope { handle value; +struct scope { + handle value; explicit scope(const handle &s) : value(s) {} }; /// Annotation for documentation -struct doc { const char *value; +struct doc { + const char *value; explicit doc(const char *value) : value(value) {} }; /// Annotation for function names -struct name { const char *value; +struct name { + const char *value; explicit name(const char *value) : value(value) {} }; /// Annotation indicating that a function is an overload associated with a given "sibling" -struct sibling { handle value; +struct sibling { + handle value; explicit sibling(const handle &value) : value(value.ptr()) {} }; /// Annotation indicating that a class derives from another given type -template struct base { +template +struct base { - PYBIND11_DEPRECATED("base() was deprecated in favor of specifying 'T' as a template argument to class_") - base() { } // NOLINT(modernize-use-equals-default): breaks MSVC 2015 when adding an attribute + PYBIND11_DEPRECATED( + "base() was deprecated in favor of specifying 'T' as a template argument to class_") + base() = default; }; /// Keep patient alive while nurse lives -template struct keep_alive { }; +template +struct keep_alive {}; /// Annotation indicating that a class is involved in a multiple inheritance relationship -struct multiple_inheritance { }; +struct multiple_inheritance {}; /// Annotation which enables dynamic attributes, i.e. adds `__dict__` to a class -struct dynamic_attr { }; +struct dynamic_attr {}; /// Annotation which enables the buffer protocol for a type -struct buffer_protocol { }; +struct buffer_protocol {}; /// Annotation which requests that a special metaclass is created for a type struct metaclass { handle value; PYBIND11_DEPRECATED("py::metaclass() is no longer required. It's turned on by default now.") - // NOLINTNEXTLINE(modernize-use-equals-default): breaks MSVC 2015 when adding an attribute - metaclass() {} + metaclass() = default; /// Override pybind11's default metaclass - explicit metaclass(handle value) : value(value) { } + explicit metaclass(handle value) : value(value) {} }; /// Specifies a custom callback with signature `void (PyHeapTypeObject*)` that @@ -99,15 +107,16 @@ struct custom_type_setup { }; /// Annotation that marks a class as local to the module: -struct module_local { const bool value; +struct module_local { + const bool value; constexpr explicit module_local(bool v = true) : value(v) {} }; /// Annotation to mark enums as an arithmetic type -struct arithmetic { }; +struct arithmetic {}; /// Mark a function for addition at the beginning of the existing overload chain instead of the end -struct prepend { }; +struct prepend {}; /** \rst A call policy which places one or more guard variables (``Ts...``) around the function call. @@ -127,9 +136,13 @@ struct prepend { }; return foo(args...); // forwarded arguments }); \endrst */ -template struct call_guard; +template +struct call_guard; -template <> struct call_guard<> { using type = detail::void_type; }; +template <> +struct call_guard<> { + using type = detail::void_type; +}; template struct call_guard { @@ -154,7 +167,8 @@ PYBIND11_NAMESPACE_BEGIN(detail) enum op_id : int; enum op_type : int; struct undefined_t; -template struct op_; +template +struct op_; void keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret); /// Internal data structure which holds metadata about a keyword argument @@ -166,15 +180,16 @@ struct argument_record { bool none : 1; ///< True if None is allowed when loading argument_record(const char *name, const char *descr, handle value, bool convert, bool none) - : name(name), descr(descr), value(value), convert(convert), none(none) { } + : name(name), descr(descr), value(value), convert(convert), none(none) {} }; -/// Internal data structure which holds metadata about a bound function (signature, overloads, etc.) +/// Internal data structure which holds metadata about a bound function (signature, overloads, +/// etc.) struct function_record { function_record() : is_constructor(false), is_new_style_constructor(false), is_stateless(false), - is_operator(false), is_method(false), has_args(false), - has_kwargs(false), prepend(false) { } + is_operator(false), is_method(false), has_args(false), has_kwargs(false), + prepend(false) {} /// Function name char *name = nullptr; /* why no C++ strings? They generate heavier code.. */ @@ -189,13 +204,13 @@ struct function_record { std::vector args; /// Pointer to lambda function which converts arguments and performs the actual call - handle (*impl) (function_call &) = nullptr; + handle (*impl)(function_call &) = nullptr; /// Storage for the wrapped function pointer and captured data, if any - void *data[3] = { }; + void *data[3] = {}; /// Pointer to custom destructor for 'data' (if needed) - void (*free_data) (function_record *ptr) = nullptr; + void (*free_data)(function_record *ptr) = nullptr; /// Return value policy associated with this function return_value_policy policy = return_value_policy::automatic; @@ -251,7 +266,7 @@ struct function_record { struct type_record { PYBIND11_NOINLINE type_record() : multiple_inheritance(false), dynamic_attr(false), buffer_protocol(false), - default_holder(true), module_local(false), is_final(false) { } + default_holder(true), module_local(false), is_final(false) {} /// Handle to the parent scope handle scope; @@ -310,42 +325,45 @@ struct type_record { /// Is the class inheritable from python classes? bool is_final : 1; - PYBIND11_NOINLINE void add_base(const std::type_info &base, void *(*caster)(void *)) { - auto base_info = detail::get_type_info(base, false); + PYBIND11_NOINLINE void add_base(const std::type_info &base, void *(*caster)(void *) ) { + auto *base_info = detail::get_type_info(base, false); if (!base_info) { std::string tname(base.name()); detail::clean_type_id(tname); - pybind11_fail("generic_type: type \"" + std::string(name) + - "\" referenced unknown base type \"" + tname + "\""); + pybind11_fail("generic_type: type \"" + std::string(name) + + "\" referenced unknown base type \"" + tname + "\""); } if (default_holder != base_info->default_holder) { std::string tname(base.name()); detail::clean_type_id(tname); - pybind11_fail("generic_type: type \"" + std::string(name) + "\" " + - (default_holder ? "does not have" : "has") + - " a non-default holder type while its base \"" + tname + "\" " + - (base_info->default_holder ? "does not" : "does")); + pybind11_fail("generic_type: type \"" + std::string(name) + "\" " + + (default_holder ? "does not have" : "has") + + " a non-default holder type while its base \"" + tname + "\" " + + (base_info->default_holder ? "does not" : "does")); } bases.append((PyObject *) base_info->type); - if (base_info->type->tp_dictoffset != 0) - dynamic_attr = true; +#if PY_VERSION_HEX < 0x030B0000 + dynamic_attr |= base_info->type->tp_dictoffset != 0; +#else + dynamic_attr |= (base_info->type->tp_flags & Py_TPFLAGS_MANAGED_DICT) != 0; +#endif - if (caster) + if (caster) { base_info->implicit_casts.emplace_back(type, caster); + } } }; -inline function_call::function_call(const function_record &f, handle p) : - func(f), parent(p) { +inline function_call::function_call(const function_record &f, handle p) : func(f), parent(p) { args.reserve(f.nargs); args_convert.reserve(f.nargs); } /// Tag for a new-style `__init__` defined in `detail/init.h` -struct is_new_style_constructor { }; +struct is_new_style_constructor {}; /** * Partial template specializations to process custom attributes provided to @@ -353,74 +371,97 @@ struct is_new_style_constructor { }; * fields in the type_record and function_record data structures or executed at * runtime to deal with custom call policies (e.g. keep_alive). */ -template struct process_attribute; +template +struct process_attribute; -template struct process_attribute_default { +template +struct process_attribute_default { /// Default implementation: do nothing - static void init(const T &, function_record *) { } - static void init(const T &, type_record *) { } - static void precall(function_call &) { } - static void postcall(function_call &, handle) { } + static void init(const T &, function_record *) {} + static void init(const T &, type_record *) {} + static void precall(function_call &) {} + static void postcall(function_call &, handle) {} }; /// Process an attribute specifying the function's name -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const name &n, function_record *r) { r->name = const_cast(n.value); } }; /// Process an attribute specifying the function's docstring -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const doc &n, function_record *r) { r->doc = const_cast(n.value); } }; /// Process an attribute specifying the function's docstring (provided as a C-style string) -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const char *d, function_record *r) { r->doc = const_cast(d); } static void init(const char *d, type_record *r) { r->doc = const_cast(d); } }; -template <> struct process_attribute : process_attribute { }; +template <> +struct process_attribute : process_attribute {}; /// Process an attribute indicating the function's return value policy -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const return_value_policy &p, function_record *r) { r->policy = p; } }; -/// Process an attribute which indicates that this is an overloaded function associated with a given sibling -template <> struct process_attribute : process_attribute_default { +/// Process an attribute which indicates that this is an overloaded function associated with a +/// given sibling +template <> +struct process_attribute : process_attribute_default { static void init(const sibling &s, function_record *r) { r->sibling = s.value; } }; /// Process an attribute which indicates that this function is a method -template <> struct process_attribute : process_attribute_default { - static void init(const is_method &s, function_record *r) { r->is_method = true; r->scope = s.class_; } +template <> +struct process_attribute : process_attribute_default { + static void init(const is_method &s, function_record *r) { + r->is_method = true; + r->scope = s.class_; + } }; /// Process an attribute which indicates the parent scope of a method -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const scope &s, function_record *r) { r->scope = s.value; } }; /// Process an attribute which indicates that this function is an operator -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const is_operator &, function_record *r) { r->is_operator = true; } }; -template <> struct process_attribute : process_attribute_default { - static void init(const is_new_style_constructor &, function_record *r) { r->is_new_style_constructor = true; } +template <> +struct process_attribute + : process_attribute_default { + static void init(const is_new_style_constructor &, function_record *r) { + r->is_new_style_constructor = true; + } }; inline void check_kw_only_arg(const arg &a, function_record *r) { - if (r->args.size() > r->nargs_pos && (!a.name || a.name[0] == '\0')) - pybind11_fail("arg(): cannot specify an unnamed argument after a kw_only() annotation or args() argument"); + if (r->args.size() > r->nargs_pos && (!a.name || a.name[0] == '\0')) { + pybind11_fail("arg(): cannot specify an unnamed argument after a kw_only() annotation or " + "args() argument"); + } } inline void append_self_arg_if_needed(function_record *r) { - if (r->is_method && r->args.empty()) - r->args.emplace_back("self", nullptr, handle(), /*convert=*/ true, /*none=*/ false); + if (r->is_method && r->args.empty()) { + r->args.emplace_back("self", nullptr, handle(), /*convert=*/true, /*none=*/false); + } } /// Process a keyword argument attribute (*without* a default value) -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const arg &a, function_record *r) { append_self_arg_if_needed(r); r->args.emplace_back(a.name, nullptr, handle(), !a.flag_noconvert, a.flag_none); @@ -430,30 +471,38 @@ template <> struct process_attribute : process_attribute_default { }; /// Process a keyword argument attribute (*with* a default value) -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const arg_v &a, function_record *r) { - if (r->is_method && r->args.empty()) - r->args.emplace_back("self", /*descr=*/ nullptr, /*parent=*/ handle(), /*convert=*/ true, /*none=*/ false); + if (r->is_method && r->args.empty()) { + r->args.emplace_back( + "self", /*descr=*/nullptr, /*parent=*/handle(), /*convert=*/true, /*none=*/false); + } if (!a.value) { -#if !defined(NDEBUG) +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) std::string descr("'"); - if (a.name) descr += std::string(a.name) + ": "; + if (a.name) { + descr += std::string(a.name) + ": "; + } descr += a.type + "'"; if (r->is_method) { - if (r->name) - descr += " in method '" + (std::string) str(r->scope) + "." + (std::string) r->name + "'"; - else + if (r->name) { + descr += " in method '" + (std::string) str(r->scope) + "." + + (std::string) r->name + "'"; + } else { descr += " in method of '" + (std::string) str(r->scope) + "'"; + } } else if (r->name) { descr += " in function '" + (std::string) r->name + "'"; } - pybind11_fail("arg(): could not convert default argument " - + descr + " into a Python object (type not registered yet?)"); + pybind11_fail("arg(): could not convert default argument " + descr + + " into a Python object (type not registered yet?)"); #else pybind11_fail("arg(): could not convert default argument " "into a Python object (type not registered yet?). " - "Compile in debug mode for more information."); + "#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for " + "more information."); #endif } r->args.emplace_back(a.name, a.descr, a.value.inc_ref(), !a.flag_noconvert, a.flag_none); @@ -463,29 +512,36 @@ template <> struct process_attribute : process_attribute_default { }; /// Process a keyword-only-arguments-follow pseudo argument -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const kw_only &, function_record *r) { append_self_arg_if_needed(r); - if (r->has_args && r->nargs_pos != static_cast(r->args.size())) - pybind11_fail("Mismatched args() and kw_only(): they must occur at the same relative argument location (or omit kw_only() entirely)"); + if (r->has_args && r->nargs_pos != static_cast(r->args.size())) { + pybind11_fail("Mismatched args() and kw_only(): they must occur at the same relative " + "argument location (or omit kw_only() entirely)"); + } r->nargs_pos = static_cast(r->args.size()); } }; /// Process a positional-only-argument maker -template <> struct process_attribute : process_attribute_default { +template <> +struct process_attribute : process_attribute_default { static void init(const pos_only &, function_record *r) { append_self_arg_if_needed(r); r->nargs_pos_only = static_cast(r->args.size()); - if (r->nargs_pos_only > r->nargs_pos) + if (r->nargs_pos_only > r->nargs_pos) { pybind11_fail("pos_only(): cannot follow a py::args() argument"); - // It also can't follow a kw_only, but a static_assert in pybind11.h checks that + } + // It also can't follow a kw_only, but a static_assert in pybind11.h checks that } }; -/// Process a parent class attribute. Single inheritance only (class_ itself already guarantees that) +/// Process a parent class attribute. Single inheritance only (class_ itself already guarantees +/// that) template -struct process_attribute::value>> : process_attribute_default { +struct process_attribute::value>> + : process_attribute_default { static void init(const handle &h, type_record *r) { r->bases.append(h); } }; @@ -498,7 +554,9 @@ struct process_attribute> : process_attribute_default> { /// Process a multiple inheritance attribute template <> struct process_attribute : process_attribute_default { - static void init(const multiple_inheritance &, type_record *r) { r->multiple_inheritance = true; } + static void init(const multiple_inheritance &, type_record *r) { + r->multiple_inheritance = true; + } }; template <> @@ -544,34 +602,41 @@ template <> struct process_attribute : process_attribute_default {}; template -struct process_attribute> : process_attribute_default> { }; +struct process_attribute> : process_attribute_default> {}; /** * Process a keep_alive call policy -- invokes keep_alive_impl during the * pre-call handler if both Nurse, Patient != 0 and use the post-call handler * otherwise */ -template struct process_attribute> : public process_attribute_default> { +template +struct process_attribute> + : public process_attribute_default> { template = 0> - static void precall(function_call &call) { keep_alive_impl(Nurse, Patient, call, handle()); } + static void precall(function_call &call) { + keep_alive_impl(Nurse, Patient, call, handle()); + } template = 0> - static void postcall(function_call &, handle) { } + static void postcall(function_call &, handle) {} template = 0> - static void precall(function_call &) { } + static void precall(function_call &) {} template = 0> - static void postcall(function_call &call, handle ret) { keep_alive_impl(Nurse, Patient, call, ret); } + static void postcall(function_call &call, handle ret) { + keep_alive_impl(Nurse, Patient, call, ret); + } }; /// Recursively iterate over variadic template arguments -template struct process_attributes { - static void init(const Args&... args, function_record *r) { +template +struct process_attributes { + static void init(const Args &...args, function_record *r) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(r); PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(r); using expander = int[]; (void) expander{ 0, ((void) process_attribute::type>::init(args, r), 0)...}; } - static void init(const Args&... args, type_record *r) { + static void init(const Args &...args, type_record *r) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(r); PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(r); using expander = int[]; @@ -603,7 +668,7 @@ using extract_guard_t = typename exactly_one_t, Extr /// Check the number of named arguments at compile time template ::value...), - size_t self = constexpr_sum(std::is_same::value...)> + size_t self = constexpr_sum(std::is_same::value...)> constexpr bool expected_num_args(size_t nargs, bool has_args, bool has_kwargs) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(nargs, has_args, has_kwargs); return named == 0 || (self + named + size_t(has_args) + size_t(has_kwargs)) == nargs; diff --git a/share/openPMD/thirdParty/pybind11/include/pybind11/buffer_info.h b/share/openPMD/thirdParty/pybind11/include/pybind11/buffer_info.h index eba68d1aa1..06120d5563 100644 --- a/share/openPMD/thirdParty/pybind11/include/pybind11/buffer_info.h +++ b/share/openPMD/thirdParty/pybind11/include/pybind11/buffer_info.h @@ -19,9 +19,11 @@ PYBIND11_NAMESPACE_BEGIN(detail) inline std::vector c_strides(const std::vector &shape, ssize_t itemsize) { auto ndim = shape.size(); std::vector strides(ndim, itemsize); - if (ndim > 0) - for (size_t i = ndim - 1; i > 0; --i) + if (ndim > 0) { + for (size_t i = ndim - 1; i > 0; --i) { strides[i - 1] = strides[i] * shape[i]; + } + } return strides; } @@ -29,8 +31,9 @@ inline std::vector c_strides(const std::vector &shape, ssize_t inline std::vector f_strides(const std::vector &shape, ssize_t itemsize) { auto ndim = shape.size(); std::vector strides(ndim, itemsize); - for (size_t i = 1; i < ndim; ++i) + for (size_t i = 1; i < ndim; ++i) { strides[i] = strides[i - 1] * shape[i - 1]; + } return strides; } @@ -41,55 +44,85 @@ struct buffer_info { void *ptr = nullptr; // Pointer to the underlying storage ssize_t itemsize = 0; // Size of individual items in bytes ssize_t size = 0; // Total number of entries - std::string format; // For homogeneous buffers, this should be set to format_descriptor::format() + std::string format; // For homogeneous buffers, this should be set to + // format_descriptor::format() ssize_t ndim = 0; // Number of dimensions std::vector shape; // Shape of the tensor (1 entry per dimension) - std::vector strides; // Number of bytes between adjacent entries (for each per dimension) + std::vector strides; // Number of bytes between adjacent entries + // (for each per dimension) bool readonly = false; // flag to indicate if the underlying storage may be written to buffer_info() = default; - buffer_info(void *ptr, ssize_t itemsize, const std::string &format, ssize_t ndim, - detail::any_container shape_in, detail::any_container strides_in, bool readonly=false) - : ptr(ptr), itemsize(itemsize), size(1), format(format), ndim(ndim), - shape(std::move(shape_in)), strides(std::move(strides_in)), readonly(readonly) { - if (ndim != (ssize_t) shape.size() || ndim != (ssize_t) strides.size()) + buffer_info(void *ptr, + ssize_t itemsize, + const std::string &format, + ssize_t ndim, + detail::any_container shape_in, + detail::any_container strides_in, + bool readonly = false) + : ptr(ptr), itemsize(itemsize), size(1), format(format), ndim(ndim), + shape(std::move(shape_in)), strides(std::move(strides_in)), readonly(readonly) { + if (ndim != (ssize_t) shape.size() || ndim != (ssize_t) strides.size()) { pybind11_fail("buffer_info: ndim doesn't match shape and/or strides length"); - for (size_t i = 0; i < (size_t) ndim; ++i) + } + for (size_t i = 0; i < (size_t) ndim; ++i) { size *= shape[i]; + } } template - buffer_info(T *ptr, detail::any_container shape_in, detail::any_container strides_in, bool readonly=false) - : buffer_info(private_ctr_tag(), ptr, sizeof(T), format_descriptor::format(), static_cast(shape_in->size()), std::move(shape_in), std::move(strides_in), readonly) { } - - buffer_info(void *ptr, ssize_t itemsize, const std::string &format, ssize_t size, bool readonly=false) - : buffer_info(ptr, itemsize, format, 1, {size}, {itemsize}, readonly) { } + buffer_info(T *ptr, + detail::any_container shape_in, + detail::any_container strides_in, + bool readonly = false) + : buffer_info(private_ctr_tag(), + ptr, + sizeof(T), + format_descriptor::format(), + static_cast(shape_in->size()), + std::move(shape_in), + std::move(strides_in), + readonly) {} + + buffer_info(void *ptr, + ssize_t itemsize, + const std::string &format, + ssize_t size, + bool readonly = false) + : buffer_info(ptr, itemsize, format, 1, {size}, {itemsize}, readonly) {} template - buffer_info(T *ptr, ssize_t size, bool readonly=false) - : buffer_info(ptr, sizeof(T), format_descriptor::format(), size, readonly) { } + buffer_info(T *ptr, ssize_t size, bool readonly = false) + : buffer_info(ptr, sizeof(T), format_descriptor::format(), size, readonly) {} template - buffer_info(const T *ptr, ssize_t size, bool readonly=true) - : buffer_info(const_cast(ptr), sizeof(T), format_descriptor::format(), size, readonly) { } + buffer_info(const T *ptr, ssize_t size, bool readonly = true) + : buffer_info( + const_cast(ptr), sizeof(T), format_descriptor::format(), size, readonly) {} explicit buffer_info(Py_buffer *view, bool ownview = true) - : buffer_info(view->buf, view->itemsize, view->format, view->ndim, + : buffer_info( + view->buf, + view->itemsize, + view->format, + view->ndim, {view->shape, view->shape + view->ndim}, /* Though buffer::request() requests PyBUF_STRIDES, ctypes objects * ignore this flag and return a view with NULL strides. * When strides are NULL, build them manually. */ view->strides - ? std::vector(view->strides, view->strides + view->ndim) - : detail::c_strides({view->shape, view->shape + view->ndim}, view->itemsize), + ? std::vector(view->strides, view->strides + view->ndim) + : detail::c_strides({view->shape, view->shape + view->ndim}, view->itemsize), (view->readonly != 0)) { + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) this->m_view = view; + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) this->ownview = ownview; } buffer_info(const buffer_info &) = delete; - buffer_info& operator=(const buffer_info &) = delete; + buffer_info &operator=(const buffer_info &) = delete; buffer_info(buffer_info &&other) noexcept { (*this) = std::move(other); } @@ -108,17 +141,28 @@ struct buffer_info { } ~buffer_info() { - if (m_view && ownview) { PyBuffer_Release(m_view); delete m_view; } + if (m_view && ownview) { + PyBuffer_Release(m_view); + delete m_view; + } } Py_buffer *view() const { return m_view; } Py_buffer *&view() { return m_view; } -private: - struct private_ctr_tag { }; - buffer_info(private_ctr_tag, void *ptr, ssize_t itemsize, const std::string &format, ssize_t ndim, - detail::any_container &&shape_in, detail::any_container &&strides_in, bool readonly) - : buffer_info(ptr, itemsize, format, ndim, std::move(shape_in), std::move(strides_in), readonly) { } +private: + struct private_ctr_tag {}; + + buffer_info(private_ctr_tag, + void *ptr, + ssize_t itemsize, + const std::string &format, + ssize_t ndim, + detail::any_container &&shape_in, + detail::any_container &&strides_in, + bool readonly) + : buffer_info( + ptr, itemsize, format, ndim, std::move(shape_in), std::move(strides_in), readonly) {} Py_buffer *m_view = nullptr; bool ownview = false; @@ -126,17 +170,22 @@ struct buffer_info { PYBIND11_NAMESPACE_BEGIN(detail) -template struct compare_buffer_info { - static bool compare(const buffer_info& b) { +template +struct compare_buffer_info { + static bool compare(const buffer_info &b) { return b.format == format_descriptor::format() && b.itemsize == (ssize_t) sizeof(T); } }; -template struct compare_buffer_info::value>> { - static bool compare(const buffer_info& b) { - return (size_t) b.itemsize == sizeof(T) && (b.format == format_descriptor::value || - ((sizeof(T) == sizeof(long)) && b.format == (std::is_unsigned::value ? "L" : "l")) || - ((sizeof(T) == sizeof(size_t)) && b.format == (std::is_unsigned::value ? "N" : "n"))); +template +struct compare_buffer_info::value>> { + static bool compare(const buffer_info &b) { + return (size_t) b.itemsize == sizeof(T) + && (b.format == format_descriptor::value + || ((sizeof(T) == sizeof(long)) + && b.format == (std::is_unsigned::value ? "L" : "l")) + || ((sizeof(T) == sizeof(size_t)) + && b.format == (std::is_unsigned::value ? "N" : "n"))); } }; diff --git a/share/openPMD/thirdParty/pybind11/include/pybind11/cast.h b/share/openPMD/thirdParty/pybind11/include/pybind11/cast.h index 165102443c..430c62f357 100644 --- a/share/openPMD/thirdParty/pybind11/include/pybind11/cast.h +++ b/share/openPMD/thirdParty/pybind11/include/pybind11/cast.h @@ -10,11 +10,12 @@ #pragma once -#include "pytypes.h" #include "detail/common.h" #include "detail/descr.h" #include "detail/type_caster_base.h" #include "detail/typeid.h" +#include "pytypes.h" + #include #include #include @@ -30,41 +31,51 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) -template class type_caster : public type_caster_base { }; -template using make_caster = type_caster>; +template +class type_caster : public type_caster_base {}; +template +using make_caster = type_caster>; // Shortcut for calling a caster's `cast_op_type` cast operator for casting a type_caster to a T -template typename make_caster::template cast_op_type cast_op(make_caster &caster) { +template +typename make_caster::template cast_op_type cast_op(make_caster &caster) { return caster.operator typename make_caster::template cast_op_type(); } -template typename make_caster::template cast_op_type::type> +template +typename make_caster::template cast_op_type::type> cast_op(make_caster &&caster) { - return std::move(caster).operator - typename make_caster::template cast_op_type::type>(); + return std::move(caster).operator typename make_caster:: + template cast_op_type::type>(); } -template class type_caster> { +template +class type_caster> { private: using caster_t = make_caster; caster_t subcaster; - using reference_t = type&; - using subcaster_cast_op_type = - typename caster_t::template cast_op_type; - - static_assert(std::is_same::type &, subcaster_cast_op_type>::value || - std::is_same::value, - "std::reference_wrapper caster requires T to have a caster with an " - "`operator T &()` or `operator const T &()`"); + using reference_t = type &; + using subcaster_cast_op_type = typename caster_t::template cast_op_type; + + static_assert( + std::is_same::type &, subcaster_cast_op_type>::value + || std::is_same::value, + "std::reference_wrapper caster requires T to have a caster with an " + "`operator T &()` or `operator const T &()`"); + public: bool load(handle src, bool convert) { return subcaster.load(src, convert); } static constexpr auto name = caster_t::name; - static handle cast(const std::reference_wrapper &src, return_value_policy policy, handle parent) { + static handle + cast(const std::reference_wrapper &src, return_value_policy policy, handle parent) { // It is definitely wrong to take ownership of this pointer, so mask that rvp - if (policy == return_value_policy::take_ownership || policy == return_value_policy::automatic) + if (policy == return_value_policy::take_ownership + || policy == return_value_policy::automatic) { policy = return_value_policy::automatic_reference; + } return caster_t::cast(&src.get(), policy, parent); } - template using cast_op_type = std::reference_wrapper; + template + using cast_op_type = std::reference_wrapper; explicit operator std::reference_wrapper() { return cast_op(subcaster); } }; @@ -74,11 +85,15 @@ protected: \ public: \ static constexpr auto name = py_name; \ - template >::value, int> = 0> \ - static handle cast(T_ *src, return_value_policy policy, handle parent) { \ + template >::value, \ + int> = 0> \ + static ::pybind11::handle cast( \ + T_ *src, ::pybind11::return_value_policy policy, ::pybind11::handle parent) { \ if (!src) \ - return none().release(); \ - if (policy == return_value_policy::take_ownership) { \ + return ::pybind11::none().release(); \ + if (policy == ::pybind11::return_value_policy::take_ownership) { \ auto h = cast(std::move(*src), policy, parent); \ delete src; \ return h; \ @@ -89,31 +104,33 @@ public: operator type &() { return value; } /* NOLINT(bugprone-macro-parentheses) */ \ operator type &&() && { return std::move(value); } /* NOLINT(bugprone-macro-parentheses) */ \ template \ - using cast_op_type = pybind11::detail::movable_cast_op_type + using cast_op_type = ::pybind11::detail::movable_cast_op_type -template using is_std_char_type = any_of< - std::is_same, /* std::string */ +template +using is_std_char_type = any_of, /* std::string */ #if defined(PYBIND11_HAS_U8STRING) - std::is_same, /* std::u8string */ + std::is_same, /* std::u8string */ #endif - std::is_same, /* std::u16string */ - std::is_same, /* std::u32string */ - std::is_same /* std::wstring */ ->; - + std::is_same, /* std::u16string */ + std::is_same, /* std::u32string */ + std::is_same /* std::wstring */ + >; template struct type_caster::value && !is_std_char_type::value>> { using _py_type_0 = conditional_t; - using _py_type_1 = conditional_t::value, _py_type_0, typename std::make_unsigned<_py_type_0>::type>; + using _py_type_1 = conditional_t::value, + _py_type_0, + typename std::make_unsigned<_py_type_0>::type>; using py_type = conditional_t::value, double, _py_type_1>; -public: +public: bool load(handle src, bool convert) { py_type py_value; - if (!src) + if (!src) { return false; + } #if !defined(PYPY_VERSION) auto index_check = [](PyObject *o) { return PyIndex_Check(o); }; @@ -124,10 +141,11 @@ struct type_caster::value && !is_std_char_t #endif if (std::is_floating_point::value) { - if (convert || PyFloat_Check(src.ptr())) + if (convert || PyFloat_Check(src.ptr())) { py_value = (py_type) PyFloat_AsDouble(src.ptr()); - else + } else { return false; + } } else if (PyFloat_Check(src.ptr()) || (!convert && !PYBIND11_LONG_CHECK(src.ptr()) && !index_check(src.ptr()))) { return false; @@ -136,14 +154,13 @@ struct type_caster::value && !is_std_char_t // PyPy: 7.3.7's 3.8 does not implement PyLong_*'s __index__ calls. #if PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION) object index; - if (!PYBIND11_LONG_CHECK(src.ptr())) { // So: index_check(src.ptr()) + if (!PYBIND11_LONG_CHECK(src.ptr())) { // So: index_check(src.ptr()) index = reinterpret_steal(PyNumber_Index(src.ptr())); if (!index) { PyErr_Clear(); if (!convert) return false; - } - else { + } else { src_or_index = index; } } @@ -152,8 +169,8 @@ struct type_caster::value && !is_std_char_t py_value = as_unsigned(src_or_index.ptr()); } else { // signed integer: py_value = sizeof(T) <= sizeof(long) - ? (py_type) PyLong_AsLong(src_or_index.ptr()) - : (py_type) PYBIND11_LONG_AS_LONGLONG(src_or_index.ptr()); + ? (py_type) PyLong_AsLong(src_or_index.ptr()) + : (py_type) PYBIND11_LONG_AS_LONGLONG(src_or_index.ptr()); } } @@ -162,12 +179,14 @@ struct type_caster::value && !is_std_char_t // Check to see if the conversion is valid (integers should match exactly) // Signed/unsigned checks happen elsewhere - if (py_err || (std::is_integral::value && sizeof(py_type) != sizeof(T) && py_value != (py_type) (T) py_value)) { + if (py_err + || (std::is_integral::value && sizeof(py_type) != sizeof(T) + && py_value != (py_type) (T) py_value)) { PyErr_Clear(); if (py_err && convert && (PyNumber_Check(src.ptr()) != 0)) { auto tmp = reinterpret_steal(std::is_floating_point::value - ? PyNumber_Float(src.ptr()) - : PyNumber_Long(src.ptr())); + ? PyNumber_Float(src.ptr()) + : PyNumber_Long(src.ptr())); PyErr_Clear(); return load(tmp, false); } @@ -178,32 +197,40 @@ struct type_caster::value && !is_std_char_t return true; } - template + template static typename std::enable_if::value, handle>::type cast(U src, return_value_policy /* policy */, handle /* parent */) { return PyFloat_FromDouble((double) src); } - template - static typename std::enable_if::value && std::is_signed::value && (sizeof(U) <= sizeof(long)), handle>::type + template + static typename std::enable_if::value && std::is_signed::value + && (sizeof(U) <= sizeof(long)), + handle>::type cast(U src, return_value_policy /* policy */, handle /* parent */) { return PYBIND11_LONG_FROM_SIGNED((long) src); } - template - static typename std::enable_if::value && std::is_unsigned::value && (sizeof(U) <= sizeof(unsigned long)), handle>::type + template + static typename std::enable_if::value && std::is_unsigned::value + && (sizeof(U) <= sizeof(unsigned long)), + handle>::type cast(U src, return_value_policy /* policy */, handle /* parent */) { return PYBIND11_LONG_FROM_UNSIGNED((unsigned long) src); } - template - static typename std::enable_if::value && std::is_signed::value && (sizeof(U) > sizeof(long)), handle>::type + template + static typename std::enable_if::value && std::is_signed::value + && (sizeof(U) > sizeof(long)), + handle>::type cast(U src, return_value_policy /* policy */, handle /* parent */) { return PyLong_FromLongLong((long long) src); } - template - static typename std::enable_if::value && std::is_unsigned::value && (sizeof(U) > sizeof(unsigned long)), handle>::type + template + static typename std::enable_if::value && std::is_unsigned::value + && (sizeof(U) > sizeof(unsigned long)), + handle>::type cast(U src, return_value_policy /* policy */, handle /* parent */) { return PyLong_FromUnsignedLongLong((unsigned long long) src); } @@ -211,22 +238,26 @@ struct type_caster::value && !is_std_char_t PYBIND11_TYPE_CASTER(T, const_name::value>("int", "float")); }; -template struct void_caster { +template +struct void_caster { public: bool load(handle src, bool) { - if (src && src.is_none()) + if (src && src.is_none()) { return true; + } return false; } static handle cast(T, return_value_policy /* policy */, handle /* parent */) { - return none().inc_ref(); + return none().release(); } PYBIND11_TYPE_CASTER(T, const_name("None")); }; -template <> class type_caster : public void_caster {}; +template <> +class type_caster : public void_caster {}; -template <> class type_caster : public type_caster { +template <> +class type_caster : public type_caster { public: using type_caster::cast; @@ -246,7 +277,7 @@ template <> class type_caster : public type_caster { } /* Check if this is a C++ type */ - auto &bases = all_type_info((PyTypeObject *) type::handle_of(h).ptr()); + const auto &bases = all_type_info((PyTypeObject *) type::handle_of(h).ptr()); if (bases.size() == 1) { // Only allowing loading from a single-value type value = values_and_holders(reinterpret_cast(h.ptr())).begin()->value_ptr(); return true; @@ -257,24 +288,31 @@ template <> class type_caster : public type_caster { } static handle cast(const void *ptr, return_value_policy /* policy */, handle /* parent */) { - if (ptr) + if (ptr) { return capsule(ptr).release(); - return none().inc_ref(); + } + return none().release(); } - template using cast_op_type = void*&; + template + using cast_op_type = void *&; explicit operator void *&() { return value; } static constexpr auto name = const_name("capsule"); + private: void *value = nullptr; }; -template <> class type_caster : public void_caster { }; +template <> +class type_caster : public void_caster {}; -template <> class type_caster { +template <> +class type_caster { public: bool load(handle src, bool convert) { - if (!src) return false; + if (!src) { + return false; + } if (src.ptr() == Py_True) { value = true; return true; @@ -288,22 +326,22 @@ template <> class type_caster { Py_ssize_t res = -1; if (src.is_none()) { - res = 0; // None is implicitly converted to False + res = 0; // None is implicitly converted to False } - #if defined(PYPY_VERSION) - // On PyPy, check that "__bool__" (or "__nonzero__" on Python 2.7) attr exists +#if defined(PYPY_VERSION) + // On PyPy, check that "__bool__" attr exists else if (hasattr(src, PYBIND11_BOOL_ATTR)) { res = PyObject_IsTrue(src.ptr()); } - #else +#else // Alternate approach for CPython: this does the same as the above, but optimized // using the CPython API so as to avoid an unneeded attribute lookup. - else if (auto tp_as_number = src.ptr()->ob_type->tp_as_number) { + else if (auto *tp_as_number = src.ptr()->ob_type->tp_as_number) { if (PYBIND11_NB_BOOL(tp_as_number)) { res = (*PYBIND11_NB_BOOL(tp_as_number))(src.ptr()); } } - #endif +#endif if (res == 0 || res == 1) { value = (res != 0); return true; @@ -319,51 +357,38 @@ template <> class type_caster { }; // Helper class for UTF-{8,16,32} C++ stl strings: -template struct string_caster { +template +struct string_caster { using CharT = typename StringType::value_type; // Simplify life by being able to assume standard char sizes (the standard only guarantees // minimums, but Python requires exact sizes) - static_assert(!std::is_same::value || sizeof(CharT) == 1, "Unsupported char size != 1"); + static_assert(!std::is_same::value || sizeof(CharT) == 1, + "Unsupported char size != 1"); #if defined(PYBIND11_HAS_U8STRING) - static_assert(!std::is_same::value || sizeof(CharT) == 1, "Unsupported char8_t size != 1"); + static_assert(!std::is_same::value || sizeof(CharT) == 1, + "Unsupported char8_t size != 1"); #endif - static_assert(!std::is_same::value || sizeof(CharT) == 2, "Unsupported char16_t size != 2"); - static_assert(!std::is_same::value || sizeof(CharT) == 4, "Unsupported char32_t size != 4"); + static_assert(!std::is_same::value || sizeof(CharT) == 2, + "Unsupported char16_t size != 2"); + static_assert(!std::is_same::value || sizeof(CharT) == 4, + "Unsupported char32_t size != 4"); // wchar_t can be either 16 bits (Windows) or 32 (everywhere else) static_assert(!std::is_same::value || sizeof(CharT) == 2 || sizeof(CharT) == 4, - "Unsupported wchar_t size != 2/4"); + "Unsupported wchar_t size != 2/4"); static constexpr size_t UTF_N = 8 * sizeof(CharT); bool load(handle src, bool) { -#if PY_MAJOR_VERSION < 3 - object temp; -#endif handle load_src = src; if (!src) { return false; } if (!PyUnicode_Check(load_src.ptr())) { -#if PY_MAJOR_VERSION >= 3 - return load_bytes(load_src); -#else - if (std::is_same::value) { - return load_bytes(load_src); - } - - // The below is a guaranteed failure in Python 3 when PyUnicode_Check returns false - if (!PYBIND11_BYTES_CHECK(load_src.ptr())) - return false; - - temp = reinterpret_steal(PyUnicode_FromObject(load_src.ptr())); - if (!temp) { PyErr_Clear(); return false; } - load_src = temp; -#endif + return load_raw(load_src); } -#if PY_VERSION_HEX >= 0x03030000 - // On Python >= 3.3, for UTF-8 we avoid the need for a temporary `bytes` - // object by using `PyUnicode_AsUTF8AndSize`. + // For UTF-8 we avoid the need for a temporary `bytes` object by using + // `PyUnicode_AsUTF8AndSize`. if (PYBIND11_SILENCE_MSVC_C4127(UTF_N == 8)) { Py_ssize_t size = -1; const auto *buffer @@ -375,13 +400,20 @@ template struct string_caster { value = StringType(buffer, static_cast(size)); return true; } -#endif - auto utfNbytes = reinterpret_steal(PyUnicode_AsEncodedString( - load_src.ptr(), UTF_N == 8 ? "utf-8" : UTF_N == 16 ? "utf-16" : "utf-32", nullptr)); - if (!utfNbytes) { PyErr_Clear(); return false; } + auto utfNbytes + = reinterpret_steal(PyUnicode_AsEncodedString(load_src.ptr(), + UTF_N == 8 ? "utf-8" + : UTF_N == 16 ? "utf-16" + : "utf-32", + nullptr)); + if (!utfNbytes) { + PyErr_Clear(); + return false; + } - const auto *buffer = reinterpret_cast(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr())); + const auto *buffer + = reinterpret_cast(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr())); size_t length = (size_t) PYBIND11_BYTES_SIZE(utfNbytes.ptr()) / sizeof(CharT); // Skip BOM for UTF-16/32 if (PYBIND11_SILENCE_MSVC_C4127(UTF_N > 8)) { @@ -391,17 +423,21 @@ template struct string_caster { value = StringType(buffer, length); // If we're loading a string_view we need to keep the encoded Python object alive: - if (IsView) + if (IsView) { loader_life_support::add_patient(utfNbytes); + } return true; } - static handle cast(const StringType &src, return_value_policy /* policy */, handle /* parent */) { + static handle + cast(const StringType &src, return_value_policy /* policy */, handle /* parent */) { const char *buffer = reinterpret_cast(src.data()); auto nbytes = ssize_t(src.size() * sizeof(CharT)); handle s = decode_utfN(buffer, nbytes); - if (!s) throw error_already_set(); + if (!s) { + throw error_already_set(); + } return s; } @@ -410,63 +446,89 @@ template struct string_caster { private: static handle decode_utfN(const char *buffer, ssize_t nbytes) { #if !defined(PYPY_VERSION) - return - UTF_N == 8 ? PyUnicode_DecodeUTF8(buffer, nbytes, nullptr) : - UTF_N == 16 ? PyUnicode_DecodeUTF16(buffer, nbytes, nullptr, nullptr) : - PyUnicode_DecodeUTF32(buffer, nbytes, nullptr, nullptr); + return UTF_N == 8 ? PyUnicode_DecodeUTF8(buffer, nbytes, nullptr) + : UTF_N == 16 ? PyUnicode_DecodeUTF16(buffer, nbytes, nullptr, nullptr) + : PyUnicode_DecodeUTF32(buffer, nbytes, nullptr, nullptr); #else - // PyPy segfaults when on PyUnicode_DecodeUTF16 (and possibly on PyUnicode_DecodeUTF32 as well), - // so bypass the whole thing by just passing the encoding as a string value, which works properly: - return PyUnicode_Decode(buffer, nbytes, UTF_N == 8 ? "utf-8" : UTF_N == 16 ? "utf-16" : "utf-32", nullptr); + // PyPy segfaults when on PyUnicode_DecodeUTF16 (and possibly on PyUnicode_DecodeUTF32 as + // well), so bypass the whole thing by just passing the encoding as a string value, which + // works properly: + return PyUnicode_Decode(buffer, + nbytes, + UTF_N == 8 ? "utf-8" + : UTF_N == 16 ? "utf-16" + : "utf-32", + nullptr); #endif } - // When loading into a std::string or char*, accept a bytes object as-is (i.e. + // When loading into a std::string or char*, accept a bytes/bytearray object as-is (i.e. // without any encoding/decoding attempt). For other C++ char sizes this is a no-op. // which supports loading a unicode from a str, doesn't take this path. template - bool load_bytes(enable_if_t::value, handle> src) { + bool load_raw(enable_if_t::value, handle> src) { if (PYBIND11_BYTES_CHECK(src.ptr())) { - // We were passed a Python 3 raw bytes; accept it into a std::string or char* + // We were passed raw bytes; accept it into a std::string or char* // without any encoding attempt. const char *bytes = PYBIND11_BYTES_AS_STRING(src.ptr()); - if (bytes) { - value = StringType(bytes, (size_t) PYBIND11_BYTES_SIZE(src.ptr())); - return true; + if (!bytes) { + pybind11_fail("Unexpected PYBIND11_BYTES_AS_STRING() failure."); } + value = StringType(bytes, (size_t) PYBIND11_BYTES_SIZE(src.ptr())); + return true; + } + if (PyByteArray_Check(src.ptr())) { + // We were passed a bytearray; accept it into a std::string or char* + // without any encoding attempt. + const char *bytearray = PyByteArray_AsString(src.ptr()); + if (!bytearray) { + pybind11_fail("Unexpected PyByteArray_AsString() failure."); + } + value = StringType(bytearray, (size_t) PyByteArray_Size(src.ptr())); + return true; } return false; } template - bool load_bytes(enable_if_t::value, handle>) { return false; } + bool load_raw(enable_if_t::value, handle>) { + return false; + } }; template -struct type_caster, enable_if_t::value>> +struct type_caster, + enable_if_t::value>> : string_caster> {}; #ifdef PYBIND11_HAS_STRING_VIEW template -struct type_caster, enable_if_t::value>> +struct type_caster, + enable_if_t::value>> : string_caster, true> {}; #endif // Type caster for C-style strings. We basically use a std::string type caster, but also add the // ability to use None as a nullptr char* (which the string caster doesn't allow). -template struct type_caster::value>> { +template +struct type_caster::value>> { using StringType = std::basic_string; - using StringCaster = type_caster; + using StringCaster = make_caster; StringCaster str_caster; bool none = false; CharT one_char = 0; + public: bool load(handle src, bool convert) { - if (!src) return false; + if (!src) { + return false; + } if (src.is_none()) { // Defer accepting None to other overloads (if we aren't in convert mode): - if (!convert) return false; + if (!convert) { + return false; + } none = true; return true; } @@ -474,14 +536,18 @@ template struct type_caster::value) { handle s = PyUnicode_DecodeLatin1((const char *) &src, 1, nullptr); - if (!s) throw error_already_set(); + if (!s) { + throw error_already_set(); + } return s; } return StringCaster::cast(StringType(1, src), policy, parent); @@ -491,19 +557,21 @@ template struct type_caster(static_cast(str_caster).c_str()); } explicit operator CharT &() { - if (none) + if (none) { throw value_error("Cannot convert None to a character"); + } auto &value = static_cast(str_caster); size_t str_len = value.size(); - if (str_len == 0) + if (str_len == 0) { throw value_error("Cannot convert empty string to a character"); + } // If we're in UTF-8 mode, we have two possible failures: one for a unicode character that - // is too high, and one for multiple unicode characters (caught later), so we need to figure - // out how long the first encoded character is in bytes to distinguish between these two - // errors. We also allow want to allow unicode characters U+0080 through U+00FF, as those - // can fit into a single char value. + // is too high, and one for multiple unicode characters (caught later), so we need to + // figure out how long the first encoded character is in bytes to distinguish between these + // two errors. We also allow want to allow unicode characters U+0080 through U+00FF, as + // those can fit into a single char value. if (PYBIND11_SILENCE_MSVC_C4127(StringCaster::UTF_N == 8) && str_len > 1 && str_len <= 4) { auto v0 = static_cast(value[0]); // low bits only: 0-127 @@ -518,7 +586,8 @@ template struct type_caster(((v0 & 3) << 6) + (static_cast(value[1]) & 0x3F)); + one_char = static_cast(((v0 & 3) << 6) + + (static_cast(value[1]) & 0x3F)); return one_char; } // Otherwise we have a single character, but it's > U+00FF @@ -531,34 +600,40 @@ template struct type_caster(value[0]); - if (one_char >= 0xD800 && one_char < 0xE000) + if (one_char >= 0xD800 && one_char < 0xE000) { throw value_error("Character code point not in range(0x10000)"); + } } - if (str_len != 1) + if (str_len != 1) { throw value_error("Expected a character, but multi-character string found"); + } one_char = value[0]; return one_char; } static constexpr auto name = const_name(PYBIND11_STRING_NAME); - template using cast_op_type = pybind11::detail::cast_op_type<_T>; + template + using cast_op_type = pybind11::detail::cast_op_type<_T>; }; // Base implementation for std::tuple and std::pair -template class Tuple, typename... Ts> class tuple_caster { +template