diff --git a/CMakeLists.txt b/CMakeLists.txt index 542601a70d..d09355ea7a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -581,6 +581,11 @@ if(openPMD_HAVE_ADIOS1) target_compile_definitions(openPMD.ADIOS1.Parallel PRIVATE openPMD_HAVE_MPI=0) endif() + target_include_directories(openPMD.ADIOS1.Serial SYSTEM PRIVATE + $) + target_include_directories(openPMD.ADIOS1.Parallel SYSTEM PRIVATE + $) + set_target_properties(openPMD.ADIOS1.Serial PROPERTIES POSITION_INDEPENDENT_CODE ON CXX_VISIBILITY_PRESET hidden @@ -772,6 +777,7 @@ set(openPMD_TEST_NAMES Auxiliary SerialIO ParallelIO + JSON ) # command line tools set(openPMD_CLI_TOOL_NAMES @@ -857,6 +863,11 @@ if(openPMD_BUILD_TESTING) else() target_link_libraries(${testname}Tests PRIVATE CatchMain) endif() + + if(${testname} STREQUAL JSON) + target_include_directories(${testname}Tests SYSTEM PRIVATE + $) + endif() endforeach() endif() diff --git a/NEWS.rst b/NEWS.rst index f8d7f10906..cf23818ae5 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -9,6 +9,10 @@ Upgrade Guide Python 3.10 is now supported. openPMD-api now depends on `toml11 `__ 3.7.0+. +The following backend-specific members of the ``Dataset`` class have been removed: ``Dataset::setChunkSize()``, ``Dataset::setCompression()``, ``Dataset::setCustomTransform()``, ``Dataset::chunkSize``, ``Dataset::compression``, ``Dataset::transform``. +They are replaced by backend-specific options in the JSON-based backend configuration. +This can be passed in ``Dataset::options``. + 0.14.0 ------ diff --git a/docs/source/details/adios1.json b/docs/source/details/adios1.json new file mode 100644 index 0000000000..5d2cb4df71 --- /dev/null +++ b/docs/source/details/adios1.json @@ -0,0 +1,7 @@ +{ + "adios2": { + "dataset": { + "transform": "blosc:compressor=zlib,shuffle=bit,lvl=1;nometa" + } + } +} diff --git a/docs/source/details/backendconfig.rst b/docs/source/details/backendconfig.rst index 43bed65b82..cbbfda4a9a 100644 --- a/docs/source/details/backendconfig.rst +++ b/docs/source/details/backendconfig.rst @@ -14,7 +14,18 @@ The fundamental structure of this JSON configuration string is given as follows: This structure allows keeping one configuration string for several backends at once, with the concrete backend configuration being chosen upon choosing the backend itself. -The configuration is read in a case-sensitive manner. +Options that can be configured via JSON are often also accessible via other means, e.g. environment variables. +The following list specifies the priority of these means, beginning with the lowest priority: + +1. Default values +2. Automatically detected options, e.g. the backend being detected by inspection of the file extension +3. Environment variables +4. JSON configuration. For JSON, a dataset-specific configuration overwrites a global, Series-wide configuration. +5. Explicit API calls such as ``setIterationEncoding()`` + +The configuration is read in a case-insensitive manner, keys as well as values. +An exception to this are string values which are forwarded to other libraries such as ADIOS1 and ADIOS2. +Those are read "as-is" and interpreted by the backend library. Generally, keys of the configuration are *lower case*. Parameters that are directly passed through to an external library and not interpreted within openPMD API (e.g. ``adios2.engine.parameters``) are unaffected by this and follow the respective library's conventions. @@ -36,6 +47,11 @@ For a consistent user interface, backends shall follow the following rules: Backend-independent JSON configuration -------------------------------------- +The openPMD backend can be chosen via the JSON key ``backend`` which recognizes the alternatives ``["hdf5", "adios1", "adios2", "json"]``. + +The iteration encoding can be chosen via the JSON key ``iteration_encoding`` which recognizes the alternatives ``["file_based", "group_based", "variable_based"]``. +Note that for file-based iteration encoding, specification of the expansion pattern in the file name (e.g. ``data_%T.json``) remains mandatory. + The key ``defer_iteration_parsing`` can be used to optimize the process of opening an openPMD Series (deferred/lazy parsing). By default, a Series is parsed eagerly, i.e. opening a Series implies reading all available iterations. Especially when a Series has many iterations, this can be a costly operation and users may wish to defer parsing of iterations to a later point adding ``{"defer_iteration_parsing": true}`` to their JSON configuration. @@ -100,6 +116,17 @@ Explanation of the single keys: ``"none"`` can be used to disable chunking. Chunking generally improves performance and only needs to be disabled in corner-cases, e.g. when heavily relying on independent, parallel I/O that non-collectively declares data records. +ADIOS1 +^^^^^^ + +ADIOS1 allows configuring custom dataset transforms via JSON: + +.. literalinclude:: adios1.json + :language: json + +This configuration can be passed globally (i.e. for the ``Series`` object) to apply for all datasets. +Alternatively, it can also be passed for single ``Dataset`` objects to only apply for single datasets. + Other backends ^^^^^^^^^^^^^^ diff --git a/examples/7_extended_write_serial.cpp b/examples/7_extended_write_serial.cpp index 5fa62add1e..02831dc11d 100644 --- a/examples/7_extended_write_serial.cpp +++ b/examples/7_extended_write_serial.cpp @@ -92,8 +92,27 @@ main() // this describes the datatype and shape of data as it should be written to disk io::Datatype dtype = io::determineDatatype(partial_mesh); auto d = io::Dataset(dtype, io::Extent{2, 5}); - d.setCompression("zlib", 9); - d.setCustomTransform("blosc:compressor=zlib,shuffle=bit,lvl=1;nometa"); + std::string datasetConfig = R"END( +{ + "adios1": { + "dataset": { + "transform": "blosc:compressor=zlib,shuffle=bit,lvl=1;nometa" + } + }, + "adios2": { + "dataset": { + "operators": [ + { + "type": "zlib", + "parameters": { + "clevel": 9 + } + } + ] + } + } +})END"; + d.options = datasetConfig; mesh["x"].resetDataset(d); io::ParticleSpecies electrons = cur_it.particles["electrons"]; diff --git a/examples/7_extended_write_serial.py b/examples/7_extended_write_serial.py index dc23d8d427..a9cdcd291e 100755 --- a/examples/7_extended_write_serial.py +++ b/examples/7_extended_write_serial.py @@ -8,6 +8,7 @@ """ from openpmd_api import Series, Access, Dataset, Mesh_Record_Component, \ Unit_Dimension +import json import numpy as np @@ -102,8 +103,24 @@ # component this describes the datatype and shape of data as it should be # written to disk d = Dataset(partial_mesh.dtype, extent=[2, 5]) - d.set_compression("zlib", 9) - d.set_custom_transform("blosc:compressor=zlib,shuffle=bit,lvl=1;nometa") + dataset_config = { + "adios1": { + "dataset": { + "transform": "blosc:compressor=zlib,shuffle=bit,lvl=1;nometa" + } + }, + "adios2": { + "dataset": { + "operators": [{ + "type": "zlib", + "parameters": { + "clevel": 9 + } + }] + } + } + } + d.options = json.dumps(dataset_config) mesh["x"].reset_dataset(d) electrons = cur_it.particles["electrons"] diff --git a/examples/8_benchmark_parallel.cpp b/examples/8_benchmark_parallel.cpp index 235576f3f0..8fef3e86b4 100644 --- a/examples/8_benchmark_parallel.cpp +++ b/examples/8_benchmark_parallel.cpp @@ -148,10 +148,14 @@ int main( // * The number of iterations. Effectively, the benchmark will be repeated for this many // times. #if openPMD_HAVE_ADIOS1 || openPMD_HAVE_ADIOS2 - benchmark.addConfiguration("", 0, "bp", dt, 10); + benchmark.addConfiguration( + R"({"adios2": {"dataset":{"operators":[{"type": "blosc"}]}}})", + "bp", + dt, + 10 ); #endif #if openPMD_HAVE_HDF5 - benchmark.addConfiguration("", 0, "h5", dt, 10); + benchmark.addConfiguration( "{}", "h5", dt, 10 ); #endif // Execute all previously configured benchmarks. Will return a MPIBenchmarkReport object diff --git a/include/openPMD/Dataset.hpp b/include/openPMD/Dataset.hpp index 88fa5a4e49..bfe41d6b63 100644 --- a/include/openPMD/Dataset.hpp +++ b/include/openPMD/Dataset.hpp @@ -49,16 +49,10 @@ class Dataset Dataset( Extent ); Dataset& extend(Extent newExtent); - Dataset& setChunkSize(Extent const&); - Dataset& setCompression(std::string const&, uint8_t const); - Dataset& setCustomTransform(std::string const&); Extent extent; Datatype dtype; uint8_t rank; - Extent chunkSize; - std::string compression; - std::string transform; std::string options = "{}"; //!< backend-dependent JSON configuration }; } // namespace openPMD diff --git a/include/openPMD/Error.hpp b/include/openPMD/Error.hpp index 0afb69998a..eea5cd56ff 100644 --- a/include/openPMD/Error.hpp +++ b/include/openPMD/Error.hpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace openPMD { @@ -62,5 +63,13 @@ namespace error public: WrongAPIUsage( std::string what ); }; + + class BackendConfigSchema : public Error + { + public: + std::vector< std::string > errorLocation; + + BackendConfigSchema( std::vector< std::string >, std::string what ); + }; } } diff --git a/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp index d80d6ddf24..4b2ff08e4a 100644 --- a/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp @@ -22,6 +22,7 @@ #include "openPMD/config.hpp" #include "openPMD/auxiliary/Export.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include @@ -42,7 +43,7 @@ namespace openPMD friend class ADIOS1IOHandlerImpl; public: - ADIOS1IOHandler(std::string path, Access); + ADIOS1IOHandler(std::string path, Access, json::TracingJSON ); ~ADIOS1IOHandler() override; std::string backendName() const override { return "ADIOS1"; } @@ -61,7 +62,7 @@ namespace openPMD friend class ADIOS1IOHandlerImpl; public: - ADIOS1IOHandler(std::string path, Access); + ADIOS1IOHandler(std::string path, Access, json::TracingJSON ); ~ADIOS1IOHandler() override; std::string backendName() const override { return "DUMMY_ADIOS1"; } diff --git a/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp b/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp index 4b06b83d6a..9b7d1e48a4 100644 --- a/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp @@ -46,7 +46,7 @@ namespace openPMD private: using Base_t = CommonADIOS1IOHandlerImpl< ADIOS1IOHandlerImpl >; public: - ADIOS1IOHandlerImpl(AbstractIOHandler*); + ADIOS1IOHandlerImpl(AbstractIOHandler*, json::TracingJSON); virtual ~ADIOS1IOHandlerImpl(); virtual void init(); diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index f35fe9883b..9cdcbf1fec 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -28,7 +28,7 @@ #include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/IO/InvalidatableFile.hpp" -#include "openPMD/auxiliary/JSON.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/auxiliary/Option.hpp" #include "openPMD/backend/Writable.hpp" #include "openPMD/config.hpp" @@ -130,14 +130,14 @@ class ADIOS2IOHandlerImpl ADIOS2IOHandlerImpl( AbstractIOHandler *, MPI_Comm, - nlohmann::json config, + json::TracingJSON config, std::string engineType ); #endif // openPMD_HAVE_MPI explicit ADIOS2IOHandlerImpl( AbstractIOHandler *, - nlohmann::json config, + json::TracingJSON config, std::string engineType ); @@ -277,15 +277,15 @@ class ADIOS2IOHandlerImpl std::vector< ParameterizedOperator > defaultOperators; - auxiliary::TracingJSON m_config; - static auxiliary::TracingJSON nullvalue; + json::TracingJSON m_config; + static json::TracingJSON nullvalue; void - init( nlohmann::json config ); + init( json::TracingJSON config ); template< typename Key > - auxiliary::TracingJSON - config( Key && key, auxiliary::TracingJSON & cfg ) + json::TracingJSON + config( Key && key, json::TracingJSON & cfg ) { if( cfg.json().is_object() && cfg.json().contains( key ) ) { @@ -298,7 +298,7 @@ class ADIOS2IOHandlerImpl } template< typename Key > - auxiliary::TracingJSON + json::TracingJSON config( Key && key ) { return config< Key >( std::forward< Key >( key ), m_config ); @@ -312,7 +312,7 @@ class ADIOS2IOHandlerImpl * operators have been configured */ auxiliary::Option< std::vector< ParameterizedOperator > > - getOperators( auxiliary::TracingJSON config ); + getOperators( json::TracingJSON config ); // use m_config auxiliary::Option< std::vector< ParameterizedOperator > > @@ -1398,7 +1398,7 @@ friend class ADIOS2IOHandlerImpl; std::string path, Access, MPI_Comm, - nlohmann::json options, + json::TracingJSON options, std::string engineType ); #endif @@ -1406,7 +1406,7 @@ friend class ADIOS2IOHandlerImpl; ADIOS2IOHandler( std::string path, Access, - nlohmann::json options, + json::TracingJSON options, std::string engineType ); std::string backendName() const override { return "ADIOS2"; } diff --git a/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp index 2dd096fc3f..8d102247ca 100644 --- a/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp @@ -27,6 +27,7 @@ #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/DerefDynamicCast.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/auxiliary/Memory.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" @@ -89,11 +90,15 @@ namespace openPMD std::unordered_map< std::shared_ptr< std::string >, ADIOS_FILE* > m_openReadFileHandles; std::unordered_map< ADIOS_FILE*, std::vector< ADIOS_SELECTION* > > m_scheduledReads; std::unordered_map< int64_t, std::unordered_map< std::string, Attribute > > m_attributeWrites; + // config options + std::string m_defaultTransform; /** * Call this function to get adios file id for a Writable. Will create one if does not exist * @return returns an adios file id. */ int64_t GetFileHandle(Writable*); + + void initJson( json::TracingJSON ); }; // ParallelADIOS1IOHandlerImpl } // openPMD diff --git a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp index 817b6f56df..9eeaefcce0 100644 --- a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp @@ -22,6 +22,7 @@ #include "openPMD/config.hpp" #include "openPMD/auxiliary/Export.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include @@ -42,9 +43,9 @@ namespace openPMD public: # if openPMD_HAVE_MPI - ParallelADIOS1IOHandler(std::string path, Access, MPI_Comm); + ParallelADIOS1IOHandler(std::string path, Access, json::TracingJSON , MPI_Comm); # else - ParallelADIOS1IOHandler(std::string path, Access); + ParallelADIOS1IOHandler(std::string path, Access, json::TracingJSON); # endif ~ParallelADIOS1IOHandler() override; diff --git a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp index f1c2a6eb0e..27ad5fae3b 100644 --- a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp +++ b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp @@ -22,6 +22,7 @@ #include "openPMD/config.hpp" #include "openPMD/auxiliary/Export.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #if openPMD_HAVE_ADIOS1 && openPMD_HAVE_MPI @@ -46,7 +47,7 @@ namespace openPMD private: using Base_t = CommonADIOS1IOHandlerImpl< ParallelADIOS1IOHandlerImpl >; public: - ParallelADIOS1IOHandlerImpl(AbstractIOHandler*, MPI_Comm); + ParallelADIOS1IOHandlerImpl(AbstractIOHandler*, json::TracingJSON, MPI_Comm); virtual ~ParallelADIOS1IOHandlerImpl(); virtual void init(); diff --git a/include/openPMD/IO/HDF5/HDF5IOHandler.hpp b/include/openPMD/IO/HDF5/HDF5IOHandler.hpp index 3dfc1f6e1b..77dbf26c37 100644 --- a/include/openPMD/IO/HDF5/HDF5IOHandler.hpp +++ b/include/openPMD/IO/HDF5/HDF5IOHandler.hpp @@ -20,10 +20,9 @@ */ #pragma once +#include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" -#include - #include #include #include @@ -36,7 +35,7 @@ class HDF5IOHandlerImpl; class HDF5IOHandler : public AbstractIOHandler { public: - HDF5IOHandler(std::string path, Access, nlohmann::json config); + HDF5IOHandler(std::string path, Access, json::TracingJSON config); ~HDF5IOHandler() override; std::string backendName() const override { return "HDF5"; } diff --git a/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp b/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp index 815c57516e..ba5247284a 100644 --- a/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp +++ b/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp @@ -24,7 +24,7 @@ #if openPMD_HAVE_HDF5 # include "openPMD/IO/AbstractIOHandlerImpl.hpp" -# include "openPMD/auxiliary/JSON.hpp" +# include "openPMD/auxiliary/JSON_internal.hpp" # include "openPMD/auxiliary/Option.hpp" # include @@ -39,7 +39,7 @@ namespace openPMD class HDF5IOHandlerImpl : public AbstractIOHandlerImpl { public: - HDF5IOHandlerImpl(AbstractIOHandler*, nlohmann::json config); + HDF5IOHandlerImpl(AbstractIOHandler*, json::TracingJSON config); ~HDF5IOHandlerImpl() override; void createFile(Writable*, Parameter< Operation::CREATE_FILE > const&) override; @@ -81,7 +81,7 @@ namespace openPMD hid_t m_H5T_CLONG_DOUBLE; private: - auxiliary::TracingJSON m_config; + json::TracingJSON m_config; std::string m_chunks = "auto"; struct File { diff --git a/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp b/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp index 70cb681f0d..71e1a13a25 100644 --- a/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp +++ b/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp @@ -21,10 +21,9 @@ #pragma once #include "openPMD/config.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" -#include - #include #include #include @@ -39,9 +38,9 @@ namespace openPMD public: #if openPMD_HAVE_MPI ParallelHDF5IOHandler( - std::string path, Access, MPI_Comm, nlohmann::json config); + std::string path, Access, MPI_Comm, json::TracingJSON config); #else - ParallelHDF5IOHandler(std::string path, Access, nlohmann::json config); + ParallelHDF5IOHandler(std::string path, Access, json::TracingJSON config); #endif ~ParallelHDF5IOHandler() override; diff --git a/include/openPMD/IO/HDF5/ParallelHDF5IOHandlerImpl.hpp b/include/openPMD/IO/HDF5/ParallelHDF5IOHandlerImpl.hpp index d0e18dc85c..843280fc55 100644 --- a/include/openPMD/IO/HDF5/ParallelHDF5IOHandlerImpl.hpp +++ b/include/openPMD/IO/HDF5/ParallelHDF5IOHandlerImpl.hpp @@ -27,7 +27,7 @@ # include # if openPMD_HAVE_HDF5 # include "openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp" -# include +# include "openPMD/auxiliary/JSON_internal.hpp" # endif #endif @@ -39,7 +39,7 @@ namespace openPMD { public: ParallelHDF5IOHandlerImpl( - AbstractIOHandler*, MPI_Comm, nlohmann::json config); + AbstractIOHandler*, MPI_Comm, json::TracingJSON config); ~ParallelHDF5IOHandlerImpl() override; MPI_Comm m_mpiComm; diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 6fb4be636d..650a9c9bbb 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -268,8 +268,7 @@ struct OPENPMDAPI_EXPORT Parameter< Operation::CREATE_DATASET > : public Abstrac Parameter() = default; Parameter(Parameter const & p) : AbstractParameter(), name(p.name), extent(p.extent), dtype(p.dtype), - chunkSize(p.chunkSize), compression(p.compression), - transform(p.transform), options(p.options) {} + options(p.options) {} std::unique_ptr< AbstractParameter > clone() const override @@ -281,10 +280,19 @@ struct OPENPMDAPI_EXPORT Parameter< Operation::CREATE_DATASET > : public Abstrac std::string name = ""; Extent extent = {}; Datatype dtype = Datatype::UNDEFINED; - Extent chunkSize = {}; - std::string compression = ""; - std::string transform = ""; std::string options = "{}"; + + /** Warn about unused JSON paramters + * + * Template parameter so we don't have to include the JSON lib here. + * This function is useful for the createDataset() methods in, + * IOHandlerImpl's, so putting that here is the simplest way to make it + * available for them. */ + template< typename TracingJSON > + static void warnUnusedParameters( + TracingJSON &, + std::string const & currentBackendName, + std::string const & warningMessage ); }; template<> diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index 89a7100b56..e73b59a5ae 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -298,9 +298,6 @@ RecordComponent::storeChunk( Offset o, Extent e, F && createBuffer ) dCreate.name = rc.m_name; dCreate.extent = getExtent(); dCreate.dtype = getDatatype(); - dCreate.chunkSize = rc.m_dataset.chunkSize; - dCreate.compression = rc.m_dataset.compression; - dCreate.transform = rc.m_dataset.transform; dCreate.options = rc.m_dataset.options; IOHandler()->enqueue(IOTask(this, dCreate)); } diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index b4488656b5..d56ad8ecb9 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -408,6 +408,17 @@ class Series : public Attributable } } std::unique_ptr< ParsedInput > parseInput(std::string); + /** + * @brief Parse non-backend-specific configuration in JSON config. + * + * Currently this parses the keys defer_iteration_parsing, backend and + * iteration_encoding. + * + * @tparam TracingJSON template parameter so we don't have + * to include the JSON lib here + */ + template< typename TracingJSON > + void parseJsonOptions( TracingJSON & options, ParsedInput & ); bool hasExpansionPattern( std::string filenameWithExtension ); bool reparseExpansionPattern( std::string filenameWithExtension ); void init(std::shared_ptr< AbstractIOHandler >, std::unique_ptr< ParsedInput >); diff --git a/include/openPMD/auxiliary/JSON.hpp b/include/openPMD/auxiliary/JSON.hpp index 0cbdad175e..eace4191d7 100644 --- a/include/openPMD/auxiliary/JSON.hpp +++ b/include/openPMD/auxiliary/JSON.hpp @@ -1,4 +1,4 @@ -/* Copyright 2020-2021 Franz Poeschel +/* Copyright 2021 Franz Poeschel * * This file is part of openPMD-api. * @@ -21,161 +21,43 @@ #pragma once -#include "openPMD/config.hpp" - -#include - -#if openPMD_HAVE_MPI -# include -#endif - -#include // std::shared_ptr -#include // std::forward +#include namespace openPMD { -namespace auxiliary +namespace json { /** - * @brief Extend nlohmann::json with tracing of which keys have been - * accessed by operator[](). - * An access is only registered if the current JSON value is a JSON object - * (not an array) and if the accessed JSON value is a leaf, i.e. anything - * but an object. This means that objects contained in arrays will not be - * traced. + * @brief Merge two JSON datasets into one. * - * If working directly with the underlying JSON value (necessary since this - * class only redefines operator[]), declareFullyRead() may be used to - * declare keys read manually. + * Merging rules: + * 1. If both `defaultValue` and `overwrite` are JSON objects, then the + * resulting JSON object will contain the union of both objects' keys. + * If a key is specified in both objects, the values corresponding to the + * key are merged recursively. + * Keys that point to a null value after this procedure will be pruned. + * 2. In any other case, the JSON dataset `defaultValue` is replaced in its + * entirety with the JSON dataset `overwrite`. * - */ - class TracingJSON - { - public: - TracingJSON(); - TracingJSON( nlohmann::json ); - - /** - * @brief Access the underlying JSON value - * - * @return nlohmann::json& - */ - inline nlohmann::json & - json() - { - return *m_positionInOriginal; - } - - template< typename Key > - TracingJSON operator[]( Key && key ); - - /** - * @brief Get the "shadow", i.e. a copy of the original JSON value - * containing all accessed object keys. - * - * @return nlohmann::json const& - */ - nlohmann::json const & - getShadow(); - - /** - * @brief Invert the "shadow", i.e. a copy of the original JSON value - * that contains exactly those values that have not been accessed yet. - * - * @return nlohmann::json - */ - nlohmann::json - invertShadow(); - - /** - * @brief Declare all keys of the current object read. - * - */ - void - declareFullyRead(); - - private: - /** - * @brief The JSON object with which this class has been initialized. - * Shared pointer shared between all instances returned by - * operator[]() in order to avoid use-after-free situations. - * - */ - std::shared_ptr< nlohmann::json > m_originalJSON; - /** - * @brief A JSON object keeping track of all accessed indices within the - * original JSON object. Initially an empty JSON object, - * gradually filled by applying each operator[]() call also to - * it. - * Shared pointer shared between all instances returned by - * operator[]() in order to avoid use-after-free situations. - * - */ - std::shared_ptr< nlohmann::json > m_shadow; - /** - * @brief The sub-expression within m_originalJSON corresponding with - * the current instance. - * - */ - nlohmann::json * m_positionInOriginal; - /** - * @brief The sub-expression within m_positionInOriginal corresponding - * with the current instance. - * - */ - nlohmann::json * m_positionInShadow; - bool m_trace = true; - - void - invertShadow( nlohmann::json & result, nlohmann::json const & shadow ); - - TracingJSON( - std::shared_ptr< nlohmann::json > originalJSON, - std::shared_ptr< nlohmann::json > shadow, - nlohmann::json * positionInOriginal, - nlohmann::json * positionInShadow, - bool trace ); - }; - - template< typename Key > - TracingJSON TracingJSON::operator[]( Key && key ) - { - nlohmann::json * newPositionInOriginal = - &m_positionInOriginal->operator[]( key ); - // If accessing a leaf in the JSON tree from an object (not an array!) - // erase the corresponding key - static nlohmann::json nullvalue; - nlohmann::json * newPositionInShadow = &nullvalue; - if( m_trace && m_positionInOriginal->is_object() ) - { - newPositionInShadow = &m_positionInShadow->operator[]( key ); - } - bool traceFurther = newPositionInOriginal->is_object(); - return TracingJSON( - m_originalJSON, - m_shadow, - newPositionInOriginal, - newPositionInShadow, - traceFurther ); - } - - /** - * Check if options points to a file (indicated by an '@' for the first - * non-whitespace character). - * If yes, return the file content, if not just parse options directly. + * Note that item 2 means that datasets of different type will replace each + * other without error. + * It also means that array types will replace each other without any notion + * of appending or merging. * - * @param options as a parsed JSON object. - */ - nlohmann::json parseOptions( std::string const & options ); - -#if openPMD_HAVE_MPI - - /** - * Parallel version of parseOptions(). MPI-collective. + * Possible use case: + * An application uses openPMD-api and wants to do the following: + * 1. Set some default backend options as JSON parameters. + * 2. Let its users specify custom backend options additionally. + * + * By using the json::merge() function, this application can then allow + * users to overwrite default options, while keeping any other ones. + * + * @param defaultValue + * @param overwrite + * @return std::string */ - nlohmann::json parseOptions( std::string const & options, MPI_Comm comm ); - -#endif - -} // namespace auxiliary -} // namespace openPMD + std::string merge( + std::string const & defaultValue, + std::string const & overwrite ); +} +} diff --git a/include/openPMD/auxiliary/JSON_internal.hpp b/include/openPMD/auxiliary/JSON_internal.hpp new file mode 100644 index 0000000000..b6b6538b4e --- /dev/null +++ b/include/openPMD/auxiliary/JSON_internal.hpp @@ -0,0 +1,235 @@ +/* Copyright 2020-2021 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#pragma once + +#include "openPMD/config.hpp" + +#include "openPMD/Error.hpp" +#include "openPMD/auxiliary/Option.hpp" + +#include + +#if openPMD_HAVE_MPI +# include +#endif + +#include // std::shared_ptr +#include // std::forward + +namespace openPMD +{ +namespace json +{ + /** + * @brief Extend nlohmann::json with tracing of which keys have been + * accessed by operator[](). + * An access is only registered if the current JSON value is a JSON object + * (not an array) and if the accessed JSON value is a leaf, i.e. anything + * but an object. This means that objects contained in arrays will not be + * traced. + * + * If working directly with the underlying JSON value (necessary since this + * class only redefines operator[]), declareFullyRead() may be used to + * declare keys read manually. + * + */ + class TracingJSON + { + public: + TracingJSON(); + TracingJSON( nlohmann::json ); + + /** + * @brief Access the underlying JSON value + * + * @return nlohmann::json& + */ + inline nlohmann::json & + json() + { + return *m_positionInOriginal; + } + + template< typename Key > + TracingJSON operator[]( Key && key ); + + /** + * @brief Get the "shadow", i.e. a copy of the original JSON value + * containing all accessed object keys. + * + * @return nlohmann::json const& + */ + nlohmann::json const & getShadow() const; + + /** + * @brief Invert the "shadow", i.e. a copy of the original JSON value + * that contains exactly those values that have not been accessed yet. + * + * @return nlohmann::json + */ + nlohmann::json invertShadow() const; + + /** + * @brief Declare all keys of the current object read. + * + * Rationale: This class does not (yet) trace array types (or anything + * contained in an array). Use this call to explicitly declare + * an array as read. + */ + void + declareFullyRead(); + + private: + /** + * @brief The JSON object with which this class has been initialized. + * Shared pointer shared between all instances returned by + * operator[]() in order to avoid use-after-free situations. + * + */ + std::shared_ptr< nlohmann::json > m_originalJSON; + /** + * @brief A JSON object keeping track of all accessed indices within the + * original JSON object. Initially an empty JSON object, + * gradually filled by applying each operator[]() call also to + * it. + * Shared pointer shared between all instances returned by + * operator[]() in order to avoid use-after-free situations. + * + */ + std::shared_ptr< nlohmann::json > m_shadow; + /** + * @brief The sub-expression within m_originalJSON corresponding with + * the current instance. + * + */ + nlohmann::json * m_positionInOriginal; + /** + * @brief The sub-expression within m_positionInOriginal corresponding + * with the current instance. + * + */ + nlohmann::json * m_positionInShadow; + bool m_trace = true; + + void invertShadow( + nlohmann::json & result, nlohmann::json const & shadow ) const; + + TracingJSON( + std::shared_ptr< nlohmann::json > originalJSON, + std::shared_ptr< nlohmann::json > shadow, + nlohmann::json * positionInOriginal, + nlohmann::json * positionInShadow, + bool trace ); + }; + + template< typename Key > + TracingJSON TracingJSON::operator[]( Key && key ) + { + nlohmann::json * newPositionInOriginal = + &m_positionInOriginal->operator[]( key ); + // If accessing a leaf in the JSON tree from an object (not an array!) + // erase the corresponding key + static nlohmann::json nullvalue; + nlohmann::json * newPositionInShadow = &nullvalue; + if( m_trace && m_positionInOriginal->is_object() ) + { + newPositionInShadow = &m_positionInShadow->operator[]( key ); + } + bool traceFurther = newPositionInOriginal->is_object(); + return TracingJSON( + m_originalJSON, + m_shadow, + newPositionInOriginal, + newPositionInShadow, + traceFurther ); + } + + /** + * Check if options points to a file (indicated by an '@' for the first + * non-whitespace character). + * If yes, return the file content, if not just parse options directly. + * + * @param options as a parsed JSON object. + * @param considerFiles If yes, check if `options` refers to a file and read + * from there. + */ + nlohmann::json + parseOptions( std::string const & options, bool considerFiles ); + +#if openPMD_HAVE_MPI + + /** + * Parallel version of parseOptions(). MPI-collective. + */ + nlohmann::json parseOptions( + std::string const & options, MPI_Comm comm, bool considerFiles ); + +#endif + + /** + * Recursively transform all keys in a JSON dataset to lower case. + * String values are unaffected. + * JSON objects at the following openPMD-defined locations are not affected: + * * `adios2.engine.parameters` + * * `adios2.dataset.operators..parameters` + * This helps us forward configurations from these locations to ADIOS2 + * "as-is". + */ + nlohmann::json & lowerCase( nlohmann::json & ); + + /** + * Read a JSON literal as a string. + * If the literal is a number, convert that number to its string + * representation. + * If it is a bool, convert it to either "0" or "1". + * If it is not a literal, return an empty option. + */ + auxiliary::Option< std::string > asStringDynamic( nlohmann::json const & ); + + /** + * Like asStringDynamic(), but convert the string to lowercase afterwards. + */ + auxiliary::Option< std::string > + asLowerCaseStringDynamic( nlohmann::json const & ); + + /** + * Vector containing the lower-case keys to the single backends' + * configurations. + */ + extern std::vector< std::string > backendKeys; + + /** + * Function that can be called after reading all global options from the + * JSON configuration (i.e. all the options that are not handled by the + * single backends). + * If any unread value persists, a warning is printed to stderr. + */ + void warnGlobalUnusedOptions( TracingJSON const & config ); + + /** + * Like merge() as defined in JSON.hpp, but this overload works directly + * on nlohmann::json values. + */ + nlohmann::json & + merge( nlohmann::json & defaultVal, nlohmann::json const & overwrite ); +} // namespace json +} // namespace openPMD diff --git a/include/openPMD/auxiliary/StringManip.hpp b/include/openPMD/auxiliary/StringManip.hpp index c7a00eb634..92b12a058d 100644 --- a/include/openPMD/auxiliary/StringManip.hpp +++ b/include/openPMD/auxiliary/StringManip.hpp @@ -21,12 +21,12 @@ #pragma once #include +#include +#include // std::tolower #include #include #include #include -#include - namespace openPMD { @@ -261,5 +261,13 @@ removeSlashes( std::string s ) return s; } +template< typename S > +S && lowerCase( S && s ) +{ + std::transform( s.begin(), s.end(), s.begin(), []( unsigned char c ) { + return std::tolower( c ); + } ); + return std::forward< S >( s ); +} } // auxiliary } // openPMD diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index ded6aabd37..fe6f82b4a0 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -131,7 +131,6 @@ class Attributable friend struct traits::GenerationPolicy; friend class Iteration; friend class Series; - friend class Series; friend class Writable; friend class WriteIterations; diff --git a/include/openPMD/backend/BaseRecord.hpp b/include/openPMD/backend/BaseRecord.hpp index dd4328e047..e32b0b6af1 100644 --- a/include/openPMD/backend/BaseRecord.hpp +++ b/include/openPMD/backend/BaseRecord.hpp @@ -56,9 +56,7 @@ class BaseRecord : public Container< T_elem > friend class ParticleSpecies; friend class PatchRecord; friend class Record; - friend class Mesh; - friend class ParticleSpecies; std::shared_ptr< internal::BaseRecordData< T_elem > > m_baseRecordData{ new internal::BaseRecordData< T_elem >() }; diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 72abb64d58..74f4a65883 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -135,7 +135,6 @@ class Container : public Attributable friend class ParticlePatches; friend class internal::SeriesData; friend class Series; - friend class Series; template< typename > friend class internal::EraseStaleEntries; protected: diff --git a/include/openPMD/benchmark/mpi/MPIBenchmark.hpp b/include/openPMD/benchmark/mpi/MPIBenchmark.hpp index 0f789d57c5..8f48f8879b 100644 --- a/include/openPMD/benchmark/mpi/MPIBenchmark.hpp +++ b/include/openPMD/benchmark/mpi/MPIBenchmark.hpp @@ -94,8 +94,7 @@ namespace openPMD ); /** - * @param compression Compression string, leave empty to disable commpression. - * @param compressionLevel Compression level. + * @param jsonConfig Backend-specific configuration. * @param backend Backend to use, specified by filename extension (eg "bp" or "h5"). * @param dt Type of data to write and read. * @param iterations The number of iterations to write and read for each @@ -104,8 +103,7 @@ namespace openPMD * @param threadSize Number of threads to use. */ void addConfiguration( - std::string compression, - uint8_t compressionLevel, + std::string jsonConfig, std::string backend, Datatype dt, typename decltype( Series::iterations )::key_type iterations, @@ -115,8 +113,7 @@ namespace openPMD /** * Version of addConfiguration() that automatically sets the number of used * threads to the MPI size. - * @param compression Compression string, leave empty to disable commpression. - * @param compressionLevel Compression level. + * @param jsonConfig Backend-specific configuration. * @param backend Backend to use, specified by filename extension (eg "bp" or "h5"). * @param dt Type of data to write and read. * @param iterations The number of iterations to write and read for each @@ -124,8 +121,7 @@ namespace openPMD * iteration, so it should create sufficient data for one iteration. */ void addConfiguration( - std::string compression, - uint8_t compressionLevel, + std::string jsonConfig, std::string backend, Datatype dt, typename decltype( Series::iterations)::key_type iterations @@ -152,7 +148,6 @@ namespace openPMD std::vector< std::tuple< std::string, - uint8_t, std::string, int, Datatype, @@ -161,8 +156,7 @@ namespace openPMD enum Config { - COMPRESSION = 0, - COMPRESSION_LEVEL, + JSON_CONFIG = 0, BACKEND, NRANKS, DTYPE, @@ -194,8 +188,7 @@ namespace openPMD /** * Execute a single read benchmark. * @tparam T Type of the dataset to write. - * @param compression Compression to use. - * @param level Compression level to use. + * @param jsonConfig Backend-specific config. * @param offset Local offset of the chunk to write. * @param extent Local extent of the chunk to write. * @param extension File extension to control the openPMD backend. @@ -207,8 +200,7 @@ namespace openPMD typename T > typename Clock::duration writeBenchmark( - std::string const & compression, - uint8_t level, + std::string const & jsonConfig, Offset & offset, Extent & extent, std::string const & extension, @@ -331,8 +323,7 @@ namespace openPMD template< typename DatasetFillerProvider > void MPIBenchmark< DatasetFillerProvider >::addConfiguration( - std::string compression, - uint8_t compressionLevel, + std::string jsonConfig, std::string backend, Datatype dt, typename decltype( Series::iterations)::key_type iterations, @@ -341,8 +332,7 @@ namespace openPMD { this->m_configurations .emplace_back( - compression, - compressionLevel, + std::move( jsonConfig ), backend, threadSize, dt, @@ -353,8 +343,7 @@ namespace openPMD template< typename DatasetFillerProvider > void MPIBenchmark< DatasetFillerProvider >::addConfiguration( - std::string compression, - uint8_t compressionLevel, + std::string jsonConfig, std::string backend, Datatype dt, typename decltype( Series::iterations)::key_type iterations @@ -366,8 +355,7 @@ namespace openPMD &size ); addConfiguration( - compression, - compressionLevel, + std::move( jsonConfig ), backend, dt, iterations, @@ -389,8 +377,7 @@ namespace openPMD template< typename T > typename Clock::duration MPIBenchmark< DatasetFillerProvider >::BenchmarkExecution< Clock >::writeBenchmark( - std::string const & compression, - uint8_t level, + std::string const & jsonConfig, Offset & offset, Extent & extent, std::string const & extension, @@ -405,7 +392,8 @@ namespace openPMD Series series = Series( m_benchmark->m_basePath + "." + extension, Access::CREATE, - m_benchmark->communicator + m_benchmark->communicator, + jsonConfig ); for( typename decltype( Series::iterations)::key_type i = 0; @@ -424,13 +412,6 @@ namespace openPMD datatype, m_benchmark->totalExtent ); - if( !compression.empty( ) ) - { - dataset.setCompression( - compression, - level - ); - } id.resetDataset( dataset ); @@ -521,15 +502,13 @@ namespace openPMD ); for( auto const & config: exec.m_benchmark->m_configurations ) { - std::string compression; - uint8_t compressionLevel; + std::string jsonConfig; std::string backend; int size; Datatype dt2; typename decltype( Series::iterations)::key_type iterations; std::tie( - compression, - compressionLevel, + jsonConfig, backend, size, dt2, @@ -551,8 +530,7 @@ namespace openPMD dsf->setNumberOfItems( blockSize ); auto writeTime = exec.writeBenchmark< T >( - compression, - compressionLevel, + jsonConfig, localCuboid.first, localCuboid.second, backend, @@ -567,8 +545,7 @@ namespace openPMD ); report.addReport( rootThread, - compression, - compressionLevel, + jsonConfig, backend, size, dt2, diff --git a/include/openPMD/benchmark/mpi/MPIBenchmarkReport.hpp b/include/openPMD/benchmark/mpi/MPIBenchmarkReport.hpp index eb43458bc2..3ee01dd948 100644 --- a/include/openPMD/benchmark/mpi/MPIBenchmarkReport.hpp +++ b/include/openPMD/benchmark/mpi/MPIBenchmarkReport.hpp @@ -53,8 +53,7 @@ namespace openPMD std::map< std::tuple< int, // rank - std::string, // compression - uint8_t, // compression level + std::string, // jsonConfig std::string, // extension int, // thread size Datatype, @@ -81,8 +80,7 @@ namespace openPMD * Add results for a certain compression strategy and level. * * @param rootThread The MPI rank which will collect the data. - * @param compression Compression strategy. - * @param level Compression level + * @param jsonConfig Compression strategy. * @param extension The openPMD filename extension. * @param threadSize The MPI size. * @param dt The openPMD datatype. @@ -91,8 +89,7 @@ namespace openPMD */ void addReport( int rootThread, - std::string compression, - uint8_t level, + std::string jsonConfig, std::string extension, int threadSize, Datatype dt, @@ -106,8 +103,7 @@ namespace openPMD /** Retrieve the time measured for a certain compression strategy. * * @param rank Which MPI rank's duration results to retrieve. - * @param compression Compression strategy. - * @param level Compression level + * @param jsonConfig Compression strategy. * @param extension The openPMD filename extension. * @param threadSize The MPI size. * @param dt The openPMD datatype. @@ -119,8 +115,7 @@ namespace openPMD Duration > getReport( int rank, - std::string compression, - uint8_t level, + std::string jsonConfig, std::string extension, int threadSize, Datatype dt, @@ -244,8 +239,7 @@ namespace openPMD template< typename Duration > void MPIBenchmarkReport< Duration >::addReport( int rootThread, - std::string compression, - uint8_t level, + std::string jsonConfig, std::string extension, int threadSize, Datatype dt, @@ -316,8 +310,7 @@ namespace openPMD .emplace( std::make_tuple( i, - compression, - level, + jsonConfig, extension, threadSize, dt, @@ -348,8 +341,7 @@ namespace openPMD Duration > MPIBenchmarkReport< Duration >::getReport( int rank, - std::string compression, - uint8_t level, + std::string jsonConfig, std::string extension, int threadSize, Datatype dt, @@ -362,8 +354,7 @@ namespace openPMD .find( std::make_tuple( rank, - compression, - level, + jsonConfig, extension, threadSize, dt, diff --git a/include/openPMD/openPMD.hpp b/include/openPMD/openPMD.hpp index b54a1810c8..55a39cf2bb 100644 --- a/include/openPMD/openPMD.hpp +++ b/include/openPMD/openPMD.hpp @@ -55,6 +55,7 @@ namespace openPMD {} #include "openPMD/auxiliary/Date.hpp" #include "openPMD/auxiliary/DerefDynamicCast.hpp" +#include "openPMD/auxiliary/JSON.hpp" #include "openPMD/auxiliary/Option.hpp" #include "openPMD/auxiliary/OutOfRangeMsg.hpp" #include "openPMD/auxiliary/ShareRaw.hpp" diff --git a/src/Dataset.cpp b/src/Dataset.cpp index 21be4fa3f2..bb89f76423 100644 --- a/src/Dataset.cpp +++ b/src/Dataset.cpp @@ -30,7 +30,6 @@ Dataset::Dataset(Datatype d, Extent e, std::string options_in) : extent{e}, dtype{d}, rank{static_cast(e.size())}, - chunkSize{e}, options{std::move(options_in)} { } @@ -50,41 +49,4 @@ Dataset::extend( Extent newExtents ) extent = newExtents; return *this; } - -Dataset& -Dataset::setChunkSize(Extent const& cs) -{ - if( extent.size() != rank ) - throw std::runtime_error("Dimensionality of extended Dataset must match the original dimensionality"); - for( size_t i = 0; i < cs.size(); ++i ) - if( cs[i] > extent[i] ) - throw std::runtime_error("Dataset chunk size must be equal or smaller than Extent"); - - chunkSize = cs; - return *this; -} - -Dataset& -Dataset::setCompression(std::string const& format, uint8_t const level) -{ - if(format == "zlib" || format == "gzip" || format == "deflate") - { - if(level > 9) - throw std::runtime_error("Compression level out of range for " + format); - } - else - std::cerr << "Unknown compression format " << format - << ". This might mean that compression will not be enabled." - << std::endl; - - compression = format + ':' + std::to_string(static_cast< int >(level)); - return *this; -} - -Dataset& -Dataset::setCustomTransform(std::string const& parameter) -{ - transform = parameter; - return *this; -} } // openPMD diff --git a/src/Error.cpp b/src/Error.cpp index c91331f52c..e6cf850279 100644 --- a/src/Error.cpp +++ b/src/Error.cpp @@ -1,5 +1,7 @@ #include "openPMD/Error.hpp" +#include + namespace openPMD { const char * Error::what() const noexcept @@ -20,5 +22,31 @@ namespace error : Error( "Wrong API usage: " + what ) { } + + static std::string concatVector( + std::vector< std::string > const & vec, + std::string const & intersperse = "." ) + { + if( vec.empty() ) + { + return ""; + } + std::stringstream res; + res << vec[ 0 ]; + for( size_t i = 1; i < vec.size(); ++i ) + { + res << intersperse << vec[ i ]; + } + return res.str(); + } + + BackendConfigSchema::BackendConfigSchema( + std::vector< std::string > errorLocation_in, std::string what ) + : Error( + "Wrong JSON schema at index '" + + concatVector( errorLocation_in ) + "': " + std::move( what ) ) + , errorLocation( std::move( errorLocation_in ) ) + { + } } } diff --git a/src/Format.cpp b/src/Format.cpp index 98defa14b3..04d68cb2bf 100644 --- a/src/Format.cpp +++ b/src/Format.cpp @@ -59,9 +59,8 @@ namespace openPMD { return Format::ADIOS2_SSC; if (auxiliary::ends_with(filename, ".json")) return Format::JSON; - if (std::string::npos != filename.find('.') /* extension is provided */ ) - throw std::runtime_error("Unknown file format. Did you append a valid filename extension?"); + // Format might still be specified via JSON return Format::DUMMY; } diff --git a/src/IO/ADIOS/ADIOS1IOHandler.cpp b/src/IO/ADIOS/ADIOS1IOHandler.cpp index 4134bcd1df..e6bce3835d 100644 --- a/src/IO/ADIOS/ADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS1IOHandler.cpp @@ -43,9 +43,11 @@ namespace openPMD # define VERIFY(CONDITION, TEXT) do{ (void)sizeof(CONDITION); } while( 0 ) # endif -ADIOS1IOHandlerImpl::ADIOS1IOHandlerImpl(AbstractIOHandler* handler) +ADIOS1IOHandlerImpl::ADIOS1IOHandlerImpl(AbstractIOHandler* handler, json::TracingJSON json) : Base_t(handler) -{ } +{ + initJson( std::move( json ) ); +} ADIOS1IOHandlerImpl::~ADIOS1IOHandlerImpl() { @@ -220,9 +222,9 @@ ADIOS1IOHandlerImpl::init() #endif #if openPMD_HAVE_ADIOS1 -ADIOS1IOHandler::ADIOS1IOHandler(std::string path, Access at) +ADIOS1IOHandler::ADIOS1IOHandler(std::string path, Access at, json::TracingJSON json) : AbstractIOHandler(std::move(path), at), - m_impl{new ADIOS1IOHandlerImpl(this)} + m_impl{new ADIOS1IOHandlerImpl(this, std::move(json))} { m_impl->init(); } @@ -317,7 +319,7 @@ ADIOS1IOHandlerImpl::initialize_group(std::string const &name) } #else -ADIOS1IOHandler::ADIOS1IOHandler(std::string path, Access at) +ADIOS1IOHandler::ADIOS1IOHandler(std::string path, Access at, json::TracingJSON) : AbstractIOHandler(std::move(path), at) { throw std::runtime_error("openPMD-api built without ADIOS1 support"); diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 46f306fd12..f32ac50bb7 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -22,6 +22,7 @@ #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" #include "openPMD/Datatype.hpp" +#include "openPMD/Error.hpp" #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" #include "openPMD/auxiliary/Environment.hpp" @@ -67,7 +68,7 @@ namespace openPMD ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( AbstractIOHandler * handler, MPI_Comm communicator, - nlohmann::json cfg, + json::TracingJSON cfg, std::string engineType ) : AbstractIOHandlerImplCommon( handler ) , m_ADIOS{ communicator, ADIOS2_DEBUG_MODE } @@ -80,7 +81,7 @@ ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( AbstractIOHandler * handler, - nlohmann::json cfg, + json::TracingJSON cfg, std::string engineType ) : AbstractIOHandlerImplCommon( handler ) , m_ADIOS{ ADIOS2_DEBUG_MODE } @@ -120,11 +121,11 @@ ADIOS2IOHandlerImpl::~ADIOS2IOHandlerImpl() } void -ADIOS2IOHandlerImpl::init( nlohmann::json cfg ) +ADIOS2IOHandlerImpl::init( json::TracingJSON cfg ) { - if( cfg.contains( "adios2" ) ) + if( cfg.json().contains( "adios2" ) ) { - m_config = std::move( cfg[ "adios2" ] ); + m_config = cfg[ "adios2" ]; if( m_config.json().contains( "schema" ) ) { @@ -140,12 +141,18 @@ ADIOS2IOHandlerImpl::init( nlohmann::json cfg ) if( !engineTypeConfig.is_null() ) { // convert to string - m_engineType = engineTypeConfig; - std::transform( - m_engineType.begin(), - m_engineType.end(), - m_engineType.begin(), - []( unsigned char c ) { return std::tolower( c ); } ); + auto maybeEngine = + json::asLowerCaseStringDynamic( engineTypeConfig ); + if( maybeEngine.has_value() ) + { + m_engineType = std::move( maybeEngine.get() ); + } + else + { + throw error::BackendConfigSchema( + {"adios2", "engine", "type"}, + "Must be convertible to string type." ); + } } } auto operators = getOperators(); @@ -159,7 +166,7 @@ ADIOS2IOHandlerImpl::init( nlohmann::json cfg ) } auxiliary::Option< std::vector< ADIOS2IOHandlerImpl::ParameterizedOperator > > -ADIOS2IOHandlerImpl::getOperators( auxiliary::TracingJSON cfg ) +ADIOS2IOHandlerImpl::getOperators( json::TracingJSON cfg ) { using ret_t = auxiliary::Option< std::vector< ParameterizedOperator > >; std::vector< ParameterizedOperator > res; @@ -188,8 +195,22 @@ ADIOS2IOHandlerImpl::getOperators( auxiliary::TracingJSON cfg ) paramIterator != params.end(); ++paramIterator ) { - adiosParams[ paramIterator.key() ] = - paramIterator.value().get< std::string >(); + auto maybeString = + json::asStringDynamic( paramIterator.value() ); + if( maybeString.has_value() ) + { + adiosParams[ paramIterator.key() ] = + std::move( maybeString.get() ); + } + else + { + throw error::BackendConfigSchema( + { "adios2", + "dataset", + "operators", + paramIterator.key() }, + "Must be convertible to string type." ); + } } } auxiliary::Option< adios2::Operator > adiosOperator = @@ -352,39 +373,25 @@ void ADIOS2IOHandlerImpl::createDataset( auto const varName = nameOfVariable( writable ); std::vector< ParameterizedOperator > operators; - nlohmann::json options = nlohmann::json::parse( parameters.options ); - if( options.contains( "adios2" ) ) + json::TracingJSON options = json::parseOptions( + parameters.options, /* considerFiles = */ false ); + if( options.json().contains( "adios2" ) ) { - auxiliary::TracingJSON datasetConfig( options[ "adios2" ] ); + json::TracingJSON datasetConfig( options[ "adios2" ] ); auto datasetOperators = getOperators( datasetConfig ); operators = datasetOperators ? std::move( datasetOperators.get() ) : defaultOperators; - - auto shadow = datasetConfig.invertShadow(); - if( shadow.size() > 0 ) - { - std::cerr << "Warning: parts of the JSON configuration for " - "ADIOS2 dataset '" - << varName << "' remain unused:\n" - << shadow << std::endl; - } } else { operators = defaultOperators; } - - if( !parameters.compression.empty() ) - { - auxiliary::Option< adios2::Operator > adiosOperator = - getCompressionOperator( parameters.compression ); - if( adiosOperator ) - { - operators.push_back( ParameterizedOperator{ - adiosOperator.get(), adios2::Params() } ); - } - } + parameters.warnUnusedParameters( + options, + "adios2", + "Warning: parts of the JSON configuration for ADIOS2 dataset '" + + varName + "' remain unused:\n" ); // cast from openPMD::Extent to adios2::Dims adios2::Dims const shape( parameters.extent.begin(), parameters.extent.end() ); @@ -1080,7 +1087,7 @@ ADIOS2IOHandlerImpl::adios2AccessMode( std::string const & fullPath ) } } -auxiliary::TracingJSON ADIOS2IOHandlerImpl::nullvalue = nlohmann::json(); +json::TracingJSON ADIOS2IOHandlerImpl::nullvalue = nlohmann::json(); std::string ADIOS2IOHandlerImpl::filePositionToString( @@ -2325,8 +2332,20 @@ namespace detail for( auto it = params.json().begin(); it != params.json().end(); it++ ) { - m_IO.SetParameter( it.key(), it.value() ); - alreadyConfigured.emplace( it.key() ); + auto maybeString = json::asStringDynamic( it.value() ); + if( maybeString.has_value() ) + { + m_IO.SetParameter( + it.key(), std::move( maybeString.get() ) ); + } + else + { + throw error::BackendConfigSchema( + {"adios2", "engine", "parameters", it.key() }, + "Must be convertible to string type." ); + } + alreadyConfigured.emplace( + auxiliary::lowerCase( std::string( it.key() ) ) ); } } auto _useAdiosSteps = @@ -2353,8 +2372,9 @@ namespace detail << shadow << std::endl; } auto notYetConfigured = - [&alreadyConfigured]( std::string const & param ) { - auto it = alreadyConfigured.find( param ); + [ &alreadyConfigured ]( std::string const & param ) { + auto it = alreadyConfigured.find( + auxiliary::lowerCase( std::string( param ) ) ); return it == alreadyConfigured.end(); }; @@ -2914,7 +2934,7 @@ ADIOS2IOHandler::ADIOS2IOHandler( std::string path, openPMD::Access at, MPI_Comm comm, - nlohmann::json options, + json::TracingJSON options, std::string engineType ) : AbstractIOHandler( std::move( path ), at, comm ) , m_impl{ this, comm, std::move( options ), std::move( engineType ) } @@ -2926,7 +2946,7 @@ ADIOS2IOHandler::ADIOS2IOHandler( ADIOS2IOHandler::ADIOS2IOHandler( std::string path, Access at, - nlohmann::json options, + json::TracingJSON options, std::string engineType ) : AbstractIOHandler( std::move( path ), at ) , m_impl{ this, std::move( options ), std::move( engineType ) } @@ -2946,7 +2966,7 @@ ADIOS2IOHandler::ADIOS2IOHandler( std::string path, Access at, MPI_Comm comm, - nlohmann::json, + json::TracingJSON, std::string ) : AbstractIOHandler( std::move( path ), at, comm ) { @@ -2957,7 +2977,7 @@ ADIOS2IOHandler::ADIOS2IOHandler( ADIOS2IOHandler::ADIOS2IOHandler( std::string path, Access at, - nlohmann::json, + json::TracingJSON, std::string ) : AbstractIOHandler( std::move( path ), at ) { diff --git a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp index aeef40e0d8..e9938ed139 100644 --- a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp @@ -23,6 +23,8 @@ #if openPMD_HAVE_ADIOS1 +#include "openPMD/auxiliary/JSON_internal.hpp" +#include "openPMD/Error.hpp" #include "openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp" #include "openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp" @@ -466,6 +468,33 @@ CommonADIOS1IOHandlerImpl< ChildClass >::createPath(Writable* writable, } } +static auxiliary::Option< std::string > datasetTransform( + json::TracingJSON config ) +{ + using ret_t = auxiliary::Option< std::string >; + if( !config.json().contains( "dataset" ) ) + { + return ret_t{}; + } + config = config[ "dataset" ]; + if( !config.json().contains( "transform" ) ) + { + return ret_t{}; + } + config = config[ "transform" ]; + auto maybeRes = json::asStringDynamic( config.json() ); + if( maybeRes.has_value() ) + { + return std::move( maybeRes.get() ); + } + else + { + throw error::BackendConfigSchema( + { "adios1", "dataset", "transform" }, + "Key must convertible to type string." ); + } +} + template< typename ChildClass > void CommonADIOS1IOHandlerImpl< ChildClass >::createDataset(Writable* writable, @@ -519,14 +548,35 @@ CommonADIOS1IOHandlerImpl< ChildClass >::createDataset(Writable* writable, chunkOffsetParam.c_str()); VERIFY(id != 0, "[ADIOS1] Internal error: Failed to define ADIOS variable during Dataset creation"); - if( !parameters.compression.empty() ) - std::cerr << "Custom compression not compatible with ADIOS1 backend. Use transform instead." - << std::endl; + std::string transform = ""; + { + json::TracingJSON options = json::parseOptions( + parameters.options, /* considerFiles = */ false ); + auto maybeTransform = datasetTransform( options ); + if( maybeTransform.has_value() ) + { + transform = maybeTransform.get(); + } + + auto shadow = options.invertShadow(); + if( shadow.size() > 0 ) + { + std::cerr << "Warning: parts of the JSON configuration for " + "ADIOS1 dataset '" + << name << "' remain unused:\n" + << shadow << std::endl; + } + } + // Fallback: global option + if( transform.empty() ) + { + transform = m_defaultTransform; + } - if( !parameters.transform.empty() ) + if( !transform.empty() ) { int status; - status = adios_set_transform(id, parameters.transform.c_str()); + status = adios_set_transform(id, transform.c_str()); VERIFY(status == err_no_error, "[ADIOS1] Internal error: Failed to set ADIOS transform during Dataset cretaion"); } @@ -1699,6 +1749,21 @@ CommonADIOS1IOHandlerImpl< ChildClass >::listAttributes(Writable* writable, } } +template< typename ChildClass > +void CommonADIOS1IOHandlerImpl< ChildClass >::initJson( + json::TracingJSON config ) +{ + if( !config.json().contains( "adios1" ) ) + { + return; + } + auto maybeTransform = datasetTransform( config[ "adios1" ] ); + if( maybeTransform.has_value() ) + { + m_defaultTransform = std::move( maybeTransform.get() ); + } +} + template class CommonADIOS1IOHandlerImpl< ADIOS1IOHandlerImpl >; #if openPMD_HAVE_MPI template class CommonADIOS1IOHandlerImpl< ParallelADIOS1IOHandlerImpl >; diff --git a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp index 91d6fcc63e..bfcc13675d 100644 --- a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp @@ -41,6 +41,7 @@ namespace openPMD # endif ParallelADIOS1IOHandlerImpl::ParallelADIOS1IOHandlerImpl(AbstractIOHandler* handler, + json::TracingJSON json, MPI_Comm comm) : Base_t{handler}, m_mpiInfo{MPI_INFO_NULL} @@ -48,6 +49,7 @@ ParallelADIOS1IOHandlerImpl::ParallelADIOS1IOHandlerImpl(AbstractIOHandler* hand int status = MPI_SUCCESS; status = MPI_Comm_dup(comm, &m_mpiComm); VERIFY(status == MPI_SUCCESS, "[ADIOS1] Internal error: Failed to duplicate MPI communicator"); + initJson( std::move( json ) ); } ParallelADIOS1IOHandlerImpl::~ParallelADIOS1IOHandlerImpl() @@ -240,9 +242,10 @@ ParallelADIOS1IOHandlerImpl::init() ParallelADIOS1IOHandler::ParallelADIOS1IOHandler(std::string path, Access at, + json::TracingJSON json, MPI_Comm comm) : AbstractIOHandler(std::move(path), at, comm), - m_impl{new ParallelADIOS1IOHandlerImpl(this, comm)} + m_impl{new ParallelADIOS1IOHandlerImpl(this, std::move(json), comm)} { m_impl->init(); } @@ -347,6 +350,7 @@ ParallelADIOS1IOHandlerImpl::initialize_group(std::string const &name) # if openPMD_HAVE_MPI ParallelADIOS1IOHandler::ParallelADIOS1IOHandler(std::string path, Access at, + json::TracingJSON, MPI_Comm comm) : AbstractIOHandler(std::move(path), at, comm) { @@ -354,7 +358,8 @@ ParallelADIOS1IOHandler::ParallelADIOS1IOHandler(std::string path, } # else ParallelADIOS1IOHandler::ParallelADIOS1IOHandler(std::string path, - Access at) + Access at, + json::TracingJSON) : AbstractIOHandler(std::move(path), at) { throw std::runtime_error("openPMD-api built without parallel ADIOS1 support"); diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index 25e2bfcd01..f4b542254a 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -27,19 +27,19 @@ #include "openPMD/IO/HDF5/HDF5IOHandler.hpp" #include "openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp" #include "openPMD/IO/JSON/JSONIOHandler.hpp" -#include "openPMD/auxiliary/JSON.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" namespace openPMD { #if openPMD_HAVE_MPI template<> std::shared_ptr< AbstractIOHandler > - createIOHandler< nlohmann::json >( + createIOHandler< json::TracingJSON >( std::string path, Access access, Format format, MPI_Comm comm, - nlohmann::json options ) + json::TracingJSON options ) { (void) options; switch( format ) @@ -49,7 +49,8 @@ namespace openPMD path, access, comm, std::move( options ) ); case Format::ADIOS1: # if openPMD_HAVE_ADIOS1 - return std::make_shared< ParallelADIOS1IOHandler >( path, access, comm ); + return std::make_shared< ParallelADIOS1IOHandler >( + path, access, std::move( options ), comm ); # else throw std::runtime_error("openPMD-api built without ADIOS1 support"); # endif @@ -71,11 +72,11 @@ namespace openPMD template<> std::shared_ptr< AbstractIOHandler > - createIOHandler< nlohmann::json >( + createIOHandler< json::TracingJSON >( std::string path, Access access, Format format, - nlohmann::json options ) + json::TracingJSON options ) { (void) options; switch( format ) @@ -85,7 +86,8 @@ namespace openPMD path, access, std::move( options ) ); case Format::ADIOS1: #if openPMD_HAVE_ADIOS1 - return std::make_shared< ADIOS1IOHandler >( path, access ); + return std::make_shared< ADIOS1IOHandler >( + path, access, std::move( options ) ); #else throw std::runtime_error("openPMD-api built without ADIOS1 support"); #endif @@ -112,6 +114,9 @@ namespace openPMD createIOHandler( std::string path, Access access, Format format ) { return createIOHandler( - std::move( path ), access, format, nlohmann::json::object() ); + std::move( path ), + access, + format, + json::TracingJSON( nlohmann::json::object() )); } } // namespace openPMD diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index af0e59c4d4..04c5993293 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -24,6 +24,7 @@ #if openPMD_HAVE_HDF5 # include "openPMD/Datatype.hpp" +# include "openPMD/Error.hpp" # include "openPMD/auxiliary/Filesystem.hpp" # include "openPMD/auxiliary/StringManip.hpp" # include "openPMD/backend/Attribute.hpp" @@ -53,7 +54,7 @@ namespace openPMD # endif HDF5IOHandlerImpl::HDF5IOHandlerImpl( - AbstractIOHandler* handler, nlohmann::json config) + AbstractIOHandler* handler, json::TracingJSON config) : AbstractIOHandlerImpl(handler), m_datasetTransferProperty{H5P_DEFAULT}, m_fileAccessProperty{H5P_DEFAULT}, @@ -87,9 +88,9 @@ HDF5IOHandlerImpl::HDF5IOHandlerImpl( m_chunks = auxiliary::getEnvString( "OPENPMD_HDF5_CHUNKS", "auto" ); // JSON option can overwrite env option: - if( config.contains( "hdf5" ) ) + if( config.json().contains( "hdf5" ) ) { - m_config = std::move( config[ "hdf5" ] ); + m_config = config[ "hdf5" ]; // check for global dataset configs if( m_config.json().contains( "dataset" ) ) @@ -97,7 +98,18 @@ HDF5IOHandlerImpl::HDF5IOHandlerImpl( auto datasetConfig = m_config[ "dataset" ]; if( datasetConfig.json().contains( "chunks" ) ) { - m_chunks = datasetConfig[ "chunks" ].json().get< std::string >(); + auto maybeChunks = json::asLowerCaseStringDynamic( + datasetConfig[ "chunks" ].json() ); + if( maybeChunks.has_value() ) + { + m_chunks = std::move( maybeChunks.get() ); + } + else + { + throw error::BackendConfigSchema( + {"hdf5", "dataset", "chunks"}, + "Must be convertible to string type." ); + } } } if( m_chunks != "auto" && m_chunks != "none" ) @@ -294,34 +306,33 @@ HDF5IOHandlerImpl::createDataset(Writable* writable, if( auxiliary::ends_with(name, '/') ) name = auxiliary::replace_last(name, "/", ""); - auto config = nlohmann::json::parse( parameters.options ); + json::TracingJSON config = json::parseOptions( + parameters.options, /* considerFiles = */ false ); // general bool is_resizable_dataset = false; - if( config.contains( "resizable" ) ) + if( config.json().contains( "resizable" ) ) { - is_resizable_dataset = config.at( "resizable" ).get< bool >(); + is_resizable_dataset = config[ "resizable" ].json().get< bool >(); } // HDF5 specific - if( config.contains( "hdf5" ) && - config[ "hdf5" ].contains( "dataset" ) ) + if( config.json().contains( "hdf5" ) && + config[ "hdf5" ].json().contains( "dataset" ) ) { - auxiliary::TracingJSON datasetConfig{ + json::TracingJSON datasetConfig{ config[ "hdf5" ][ "dataset" ] }; /* * @todo Read more options from config here. */ - auto shadow = datasetConfig.invertShadow(); - if( shadow.size() > 0 ) - { - std::cerr << "Warning: parts of the JSON configuration for " - "HDF5 dataset '" - << name << "' remain unused:\n" - << shadow << std::endl; - } + ( void )datasetConfig; } + parameters.warnUnusedParameters( + config, + "hdf5", + "Warning: parts of the JSON configuration for HDF5 dataset '" + + name + "' remain unused:\n" ); hid_t gapl = H5Pcreate(H5P_GROUP_ACCESS); #if H5_VERSION_GE(1,10,0) && openPMD_HAVE_MPI @@ -385,7 +396,7 @@ HDF5IOHandlerImpl::createDataset(Writable* writable, VERIFY(status == 0, "[HDF5] Internal error: Failed to set chunk size during dataset creation"); } - std::string const& compression = parameters.compression; + std::string const& compression = ""; // @todo read from JSON if( !compression.empty() ) std::cerr << "[HDF5] Compression not yet implemented in HDF5 backend." << std::endl; @@ -409,11 +420,6 @@ HDF5IOHandlerImpl::createDataset(Writable* writable, } */ - std::string const& transform = parameters.transform; - if( !transform.empty() ) - std::cerr << "[HDF5] Custom transform not yet implemented in HDF5 backend." - << std::endl; - GetH5DataType getH5DataType({ { typeid(bool).name(), m_H5T_BOOL_ENUM }, { typeid(std::complex< float >).name(), m_H5T_CFLOAT }, @@ -2018,7 +2024,7 @@ HDF5IOHandlerImpl::getFile( Writable * writable ) #endif #if openPMD_HAVE_HDF5 -HDF5IOHandler::HDF5IOHandler(std::string path, Access at, nlohmann::json config) +HDF5IOHandler::HDF5IOHandler(std::string path, Access at, json::TracingJSON config) : AbstractIOHandler(std::move(path), at), m_impl{new HDF5IOHandlerImpl(this, std::move(config))} { } @@ -2031,7 +2037,7 @@ HDF5IOHandler::flush() return m_impl->flush(); } #else -HDF5IOHandler::HDF5IOHandler(std::string path, Access at, nlohmann::json /* config */) +HDF5IOHandler::HDF5IOHandler(std::string path, Access at, json::TracingJSON /* config */) : AbstractIOHandler(std::move(path), at) { throw std::runtime_error("openPMD-api built without HDF5 support"); diff --git a/src/IO/HDF5/ParallelHDF5IOHandler.cpp b/src/IO/HDF5/ParallelHDF5IOHandler.cpp index 19c9621393..59178825c5 100644 --- a/src/IO/HDF5/ParallelHDF5IOHandler.cpp +++ b/src/IO/HDF5/ParallelHDF5IOHandler.cpp @@ -40,7 +40,7 @@ namespace openPMD # endif ParallelHDF5IOHandler::ParallelHDF5IOHandler( - std::string path, Access at, MPI_Comm comm, nlohmann::json config ) + std::string path, Access at, MPI_Comm comm, json::TracingJSON config ) : AbstractIOHandler(std::move(path), at, comm), m_impl{new ParallelHDF5IOHandlerImpl(this, comm, std::move(config))} { } @@ -54,7 +54,7 @@ ParallelHDF5IOHandler::flush() } ParallelHDF5IOHandlerImpl::ParallelHDF5IOHandlerImpl( - AbstractIOHandler* handler, MPI_Comm comm, nlohmann::json config ) + AbstractIOHandler* handler, MPI_Comm comm, json::TracingJSON config ) : HDF5IOHandlerImpl{handler, std::move(config)}, m_mpiComm{comm}, m_mpiInfo{MPI_INFO_NULL} /* MPI 3.0+: MPI_INFO_ENV */ @@ -150,7 +150,7 @@ ParallelHDF5IOHandlerImpl::~ParallelHDF5IOHandlerImpl() ParallelHDF5IOHandler::ParallelHDF5IOHandler(std::string path, Access at, MPI_Comm comm, - nlohmann::json /* config */) + json::TracingJSON /* config */) : AbstractIOHandler(std::move(path), at, comm) { throw std::runtime_error("openPMD-api built without HDF5 support"); @@ -158,7 +158,7 @@ ParallelHDF5IOHandler::ParallelHDF5IOHandler(std::string path, # else ParallelHDF5IOHandler::ParallelHDF5IOHandler(std::string path, Access at, - nlohmann::json /* config */) + json::TracingJSON /* config */) : AbstractIOHandler(std::move(path), at) { throw std::runtime_error("openPMD-api built without parallel support and without HDF5 support"); diff --git a/src/IO/IOTask.cpp b/src/IO/IOTask.cpp index 5ff2a31b8d..b7b54ab860 100644 --- a/src/IO/IOTask.cpp +++ b/src/IO/IOTask.cpp @@ -19,12 +19,47 @@ * If not, see . */ #include "openPMD/IO/IOTask.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/backend/Attributable.hpp" +#include // std::cerr + namespace openPMD { Writable* getWritable(Attributable* a) { return &a->writable(); } + +template<> +void Parameter< Operation::CREATE_DATASET >::warnUnusedParameters< + json::TracingJSON >( + json::TracingJSON & config, + std::string const & currentBackendName, + std::string const & warningMessage ) +{ + /* + * Fake-read non-backend-specific options. Some backends don't read those + * and we don't want to have warnings for them. + */ + for( std::string const & key : { "resizable" } ) + { + config[ key ]; + } + + auto shadow = config.invertShadow(); + // The backends are supposed to deal with this + // Only global options here + for( auto const & backendKey : json::backendKeys ) + { + if( backendKey != currentBackendName ) + { + shadow.erase( backendKey ); + } + } + if( shadow.size() > 0 ) + { + std::cerr << warningMessage << shadow.dump() << std::endl; + } +} } // openPMD diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index b1727b8c7d..305a0cab4b 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -256,9 +256,6 @@ RecordComponent::flush(std::string const& name) dCreate.name = name; dCreate.extent = getExtent(); dCreate.dtype = getDatatype(); - dCreate.chunkSize = rc.m_dataset.chunkSize; - dCreate.compression = rc.m_dataset.compression; - dCreate.transform = rc.m_dataset.transform; dCreate.options = rc.m_dataset.options; IOHandler()->enqueue(IOTask(this, dCreate)); } diff --git a/src/Series.cpp b/src/Series.cpp index 91bcc12f48..519710aa3f 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -20,7 +20,7 @@ */ #include "openPMD/auxiliary/Date.hpp" #include "openPMD/auxiliary/Filesystem.hpp" -#include "openPMD/auxiliary/JSON.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerHelper.hpp" @@ -38,6 +38,7 @@ #include #include #include +#include namespace openPMD @@ -1458,21 +1459,110 @@ void Series::openIteration( uint64_t index, Iteration iteration ) namespace { -template< typename T > -void getJsonOption( - nlohmann::json const & config, std::string const & key, T & dest ) -{ - if( config.contains( key ) ) + /** + * Look up if the specified key is contained in the JSON dataset. + * If yes, read it into the specified location. + */ + template< typename From, typename Dest = From > + void getJsonOption( + json::TracingJSON & config, std::string const & key, Dest & dest ) { - dest = config.at( key ).get< T >(); + if( config.json().contains( key ) ) + { + dest = config[ key ].json().get< From >(); + } + } + + /** + * Like getJsonOption(), but for string types. + * Numbers and booleans are converted to their string representation. + * The string is converted to lower case. + */ + template< typename Dest = std::string > + void getJsonOptionLowerCase( + json::TracingJSON & config, std::string const & key, Dest & dest ) + { + if( config.json().contains( key ) ) + { + auto maybeString = + json::asLowerCaseStringDynamic( config[ key ].json() ); + if( maybeString.has_value() ) + { + dest = std::move( maybeString.get() ); + } + else + { + throw error::BackendConfigSchema( + { key }, "Must be convertible to string type." ); + } + } } } -void parseJsonOptions( - internal::SeriesData & series, nlohmann::json const & options ) +template< typename TracingJSON > +void Series::parseJsonOptions( + TracingJSON & options, ParsedInput & input ) { - getJsonOption( options, "defer_iteration_parsing", series.m_parseLazily ); -} + auto & series = get(); + getJsonOption< bool >( + options, "defer_iteration_parsing", series.m_parseLazily ); + // backend key + { + std::map< std::string, Format > const backendDescriptors{ + { "hdf5", Format::HDF5 }, + { "adios1", Format::ADIOS1 }, + { "adios2", Format::ADIOS2 }, + { "json", Format::JSON } }; + std::string backend; + getJsonOptionLowerCase( options, "backend", backend ); + if( !backend.empty() ) + { + auto it = backendDescriptors.find( backend ); + if( it != backendDescriptors.end() ) + { + if( input.format != Format::DUMMY && + suffix( input.format ) != suffix( it->second ) ) + { + std::cerr << "[Warning] Supplied filename extension '" + << suffix( input.format ) + << "' contradicts the backend specified via the " + "'backend' key. Will go on with backend " + << it->first << "." << std::endl; + } + input.format = it->second; + } + else + { + throw error::BackendConfigSchema( + { "backend" }, "Unknown backend specified: " + backend ); + } + } + } + // iteration_encoding key + { + std::map< std::string, IterationEncoding > const ieDescriptors{ + { "file_based", IterationEncoding::fileBased }, + { "group_based", IterationEncoding::groupBased }, + { "variable_based", IterationEncoding::variableBased } }; + std::string iterationEncoding; + getJsonOptionLowerCase( + options, "iteration_encoding", iterationEncoding ); + if( !iterationEncoding.empty() ) + { + auto it = ieDescriptors.find( iterationEncoding ); + if( it != ieDescriptors.end() ) + { + input.iterationEncoding = it->second; + } + else + { + throw error::BackendConfigSchema( + { "iteration_encoding" }, + "Unknown iteration encoding specified: " + + iterationEncoding ); + } + } + } } namespace internal @@ -1531,12 +1621,14 @@ Series::Series( { Attributable::setData( m_series ); iterations = m_series->iterations; - nlohmann::json optionsJson = auxiliary::parseOptions( options, comm ); - parseJsonOptions( get(), optionsJson ); + json::TracingJSON optionsJson = json::parseOptions( + options, comm, /* considerFiles = */ true ); auto input = parseInput( filepath ); + parseJsonOptions( optionsJson, *input ); auto handler = createIOHandler( - input->path, at, input->format, comm, std::move( optionsJson ) ); + input->path, at, input->format, comm, optionsJson ); init( handler, std::move( input ) ); + json::warnGlobalUnusedOptions( optionsJson ); } #endif @@ -1547,12 +1639,14 @@ Series::Series( { Attributable::setData( m_series ); iterations = m_series->iterations; - nlohmann::json optionsJson = auxiliary::parseOptions( options ); - parseJsonOptions( get(), optionsJson ); + json::TracingJSON optionsJson = json::parseOptions( + options, /* considerFiles = */ true ); auto input = parseInput( filepath ); + parseJsonOptions( optionsJson, *input ); auto handler = createIOHandler( - input->path, at, input->format, std::move( optionsJson ) ); + input->path, at, input->format, optionsJson ); init( handler, std::move( input ) ); + json::warnGlobalUnusedOptions( optionsJson ); } Series::operator bool() const diff --git a/src/auxiliary/JSON.cpp b/src/auxiliary/JSON.cpp index d0e844c525..ae3e1dfc58 100644 --- a/src/auxiliary/JSON.cpp +++ b/src/auxiliary/JSON.cpp @@ -20,19 +20,24 @@ */ #include "openPMD/auxiliary/JSON.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/Option.hpp" #include "openPMD/auxiliary/StringManip.hpp" +#include #include // std::isspace #include +#include // std::cerr +#include #include +#include // std::forward #include namespace openPMD { -namespace auxiliary +namespace json { TracingJSON::TracingJSON() : TracingJSON( nlohmann::json() ) { @@ -47,24 +52,20 @@ namespace auxiliary { } - nlohmann::json const & - TracingJSON::getShadow() + nlohmann::json const & TracingJSON::getShadow() const { return *m_positionInShadow; } - nlohmann::json - TracingJSON::invertShadow() + nlohmann::json TracingJSON::invertShadow() const { nlohmann::json inverted = *m_positionInOriginal; invertShadow( inverted, *m_positionInShadow ); return inverted; } - void - TracingJSON::invertShadow( - nlohmann::json & result, - nlohmann::json const & shadow ) + void TracingJSON::invertShadow( + nlohmann::json & result, nlohmann::json const & shadow ) const { if( !shadow.is_object() ) { @@ -123,7 +124,7 @@ namespace auxiliary { std::string trimmed = auxiliary::trim( unparsed, []( char c ) { return std::isspace( c ); } ); - if( trimmed.at( 0 ) == '@' ) + if( !trimmed.empty() && trimmed.at( 0 ) == '@' ) { trimmed = trimmed.substr( 1 ); trimmed = auxiliary::trim( @@ -138,44 +139,247 @@ namespace auxiliary } nlohmann::json - parseOptions( std::string const & options ) + parseOptions( std::string const & options, bool considerFiles ) { - auto filename = extractFilename( options ); - if( filename.has_value() ) + if( considerFiles ) { - std::fstream handle; - handle.open( filename.get(), std::ios_base::in ); - nlohmann::json res; - handle >> res; - if( !handle.good() ) + auto filename = extractFilename( options ); + if( filename.has_value() ) { - throw std::runtime_error( - "Failed reading JSON config from file " + filename.get() + - "." ); + std::fstream handle; + handle.open( filename.get(), std::ios_base::in ); + nlohmann::json res; + handle >> res; + if( !handle.good() ) + { + throw std::runtime_error( + "Failed reading JSON config from file " + + filename.get() + "." ); + } + lowerCase( res ); + return res; } - return res; } - else + auto res = nlohmann::json::parse( options ); + lowerCase( res ); + return res; + } + +#if openPMD_HAVE_MPI + nlohmann::json parseOptions( + std::string const & options, MPI_Comm comm, bool considerFiles ) + { + if( considerFiles ) { - return nlohmann::json::parse( options ); + auto filename = extractFilename( options ); + if( filename.has_value() ) + { + auto res = nlohmann::json::parse( + auxiliary::collective_file_read( filename.get(), comm ) ); + lowerCase( res ); + return res; + } } + auto res = nlohmann::json::parse( options ); + lowerCase( res ); + return res; } +#endif -#if openPMD_HAVE_MPI - nlohmann::json - parseOptions( std::string const & options, MPI_Comm comm ) + template< typename F > + static nlohmann::json & lowerCase( + nlohmann::json & json, + std::vector< std::string > & currentPath, + F const & ignoreCurrentPath ) + { + auto transFormCurrentObject = [ ¤tPath ]( + nlohmann::json::object_t & val ) { + // somekey -> SomeKey + std::map< std::string, std::string > originalKeys; + for( auto & pair : val ) + { + std::string lower = + auxiliary::lowerCase( std::string( pair.first ) ); + auto findEntry = originalKeys.find( lower ); + if( findEntry != originalKeys.end() ) + { + // double entry found + std::vector< std::string > copyCurrentPath{ currentPath }; + copyCurrentPath.push_back( lower ); + throw error::BackendConfigSchema( + std::move( copyCurrentPath ), + "JSON config: duplicate keys." ); + } + originalKeys.emplace_hint( + findEntry, std::move( lower ), pair.first ); + } + + nlohmann::json::object_t newObject; + for( auto & pair : originalKeys ) + { + newObject[ pair.first ] = std::move( val[ pair.second ] ); + } + val = newObject; + }; + + if( json.is_object() ) + { + auto & val = json.get_ref< nlohmann::json::object_t & >(); + + if( !ignoreCurrentPath( currentPath ) ) + { + transFormCurrentObject( val ); + } + + // now recursively + for( auto & pair : val ) + { + // ensure that the path consists only of lowercase strings, + // even if ignoreCurrentPath() was true + currentPath.push_back( + auxiliary::lowerCase( std::string( pair.first ) ) ); + lowerCase( pair.second, currentPath, ignoreCurrentPath ); + currentPath.pop_back(); + } + } + else if( json.is_array() ) + { + for( auto & val : json ) + { + currentPath.emplace_back( "\vnum" ); + lowerCase( val, currentPath, ignoreCurrentPath ); + currentPath.pop_back(); + } + } + return json; + } + + nlohmann::json & lowerCase( nlohmann::json & json ) + { + std::vector< std::string > currentPath; + // that's as deep as our config currently goes, +1 for good measure + currentPath.reserve( 7 ); + return lowerCase( + json, currentPath, []( std::vector< std::string > const & path ) { + std::vector< std::string > const ignoredPaths[] = { + { "adios2", "engine", "parameters" }, + { "adios2", + "dataset", + "operators", + /* + * We use "\vnum" to indicate "any array index". + */ + "\vnum", + "parameters" } }; + for( auto const & ignored : ignoredPaths ) + { + if( ignored == path ) + { + return true; + } + } + return false; + } ); + } + + auxiliary::Option< std::string > + asStringDynamic( nlohmann::json const & value ) + { + if( value.is_string() ) + { + return value.get< std::string >(); + } + else if( value.is_number_integer() ) + { + return std::to_string( value.get< long long >() ); + } + else if( value.is_number_float() ) + { + return std::to_string( value.get< long double >() ); + } + else if( value.is_boolean() ) + { + return std::string( value.get< bool >() ? "1" : "0" ); + } + return auxiliary::Option< std::string >{}; + } + + auxiliary::Option< std::string > + asLowerCaseStringDynamic( nlohmann::json const & value ) { - auto filename = extractFilename( options ); - if( filename.has_value() ) + auto maybeString = asStringDynamic( value ); + if( maybeString.has_value() ) { - return nlohmann::json::parse( - auxiliary::collective_file_read( filename.get(), comm ) ); + auxiliary::lowerCase( maybeString.get() ); + } + return maybeString; + } + + std::vector< std::string > backendKeys{ + "adios1", "adios2", "json", "hdf5" }; + + void warnGlobalUnusedOptions( TracingJSON const & config ) + { + auto shadow = config.invertShadow(); + // The backends are supposed to deal with this + // Only global options here + for( auto const & backendKey : json::backendKeys ) + { + shadow.erase( backendKey ); + } + if( shadow.size() > 0 ) + { + std::cerr + << "[Series] The following parts of the global JSON config " + "remains unused:\n" + << shadow.dump() << std::endl; + } + } + + nlohmann::json & + merge( nlohmann::json & defaultVal, nlohmann::json const & overwrite ) + { + if( defaultVal.is_object() && overwrite.is_object() ) + { + std::vector< std::string > prunedKeys; + for( auto it = overwrite.begin(); it != overwrite.end(); ++it ) + { + auto & valueInDefault = defaultVal[ it.key() ]; + merge( valueInDefault, it.value() ); + if( valueInDefault.is_null() ) + { + prunedKeys.emplace_back( it.key() ); + } + } + for( auto const & key : prunedKeys ) + { + defaultVal.erase( key ); + } } else { - return nlohmann::json::parse( options ); + /* + * Anything else, just overwrite. + * Note: There's no clear generic way to merge arrays: + * Should we concatenate? Or should we merge at the same indices? + * From the user side, this means: + * An application can specify a number of default compression + * operators, e.g. in adios2.dataset.operators, but a user can + * overwrite the operators. Neither appending nor pointwise update + * are quite useful here. + */ + defaultVal = overwrite; } + return defaultVal; } -#endif -} // namespace auxiliary + + std::string merge( + std::string const & defaultValue, + std::string const & overwrite ) + { + auto res = parseOptions( defaultValue, /* considerFiles = */ false ); + merge( res, parseOptions( overwrite, /* considerFiles = */ false ) ); + return res.dump(); + } +} // namespace json } // namespace openPMD diff --git a/src/backend/PatchRecordComponent.cpp b/src/backend/PatchRecordComponent.cpp index ed705f1cc5..9331e65b90 100644 --- a/src/backend/PatchRecordComponent.cpp +++ b/src/backend/PatchRecordComponent.cpp @@ -102,9 +102,6 @@ PatchRecordComponent::flush(std::string const& name) dCreate.name = name; dCreate.extent = getExtent(); dCreate.dtype = getDatatype(); - dCreate.chunkSize = getExtent(); - dCreate.compression = rc.m_dataset.compression; - dCreate.transform = rc.m_dataset.transform; dCreate.options = rc.m_dataset.options; IOHandler()->enqueue(IOTask(this, dCreate)); } diff --git a/src/binding/python/Dataset.cpp b/src/binding/python/Dataset.cpp index 3979244e45..16d2ef4cf3 100644 --- a/src/binding/python/Dataset.cpp +++ b/src/binding/python/Dataset.cpp @@ -61,12 +61,6 @@ void init_Dataset(py::module &m) { .def_readonly("extent", &Dataset::extent) .def("extend", &Dataset::extend) - .def_readonly("chunk_size", &Dataset::chunkSize) - .def("set_chunk_size", &Dataset::setChunkSize) - .def_readonly("compression", &Dataset::compression) - .def("set_compression", &Dataset::setCompression) - .def_readonly("transform", &Dataset::transform) - .def("set_custom_transform", &Dataset::setCustomTransform) .def_readonly("rank", &Dataset::rank) .def_property_readonly("dtype", [](const Dataset &d) { return dtype_to_numpy( d.dtype ); diff --git a/src/binding/python/Error.cpp b/src/binding/python/Error.cpp index 6eb5c18aa3..056faab956 100644 --- a/src/binding/python/Error.cpp +++ b/src/binding/python/Error.cpp @@ -12,6 +12,8 @@ void init_Error( py::module & m ) m, "ErrorOperationUnsupportedInBackend", baseError ); py::register_exception< error::WrongAPIUsage >( m, "ErrorWrongAPIUsage", baseError ); + py::register_exception< error::BackendConfigSchema >( + m, "ErrorBackendConfigSchema", baseError ); #ifndef NDEBUG m.def( "test_throw", []( std::string description ) { diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 63aa300800..93205a8acc 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -5,17 +5,22 @@ #endif #include "openPMD/openPMD.hpp" +#include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/auxiliary/JSON.hpp" + #include #include -#include -#include #include #include #include #include -#include #include +#include +// cstdlib does not have setenv +#include // NOLINT(modernize-deprecated-headers) +#include +#include using namespace openPMD; @@ -866,6 +871,118 @@ TEST_CASE( "no_file_ending", "[core]" ) Catch::Equals("Unknown file format! Did you specify a file ending?")); REQUIRE_THROWS_WITH(Series("./new_openpmd_output_%05T", Access::CREATE), Catch::Equals("Unknown file format! Did you specify a file ending?")); + { + Series( + "../samples/no_extension_specified", + Access::CREATE, + R"({"backend": "json"})" ); + } + REQUIRE( + auxiliary::file_exists( "../samples/no_extension_specified.json" ) ); +} + +TEST_CASE( "backend_via_json", "[core]" ) +{ + std::string encodingVariableBased = + R"({"backend": "json", "iteration_encoding": "variable_based"})"; + { + Series series( + "../samples/optionsViaJson", + Access::CREATE, + encodingVariableBased ); + REQUIRE( series.backend() == "JSON" ); + REQUIRE( + series.iterationEncoding() == IterationEncoding::variableBased ); + } +#if openPMD_HAVE_ADIOS2 + { + /* + * JSON backend should be chosen even if ending .bp is given + * {"backend": "json"} overwrites automatic detection + */ + Series series( + "../samples/optionsViaJson.bp", + Access::CREATE, + encodingVariableBased ); + REQUIRE( series.backend() == "JSON" ); + REQUIRE( + series.iterationEncoding() == IterationEncoding::variableBased ); + } + + { + /* + * BP4 engine should be selected even if ending .sst is given + */ + Series series( + "../samples/optionsViaJsonOverwritesAutomaticDetection.sst", + Access::CREATE, + R"({"adios2": {"engine": {"type": "bp4"}}})" ); + } + REQUIRE( auxiliary::directory_exists( + "../samples/optionsViaJsonOverwritesAutomaticDetection.bp" ) ); + +#if openPMD_HAVE_ADIOS1 + setenv( "OPENPMD_BP_BACKEND", "ADIOS1", 1 ); + { + /* + * ADIOS2 backend should be selected even if OPENPMD_BP_BACKEND is set + * as ADIOS1 + * JSON config overwrites environment variables + */ + Series series( + "../samples/optionsPreferJsonOverEnvVar.bp", + Access::CREATE, + R"({"backend": "ADIOS2"})" ); + REQUIRE( series.backend() == "ADIOS2" ); + } + // unset again + unsetenv( "OPENPMD_BP_BACKEND" ); + REQUIRE( auxiliary::directory_exists( + "../samples/optionsPreferJsonOverEnvVar.bp" ) ); +#endif +#endif + std::string encodingFileBased = + R"({"backend": "json", "iteration_encoding": "file_based"})"; + { + /* + * File-based iteration encoding can only be chosen if an expansion + * pattern is detected in the filename. + */ + REQUIRE_THROWS_AS( + [ & ]() { + Series series( + "../samples/optionsViaJson", + Access::CREATE, + encodingFileBased ); + }(), + error::WrongAPIUsage ); + } + { + /* + * ... but specifying both the pattern and the option in JSON should work. + */ + Series series( + "../samples/optionsViaJson%06T", + Access::CREATE, + encodingFileBased ); + series.iterations[1456]; + } + std::string encodingGroupBased = + R"({"backend": "json", "iteration_encoding": "group_based"})"; + { + /* + * ... and if a pattern is detected, but the JSON config says to use + * an iteration encoding that is not file-based, the pattern should + * be ignored. + */ + Series series( + "../samples/optionsViaJsonPseudoFilebased%T.json", + Access::CREATE, + encodingGroupBased ); + REQUIRE( series.iterationEncoding() == IterationEncoding::groupBased ); + } + REQUIRE( auxiliary::file_exists( + "../samples/optionsViaJsonPseudoFilebased%T.json" ) ); } TEST_CASE( "custom_geometries", "[core]" ) diff --git a/test/JSONTest.cpp b/test/JSONTest.cpp new file mode 100644 index 0000000000..592f390733 --- /dev/null +++ b/test/JSONTest.cpp @@ -0,0 +1,198 @@ +#include "openPMD/auxiliary/JSON.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" + +#include + +using namespace openPMD; + +TEST_CASE( "json_parsing", "[auxiliary]" ) +{ + std::string wrongValue = R"END( +{ + "ADIOS2": { + "duplicate key": 1243, + "DUPLICATE KEY": 234 + } +})END"; + REQUIRE_THROWS_WITH( + json::parseOptions( wrongValue, false ), + error::BackendConfigSchema( + { "adios2", "duplicate key" }, "JSON config: duplicate keys." ) + .what() ); + std::string same1 = R"( +{ + "ADIOS2": { + "type": "nullcore", + "engine": { + "type": "bp4", + "usesteps": true + } + } +})"; + std::string same2 = R"( +{ + "adios2": { + "type": "nullcore", + "ENGINE": { + "type": "bp4", + "usesteps": true + } + } +})"; + std::string different = R"( +{ + "adios2": { + "type": "NULLCORE", + "ENGINE": { + "type": "bp4", + "usesteps": true + } + } +})"; + REQUIRE( + json::parseOptions( same1, false ).dump() == + json::parseOptions( same2, false ).dump() ); + // Only keys should be transformed to lower case, values must stay the same + REQUIRE( + json::parseOptions( same1, false ).dump() != + json::parseOptions( different, false ).dump() ); + + // Keys forwarded to ADIOS2 should remain untouched + std::string upper = R"END( +{ + "ADIOS2": { + "ENGINE": { + "TYPE": "BP3", + "UNUSED": "PARAMETER", + "PARAMETERS": { + "BUFFERGROWTHFACTOR": "2.0", + "PROFILE": "ON" + } + }, + "UNUSED": "AS WELL", + "DATASET": { + "OPERATORS": [ + { + "TYPE": "BLOSC", + "PARAMETERS": { + "CLEVEL": "1", + "DOSHUFFLE": "BLOSC_BITSHUFFLE" + } + } + ] + } + } +} +)END"; + std::string lower = R"END( +{ + "adios2": { + "engine": { + "type": "BP3", + "unused": "PARAMETER", + "parameters": { + "BUFFERGROWTHFACTOR": "2.0", + "PROFILE": "ON" + } + }, + "unused": "AS WELL", + "dataset": { + "operators": [ + { + "type": "BLOSC", + "parameters": { + "CLEVEL": "1", + "DOSHUFFLE": "BLOSC_BITSHUFFLE" + } + } + ] + } + } +} +)END"; + nlohmann::json jsonUpper = nlohmann::json::parse( upper ); + nlohmann::json jsonLower = nlohmann::json::parse( lower ); + REQUIRE( jsonUpper.dump() != jsonLower.dump() ); + json::lowerCase( jsonUpper ); + REQUIRE( jsonUpper.dump() == jsonLower.dump() ); +} + +TEST_CASE( "json_merging", "auxiliary" ) +{ + std::string defaultVal = R"END( +{ + "mergeRecursively": { + "changed": 43, + "unchanged": true, + "delete_me": "adsf" + }, + "dontmergearrays": [ + 1, + 2, + 3, + 4, + 5 + ], + "delete_me": [345,2345,36] +} +)END"; + + std::string overwrite = R"END( +{ + "mergeRecursively": { + "changed": "new value", + "newValue": "44", + "delete_me": null + }, + "dontmergearrays": [ + 5, + 6, + 7 + ], + "delete_me": null +} +)END"; + + std::string expect = R"END( +{ + "mergeRecursively": { + "changed": "new value", + "unchanged": true, + "newValue": "44" + }, + "dontmergearrays": [ + 5, + 6, + 7 + ] +})END"; + REQUIRE( + json::merge( defaultVal, overwrite ) == + json::parseOptions( expect, false ).dump() ); +} + +TEST_CASE( "optional", "[auxiliary]" ) { + using namespace auxiliary; + + Option opt; + + REQUIRE_THROWS_AS(opt.get(), variantSrc::bad_variant_access); + REQUIRE_THROWS_AS(opt.get() = 42, variantSrc::bad_variant_access); + REQUIRE(!opt); + REQUIRE(!opt.has_value()); + + opt = 43; + REQUIRE(opt); + REQUIRE(opt.has_value()); + REQUIRE(opt.get() == 43); + + Option opt2{ opt }; + REQUIRE(opt2); + REQUIRE(opt2.has_value()); + REQUIRE(opt2.get() == 43); + + Option opt3 = makeOption( 3 ); + REQUIRE(opt3); + REQUIRE(opt3.has_value()); + REQUIRE(opt3.get() == 3); +} diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index da93297358..62968fa7e4 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -374,9 +374,11 @@ TEST_CASE( "available_chunks_test", "[parallel][adios]" ) { available_chunks_test( "bp" ); } +#endif +#if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI void -extendDataset( std::string const & ext ) +extendDataset( std::string const & ext, std::string const & jsonConfig ) { std::string filename = "../samples/parallelExtendDataset." + ext; int r_mpi_rank{ -1 }, r_mpi_size{ -1 }; @@ -389,7 +391,7 @@ extendDataset( std::string const & ext ) std::iota( data1.begin(), data1.end(), 0 ); std::iota( data2.begin(), data2.end(), 25 ); { - Series write( filename, Access::CREATE, MPI_COMM_WORLD ); + Series write( filename, Access::CREATE, MPI_COMM_WORLD, jsonConfig ); if( ext == "bp" && write.backend() != "ADIOS2" ) { // dataset resizing unsupported in ADIOS1 @@ -413,7 +415,7 @@ extendDataset( std::string const & ext ) MPI_Barrier( MPI_COMM_WORLD ); { - Series read( filename, Access::READ_ONLY ); + Series read( filename, Access::READ_ONLY, jsonConfig ); auto E_x = read.iterations[ 0 ].meshes[ "E" ][ "x" ]; REQUIRE( E_x.getExtent() == Extent{ mpi_size, 50 } ); auto chunk = E_x.loadChunk< int >( { 0, 0 }, { mpi_size, 50 } ); @@ -430,7 +432,7 @@ extendDataset( std::string const & ext ) TEST_CASE( "extend_dataset", "[parallel]" ) { - extendDataset( "bp" ); + extendDataset( "bp", R"({"backend": "adios2"})" ); } #endif @@ -804,11 +806,15 @@ hipace_like_write( std::string file_ending ) bool const isHDF5 = file_ending == "h5"; std::string options = "{}"; if( isHDF5 ) + /* + * some keys and values capitalized randomly to check whether + * capitalization-insensitivity is working. + */ options = R"( { - "hdf5": { + "HDF5": { "dataset": { - "chunks": "none" + "chunks": "NONE" } } })"; diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index fff67243e2..0f9a70f846 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -32,6 +32,42 @@ using namespace openPMD; +struct BackendSelection +{ + std::string backendName; + std::string extension; + + inline std::string jsonBaseConfig() const + { + return R"({"backend": ")" + backendName + "\"}"; + } +}; + +std::vector< BackendSelection > testedBackends() +{ + auto variants = getVariants(); + std::map< std::string, std::string > extensions{ + { "json", "json" }, + { "adios1", "bp1" }, + { "adios2", "bp" }, + { "hdf5", "h5" } }; + std::vector< BackendSelection > res; + for( auto const & pair : variants ) + { + if( pair.second ) + { + auto lookup = extensions.find( pair.first ); + if( lookup != extensions.end() ) + { + std::string extension = lookup->second; + res.push_back( + { std::move( pair.first ), std::move( extension ) } ); + } + } + } + return res; +} + std::vector< std::string > testedFileExtensions() { auto allExtensions = getFileExtensions(); @@ -2057,14 +2093,16 @@ void optional_paths_110_test(const std::string & backend) void git_early_chunk_query( std::string const filename, std::string const species, - int const step + int const step, + std::string const & jsonConfig = "{}" ) { try { Series s = Series( filename, - Access::READ_ONLY + Access::READ_ONLY, + jsonConfig ); auto electrons = s.iterations[step].particles[species]; @@ -3304,7 +3342,11 @@ TEST_CASE( "no_serial_adios1", "[serial][adios]") #if openPMD_HAVE_ADIOS2 TEST_CASE( "git_adios2_early_chunk_query", "[serial][adios2]" ) { - git_early_chunk_query("../samples/git-sample/3d-bp4/example-3d-bp4_%T.bp", "e", 600); + git_early_chunk_query( + "../samples/git-sample/3d-bp4/example-3d-bp4_%T.bp", + "e", + 600, + R"({"backend": "adios2"})" ); } TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) @@ -3316,12 +3358,16 @@ TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) } std::string writeConfigBP3 = R"END( { + "unused": "global parameter", + "hdf5": { + "unused": "hdf5 parameter please do not warn" + }, "adios2": { "engine": { "type": "bp3", "unused": "parameter", "parameters": { - "BufferGrowthFactor": "2.0", + "BufferGrowthFactor": 2, "Profile": "On" } }, @@ -3347,7 +3393,7 @@ TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) "type": "bp4", "unused": "parameter", "parameters": { - "BufferGrowthFactor": "2.0", + "BufferGrowthFactor": 2.0, "Profile": "On" } }, @@ -3357,7 +3403,7 @@ TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) { "type": "blosc", "parameters": { - "clevel": "1", + "clevel": 1, "doshuffle": "BLOSC_BITSHUFFLE" } } @@ -3392,8 +3438,25 @@ TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) } } )END"; + /* + * Notes on the upcoming dataset JSON configuration: + * * The resizable key is needed by some backends (HDF5) so the backend can + * create a dataset that can later be resized. + * * The asdf key should lead to a warning about unused parameters. + * * Inside the hdf5 configuration, there are unused keys ("this"). + * However, since this configuration is used by the ADIOS2 backend, there + * will be no warning for it. + * + * In the end, this config should lead to a warning similar to: + * > Warning: parts of the JSON configuration for ADIOS2 dataset + * > '/data/0/meshes/E/y' remain unused: + * > {"adios2":{"dataset":{"unused":"too"},"unused":"dataset parameter"}, + * > "asdf":"asdf"} + */ std::string datasetConfig = R"END( { + "resizable": true, + "asdf": "asdf", "adios2": { "unused": "dataset parameter", "dataset": { @@ -3402,12 +3465,15 @@ TEST_CASE( "serial_adios2_json_config", "[serial][adios2]" ) { "type": "blosc", "parameters": { - "clevel": "3", - "doshuffle": "BLOSC_BITSHUFFLE" + "clevel": 3, + "doshuffle": "BLOSC_BITSHUFFLE" } } ] } + }, + "hdf5": { + "this": "should not warn" } } )END"; @@ -3547,7 +3613,7 @@ TEST_CASE( "bp4_steps", "[serial][adios2]" ) { std::string useSteps = R"( { - "adios2": { + "ADIOS2": { "engine": { "type": "bp4", "usesteps": true @@ -3559,7 +3625,7 @@ TEST_CASE( "bp4_steps", "[serial][adios2]" ) { "adios2": { "type": "nullcore", - "engine": { + "ENGINE": { "type": "bp4", "usesteps": true } @@ -3571,7 +3637,7 @@ TEST_CASE( "bp4_steps", "[serial][adios2]" ) "adios2": { "engine": { "type": "bp4", - "usesteps": false + "UseSteps": false } } } @@ -3673,8 +3739,13 @@ variableBasedSingleIteration( std::string const & file ) { constexpr Extent::value_type extent = 1000; { - Series writeSeries( file, Access::CREATE ); - writeSeries.setIterationEncoding( IterationEncoding::variableBased ); + Series writeSeries( + file, + Access::CREATE, + R"({"iteration_encoding": "variable_based"})" ); + REQUIRE( + writeSeries.iterationEncoding() == + IterationEncoding::variableBased ); auto iterations = writeSeries.writeIterations(); auto iteration = writeSeries.iterations[ 0 ]; auto E_x = iteration.meshes[ "E" ][ "x" ]; @@ -3764,7 +3835,7 @@ TEST_CASE( "git_adios2_sample_test", "[serial][adios2]" ) << samplePath << "' not accessible \n"; return; } - Series o( samplePath, Access::READ_ONLY ); + Series o( samplePath, Access::READ_ONLY, R"({"backend": "adios2"})" ); REQUIRE( o.openPMD() == "1.1.0" ); REQUIRE( o.openPMDextension() == 0 ); REQUIRE( o.basePath() == "/data/%T/" ); @@ -4018,16 +4089,13 @@ TEST_CASE( "git_adios2_sample_test", "[serial][adios2]" ) void variableBasedSeries( std::string const & file ) { + std::string selectADIOS2 = R"({"backend": "adios2"})"; constexpr Extent::value_type extent = 1000; { - Series writeSeries( file, Access::CREATE ); + Series writeSeries( file, Access::CREATE, selectADIOS2 ); writeSeries.setIterationEncoding( IterationEncoding::variableBased ); REQUIRE( writeSeries.iterationEncoding() == IterationEncoding::variableBased ); - if( writeSeries.backend() == "ADIOS1" ) - { - return; - } auto iterations = writeSeries.writeIterations(); for( size_t i = 0; i < 10; ++i ) { @@ -4065,9 +4133,10 @@ void variableBasedSeries( std::string const & file ) REQUIRE( auxiliary::directory_exists( file ) ); - auto testRead = [ &file, &extent ]( std::string const & jsonConfig ) - { - Series readSeries( file, Access::READ_ONLY, jsonConfig ); + auto testRead = [ &file, &extent, &selectADIOS2 ]( + std::string const & jsonConfig ) { + Series readSeries( + file, Access::READ_ONLY, json::merge( selectADIOS2, jsonConfig ) ); size_t last_iteration_index = 0; for( auto iteration : readSeries.readIterations() ) @@ -4123,10 +4192,12 @@ void variableBasedSeries( std::string const & file ) testRead( "{\"defer_iteration_parsing\": false}" ); } +#if openPMD_HAVE_ADIOS2 TEST_CASE( "variableBasedSeries", "[serial][adios2]" ) { variableBasedSeries( "../samples/variableBasedSeries.bp" ); } +#endif void variableBasedParticleData() { @@ -4210,19 +4281,14 @@ TEST_CASE( "variableBasedParticleData", "[serial][adios2]" ) #endif // @todo Upon switching to ADIOS2 2.7.0, test this the other way around also -void -iterate_nonstreaming_series( - std::string const & file, bool variableBasedLayout ) +void iterate_nonstreaming_series( + std::string const & file, bool variableBasedLayout, std::string jsonConfig ) { constexpr size_t extent = 100; { - Series writeSeries( file, Access::CREATE ); + Series writeSeries( file, Access::CREATE, jsonConfig ); if( variableBasedLayout ) { - if( writeSeries.backend() != "ADIOS2" ) - { - return; - } writeSeries.setIterationEncoding( IterationEncoding::variableBased ); } @@ -4296,7 +4362,10 @@ iterate_nonstreaming_series( } } - Series readSeries( file, Access::READ_ONLY, "{\"defer_iteration_parsing\": true}" ); + Series readSeries( + file, + Access::READ_ONLY, + json::merge( jsonConfig, R"({"defer_iteration_parsing": true})" ) ); size_t last_iteration_index = 0; // conventionally written Series must be readable with streaming-aware API! @@ -4332,19 +4401,28 @@ iterate_nonstreaming_series( TEST_CASE( "iterate_nonstreaming_series", "[serial][adios2]" ) { - for( auto const & t : testedFileExtensions() ) + for( auto const & backend : testedBackends() ) { iterate_nonstreaming_series( - "../samples/iterate_nonstreaming_series_filebased_%T." + t, false ); - iterate_nonstreaming_series( - "../samples/iterate_nonstreaming_series_groupbased." + t, false ); + "../samples/iterate_nonstreaming_series_filebased_%T." + + backend.extension, + false, + backend.jsonBaseConfig() ); iterate_nonstreaming_series( - "../samples/iterate_nonstreaming_series_variablebased." + t, true ); + "../samples/iterate_nonstreaming_series_groupbased." + + backend.extension, + false, + backend.jsonBaseConfig() ); } +#if openPMD_HAVE_ADIOS2 + iterate_nonstreaming_series( + "../samples/iterate_nonstreaming_series_variablebased.bp", + true, + R"({"backend": "adios2"})" ); +#endif } -void -extendDataset( std::string const & ext ) +void extendDataset( std::string const & ext, std::string const & jsonConfig ) { std::string filename = "../samples/extendDataset." + ext; std::vector< int > data1( 25 ); @@ -4352,16 +4430,14 @@ extendDataset( std::string const & ext ) std::iota( data1.begin(), data1.end(), 0 ); std::iota( data2.begin(), data2.end(), 25 ); { - Series write( filename, Access::CREATE ); - if( ext == "bp" && write.backend() != "ADIOS2" ) - { - // dataset resizing unsupported in ADIOS1 - return; - } + Series write( filename, Access::CREATE, jsonConfig ); // only one iteration written anyway write.setIterationEncoding( IterationEncoding::variableBased ); - Dataset ds1{ Datatype::INT, { 5, 5 }, "{ \"resizable\": true }" }; + Dataset ds1{ + Datatype::INT, + { 5, 5 }, + R"({ "resizable": true, "resizeble": "typo" })" }; Dataset ds2{ Datatype::INT, { 10, 5 } }; // array record component -> array record component @@ -4443,7 +4519,7 @@ extendDataset( std::string const & ext ) } { - Series read( filename, Access::READ_ONLY ); + Series read( filename, Access::READ_ONLY, jsonConfig ); auto E_x = read.iterations[ 0 ].meshes[ "E" ][ "x" ]; REQUIRE( E_x.getExtent() == Extent{ 10, 5 } ); auto chunk = E_x.loadChunk< int >( { 0, 0 }, { 10, 5 } ); @@ -4473,21 +4549,20 @@ extendDataset( std::string const & ext ) TEST_CASE( "extend_dataset", "[serial]" ) { - extendDataset( "json" ); + extendDataset( "json", R"({"backend": "json"})" ); #if openPMD_HAVE_ADIOS2 - extendDataset( "bp" ); + extendDataset( "bp", R"({"backend": "adios2"})" ); #endif #if openPMD_HAVE_HDF5 // extensible datasets require chunking // skip this test for if chunking is disabled if( auxiliary::getEnvString( "OPENPMD_HDF5_CHUNKS", "auto" ) != "none" ) { - extendDataset("h5"); + extendDataset( "h5", R"({"backend": "hdf5"})" ); } #endif } - void deferred_parsing( std::string const & extension ) { if( auxiliary::directory_exists( "../samples/lazy_parsing" ) )