diff --git a/bindings/C/adios2/c/adios2_c_engine.cpp b/bindings/C/adios2/c/adios2_c_engine.cpp index 1836893657..9d6cecf32f 100644 --- a/bindings/C/adios2/c/adios2_c_engine.cpp +++ b/bindings/C/adios2/c/adios2_c_engine.cpp @@ -34,6 +34,9 @@ adios2::Mode adios2_ToMode(const adios2_mode mode, const std::string &hint) case adios2_mode_readRandomAccess: modeCpp = adios2::Mode::ReadRandomAccess; break; + case adios2_mode_readFlattenSteps: + modeCpp = adios2::Mode::ReadFlattenSteps; + break; case adios2_mode_deferred: modeCpp = adios2::Mode::Deferred; break; @@ -63,6 +66,9 @@ adios2_mode adios2_fromMode(const adios2::Mode mode, const std::string &hint) case adios2::Mode::ReadRandomAccess: modeC = adios2_mode_readRandomAccess; break; + case adios2::Mode::ReadFlattenSteps: + modeC = adios2_mode_readFlattenSteps; + break; case adios2::Mode::Deferred: modeC = adios2_mode_deferred; break; diff --git a/bindings/C/adios2/c/adios2_c_io.h b/bindings/C/adios2/c/adios2_c_io.h index 09fa10e256..a622328e38 100644 --- a/bindings/C/adios2/c/adios2_c_io.h +++ b/bindings/C/adios2/c/adios2_c_io.h @@ -329,8 +329,8 @@ adios2_error adios2_remove_all_attributes(adios2_io *io); * MPI Collective function as it calls MPI_Comm_dup * @param io engine owner * @param name unique engine identifier - * @param mode adios2_mode_write, adios2_mode_read, adios2_mode_append, and - * adios2_mode_readRandomAccess + * @param mode adios2_mode_write, adios2_mode_read, adios2_mode_append, + * adios2_mode_readRandomAccess and adios2_mode_readFlattenSteps * @return success: handler, failure: NULL */ adios2_engine *adios2_open(adios2_io *io, const char *name, const adios2_mode mode); @@ -341,8 +341,8 @@ adios2_engine *adios2_open(adios2_io *io, const char *name, const adios2_mode mo * MPI Collective function as it calls MPI_Comm_dup * @param io engine owner * @param name unique engine identifier - * @param mode adios2_mode_write, adios2_mode_read, adios2_mode_append, and - * adios2_mode_readRandomAccess + * @param mode adios2_mode_write, adios2_mode_read, adios2_mode_append, + * adios2_mode_readRandomAccess and adios2_mode_readFlattenSteps * @param comm communicator other than adios' handler comm. MPI only. * @return success: handler, failure: NULL */ diff --git a/bindings/C/adios2/c/adios2_c_io.tcc b/bindings/C/adios2/c/adios2_c_io.tcc index 29607b87d8..9c9000cd44 100644 --- a/bindings/C/adios2/c/adios2_c_io.tcc +++ b/bindings/C/adios2/c/adios2_c_io.tcc @@ -36,6 +36,10 @@ adios2::Mode adios2_ToOpenMode(const adios2_mode modeC) mode = adios2::Mode::ReadRandomAccess; break; + case adios2_mode_readFlattenSteps: + mode = adios2::Mode::ReadFlattenSteps; + break; + default: break; } diff --git a/bindings/C/adios2/c/adios2_c_types.h b/bindings/C/adios2/c/adios2_c_types.h index fb102df4cb..b28bc7f21e 100644 --- a/bindings/C/adios2/c/adios2_c_types.h +++ b/bindings/C/adios2/c/adios2_c_types.h @@ -103,6 +103,7 @@ typedef enum adios2_mode_read = 2, adios2_mode_append = 3, adios2_mode_readRandomAccess = 6, + adios2_mode_readFlattenSteps = 7, adios2_mode_deferred = 4, adios2_mode_sync = 5 diff --git a/bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.cpp b/bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.cpp index 03b881e7fa..12d083beb2 100644 --- a/bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.cpp +++ b/bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.cpp @@ -89,6 +89,9 @@ adios2::Mode fstream::ToMode(const openmode mode) const noexcept case (openmode::in_random_access): modeCpp = adios2::Mode::ReadRandomAccess; break; + case (openmode::in_flatten_steps): + modeCpp = adios2::Mode::ReadFlattenSteps; + break; case (openmode::app): modeCpp = adios2::Mode::Append; break; diff --git a/bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.h b/bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.h index 546bf54ccb..d96a7a749b 100644 --- a/bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.h +++ b/bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.h @@ -48,6 +48,7 @@ class fstream out, //!< write in, //!< read in_random_access, //!< read_random_access + in_flatten_steps, //!< flatten all input steps to 1 app //!< append, not yet supported }; diff --git a/bindings/Fortran/modules/adios2_parameters_mod.f90 b/bindings/Fortran/modules/adios2_parameters_mod.f90 index bf0dd18c9b..f73561702e 100644 --- a/bindings/Fortran/modules/adios2_parameters_mod.f90 +++ b/bindings/Fortran/modules/adios2_parameters_mod.f90 @@ -56,6 +56,7 @@ module adios2_parameters_mod integer, parameter :: adios2_mode_read = 2 integer, parameter :: adios2_mode_append = 3 integer, parameter :: adios2_mode_readRandomAccess = 6 + integer, parameter :: adios2_mode_readFlattenSteps = 7 integer, parameter :: adios2_mode_deferred = 4 integer, parameter :: adios2_mode_sync = 5 diff --git a/bindings/Python/py11glue.cpp b/bindings/Python/py11glue.cpp index 2c40280f99..b1989f7ece 100644 --- a/bindings/Python/py11glue.cpp +++ b/bindings/Python/py11glue.cpp @@ -98,6 +98,7 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) .value("Write", adios2::Mode::Write) .value("Read", adios2::Mode::Read) .value("ReadRandomAccess", adios2::Mode::ReadRandomAccess) + .value("ReadFlattenSteps", adios2::Mode::ReadFlattenSteps) .value("Append", adios2::Mode::Append) .value("Deferred", adios2::Mode::Deferred) .value("Sync", adios2::Mode::Sync) diff --git a/docs/user_guide/source/components/anatomy.rst b/docs/user_guide/source/components/anatomy.rst index 0832b6dc3c..2aa8c7f89a 100644 --- a/docs/user_guide/source/components/anatomy.rst +++ b/docs/user_guide/source/components/anatomy.rst @@ -114,7 +114,10 @@ named `adios2::Mode::ReadRandomAccess`. `adios2::Mode::Read` mode allows data ac current timestep. `ReadRandomAccess` can only be used with file engines and involves loading all the file metadata at once. So it can be more memory intensive than `adios2::Mode::Read` mode, but allows reading data from any timestep using `SetStepSelection()`. If you use `adios2::Mode::ReadRandomAccess` mode, be sure to allocate enough memory to hold -multiple steps of the variable content. +multiple steps of the variable content. Note that ADIOS streaming +engines (like SST, DataMan, etc.) do not support `ReadRandomAccess` +mode. Also newer file Engines like BP5 to not allow +`BeginStep/EndStep` calls in `ReadRandomAccess` mode. .. code:: C++ @@ -134,3 +137,45 @@ multiple steps of the variable content. | |--> IO goes out of scope | |--> ADIOS goes out of scope or adios2_finalize() + +Previously we explored how to read using the input mode `adios2::Mode::Read`. Nonetheless, ADIOS has another input mode +named `adios2::Mode::ReadRandomAccess`. `adios2::Mode::Read` mode allows data access only timestep by timestep using +`BeginStep/EndStep`, but generally it is more memory efficient as ADIOS is only required to load metadata for the +current timestep. `ReadRandomAccess` can only be used with file engines and involves loading all the file metadata at +once. So it can be more memory intensive than `adios2::Mode::Read` mode, but allows reading data from any timestep using +`SetStepSelection()`. If you use `adios2::Mode::ReadRandomAccess` mode, be sure to allocate enough memory to hold +multiple steps of the variable content. Note that ADIOS streaming +engines (like SST, DataMan, etc.) do not support `ReadRandomAccess` +mode. Also newer file Engines like BP5 to not allow +`BeginStep/EndStep` calls in `ReadRandomAccess` mode. + +.. code:: C++ + + ADIOS adios("config.xml", MPI_COMM_WORLD); + | + | IO io = adios.DeclareIO(...); + | | + | | Engine e = io.Open("InputFileName.bp", adios2::Mode::ReadRandomAccess); + | | | + | | | Variable var = io.InquireVariable(...) + | | | | var.SetStepSelection() + | | | | e.Get(var, datapointer); + | | | | + | | | + | | e.Close(); + | | + | |--> IO goes out of scope + | + |--> ADIOS goes out of scope or adios2_finalize() + + +In addition to the two read modes discussed above, ADIOS has another +input mode named `adios2::Mode::ReadFlattenSteps`. This is a highly +specialized mode built that is unlikely to be of general utility, but +we describe it for completeness. In `ReadFlattenSteps` mode, ADIOS +loads all the metadata in the file upon Open (just like +`ReadRandomAccess` mode, but everything that was written appears that +it was output on the same step, regardless of how many steps actually +appear in the file. This affects the operation of many reader-side +ADIOS functions, including Steps(), BlocksInfo(), Get(), etc. + diff --git a/python/adios2/stream.py b/python/adios2/stream.py index 0285507cc9..39123d729d 100644 --- a/python/adios2/stream.py +++ b/python/adios2/stream.py @@ -37,6 +37,9 @@ def string_to_mode(mode: str) -> [bindings.Mode, bool]: elif mode == "rra": bmode = bindings.Mode.ReadRandomAccess read_mode = True + elif mode == "rfs": + bmode = bindings.Mode.ReadFlattenSteps + read_mode = True elif mode == "w": bmode = bindings.Mode.Write elif mode == "a": diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h index fad51e8fd9..111ebe4a1d 100644 --- a/source/adios2/common/ADIOSTypes.h +++ b/source/adios2/common/ADIOSTypes.h @@ -80,6 +80,7 @@ enum class Mode Read, Append, ReadRandomAccess, // reader random access mode + ReadFlattenSteps, // reader flatten steps to one // launch execution modes Sync, Deferred diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp index 72ab54d4ac..56e6f02c08 100644 --- a/source/adios2/core/Engine.cpp +++ b/source/adios2/core/Engine.cpp @@ -135,7 +135,8 @@ void Engine::Put(VariableStruct &variable, const void *data, const Mode launch) void Engine::Get(VariableStruct &variable, void *data, const Mode launch) { - CommonChecks(variable, data, {Mode::Read, Mode::ReadRandomAccess}, "in call to Get"); + CommonChecks(variable, data, {Mode::Read, Mode::ReadRandomAccess, Mode::ReadFlattenSteps}, + "in call to Get"); switch (launch) { diff --git a/source/adios2/core/Engine.tcc b/source/adios2/core/Engine.tcc index b06050b86c..656b2fc30a 100644 --- a/source/adios2/core/Engine.tcc +++ b/source/adios2/core/Engine.tcc @@ -87,7 +87,8 @@ void Engine::Put(const std::string &variableName, const T &datum, const Mode /*l template void Engine::Get(Variable &variable, T *data, const Mode launch) { - CommonChecks(variable, data, {Mode::Read, Mode::ReadRandomAccess}, "in call to Get"); + CommonChecks(variable, data, {Mode::Read, Mode::ReadRandomAccess, Mode::ReadFlattenSteps}, + "in call to Get"); switch (launch) { diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index 83bf0f3781..eaf4394351 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -150,6 +150,14 @@ const std::unordered_map ReadRandomAccess_Supported = { {"campaign", true}, }; +const std::unordered_map ReadFlattenSteps_Supported = { + {"bp3", false}, {"bp4", false}, {"bp5", true}, {"dataman", false}, + {"ssc", false}, {"mhs", false}, {"sst", false}, {"daos", false}, + {"effis", false}, {"dataspaces", false}, {"hdf5", false}, {"skeleton", false}, + {"inline", false}, {"null", true}, {"nullcore", true}, {"plugin", false}, + {"campaign", true}, +}; + // Synchronize access to the factory in case one thread is // looking up while another registers additional entries. std::mutex FactoryMutex; @@ -560,7 +568,8 @@ Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm) { engineTypeLC = "campaign"; } - else if ((mode_to_use == Mode::Read) || (mode_to_use == Mode::ReadRandomAccess)) + else if ((mode_to_use == Mode::Read) || (mode_to_use == Mode::ReadRandomAccess) || + (mode_to_use == Mode::ReadFlattenSteps)) { if (adios2sys::SystemTools::FileIsDirectory(name)) { @@ -668,10 +677,26 @@ Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm) } } + if (mode_to_use == Mode::ReadFlattenSteps) + { + // older engines don't know about ReadFlattenSteps Mode, throw an exception + auto it = ReadFlattenSteps_Supported.find(engineTypeLC); + if (it != ReadFlattenSteps_Supported.end()) + { + if (!it->second) + { + helper::Throw("Core", "IO", "Open", + "Engine " + engineTypeLC + + " doesn't support ReadFlattenSteps mode"); + } + } + } + auto f = FactoryLookup(engineTypeLC); if (f != Factory.end()) { - if ((mode_to_use == Mode::Read) || (mode_to_use == Mode::ReadRandomAccess)) + if ((mode_to_use == Mode::Read) || (mode_to_use == Mode::ReadRandomAccess) || + (mode_to_use == Mode::ReadFlattenSteps)) { engine = f->second.MakeReader(*this, name, mode_to_use, std::move(comm)); } diff --git a/source/adios2/core/Stream.cpp b/source/adios2/core/Stream.cpp index 43abacc703..2b923886ae 100644 --- a/source/adios2/core/Stream.cpp +++ b/source/adios2/core/Stream.cpp @@ -23,7 +23,8 @@ Stream::Stream(const std::string &name, const Mode mode, helper::Comm comm, : m_ADIOS(std::make_shared(std::move(comm), hostLanguage)), m_IO(&m_ADIOS->DeclareIO(name)), m_Name(name), m_Mode(mode), m_EngineType(engineType) { - if ((mode == adios2::Mode::Read) || (mode == adios2::Mode::ReadRandomAccess)) + if ((mode == adios2::Mode::Read) || (mode == adios2::Mode::ReadRandomAccess) || + (mode == adios2::Mode::ReadFlattenSteps)) { CheckOpen(); } @@ -41,7 +42,8 @@ Stream::Stream(const std::string &name, const Mode mode, helper::Comm comm, : m_ADIOS(std::make_shared(configFile, std::move(comm), hostLanguage)), m_IO(&m_ADIOS->DeclareIO(ioInConfigFile)), m_Name(name), m_Mode(mode) { - if ((mode == adios2::Mode::Read) || (mode == adios2::Mode::ReadRandomAccess)) + if ((mode == adios2::Mode::Read) || (mode == adios2::Mode::ReadRandomAccess) || + (mode == adios2::Mode::ReadFlattenSteps)) { CheckOpen(); } diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 094821bc16..5f6ad0d030 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -72,7 +72,7 @@ void BP5Reader::InstallMetadataForTimestep(size_t Step) size_t ThisMDSize = helper::ReadValue(m_Metadata.Data(), Position, m_Minifooter.IsLittleEndian); char *ThisMD = m_Metadata.Data() + MDPosition; - if (m_OpenMode == Mode::ReadRandomAccess) + if ((m_OpenMode == Mode::ReadRandomAccess) || (m_OpenMode == Mode::ReadFlattenSteps)) { m_BP5Deserializer->InstallMetaData(ThisMD, ThisMDSize, WriterRank, Step); } @@ -98,7 +98,7 @@ StepStatus BP5Reader::BeginStep(StepMode mode, const float timeoutSeconds) { PERFSTUBS_SCOPED_TIMER("BP5Reader::BeginStep"); - if (m_OpenMode == Mode::ReadRandomAccess) + if (m_OpenMode != Mode::Read) { helper::Throw("Engine", "BP5Reader", "BeginStep", "BeginStep called in random access mode"); @@ -184,7 +184,7 @@ size_t BP5Reader::CurrentStep() const { return m_CurrentStep; } void BP5Reader::EndStep() { - if (m_OpenMode == Mode::ReadRandomAccess) + if (m_OpenMode != Mode::Read) { helper::Throw("Engine", "BP5Reader", "EndStep", "EndStep called in random access mode"); @@ -475,12 +475,14 @@ void BP5Reader::PerformLocalGets() // PRIVATE void BP5Reader::Init() { - if ((m_OpenMode != Mode::Read) && (m_OpenMode != Mode::ReadRandomAccess)) + if ((m_OpenMode != Mode::Read) && (m_OpenMode != Mode::ReadRandomAccess) && + (m_OpenMode != Mode::ReadFlattenSteps)) { - helper::Throw("Engine", "BP5Reader", "Init", - "BPFileReader only supports OpenMode::Read or " - "OpenMode::ReadRandomAccess from" + - m_Name); + helper::Throw( + "Engine", "BP5Reader", "Init", + "BPFileReader only supports OpenMode::Read, " + "OpenMode::ReadRandomAccess, or OpenMode::ReadFlattenSteps from" + + m_Name); } // if IO was involved in reading before this flag may be true now @@ -519,7 +521,7 @@ void BP5Reader::InitParameters() ParseParams(m_IO, m_Parameters); if (m_Parameters.OpenTimeoutSecs < 0.0f) { - if (m_OpenMode == Mode::ReadRandomAccess) + if ((m_OpenMode == Mode::ReadRandomAccess) || (m_OpenMode == Mode::ReadFlattenSteps)) { m_Parameters.OpenTimeoutSecs = 0.0f; } @@ -808,7 +810,8 @@ void BP5Reader::UpdateBuffer(const TimePoint &timeoutInstant, const Seconds &pol if (!m_BP5Deserializer) { m_BP5Deserializer = new format::BP5Deserializer(m_WriterIsRowMajor, m_ReaderIsRowMajor, - (m_OpenMode == Mode::ReadRandomAccess)); + (m_OpenMode != Mode::Read), + (m_OpenMode == Mode::ReadFlattenSteps)); m_BP5Deserializer->m_Engine = this; } } @@ -905,7 +908,7 @@ void BP5Reader::UpdateBuffer(const TimePoint &timeoutInstant, const Seconds &pol m_Comm.Bcast(m_Metadata.Data(), inputSize, 0); - if (m_OpenMode == Mode::ReadRandomAccess) + if ((m_OpenMode == Mode::ReadRandomAccess) || (m_OpenMode == Mode::ReadFlattenSteps)) { for (size_t Step = 0; Step < m_MetadataIndexTable.size(); Step++) { @@ -1250,7 +1253,7 @@ void BP5Reader::DoGetStructDeferred(VariableStruct &variable, void *data) void BP5Reader::DoClose(const int transportIndex) { PERFSTUBS_SCOPED_TIMER("BP5Reader::Close"); - if (m_OpenMode == Mode::ReadRandomAccess) + if ((m_OpenMode == Mode::ReadRandomAccess) || (m_OpenMode == Mode::ReadFlattenSteps)) { PerformGets(); } @@ -1311,7 +1314,13 @@ void BP5Reader::FlushProfiler() } } -size_t BP5Reader::DoSteps() const { return m_StepsCount; } +size_t BP5Reader::DoSteps() const +{ + if (m_OpenMode == Mode::ReadFlattenSteps) + return 1; + else + return m_StepsCount; +} void BP5Reader::NotifyEngineNoVarsQuery() { diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp index d4315a04af..46d272a191 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp @@ -980,7 +980,13 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, size { VarRec->FirstTSSeen = Step; } - if (m_RandomAccessMode && (VarRec->LastTSAdded != Step)) + if (m_FlattenSteps) + { + static_cast(VarRec->Variable)->m_AvailableStepsCount = 1; + VarRec->LastTSAdded = 0; + VarRec->FirstTSSeen = 0; + } + else if (m_RandomAccessMode && (VarRec->LastTSAdded != Step)) { static_cast(VarRec->Variable)->m_AvailableStepsCount++; VarRec->LastTSAdded = Step; @@ -1545,198 +1551,223 @@ BP5Deserializer::GenerateReadRequests(const bool doAllocTempBuffers, size_t *max { std::vector Ret; *maxReadSize = 0; + size_t StepLoopStart, StepLoopEnd; for (size_t ReqIndex = 0; ReqIndex < PendingGetRequests.size(); ReqIndex++) { auto Req = &PendingGetRequests[ReqIndex]; auto VarRec = (struct BP5VarRec *)Req->VarRec; VariableBase *VB = static_cast(VarRec->Variable); + if (m_FlattenSteps) + { + StepLoopStart = 0; + StepLoopEnd = m_ControlArray.size(); + } + else + { + StepLoopStart = Req->Step; + StepLoopEnd = Req->Step + 1; + } + if (Req->RequestType == Local) { - const size_t writerCohortSize = WriterCohortSize(Req->Step); size_t NodeFirstBlock = 0; - for (size_t WriterRank = 0; WriterRank < writerCohortSize; WriterRank++) + for (size_t Step = StepLoopStart; Step < StepLoopEnd; Step++) { - MetaArrayRecOperator *writer_meta_base = (MetaArrayRecOperator *)GetMetadataBase( - (struct BP5VarRec *)Req->VarRec, Req->Step, WriterRank); - if (!writer_meta_base) + const size_t writerCohortSize = WriterCohortSize(Step); + for (size_t WriterRank = 0; WriterRank < writerCohortSize; WriterRank++) { - continue; // Not writen on this step - } - size_t NodeLastBlock = NodeFirstBlock + writer_meta_base->BlockCount - 1; - if ((NodeFirstBlock <= Req->BlockID) && (NodeLastBlock >= Req->BlockID)) - { - // block is here - size_t NeededBlock = Req->BlockID - NodeFirstBlock; - size_t StartDim = NeededBlock * VarRec->DimCount; - ReadRequest RR; - RR.Timestep = Req->Step; - RR.WriterRank = WriterRank; - RR.StartOffset = writer_meta_base->DataBlockLocation[NeededBlock]; - if (RR.StartOffset == (size_t)-1) - throw std::runtime_error("No data exists for this variable"); - if (Req->MemSpace != MemorySpace::Host) - RR.DirectToAppMemory = false; - else if (VarRec->Operator != NULL) - RR.DirectToAppMemory = false; - else - RR.DirectToAppMemory = - IsContiguousTransfer(Req, &writer_meta_base->Offsets[StartDim], - &writer_meta_base->Count[StartDim]); - if (VarRec->Operator) - { - // have to have the whole thing - RR.ReadLength = writer_meta_base->DataBlockSize[NeededBlock]; - } - else + MetaArrayRecOperator *writer_meta_base = + (MetaArrayRecOperator *)GetMetadataBase((struct BP5VarRec *)Req->VarRec, + Step, WriterRank); + if (!writer_meta_base) { - RR.ReadLength = - helper::GetDataTypeSize(VarRec->Type) * - CalcBlockLength(VarRec->DimCount, &writer_meta_base->Count[StartDim]); + continue; // Not writen on this step } - RR.OffsetInBlock = 0; - if (RR.DirectToAppMemory) + size_t NodeLastBlock = NodeFirstBlock + writer_meta_base->BlockCount - 1; + if ((NodeFirstBlock <= Req->BlockID) && (NodeLastBlock >= Req->BlockID)) { - RR.DestinationAddr = (char *)Req->Data; - if (Req->Start.size() != 0) + // block is here + size_t NeededBlock = Req->BlockID - NodeFirstBlock; + size_t StartDim = NeededBlock * VarRec->DimCount; + ReadRequest RR; + RR.Timestep = Req->Step; + RR.WriterRank = WriterRank; + RR.StartOffset = writer_meta_base->DataBlockLocation[NeededBlock]; + if (RR.StartOffset == (size_t)-1) + throw std::runtime_error("No data exists for this variable"); + if (Req->MemSpace != MemorySpace::Host) + RR.DirectToAppMemory = false; + else if (VarRec->Operator != NULL) + RR.DirectToAppMemory = false; + else + RR.DirectToAppMemory = + IsContiguousTransfer(Req, &writer_meta_base->Offsets[StartDim], + &writer_meta_base->Count[StartDim]); + if (VarRec->Operator) + { + // have to have the whole thing + RR.ReadLength = writer_meta_base->DataBlockSize[NeededBlock]; + } + else { RR.ReadLength = helper::GetDataTypeSize(VarRec->Type) * - CalcBlockLength(VarRec->DimCount, Req->Count.data()); - /* DirectToAppMemory handles only 1D, so offset calc - * is 1D only for the moment */ - RR.StartOffset += helper::GetDataTypeSize(VarRec->Type) * Req->Start[0]; + CalcBlockLength(VarRec->DimCount, + &writer_meta_base->Count[StartDim]); } - } - else - { - RR.DestinationAddr = nullptr; - if (doAllocTempBuffers) + RR.OffsetInBlock = 0; + if (RR.DirectToAppMemory) { - RR.DestinationAddr = (char *)malloc(RR.ReadLength); + RR.DestinationAddr = (char *)Req->Data; + if (Req->Start.size() != 0) + { + RR.ReadLength = + helper::GetDataTypeSize(VarRec->Type) * + CalcBlockLength(VarRec->DimCount, Req->Count.data()); + /* DirectToAppMemory handles only 1D, so offset calc + * is 1D only for the moment */ + RR.StartOffset += + helper::GetDataTypeSize(VarRec->Type) * Req->Start[0]; + } } - *maxReadSize = - (*maxReadSize < RR.ReadLength ? RR.ReadLength : *maxReadSize); + else + { + RR.DestinationAddr = nullptr; + if (doAllocTempBuffers) + { + RR.DestinationAddr = (char *)malloc(RR.ReadLength); + } + *maxReadSize = + (*maxReadSize < RR.ReadLength ? RR.ReadLength : *maxReadSize); + } + RR.ReqIndex = ReqIndex; + RR.BlockID = NeededBlock; + Ret.push_back(RR); + break; } - RR.ReqIndex = ReqIndex; - RR.BlockID = NeededBlock; - Ret.push_back(RR); - break; + NodeFirstBlock += writer_meta_base->BlockCount; } - NodeFirstBlock += writer_meta_base->BlockCount; } } else { /* global case */ - const size_t writerCohortSize = WriterCohortSize(Req->Step); - for (size_t WriterRank = 0; WriterRank < writerCohortSize; WriterRank++) + for (size_t Step = StepLoopStart; Step < StepLoopEnd; Step++) { - MetaArrayRecOperator *writer_meta_base = (MetaArrayRecOperator *)GetMetadataBase( - (struct BP5VarRec *)Req->VarRec, Req->Step, WriterRank); - if (!writer_meta_base) - continue; // Not writen on this step - - for (size_t Block = 0; Block < writer_meta_base->BlockCount; Block++) + const size_t writerCohortSize = WriterCohortSize(Step); + for (size_t WriterRank = 0; WriterRank < writerCohortSize; WriterRank++) { - std::array intersectionstart; - std::array intersectionend; - std::array intersectioncount; - - size_t StartDim = Block * VarRec->DimCount; - if (IntersectionStartCount(VarRec->DimCount, Req->Start.data(), - Req->Count.data(), - &writer_meta_base->Offsets[StartDim], - &writer_meta_base->Count[StartDim], - &intersectionstart[0], &intersectioncount[0])) + MetaArrayRecOperator *writer_meta_base = + (MetaArrayRecOperator *)GetMetadataBase((struct BP5VarRec *)Req->VarRec, + Step, WriterRank); + if (!writer_meta_base) + continue; // Not writen on this step + + for (size_t Block = 0; Block < writer_meta_base->BlockCount; Block++) { - if (VarRec->Operator != NULL) - { - // need the whole thing for decompression anyway - ReadRequest RR; - RR.Timestep = Req->Step; - RR.WriterRank = WriterRank; - RR.StartOffset = writer_meta_base->DataBlockLocation[Block]; - RR.ReadLength = writer_meta_base->DataBlockSize[Block]; - RR.DestinationAddr = nullptr; - if (RR.StartOffset == (size_t)-1) - throw std::runtime_error("No data exists for this variable"); - if (doAllocTempBuffers) - { - RR.DestinationAddr = (char *)malloc(RR.ReadLength); - } - *maxReadSize = - (*maxReadSize < RR.ReadLength ? RR.ReadLength : *maxReadSize); - RR.DirectToAppMemory = false; - RR.ReqIndex = ReqIndex; - RR.BlockID = Block; - RR.OffsetInBlock = 0; - Ret.push_back(RR); - } - else + std::array intersectionstart; + std::array intersectionend; + std::array intersectioncount; + + size_t StartDim = Block * VarRec->DimCount; + if (IntersectionStartCount(VarRec->DimCount, Req->Start.data(), + Req->Count.data(), + &writer_meta_base->Offsets[StartDim], + &writer_meta_base->Count[StartDim], + &intersectionstart[0], &intersectioncount[0])) { - for (size_t Dim = 0; Dim < VarRec->DimCount; Dim++) - { - intersectionstart[Dim] -= writer_meta_base->Offsets[StartDim + Dim]; - } - size_t StartOffsetInBlock = - VB->m_ElementSize * - LinearIndex(VarRec->DimCount, &writer_meta_base->Count[StartDim], - &intersectionstart[0], m_ReaderIsRowMajor); - for (size_t Dim = 0; Dim < VarRec->DimCount; Dim++) - { - intersectionend[Dim] = - intersectionstart[Dim] + intersectioncount[Dim] - 1; - } - size_t EndOffsetInBlock = - VB->m_ElementSize * - (LinearIndex(VarRec->DimCount, &writer_meta_base->Count[StartDim], - &intersectionend[0], m_ReaderIsRowMajor) + - 1); - ReadRequest RR; - RR.Timestep = Req->Step; - RR.WriterRank = WriterRank; - RR.StartOffset = - writer_meta_base->DataBlockLocation[Block] + StartOffsetInBlock; - if (writer_meta_base->DataBlockLocation[Block] == (size_t)-1) - throw std::runtime_error("No data exists for this variable"); - RR.ReadLength = EndOffsetInBlock - StartOffsetInBlock; - if (Req->MemSpace != MemorySpace::Host) - RR.DirectToAppMemory = false; - else - RR.DirectToAppMemory = - IsContiguousTransfer(Req, &writer_meta_base->Offsets[StartDim], - &writer_meta_base->Count[StartDim]); - if (RR.DirectToAppMemory) - { - /* - * DirectToAppMemory handles only 1D, so offset - * calc is 1D only for the moment ContigOffset - * handles the case where our destination is not - * the start of the destination memory (because - * some other block filled in that start) - */ - - ssize_t ContigOffset = - (writer_meta_base->Offsets[StartDim + 0] - Req->Start[0]) * - VB->m_ElementSize; - if (ContigOffset < 0) - ContigOffset = 0; - RR.DestinationAddr = (char *)Req->Data + ContigOffset; - } - else + if (VarRec->Operator != NULL) { + // need the whole thing for decompression anyway + ReadRequest RR; + RR.Timestep = Step; + RR.WriterRank = WriterRank; + RR.StartOffset = writer_meta_base->DataBlockLocation[Block]; + RR.ReadLength = writer_meta_base->DataBlockSize[Block]; RR.DestinationAddr = nullptr; + if (RR.StartOffset == (size_t)-1) + throw std::runtime_error("No data exists for this variable"); if (doAllocTempBuffers) { RR.DestinationAddr = (char *)malloc(RR.ReadLength); } *maxReadSize = (*maxReadSize < RR.ReadLength ? RR.ReadLength : *maxReadSize); + RR.DirectToAppMemory = false; + RR.ReqIndex = ReqIndex; + RR.BlockID = Block; + RR.OffsetInBlock = 0; + Ret.push_back(RR); + } + else + { + for (size_t Dim = 0; Dim < VarRec->DimCount; Dim++) + { + intersectionstart[Dim] -= + writer_meta_base->Offsets[StartDim + Dim]; + } + size_t StartOffsetInBlock = + VB->m_ElementSize * + LinearIndex(VarRec->DimCount, + &writer_meta_base->Count[StartDim], + &intersectionstart[0], m_ReaderIsRowMajor); + for (size_t Dim = 0; Dim < VarRec->DimCount; Dim++) + { + intersectionend[Dim] = + intersectionstart[Dim] + intersectioncount[Dim] - 1; + } + size_t EndOffsetInBlock = + VB->m_ElementSize * + (LinearIndex(VarRec->DimCount, + &writer_meta_base->Count[StartDim], + &intersectionend[0], m_ReaderIsRowMajor) + + 1); + ReadRequest RR; + RR.Timestep = Step; + RR.WriterRank = WriterRank; + RR.StartOffset = + writer_meta_base->DataBlockLocation[Block] + StartOffsetInBlock; + if (writer_meta_base->DataBlockLocation[Block] == (size_t)-1) + throw std::runtime_error("No data exists for this variable"); + RR.ReadLength = EndOffsetInBlock - StartOffsetInBlock; + if (Req->MemSpace != MemorySpace::Host) + RR.DirectToAppMemory = false; + else + RR.DirectToAppMemory = IsContiguousTransfer( + Req, &writer_meta_base->Offsets[StartDim], + &writer_meta_base->Count[StartDim]); + if (RR.DirectToAppMemory) + { + /* + * DirectToAppMemory handles only 1D, so offset + * calc is 1D only for the moment ContigOffset + * handles the case where our destination is not + * the start of the destination memory (because + * some other block filled in that start) + */ + + ssize_t ContigOffset = + (writer_meta_base->Offsets[StartDim + 0] - Req->Start[0]) * + VB->m_ElementSize; + if (ContigOffset < 0) + ContigOffset = 0; + RR.DestinationAddr = (char *)Req->Data + ContigOffset; + } + else + { + RR.DestinationAddr = nullptr; + if (doAllocTempBuffers) + { + RR.DestinationAddr = (char *)malloc(RR.ReadLength); + } + *maxReadSize = (*maxReadSize < RR.ReadLength ? RR.ReadLength + : *maxReadSize); + } + RR.OffsetInBlock = StartOffsetInBlock; + RR.ReqIndex = ReqIndex; + RR.BlockID = Block; + Ret.push_back(RR); } - RR.OffsetInBlock = StartOffsetInBlock; - RR.ReqIndex = ReqIndex; - RR.BlockID = Block; - Ret.push_back(RR); } } } @@ -1756,7 +1787,7 @@ void BP5Deserializer::FinalizeGet(const ReadRequest &Read, const bool freeAddr) int ElementSize = ((struct BP5VarRec *)Req.VarRec)->ElementSize; MetaArrayRec *writer_meta_base = (MetaArrayRec *)GetMetadataBase( - ((struct BP5VarRec *)Req.VarRec), Req.Step, Read.WriterRank); + ((struct BP5VarRec *)Req.VarRec), Read.Timestep, Read.WriterRank); size_t *GlobalDimensions = writer_meta_base->Shape; auto DimCount = writer_meta_base->Dims; @@ -1966,9 +1997,9 @@ int BP5Deserializer::FindOffset(size_t Dims, const size_t *Size, const size_t *I */ BP5Deserializer::BP5Deserializer(bool WriterIsRowMajor, bool ReaderIsRowMajor, - bool RandomAccessMode) + bool RandomAccessMode, bool FlattenSteps) : m_WriterIsRowMajor{WriterIsRowMajor}, m_ReaderIsRowMajor{ReaderIsRowMajor}, - m_RandomAccessMode{RandomAccessMode} + m_RandomAccessMode{RandomAccessMode}, m_FlattenSteps{FlattenSteps} { FMContext Tmp = create_local_FMcontext(); ReaderFFSContext = create_FFSContext_FM(Tmp); @@ -2065,17 +2096,57 @@ void *BP5Deserializer::GetMetadataBase(BP5VarRec *VarRec, size_t Step, size_t Wr MinVarInfo *BP5Deserializer::MinBlocksInfo(const VariableBase &Var, size_t RelStep) { + auto PossiblyAddValueBlocks = [this](MinVarInfo *MV, BP5VarRec *VarRec, size_t &Id, + const size_t AbsStep) { + const size_t writerCohortSize = WriterCohortSize(AbsStep); + for (size_t WriterRank = 0; WriterRank < writerCohortSize; WriterRank++) + { + MetaArrayRec *writer_meta_base = + (MetaArrayRec *)GetMetadataBase(VarRec, AbsStep, WriterRank); + if (writer_meta_base) + { + MinBlockInfo Blk; + Blk.MinMax.Init(VarRec->Type); + Blk.WriterID = (int)WriterRank; + Blk.BlockID = Id++; + Blk.BufferP = writer_meta_base; + Blk.Start = NULL; + Blk.Count = NULL; + if (VarRec->OrigShapeID == ShapeID::LocalValue) + { + Blk.Count = (size_t *)1; + Blk.Start = (size_t *)WriterRank; + } + if (writer_meta_base) + { + ApplyElementMinMax(Blk.MinMax, VarRec->Type, writer_meta_base); + } + MV->BlocksInfo.push_back(Blk); + } + } + }; + BP5VarRec *VarRec = LookupVarByKey((void *)&Var); MinVarInfo *MV = new MinVarInfo((int)VarRec->DimCount, VarRec->GlobalDims); size_t AbsStep = RelStep; + size_t StepLoopStart, StepLoopEnd; if (m_RandomAccessMode) { AbsStep = VarRec->AbsStepFromRel[RelStep]; } - const size_t writerCohortSize = WriterCohortSize(AbsStep); + if (m_FlattenSteps) + { + StepLoopStart = 0; + StepLoopEnd = m_ControlArray.size(); + } + else + { + StepLoopStart = AbsStep; + StepLoopEnd = AbsStep + 1; + } size_t Id = 0; MV->Step = RelStep; MV->Dims = (int)VarRec->DimCount; @@ -2086,6 +2157,7 @@ MinVarInfo *BP5Deserializer::MinBlocksInfo(const VariableBase &Var, size_t RelSt if ((VarRec->OrigShapeID == ShapeID::LocalValue) || (VarRec->OrigShapeID == ShapeID::GlobalValue)) { + const size_t writerCohortSize = WriterCohortSize(AbsStep); if (VarRec->OrigShapeID == ShapeID::LocalValue) { // appear as an array locally @@ -2099,88 +2171,75 @@ MinVarInfo *BP5Deserializer::MinBlocksInfo(const VariableBase &Var, size_t RelSt } MV->BlocksInfo.reserve(writerCohortSize); - for (size_t WriterRank = 0; WriterRank < writerCohortSize; WriterRank++) + for (size_t Step = StepLoopStart; Step < StepLoopEnd; Step++) { - MetaArrayRec *writer_meta_base = - (MetaArrayRec *)GetMetadataBase(VarRec, AbsStep, WriterRank); - if (writer_meta_base) - { - MinBlockInfo Blk; - Blk.MinMax.Init(VarRec->Type); - Blk.WriterID = (int)WriterRank; - Blk.BlockID = Id++; - Blk.BufferP = writer_meta_base; - Blk.Start = NULL; - Blk.Count = NULL; - if (VarRec->OrigShapeID == ShapeID::LocalValue) - { - Blk.Count = (size_t *)1; - Blk.Start = (size_t *)WriterRank; - } - if (writer_meta_base) - { - ApplyElementMinMax(Blk.MinMax, VarRec->Type, writer_meta_base); - } - MV->BlocksInfo.push_back(Blk); - } + PossiblyAddValueBlocks(MV, VarRec, Id, Step); } return MV; } - for (size_t WriterRank = 0; WriterRank < writerCohortSize; WriterRank++) + for (size_t Step = StepLoopStart; Step < StepLoopEnd; Step++) { - MetaArrayRec *writer_meta_base = - (MetaArrayRec *)GetMetadataBase(VarRec, AbsStep, WriterRank); - if (writer_meta_base) + const size_t writerCohortSize = WriterCohortSize(Step); + for (size_t WriterRank = 0; WriterRank < writerCohortSize; WriterRank++) { - if (MV->Shape == NULL) + MetaArrayRec *writer_meta_base = + (MetaArrayRec *)GetMetadataBase(VarRec, Step, WriterRank); + if (writer_meta_base) { - MV->Shape = writer_meta_base->Shape; + if (MV->Shape == NULL) + { + MV->Shape = writer_meta_base->Shape; + } + size_t WriterBlockCount = + writer_meta_base->Dims ? writer_meta_base->DBCount / writer_meta_base->Dims : 1; + Id += WriterBlockCount; } - size_t WriterBlockCount = - writer_meta_base->Dims ? writer_meta_base->DBCount / writer_meta_base->Dims : 1; - Id += WriterBlockCount; } } MV->BlocksInfo.reserve(Id); Id = 0; - for (size_t WriterRank = 0; WriterRank < writerCohortSize; WriterRank++) + for (size_t Step = StepLoopStart; Step < StepLoopEnd; Step++) { - MetaArrayRec *writer_meta_base = - (MetaArrayRec *)GetMetadataBase(VarRec, AbsStep, WriterRank); + const size_t writerCohortSize = WriterCohortSize(Step); + for (size_t WriterRank = 0; WriterRank < writerCohortSize; WriterRank++) + { + MetaArrayRec *writer_meta_base = + (MetaArrayRec *)GetMetadataBase(VarRec, Step, WriterRank); - if (!writer_meta_base) - continue; - size_t WriterBlockCount = MV->Dims ? writer_meta_base->DBCount / MV->Dims : 1; - MinMaxStruct *MMs = NULL; - if (VarRec->MinMaxOffset != SIZE_MAX) - { - MMs = *(MinMaxStruct **)(((char *)writer_meta_base) + VarRec->MinMaxOffset); - } - for (size_t i = 0; i < WriterBlockCount; i++) - { - size_t *Offsets = NULL; - size_t *Count = NULL; - if (writer_meta_base->Offsets) - Offsets = writer_meta_base->Offsets + (i * MV->Dims); - if (writer_meta_base->Count) - Count = writer_meta_base->Count + (i * MV->Dims); - MinBlockInfo Blk; - Blk.WriterID = (int)WriterRank; - Blk.BlockID = Id++; - Blk.Start = Offsets; - Blk.Count = Count; - Blk.MinMax.Init(VarRec->Type); - if (MMs) + if (!writer_meta_base) + continue; + size_t WriterBlockCount = MV->Dims ? writer_meta_base->DBCount / MV->Dims : 1; + MinMaxStruct *MMs = NULL; + if (VarRec->MinMaxOffset != SIZE_MAX) { + MMs = *(MinMaxStruct **)(((char *)writer_meta_base) + VarRec->MinMaxOffset); + } + for (size_t i = 0; i < WriterBlockCount; i++) + { + size_t *Offsets = NULL; + size_t *Count = NULL; + if (writer_meta_base->Offsets) + Offsets = writer_meta_base->Offsets + (i * MV->Dims); + if (writer_meta_base->Count) + Count = writer_meta_base->Count + (i * MV->Dims); + MinBlockInfo Blk; + Blk.WriterID = (int)WriterRank; + Blk.BlockID = Id++; + Blk.Start = Offsets; + Blk.Count = Count; + Blk.MinMax.Init(VarRec->Type); + if (MMs) + { - char *BlockMinAddr = (((char *)MMs) + 2 * i * VarRec->ElementSize); - char *BlockMaxAddr = (((char *)MMs) + (2 * i + 1) * VarRec->ElementSize); - ApplyElementMinMax(Blk.MinMax, VarRec->Type, (void *)BlockMinAddr); - ApplyElementMinMax(Blk.MinMax, VarRec->Type, (void *)BlockMaxAddr); + char *BlockMinAddr = (((char *)MMs) + 2 * i * VarRec->ElementSize); + char *BlockMaxAddr = (((char *)MMs) + (2 * i + 1) * VarRec->ElementSize); + ApplyElementMinMax(Blk.MinMax, VarRec->Type, (void *)BlockMinAddr); + ApplyElementMinMax(Blk.MinMax, VarRec->Type, (void *)BlockMaxAddr); + } + // Blk.BufferP + MV->BlocksInfo.push_back(Blk); } - // Blk.BufferP - MV->BlocksInfo.push_back(Blk); } } return MV; diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.h b/source/adios2/toolkit/format/bp5/BP5Deserializer.h index 1dbb3594a2..ebff1ab82f 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.h +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.h @@ -35,7 +35,8 @@ class BP5Deserializer : virtual public BP5Base { public: - BP5Deserializer(bool WriterIsRowMajor, bool ReaderIsRowMajor, bool RandomAccessMode = false); + BP5Deserializer(bool WriterIsRowMajor, bool ReaderIsRowMajor, bool RandomAccessMode = false, + bool FlattenSteps = false); ~BP5Deserializer(); @@ -174,6 +175,7 @@ class BP5Deserializer : virtual public BP5Base FFSContext ReaderFFSContext; const bool m_RandomAccessMode; + const bool m_FlattenSteps; std::vector m_WriterCohortSize; // per step, in random mode size_t m_CurrentWriterCohortSize; // valid in streaming mode diff --git a/source/adios2/toolkit/remote/Remote.cpp b/source/adios2/toolkit/remote/Remote.cpp index 29c0da3184..a48fa1610b 100644 --- a/source/adios2/toolkit/remote/Remote.cpp +++ b/source/adios2/toolkit/remote/Remote.cpp @@ -109,6 +109,9 @@ void Remote::Open(const std::string hostname, const int32_t port, const std::str case Mode::ReadRandomAccess: open_msg.Mode = RemoteCommon::RemoteFileMode::RemoteOpenRandomAccess; break; + case Mode::ReadFlattenSteps: + open_msg.Mode = RemoteCommon::RemoteFileMode::RemoteOpenFlattenSteps; + break; default: break; } diff --git a/source/adios2/toolkit/remote/remote_common.h b/source/adios2/toolkit/remote/remote_common.h index 0d78bd290a..ba98cb383f 100644 --- a/source/adios2/toolkit/remote/remote_common.h +++ b/source/adios2/toolkit/remote/remote_common.h @@ -15,6 +15,7 @@ enum RemoteFileMode { RemoteOpen, RemoteOpenRandomAccess, + RemoteOpenFlattenSteps, }; /* */ diff --git a/source/adios2/toolkit/remote/remote_server.cpp b/source/adios2/toolkit/remote/remote_server.cpp index 5108ed421e..4b79d787e6 100644 --- a/source/adios2/toolkit/remote/remote_server.cpp +++ b/source/adios2/toolkit/remote/remote_server.cpp @@ -92,6 +92,8 @@ class AnonADIOSFile m_mode = mode; if (m_mode == RemoteOpenRandomAccess) adios_read_mode = adios2::Mode::ReadRandomAccess; + if (m_mode == RemoteOpenFlattenSteps) + adios_read_mode = adios2::Mode::ReadFlattenSteps; m_engine = &m_io->Open(FileName, adios_read_mode); memcpy(&m_ID, m_IOname.c_str(), sizeof(m_ID)); } @@ -175,6 +177,8 @@ static void OpenHandler(CManager cm, CMConnection conn, void *vevent, void *clie std::string strMode = "Streaming"; if (open_msg->Mode == RemoteOpenRandomAccess) strMode = "RandomAccess"; + if (open_msg->Mode == RemoteOpenFlattenSteps) + strMode = "FlattenSteps"; std::cout << "Got an open request (mode " << strMode << ") for file " << open_msg->FileName << std::endl; AnonADIOSFile *f = diff --git a/source/utils/bpls/bpls.cpp b/source/utils/bpls/bpls.cpp index 2b2e6a26bc..b390c6b3cd 100644 --- a/source/utils/bpls/bpls.cpp +++ b/source/utils/bpls/bpls.cpp @@ -96,6 +96,7 @@ bool listmeshes; // do list meshes too bool attrsonly; // do list attributes only bool longopt; // -l is turned on bool timestep; // read step by step +bool flatten; // flatten steps to one bool filestream = false; // are we using an engine through FileStream? bool noindex; // do no print array indices with data bool printByteAsChar; // print 8 bit integer arrays as string @@ -146,6 +147,7 @@ void display_help() */ " --timestep | -t Read content step by step (stream " "reading)\n" + " --flatten Flatten Steps into one step (open in flatten mode)\n" " --dump | -d Dump matched variables/attributes\n" " To match attributes too, add option " "-a\n" @@ -620,6 +622,7 @@ int bplsMain(int argc, char *argv[]) arg.AddBooleanArgument("--noindex", &noindex, " | -y Print data without array indices"); arg.AddBooleanArgument("-y", &noindex, ""); arg.AddBooleanArgument("--timestep", ×tep, " | -t Print values of timestep elements"); + arg.AddBooleanArgument("--flatten", &flatten, " Flatten steps to one"); arg.AddBooleanArgument("-t", ×tep, ""); arg.AddBooleanArgument("--attrs", &listattrs, " | -a List/match attributes too"); arg.AddBooleanArgument("-a", &listattrs, ""); @@ -770,6 +773,7 @@ void init_globals() output_xml = false; noindex = false; timestep = false; + flatten = false; sortnames = false; listattrs = false; listmeshes = false; @@ -863,6 +867,8 @@ void printSettings(void) printf(" -V : show binary version info of file\n"); if (timestep) printf(" -t : read step-by-step\n"); + if (flatten) + printf(" --flatten : flatten steps into one\n"); if (hidden_attrs) { @@ -1677,6 +1683,10 @@ int doList(std::string path) { fp = &io.Open(path, Mode::Read); } + else if (flatten) + { + fp = &io.Open(path, Mode::ReadFlattenSteps); + } else { fp = &io.Open(path, Mode::ReadRandomAccess); @@ -2923,7 +2933,8 @@ bool print_data_xml(const char *s, const size_t length) return false; } -int print_data(const void *data, int item, DataType adiosvartype, bool allowformat) +int print_data(const void *data, int item, DataType adiosvartype, bool allowformat, + bool char_star_string) { bool f = format.size() && allowformat; const char *fmt = format.c_str(); @@ -2946,9 +2957,15 @@ int print_data(const void *data, int item, DataType adiosvartype, bool allowform break; case DataType::String: { - // fprintf(outf, (f ? fmt : "\"%s\""), ((char *)data) + item); - const std::string *dataStr = reinterpret_cast(data); - fprintf(outf, (f ? fmt : "\"%s\""), dataStr[item].c_str()); + if (char_star_string) + { + fprintf(outf, (f ? fmt : "\"%s\""), *((char **)data)); + } + else + { + const std::string *dataStr = reinterpret_cast(data); + fprintf(outf, (f ? fmt : "\"%s\""), dataStr[item].c_str()); + } break; } @@ -3397,7 +3414,7 @@ void print_decomp(core::Engine *fp, core::IO *io, core::Variable *variable) if (blocks.size() == 1) { fprintf(outf, " = "); - print_data(blocks[0].BufferP, 0, adiosvartype, true); + print_data(blocks[0].BufferP, 0, adiosvartype, true, /* MBI */ true); fprintf(outf, "\n"); } else @@ -3410,7 +3427,7 @@ void print_decomp(core::Engine *fp, core::IO *io, core::Variable *variable) int col = 0; for (size_t j = 0; j < blocks.size(); j++) { - print_data(blocks[j].BufferP, 0, adiosvartype, true); + print_data(blocks[j].BufferP, 0, adiosvartype, true, /* MBI */ true); ++col; if (j < blocks.size() - 1) { diff --git a/source/utils/bpls/bpls.h b/source/utils/bpls/bpls.h index 8d5c9be035..afd1add048 100644 --- a/source/utils/bpls/bpls.h +++ b/source/utils/bpls/bpls.h @@ -88,7 +88,8 @@ bool matchesAMask(const char *name); int print_start(const std::string &fnamestr); void print_slice_info(core::VariableBase *variable, bool timed, uint64_t *s, uint64_t *c, Dims count); -int print_data(const void *data, int item, DataType adiosvartypes, bool allowformat); +int print_data(const void *data, int item, DataType adiosvartypes, bool allowformat, + bool char_star_string = false); /* s is a character array not necessarily null terminated. * return false on OK print, true if it not XML (not printed)*/ diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt index bf99106bfa..707b9e4ee8 100644 --- a/testing/adios2/engine/bp/CMakeLists.txt +++ b/testing/adios2/engine/bp/CMakeLists.txt @@ -91,6 +91,8 @@ set(CTEST_TEST_TIMEOUT 10) bp_gtest_add_tests_helper(WriteReadADIOS2 MPI_ALLOW) async_gtest_add_tests_helper(WriteReadADIOS2 MPI_ALLOW) +gtest_add_tests_helper(WriteReadFlatten MPI_ONLY BP Engine.BP. .BP5 WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5" ) + bp_gtest_add_tests_helper(WriteReadADIOS2fstream MPI_ALLOW) bp_gtest_add_tests_helper(WriteReadADIOS2stdio MPI_ALLOW) bp_gtest_add_tests_helper(WriteReadAsStreamADIOS2 MPI_ALLOW) diff --git a/testing/adios2/engine/bp/TestBPWriteReadFlatten.cpp b/testing/adios2/engine/bp/TestBPWriteReadFlatten.cpp new file mode 100644 index 0000000000..5f5b9e50fb --- /dev/null +++ b/testing/adios2/engine/bp/TestBPWriteReadFlatten.cpp @@ -0,0 +1,1338 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ +#include +#include + +#include +#include //std::iota +#include + +#include + +#include + +#include "../SmallTestData.h" + +std::string engineName; // comes from command line +std::string engineParameters; // comes from command line + +class BPWriteReadTestFlatten : public ::testing::Test +{ +public: + BPWriteReadTestFlatten() = default; + + SmallTestData m_TestData; +}; + +//****************************************************************************** +// 1D 1x8 test data +//****************************************************************************** + +// Flatten BP write and read 1D arrays +TEST_F(BPWriteReadTestFlatten, FlattenBPWriteRead1D8) +{ + // Each process would write a 1x8 array and all processes would + // form a mpiSize * Nx 1D array + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const size_t Nx = 8; + +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("FlattenBPWriteRead1D8_MPI.bp"); +#else + const std::string fname("FlattenBPWriteRead1D8.bp"); +#endif + + // Write test data using BP + +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + { + adios2::IO io = adios.DeclareIO("TestIO"); + + // Declare 1D variables (NumOfProcesses * Nx) + // The local process' part (start, count) can be defined now or later + // before Write(). + { + const adios2::Dims shape{static_cast(Nx * mpiSize)}; + const adios2::Dims start{static_cast(Nx * mpiRank)}; + const adios2::Dims count{Nx}; + + auto var_char = io.DefineVariable("ch", shape, start, count); + EXPECT_TRUE(var_char); + auto var_iString = io.DefineVariable("iString"); + EXPECT_TRUE(var_iString); + auto var_i8 = io.DefineVariable("i8", shape, start, count); + EXPECT_TRUE(var_i8); + auto var_i16 = io.DefineVariable("i16", shape, start, count); + EXPECT_TRUE(var_i16); + auto var_i32 = io.DefineVariable("i32", shape, start, count); + EXPECT_TRUE(var_i32); + auto var_i64 = io.DefineVariable("i64", shape, start, count); + EXPECT_TRUE(var_i64); + auto var_u8 = io.DefineVariable("u8", shape, start, count); + EXPECT_TRUE(var_u8); + auto var_u16 = io.DefineVariable("u16", shape, start, count); + EXPECT_TRUE(var_u16); + auto var_u32 = io.DefineVariable("u32", shape, start, count); + EXPECT_TRUE(var_u32); + auto var_u64 = io.DefineVariable("u64", shape, start, count); + EXPECT_TRUE(var_u64); + auto var_r32 = io.DefineVariable("r32", shape, start, count); + EXPECT_TRUE(var_r32); + auto var_r64 = io.DefineVariable("r64", shape, start, count); + EXPECT_TRUE(var_r64); + } + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + // Create the BP Engine + io.SetEngine("BPFile"); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + io.AddTransport("file"); + + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + EXPECT_EQ(bpWriter.OpenMode(), adios2::Mode::Write); + + for (size_t step = 0; step < (size_t)mpiSize; ++step) + { + // Generate test data for each process uniquely (all as if for step 0) + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, static_cast(0), mpiRank, mpiSize); + + // Retrieve the variables that previously went out of scope + auto var_char = io.InquireVariable("ch"); + auto var_iString = io.InquireVariable("iString"); + auto var_i8 = io.InquireVariable("i8"); + auto var_i16 = io.InquireVariable("i16"); + auto var_i32 = io.InquireVariable("i32"); + auto var_i64 = io.InquireVariable("i64"); + auto var_u8 = io.InquireVariable("u8"); + auto var_u16 = io.InquireVariable("u16"); + auto var_u32 = io.InquireVariable("u32"); + auto var_u64 = io.InquireVariable("u64"); + auto var_r32 = io.InquireVariable("r32"); + auto var_r64 = io.InquireVariable("r64"); + + // Make a 1D selection to describe the local dimensions of the + // variable we write and its offsets in the global spaces + adios2::Box sel({mpiRank * Nx}, {Nx}); + + var_char.SetSelection(sel); + EXPECT_THROW(var_iString.SetSelection(sel), std::invalid_argument); + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + // Write each one + // fill in the variable with values from starting index to + // starting index + count + bpWriter.BeginStep(); + + if (step == (size_t)mpiRank) + { + bpWriter.Put(var_char, currentTestData.CHAR.data()); + bpWriter.Put(var_iString, currentTestData.S1); + bpWriter.Put(var_i8, currentTestData.I8.data()); + bpWriter.Put(var_i16, currentTestData.I16.data()); + bpWriter.Put(var_i32, currentTestData.I32.data()); + bpWriter.Put(var_i64, currentTestData.I64.data()); + bpWriter.Put(var_u8, currentTestData.U8.data()); + bpWriter.Put(var_u16, currentTestData.U16.data()); + bpWriter.Put(var_u32, currentTestData.U32.data()); + bpWriter.Put(var_u64, currentTestData.U64.data()); + bpWriter.Put(var_r32, currentTestData.R32.data()); + bpWriter.Put(var_r64, currentTestData.R64.data()); + } + bpWriter.EndStep(); + } + + // Close the file + bpWriter.Close(); + } + + { + adios2::IO io = adios.DeclareIO("ReadIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + adios2::Engine bpReader = io.Open(fname, adios2::Mode::ReadFlattenSteps); + + EXPECT_EQ(bpReader.Steps(), 1); + + auto var_char = io.InquireVariable("ch"); + EXPECT_TRUE(var_char); + ASSERT_EQ(var_char.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_char.Steps(), 1); + ASSERT_EQ(var_char.Shape()[0], mpiSize * Nx); + + auto var_iString = io.InquireVariable("iString"); + EXPECT_TRUE(var_iString); + ASSERT_EQ(var_iString.Shape().size(), 0); + ASSERT_EQ(var_iString.Steps(), 1); + + auto var_i8 = io.InquireVariable("i8"); + EXPECT_TRUE(var_i8); + ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8.Steps(), 1); + ASSERT_EQ(var_i8.Shape()[0], mpiSize * Nx); + + auto var_i16 = io.InquireVariable("i16"); + EXPECT_TRUE(var_i16); + ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16.Steps(), 1); + ASSERT_EQ(var_i16.Shape()[0], mpiSize * Nx); + + auto var_i32 = io.InquireVariable("i32"); + EXPECT_TRUE(var_i32); + ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32.Steps(), 1); + ASSERT_EQ(var_i32.Shape()[0], mpiSize * Nx); + + auto var_i64 = io.InquireVariable("i64"); + EXPECT_TRUE(var_i64); + ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64.Steps(), 1); + ASSERT_EQ(var_i64.Shape()[0], mpiSize * Nx); + + auto var_u8 = io.InquireVariable("u8"); + EXPECT_TRUE(var_u8); + ASSERT_EQ(var_u8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8.Steps(), 1); + ASSERT_EQ(var_u8.Shape()[0], mpiSize * Nx); + + auto var_u16 = io.InquireVariable("u16"); + EXPECT_TRUE(var_u16); + ASSERT_EQ(var_u16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16.Steps(), 1); + ASSERT_EQ(var_u16.Shape()[0], mpiSize * Nx); + + auto var_u32 = io.InquireVariable("u32"); + EXPECT_TRUE(var_u32); + ASSERT_EQ(var_u32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32.Steps(), 1); + ASSERT_EQ(var_u32.Shape()[0], mpiSize * Nx); + + auto var_u64 = io.InquireVariable("u64"); + EXPECT_TRUE(var_u64); + ASSERT_EQ(var_u64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64.Steps(), 1); + ASSERT_EQ(var_u64.Shape()[0], mpiSize * Nx); + + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32.Steps(), 1); + ASSERT_EQ(var_r32.Shape()[0], mpiSize * Nx); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64.Steps(), 1); + ASSERT_EQ(var_r64.Shape()[0], mpiSize * Nx); + + // TODO: other types + + SmallTestData testData; + + std::string IString; + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + std::array CHAR; + + const adios2::Dims start{mpiRank * Nx}; + const adios2::Dims count{Nx}; + + const adios2::Box sel(start, count); + + for (size_t t = 0; t < 1; ++t) + { + var_char.SetSelection(sel); + + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + // default step selection should be 0, 1, so no need for that + + // Generate test data for each rank uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, static_cast(0), mpiRank, mpiSize); + + bpReader.Get(var_char, CHAR.data()); + bpReader.Get(var_iString, IString); + bpReader.Get(var_i8, I8.data()); + bpReader.Get(var_i16, I16.data()); + bpReader.Get(var_i32, I32.data()); + bpReader.Get(var_i64, I64.data()); + bpReader.Get(var_u8, U8.data()); + bpReader.Get(var_u16, U16.data()); + bpReader.Get(var_u32, U32.data()); + bpReader.Get(var_u64, U64.data()); + bpReader.Get(var_r32, R32.data()); + bpReader.Get(var_r64, R64.data()); + + bpReader.PerformGets(); + + EXPECT_EQ(IString, currentTestData.S1) << "rank=" << mpiRank; + + for (size_t i = 0; i < Nx; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + // EXPECT_EQ(TF[i], currentTestData.TF[i]) << msg; + EXPECT_EQ(CHAR[i], currentTestData.CHAR[i]) << msg; + EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; + EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; + EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; + EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; + EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; + EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; + EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; + EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; + EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; + EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; + } + } + bpReader.Close(); + } +} + +//****************************************************************************** +// 2D 2x4 test data +//****************************************************************************** + +// ADIOS2 BP write and read 2D array +TEST_F(BPWriteReadTestFlatten, FlattenBPWriteRead2D2x4) +{ + // Each process would write a 2x4 array and all processes would + // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const std::size_t Nx = 4; + + // Number of rows + const std::size_t Ny = 2; + +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("FlattenBPWriteRead2D2x4Test_MPI.bp"); +#else + const std::string fname("FlattenBPWriteRead2D2x4Test.bp"); +#endif + + // Write test data using ADIOS2 + +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + { + adios2::IO io = adios.DeclareIO("TestIO"); + + // Declare 2D variables (Ny * (NumOfProcesses * Nx)) + // The local process' part (start, count) can be defined now or later + // before Write(). + { + const adios2::Dims shape{Ny, static_cast(Nx * mpiSize)}; + const adios2::Dims start{0, static_cast(mpiRank * Nx)}; + const adios2::Dims count{Ny, Nx}; + + auto var_iString = io.DefineVariable("iString"); + EXPECT_TRUE(var_iString); + auto var_i8 = io.DefineVariable("i8", shape, start, count); + EXPECT_TRUE(var_i8); + auto var_i16 = io.DefineVariable("i16", shape, start, count); + EXPECT_TRUE(var_i16); + auto var_i32 = io.DefineVariable("i32", shape, start, count); + EXPECT_TRUE(var_i32); + auto var_i64 = io.DefineVariable("i64", shape, start, count); + EXPECT_TRUE(var_i64); + auto var_u8 = io.DefineVariable("u8", shape, start, count); + EXPECT_TRUE(var_u8); + auto var_u16 = io.DefineVariable("u16", shape, start, count); + EXPECT_TRUE(var_u16); + auto var_u32 = io.DefineVariable("u32", shape, start, count); + EXPECT_TRUE(var_u32); + auto var_u64 = io.DefineVariable("u64", shape, start, count); + EXPECT_TRUE(var_u64); + auto var_r32 = io.DefineVariable("r32", shape, start, count); + EXPECT_TRUE(var_r32); + auto var_r64 = io.DefineVariable("r64", shape, start, count); + EXPECT_TRUE(var_r64); + } + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + // Create the BP Engine + io.SetEngine("BPFile"); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + io.AddTransport("file"); + + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < (size_t)mpiSize; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, static_cast(0), mpiRank, mpiSize); + + // Retrieve the variables that previously went out of scope + auto var_iString = io.InquireVariable("iString"); + auto var_i8 = io.InquireVariable("i8"); + auto var_i16 = io.InquireVariable("i16"); + auto var_i32 = io.InquireVariable("i32"); + auto var_i64 = io.InquireVariable("i64"); + auto var_u8 = io.InquireVariable("u8"); + auto var_u16 = io.InquireVariable("u16"); + auto var_u32 = io.InquireVariable("u32"); + auto var_u64 = io.InquireVariable("u64"); + auto var_r32 = io.InquireVariable("r32"); + auto var_r64 = io.InquireVariable("r64"); + + // Make a 2D selection to describe the local dimensions of the + // variable we write and its offsets in the global spaces + adios2::Box sel({0, static_cast(mpiRank * Nx)}, {Ny, Nx}); + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + // Write each one + // fill in the variable with values from starting index to + // starting index + count + bpWriter.BeginStep(); + if (step == (size_t)mpiRank) + { + bpWriter.Put(var_iString, currentTestData.S1); + bpWriter.Put(var_i8, currentTestData.I8.data()); + bpWriter.Put(var_i16, currentTestData.I16.data()); + bpWriter.Put(var_i32, currentTestData.I32.data()); + bpWriter.Put(var_i64, currentTestData.I64.data()); + bpWriter.Put(var_u8, currentTestData.U8.data()); + bpWriter.Put(var_u16, currentTestData.U16.data()); + bpWriter.Put(var_u32, currentTestData.U32.data()); + bpWriter.Put(var_u64, currentTestData.U64.data()); + bpWriter.Put(var_r32, currentTestData.R32.data()); + bpWriter.Put(var_r64, currentTestData.R64.data()); + } + bpWriter.EndStep(); + } + + // Close the file + bpWriter.Close(); + } + + { + adios2::IO io = adios.DeclareIO("ReadIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + adios2::Engine bpReader = io.Open(fname, adios2::Mode::ReadFlattenSteps); + + EXPECT_EQ(bpReader.Steps(), 1); + auto var_iString = io.InquireVariable("iString"); + EXPECT_TRUE(var_iString); + ASSERT_EQ(var_iString.Shape().size(), 0); + ASSERT_EQ(var_iString.Steps(), 1); + + auto var_i8 = io.InquireVariable("i8"); + EXPECT_TRUE(var_i8); + ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8.Steps(), 1); + ASSERT_EQ(var_i8.Shape()[0], Ny); + ASSERT_EQ(var_i8.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_i16 = io.InquireVariable("i16"); + EXPECT_TRUE(var_i16); + ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16.Steps(), 1); + ASSERT_EQ(var_i16.Shape()[0], Ny); + ASSERT_EQ(var_i16.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_i32 = io.InquireVariable("i32"); + EXPECT_TRUE(var_i32); + ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32.Steps(), 1); + ASSERT_EQ(var_i32.Shape()[0], Ny); + ASSERT_EQ(var_i32.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_i64 = io.InquireVariable("i64"); + EXPECT_TRUE(var_i64); + ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64.Steps(), 1); + ASSERT_EQ(var_i64.Shape()[0], Ny); + ASSERT_EQ(var_i64.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u8 = io.InquireVariable("u8"); + EXPECT_TRUE(var_u8); + ASSERT_EQ(var_u8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8.Steps(), 1); + ASSERT_EQ(var_u8.Shape()[0], Ny); + ASSERT_EQ(var_u8.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u16 = io.InquireVariable("u16"); + EXPECT_TRUE(var_u16); + ASSERT_EQ(var_u16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16.Steps(), 1); + ASSERT_EQ(var_u16.Shape()[0], Ny); + ASSERT_EQ(var_u16.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u32 = io.InquireVariable("u32"); + EXPECT_TRUE(var_u32); + ASSERT_EQ(var_u32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32.Steps(), 1); + ASSERT_EQ(var_u32.Shape()[0], Ny); + ASSERT_EQ(var_u32.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u64 = io.InquireVariable("u64"); + EXPECT_TRUE(var_u64); + ASSERT_EQ(var_u64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64.Steps(), 1); + ASSERT_EQ(var_u64.Shape()[0], Ny); + ASSERT_EQ(var_u64.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32.Steps(), 1); + ASSERT_EQ(var_r32.Shape()[0], Ny); + ASSERT_EQ(var_r32.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64.Steps(), 1); + ASSERT_EQ(var_r64.Shape()[0], Ny); + ASSERT_EQ(var_r64.Shape()[1], static_cast(mpiSize * Nx)); + + std::string IString; + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + + const adios2::Dims start{0, static_cast(mpiRank * Nx)}; + const adios2::Dims count{Ny, Nx}; + + const adios2::Box sel(start, count); + + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + for (size_t t = 0; t < 1; ++t) + { + var_i8.SetStepSelection({t, 1}); + var_i16.SetStepSelection({t, 1}); + var_i32.SetStepSelection({t, 1}); + var_i64.SetStepSelection({t, 1}); + + var_u8.SetStepSelection({t, 1}); + var_u16.SetStepSelection({t, 1}); + var_u32.SetStepSelection({t, 1}); + var_u64.SetStepSelection({t, 1}); + + var_r32.SetStepSelection({t, 1}); + var_r64.SetStepSelection({t, 1}); + + bpReader.Get(var_iString, IString); + + bpReader.Get(var_i8, I8.data()); + bpReader.Get(var_i16, I16.data()); + bpReader.Get(var_i32, I32.data()); + bpReader.Get(var_i64, I64.data()); + + bpReader.Get(var_u8, U8.data()); + bpReader.Get(var_u16, U16.data()); + bpReader.Get(var_u32, U32.data()); + bpReader.Get(var_u64, U64.data()); + + bpReader.Get(var_r32, R32.data()); + bpReader.Get(var_r64, R64.data()); + + bpReader.PerformGets(); + // Generate test data for each rank uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, static_cast(0), mpiRank, mpiSize); + + EXPECT_EQ(IString, currentTestData.S1); + + for (size_t i = 0; i < Nx * Ny; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; + EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; + EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; + EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; + EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; + EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; + EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; + EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; + EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; + EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; + } + } + bpReader.Close(); + } +} + +//****************************************************************************** +// 2D 4x2 test data +//****************************************************************************** + +TEST_F(BPWriteReadTestFlatten, FlattenBPWriteRead2D4x2) +{ + // Each process would write a 4x2 array and all processes would + // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const std::size_t Nx = 2; + // Number of cols + const std::size_t Ny = 4; + +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("FlattenBPWriteRead2D4x2Test_MPI.bp"); +#else + const std::string fname("FlattenBPWriteRead2D4x2Test.bp"); +#endif + + // Write test data using ADIOS2 + +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + { + adios2::IO io = adios.DeclareIO("TestIO"); + + // Declare 2D variables (4 * (NumberOfProcess * Nx)) + // The local process' part (start, count) can be defined now or later + // before Write(). + { + adios2::Dims shape{static_cast(Ny), + static_cast(mpiSize * Nx)}; + adios2::Dims start{static_cast(0), + static_cast(mpiRank * Nx)}; + adios2::Dims count{static_cast(Ny), static_cast(Nx)}; + auto var_i8 = io.DefineVariable("i8", shape, start, count); + EXPECT_TRUE(var_i8); + auto var_i16 = io.DefineVariable("i16", shape, start, count); + EXPECT_TRUE(var_i16); + auto var_i32 = io.DefineVariable("i32", shape, start, count); + EXPECT_TRUE(var_i32); + auto var_i64 = io.DefineVariable("i64", shape, start, count); + EXPECT_TRUE(var_i64); + auto var_u8 = io.DefineVariable("u8", shape, start, count); + EXPECT_TRUE(var_u8); + auto var_u16 = io.DefineVariable("u16", shape, start, count); + EXPECT_TRUE(var_u16); + auto var_u32 = io.DefineVariable("u32", shape, start, count); + EXPECT_TRUE(var_u32); + auto var_u64 = io.DefineVariable("u64", shape, start, count); + EXPECT_TRUE(var_u64); + auto var_r32 = io.DefineVariable("r32", shape, start, count); + EXPECT_TRUE(var_r32); + auto var_r64 = io.DefineVariable("r64", shape, start, count); + EXPECT_TRUE(var_r64); + } + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + // Create the BP Engine + io.SetEngine("BPFile"); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + io.AddTransport("file"); + + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < (size_t)mpiSize; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, static_cast(0), mpiRank, mpiSize); + + // Retrieve the variables that previously went out of scope + auto var_i8 = io.InquireVariable("i8"); + auto var_i16 = io.InquireVariable("i16"); + auto var_i32 = io.InquireVariable("i32"); + auto var_i64 = io.InquireVariable("i64"); + auto var_u8 = io.InquireVariable("u8"); + auto var_u16 = io.InquireVariable("u16"); + auto var_u32 = io.InquireVariable("u32"); + auto var_u64 = io.InquireVariable("u64"); + auto var_r32 = io.InquireVariable("r32"); + auto var_r64 = io.InquireVariable("r64"); + + // Make a 2D selection to describe the local dimensions of the + // variable we write and its offsets in the global spaces + adios2::Box sel({0, static_cast(mpiRank * Nx)}, {Ny, Nx}); + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + // Write each one + // fill in the variable with values from starting index to + // starting index + count + bpWriter.BeginStep(); + if (step == (size_t)mpiRank) + { + bpWriter.Put(var_i8, currentTestData.I8.data()); + bpWriter.Put(var_i16, currentTestData.I16.data()); + bpWriter.Put(var_i32, currentTestData.I32.data()); + bpWriter.Put(var_i64, currentTestData.I64.data()); + bpWriter.Put(var_u8, currentTestData.U8.data()); + bpWriter.Put(var_u16, currentTestData.U16.data()); + bpWriter.Put(var_u32, currentTestData.U32.data()); + bpWriter.Put(var_u64, currentTestData.U64.data()); + bpWriter.Put(var_r32, currentTestData.R32.data()); + bpWriter.Put(var_r64, currentTestData.R64.data()); + } + bpWriter.EndStep(); + } + + // Close the file + bpWriter.Close(); + } + + { + adios2::IO io = adios.DeclareIO("ReadIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + adios2::Engine bpReader = io.Open(fname, adios2::Mode::ReadFlattenSteps); + + EXPECT_EQ(bpReader.Steps(), 1); + + auto var_i8 = io.InquireVariable("i8"); + EXPECT_TRUE(var_i8); + ASSERT_EQ(var_i8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8.Steps(), 1); + ASSERT_EQ(var_i8.Shape()[0], Ny); + ASSERT_EQ(var_i8.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_i16 = io.InquireVariable("i16"); + EXPECT_TRUE(var_i16); + ASSERT_EQ(var_i16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16.Steps(), 1); + ASSERT_EQ(var_i16.Shape()[0], Ny); + ASSERT_EQ(var_i16.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_i32 = io.InquireVariable("i32"); + EXPECT_TRUE(var_i32); + ASSERT_EQ(var_i32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32.Steps(), 1); + ASSERT_EQ(var_i32.Shape()[0], Ny); + ASSERT_EQ(var_i32.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_i64 = io.InquireVariable("i64"); + EXPECT_TRUE(var_i64); + ASSERT_EQ(var_i64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64.Steps(), 1); + ASSERT_EQ(var_i64.Shape()[0], Ny); + ASSERT_EQ(var_i64.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u8 = io.InquireVariable("u8"); + EXPECT_TRUE(var_u8); + ASSERT_EQ(var_u8.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8.Steps(), 1); + ASSERT_EQ(var_u8.Shape()[0], Ny); + ASSERT_EQ(var_u8.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u16 = io.InquireVariable("u16"); + EXPECT_TRUE(var_u16); + ASSERT_EQ(var_u16.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16.Steps(), 1); + ASSERT_EQ(var_u16.Shape()[0], Ny); + ASSERT_EQ(var_u16.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u32 = io.InquireVariable("u32"); + EXPECT_TRUE(var_u32); + ASSERT_EQ(var_u32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32.Steps(), 1); + ASSERT_EQ(var_u32.Shape()[0], Ny); + ASSERT_EQ(var_u32.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_u64 = io.InquireVariable("u64"); + EXPECT_TRUE(var_u64); + ASSERT_EQ(var_u64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64.Steps(), 1); + ASSERT_EQ(var_u64.Shape()[0], Ny); + ASSERT_EQ(var_u64.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32.Steps(), 1); + ASSERT_EQ(var_r32.Shape()[0], Ny); + ASSERT_EQ(var_r32.Shape()[1], static_cast(mpiSize * Nx)); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64.Steps(), 1); + ASSERT_EQ(var_r64.Shape()[0], Ny); + ASSERT_EQ(var_r64.Shape()[1], static_cast(mpiSize * Nx)); + + // If the size of the array is smaller than the data + // the result is weird... double and uint64_t would get + // completely garbage data + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + + const adios2::Dims start{0, static_cast(mpiRank * Nx)}; + const adios2::Dims count{Ny, Nx}; + + const adios2::Box sel(start, count); + + var_i8.SetSelection(sel); + var_i16.SetSelection(sel); + var_i32.SetSelection(sel); + var_i64.SetSelection(sel); + + var_u8.SetSelection(sel); + var_u16.SetSelection(sel); + var_u32.SetSelection(sel); + var_u64.SetSelection(sel); + + var_r32.SetSelection(sel); + var_r64.SetSelection(sel); + + for (size_t t = 0; t < 1; ++t) + { + var_i8.SetStepSelection({t, 1}); + var_i16.SetStepSelection({t, 1}); + var_i32.SetStepSelection({t, 1}); + var_i64.SetStepSelection({t, 1}); + + var_u8.SetStepSelection({t, 1}); + var_u16.SetStepSelection({t, 1}); + var_u32.SetStepSelection({t, 1}); + var_u64.SetStepSelection({t, 1}); + + var_r32.SetStepSelection({t, 1}); + var_r64.SetStepSelection({t, 1}); + + bpReader.Get(var_i8, I8.data()); + bpReader.Get(var_i16, I16.data()); + bpReader.Get(var_i32, I32.data()); + bpReader.Get(var_i64, I64.data()); + + bpReader.Get(var_u8, U8.data()); + bpReader.Get(var_u16, U16.data()); + bpReader.Get(var_u32, U32.data()); + bpReader.Get(var_u64, U64.data()); + + bpReader.Get(var_r32, R32.data()); + bpReader.Get(var_r64, R64.data()); + + bpReader.PerformGets(); + + // Generate test data for each rank uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, static_cast(t), mpiRank, mpiSize); + + for (size_t i = 0; i < Nx * Ny; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(I8[i], currentTestData.I8[i]) << msg; + EXPECT_EQ(I16[i], currentTestData.I16[i]) << msg; + EXPECT_EQ(I32[i], currentTestData.I32[i]) << msg; + EXPECT_EQ(I64[i], currentTestData.I64[i]) << msg; + EXPECT_EQ(U8[i], currentTestData.U8[i]) << msg; + EXPECT_EQ(U16[i], currentTestData.U16[i]) << msg; + EXPECT_EQ(U32[i], currentTestData.U32[i]) << msg; + EXPECT_EQ(U64[i], currentTestData.U64[i]) << msg; + EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; + EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; + } + } + bpReader.Close(); + } +} + +TEST_F(BPWriteReadTestFlatten, FlattenBPWriteRead10D2x2) +{ + // Each process would write a 2x2x...x2 9D array and all processes would + // form a 10D NumberOfProcess x 2 x ... x 2) array + + int mpiRank = 0, mpiSize = 1; + +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string fname("FlattenBPWriteRead10D2x2Test_MPI.bp"); +#else + const std::string fname("FlattenBPWriteRead10D2x2Test.bp"); +#endif + + size_t NX = static_cast(mpiSize); + size_t OX = static_cast(mpiRank); + const adios2::Dims shape{NX, 2, 2, 2, 2, 2, 2, 2, 2, 2}; + const adios2::Dims start{OX, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + const adios2::Dims count{1, 2, 2, 2, 2, 2, 2, 2, 2, 2}; + + std::array R64w, R64r; + std::array, 512> CR64w, CR64r; + + // Write test data using ADIOS2 + +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + { + adios2::IO io = adios.DeclareIO("TestIO"); + + // Declare 10D variables + { + auto var_r64 = io.DefineVariable("r64", shape, start, count); + EXPECT_TRUE(var_r64); + auto var_c64 = io.DefineVariable>("cr64", shape, start, count); + EXPECT_TRUE(var_c64); + } + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + // Create the BP Engine + io.SetEngine("BPFile"); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + io.AddTransport("file"); + + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < (size_t)mpiSize; ++step) + { + // double d = mpiRank + 1 + step / 10.0; + double d = mpiRank + 1 / 10.0; // every step is the same + // Generate test data for each process uniquely + std::for_each(R64w.begin(), R64w.end(), [&](double &v) { + v = d; + d += 0.0001; + }); + std::for_each(CR64w.begin(), CR64w.end(), [&](std::complex &v) { + v.real(d); + v.imag(d); + }); + + // Retrieve the variables that previously went out of scope + auto var_r64 = io.InquireVariable("r64"); + auto var_cr64 = io.InquireVariable>("cr64"); + + // Make a 2D selection to describe the local dimensions of the + // variable we write and its offsets in the global spaces + adios2::Box sel({start, count}); + var_r64.SetSelection(sel); + var_cr64.SetSelection(sel); + + // Write each one + // fill in the variable with values from starting index to + // starting index + count + bpWriter.BeginStep(); + // write ranks in reverse, end down + if (step == (size_t)(mpiSize - mpiRank - 1)) + { + bpWriter.Put(var_r64, R64w.data()); + bpWriter.Put(var_cr64, CR64w.data()); + } + bpWriter.EndStep(); + } + + // Close the file + bpWriter.Close(); + } + + { + adios2::IO io = adios.DeclareIO("ReadIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + adios2::Engine bpReader = io.Open(fname, adios2::Mode::ReadFlattenSteps); + + EXPECT_EQ(bpReader.Steps(), 1); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64.Steps(), 1); + ASSERT_EQ(var_r64.Shape().size(), 10); + ASSERT_EQ(var_r64.Shape()[0], NX); + ASSERT_EQ(var_r64.Shape()[1], 2); + ASSERT_EQ(var_r64.Shape()[2], 2); + ASSERT_EQ(var_r64.Shape()[3], 2); + ASSERT_EQ(var_r64.Shape()[4], 2); + ASSERT_EQ(var_r64.Shape()[5], 2); + ASSERT_EQ(var_r64.Shape()[6], 2); + ASSERT_EQ(var_r64.Shape()[7], 2); + ASSERT_EQ(var_r64.Shape()[8], 2); + ASSERT_EQ(var_r64.Shape()[9], 2); + + auto var_cr64 = io.InquireVariable>("cr64"); + EXPECT_TRUE(var_cr64); + ASSERT_EQ(var_cr64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_cr64.Steps(), 1); + ASSERT_EQ(var_cr64.Shape().size(), 10); + ASSERT_EQ(var_cr64.Shape()[0], NX); + ASSERT_EQ(var_cr64.Shape()[1], 2); + ASSERT_EQ(var_cr64.Shape()[2], 2); + ASSERT_EQ(var_cr64.Shape()[3], 2); + ASSERT_EQ(var_cr64.Shape()[4], 2); + ASSERT_EQ(var_cr64.Shape()[5], 2); + ASSERT_EQ(var_cr64.Shape()[6], 2); + ASSERT_EQ(var_cr64.Shape()[7], 2); + ASSERT_EQ(var_cr64.Shape()[8], 2); + ASSERT_EQ(var_cr64.Shape()[9], 2); + + const adios2::Box sel(start, count); + + var_r64.SetSelection(sel); + var_cr64.SetSelection(sel); + + for (size_t step = 0; step < 1; ++step) + { + var_r64.SetStepSelection({step, 1}); + var_cr64.SetStepSelection({step, 1}); + bpReader.Get(var_r64, R64r.data()); + bpReader.Get(var_cr64, CR64r.data()); + bpReader.PerformGets(); + + // double d = mpiRank + 1 + step / 10.0; + double d = mpiRank + 1 / 10.0; + // Re-generate test data for each process uniquely that was written + std::for_each(R64w.begin(), R64w.end(), [&](double &v) { + v = d; + d += 0.0001; + }); + std::for_each(CR64w.begin(), CR64w.end(), [&](std::complex &v) { + v.real(d); + v.imag(d); + }); + + for (size_t i = 0; i < 512; ++i) + { + std::stringstream ss; + ss << "t=" << step << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(R64r[i], R64w[i]) << msg; + EXPECT_EQ(CR64r[i], CR64w[i]) << msg; + } + } + bpReader.Close(); + } +} + +// ADIOS2 BP write and read 1D arrays +TEST_F(BPWriteReadTestFlatten, FlattenBPWriteReadEmptyProcess) +{ +#if ADIOS2_USE_MPI + // Each process, except rank 0 would write a 1x8 array and all + // processes would form a (mpiSize-1) * Nx 1D array + const std::string fname("FlattenBPWriteReadEmptyProces.bp"); + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const size_t Nx = 8; + + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + + // Number of steps + const size_t NSteps = mpiSize; + + /* This is a parallel test, do not run in serial */ + adios2::ADIOS adios(MPI_COMM_WORLD); + { + adios2::IO io = adios.DeclareIO("TestIO"); + // Declare 1D variables (NumOfProcesses * Nx) + // The local process' part (start, count) can be defined now or later + // before Write(). + + adios2::Dims shape{static_cast(Nx * (mpiSize - 1))}; + adios2::Dims start{static_cast(Nx * (mpiRank - 1))}; + adios2::Dims count{Nx}; + if (!mpiRank) + { + count[0] = 0; + start[0] = 0; + } + + auto var_r32 = io.DefineVariable("r32", shape, start, count); + EXPECT_TRUE(var_r32); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + // Create the BP Engine + io.SetEngine("BPFile"); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, static_cast(0), mpiRank, mpiSize); + + bpWriter.BeginStep(); + if (step == (size_t)mpiRank) + { + if (mpiRank != 0) + { + bpWriter.Put(var_r32, currentTestData.R32.data()); + } + } + bpWriter.EndStep(); + } + + // Close the file + bpWriter.Close(); + } + + { + adios2::IO io = adios.DeclareIO("ReadIO"); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + adios2::Engine bpReader = io.Open(fname, adios2::Mode::ReadFlattenSteps); + + for (size_t step = 0; step < 1; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, static_cast(0), mpiRank + 1, mpiSize); + + auto var_r32 = io.InquireVariable("r32"); + EXPECT_TRUE(var_r32); + ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32.Shape()[0], (mpiSize - 1) * Nx); + + SmallTestData testData; + std::array R32; + + // last process does not read + // readers 0..N-2, while data was produced by 1..N-1 + adios2::Dims start{mpiRank * Nx}; + adios2::Dims count{Nx}; + + if (mpiRank == mpiSize - 1) + { + count[0] = 0; + start[0] = 0; + } + + const adios2::Box sel(start, count); + var_r32.SetSelection(sel); + + if (mpiRank < mpiSize - 1) + { + bpReader.Get(var_r32, R32.data(), adios2::Mode::Sync); + for (size_t i = 0; i < Nx; ++i) + { + std::stringstream ss; + ss << "t=" << step << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + EXPECT_EQ(R32[i], currentTestData.R32[i]) << msg; + } + } + } + bpReader.Close(); + } +#else + return; +#endif +} + +//****************************************************************************** +// main +//****************************************************************************** + +int main(int argc, char **argv) +{ +#if ADIOS2_USE_MPI + int provided; + + // MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); +#endif + + int result; + ::testing::InitGoogleTest(&argc, argv); + + if (argc > 1) + { + engineName = std::string(argv[1]); + } + if (argc > 2) + { + engineParameters = std::string(argv[2]); + } + result = RUN_ALL_TESTS(); + +#if ADIOS2_USE_MPI + MPI_Finalize(); +#endif + + return result; +} diff --git a/testing/utils/cwriter/TestUtilsCWriter.bplsh.expected.txt b/testing/utils/cwriter/TestUtilsCWriter.bplsh.expected.txt index 4da5fd17a3..ba3fecafca 100644 --- a/testing/utils/cwriter/TestUtilsCWriter.bplsh.expected.txt +++ b/testing/utils/cwriter/TestUtilsCWriter.bplsh.expected.txt @@ -11,6 +11,7 @@ The time dimension is the first dimension then. --attrsonly | -A List attributes only --meshes | -m List meshes --timestep | -t Read content step by step (stream reading) + --flatten Flatten Steps into one step (open in flatten mode) --dump | -d Dump matched variables/attributes To match attributes too, add option -a --regexp | -e Treat masks as extended regular expressions