ReadModeFlatten (#4147)

Add a new external mode for opening files, ReadFlattenSteps.  This is like ReadRandomAccess (in that it reads all metadata immediately and you can't use BeginStep), but it causes *all* the timesteps in the file to be Flattened into a single step, as if each Put() has been done in that step.

bindings/C/adios2/c/adios2_c_engine.cpp

@@ -34,6 +34,9 @@ adios2::Mode adios2_ToMode(const adios2_mode mode, const std::string &hint)
     case adios2_mode_readRandomAccess:
         modeCpp = adios2::Mode::ReadRandomAccess;
         break;
+    case adios2_mode_readFlattenSteps:
+        modeCpp = adios2::Mode::ReadFlattenSteps;
+        break;
     case adios2_mode_deferred:
         modeCpp = adios2::Mode::Deferred;
         break;
@@ -63,6 +66,9 @@ adios2_mode adios2_fromMode(const adios2::Mode mode, const std::string &hint)
     case adios2::Mode::ReadRandomAccess:
         modeC = adios2_mode_readRandomAccess;
         break;
+    case adios2::Mode::ReadFlattenSteps:
+        modeC = adios2_mode_readFlattenSteps;
+        break;
     case adios2::Mode::Deferred:
         modeC = adios2_mode_deferred;
         break;

bindings/C/adios2/c/adios2_c_io.h

@@ -329,8 +329,8 @@ adios2_error adios2_remove_all_attributes(adios2_io *io);
  * MPI Collective function as it calls MPI_Comm_dup
  * @param io engine owner
  * @param name unique engine identifier
- * @param mode adios2_mode_write, adios2_mode_read, adios2_mode_append, and
- * adios2_mode_readRandomAccess
+ * @param mode adios2_mode_write, adios2_mode_read, adios2_mode_append,
+ * adios2_mode_readRandomAccess and adios2_mode_readFlattenSteps
  * @return success: handler, failure: NULL
  */
 adios2_engine *adios2_open(adios2_io *io, const char *name, const adios2_mode mode);
@@ -341,8 +341,8 @@ adios2_engine *adios2_open(adios2_io *io, const char *name, const adios2_mode mo
  * MPI Collective function as it calls MPI_Comm_dup
  * @param io engine owner
  * @param name unique engine identifier
- * @param mode adios2_mode_write, adios2_mode_read, adios2_mode_append, and
- * adios2_mode_readRandomAccess
+ * @param mode adios2_mode_write, adios2_mode_read, adios2_mode_append,
+ * adios2_mode_readRandomAccess and adios2_mode_readFlattenSteps
  * @param comm communicator other than adios' handler comm. MPI only.
  * @return success: handler, failure: NULL
  */

bindings/C/adios2/c/adios2_c_io.tcc

@@ -36,6 +36,10 @@ adios2::Mode adios2_ToOpenMode(const adios2_mode modeC)
         mode = adios2::Mode::ReadRandomAccess;
         break;
 
+    case adios2_mode_readFlattenSteps:
+        mode = adios2::Mode::ReadFlattenSteps;
+        break;
+
     default:
         break;
     }

bindings/C/adios2/c/adios2_c_types.h

@@ -103,6 +103,7 @@ typedef enum
     adios2_mode_read = 2,
     adios2_mode_append = 3,
     adios2_mode_readRandomAccess = 6,
+    adios2_mode_readFlattenSteps = 7,
 
     adios2_mode_deferred = 4,
     adios2_mode_sync = 5

bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.cpp

@@ -89,6 +89,9 @@ adios2::Mode fstream::ToMode(const openmode mode) const noexcept
     case (openmode::in_random_access):
         modeCpp = adios2::Mode::ReadRandomAccess;
         break;
+    case (openmode::in_flatten_steps):
+        modeCpp = adios2::Mode::ReadFlattenSteps;
+        break;
     case (openmode::app):
         modeCpp = adios2::Mode::Append;
         break;

bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.h

@@ -48,6 +48,7 @@ class fstream
         out,              //!< write
         in,               //!< read
         in_random_access, //!< read_random_access
+        in_flatten_steps, //!< flatten all input steps to 1
         app               //!< append, not yet supported
     };
 

bindings/Fortran/modules/adios2_parameters_mod.f90

@@ -56,6 +56,7 @@ module adios2_parameters_mod
    integer, parameter :: adios2_mode_read = 2
    integer, parameter :: adios2_mode_append = 3
    integer, parameter :: adios2_mode_readRandomAccess = 6
+   integer, parameter :: adios2_mode_readFlattenSteps = 7
 
    integer, parameter :: adios2_mode_deferred = 4
    integer, parameter :: adios2_mode_sync = 5

bindings/Python/py11glue.cpp

@@ -98,6 +98,7 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
         .value("Write", adios2::Mode::Write)
         .value("Read", adios2::Mode::Read)
         .value("ReadRandomAccess", adios2::Mode::ReadRandomAccess)
+        .value("ReadFlattenSteps", adios2::Mode::ReadFlattenSteps)
         .value("Append", adios2::Mode::Append)
         .value("Deferred", adios2::Mode::Deferred)
         .value("Sync", adios2::Mode::Sync)

docs/user_guide/source/components/anatomy.rst

@@ -114,7 +114,10 @@ named `adios2::Mode::ReadRandomAccess`. `adios2::Mode::Read` mode allows data ac
 current timestep. `ReadRandomAccess` can only be used with file engines and involves loading all the file metadata at
 once. So it can be more memory intensive than `adios2::Mode::Read` mode, but allows reading data from any timestep using
 `SetStepSelection()`. If you use `adios2::Mode::ReadRandomAccess` mode, be sure to allocate enough memory to hold
-multiple steps of the variable content.
+multiple steps of the variable content.  Note that ADIOS streaming
+engines (like SST, DataMan, etc.) do not support `ReadRandomAccess`
+mode.  Also newer file Engines like BP5 to not allow
+`BeginStep/EndStep` calls in `ReadRandomAccess` mode.
 
 .. code:: C++
 
@@ -134,3 +137,45 @@ multiple steps of the variable content.
     |   |--> IO goes out of scope
     |
     |--> ADIOS goes out of scope or adios2_finalize()
+
+Previously we explored how to read using the input mode `adios2::Mode::Read`. Nonetheless, ADIOS has another input mode
+named `adios2::Mode::ReadRandomAccess`. `adios2::Mode::Read` mode allows data access only timestep by timestep using
+`BeginStep/EndStep`, but generally it is more memory efficient as ADIOS is only required to load metadata for the
+current timestep. `ReadRandomAccess` can only be used with file engines and involves loading all the file metadata at
+once. So it can be more memory intensive than `adios2::Mode::Read` mode, but allows reading data from any timestep using
+`SetStepSelection()`. If you use `adios2::Mode::ReadRandomAccess` mode, be sure to allocate enough memory to hold
+multiple steps of the variable content.  Note that ADIOS streaming
+engines (like SST, DataMan, etc.) do not support `ReadRandomAccess`
+mode.  Also newer file Engines like BP5 to not allow
+`BeginStep/EndStep` calls in `ReadRandomAccess` mode.
+
+.. code:: C++
+
+    ADIOS adios("config.xml", MPI_COMM_WORLD);
+    |
+    |   IO io = adios.DeclareIO(...);
+    |   |
+    |   |   Engine e = io.Open("InputFileName.bp", adios2::Mode::ReadRandomAccess);
+    |   |   |
+    |   |   |   Variable var = io.InquireVariable(...)
+    |   |   |   |   var.SetStepSelection()
+    |   |   |   |   e.Get(var, datapointer);
+    |   |   |   |
+    |   |   |
+    |   |   e.Close();
+    |   |
+    |   |--> IO goes out of scope
+    |
+    |--> ADIOS goes out of scope or adios2_finalize()
+    
+
+In addition to the two read modes discussed above, ADIOS has another
+input mode named `adios2::Mode::ReadFlattenSteps`. This is a highly
+specialized mode built that is unlikely to be of general utility, but
+we describe it for completeness.  In `ReadFlattenSteps` mode, ADIOS
+loads all the metadata in the file upon Open (just like
+`ReadRandomAccess` mode, but everything that was written appears that
+it was output on the same step, regardless of how many steps actually
+appear in the file.  This affects the operation of many reader-side
+ADIOS functions, including Steps(), BlocksInfo(), Get(), etc.
+

python/adios2/stream.py

@@ -37,6 +37,9 @@ def string_to_mode(mode: str) -> [bindings.Mode, bool]:
     elif mode == "rra":
         bmode = bindings.Mode.ReadRandomAccess
         read_mode = True
+    elif mode == "rfs":
+        bmode = bindings.Mode.ReadFlattenSteps
+        read_mode = True
     elif mode == "w":
         bmode = bindings.Mode.Write
     elif mode == "a":

source/adios2/common/ADIOSTypes.h

@@ -80,6 +80,7 @@ enum class Mode
     Read,
     Append,
     ReadRandomAccess, // reader random access mode
+    ReadFlattenSteps, // reader flatten steps to one
     // launch execution modes
     Sync,
     Deferred

source/adios2/core/Engine.cpp

@@ -135,7 +135,8 @@ void Engine::Put(VariableStruct &variable, const void *data, const Mode launch)
 
 void Engine::Get(VariableStruct &variable, void *data, const Mode launch)
 {
-    CommonChecks(variable, data, {Mode::Read, Mode::ReadRandomAccess}, "in call to Get");
+    CommonChecks(variable, data, {Mode::Read, Mode::ReadRandomAccess, Mode::ReadFlattenSteps},
+                 "in call to Get");
 
     switch (launch)
     {

source/adios2/core/Engine.tcc

@@ -87,7 +87,8 @@ void Engine::Put(const std::string &variableName, const T &datum, const Mode /*l
 template <class T>
 void Engine::Get(Variable<T> &variable, T *data, const Mode launch)
 {
-    CommonChecks(variable, data, {Mode::Read, Mode::ReadRandomAccess}, "in call to Get");
+    CommonChecks(variable, data, {Mode::Read, Mode::ReadRandomAccess, Mode::ReadFlattenSteps},
+                 "in call to Get");
 
     switch (launch)
     {

source/adios2/core/IO.cpp

@@ -150,6 +150,14 @@ const std::unordered_map<std::string, bool> ReadRandomAccess_Supported = {
     {"campaign", true},
 };
 
+const std::unordered_map<std::string, bool> ReadFlattenSteps_Supported = {
+    {"bp3", false},     {"bp4", false},        {"bp5", true},      {"dataman", false},
+    {"ssc", false},     {"mhs", false},        {"sst", false},     {"daos", false},
+    {"effis", false},   {"dataspaces", false}, {"hdf5", false},    {"skeleton", false},
+    {"inline", false},  {"null", true},        {"nullcore", true}, {"plugin", false},
+    {"campaign", true},
+};
+
 // Synchronize access to the factory in case one thread is
 // looking up while another registers additional entries.
 std::mutex FactoryMutex;
@@ -560,7 +568,8 @@ Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm)
         {
             engineTypeLC = "campaign";
         }
-        else if ((mode_to_use == Mode::Read) || (mode_to_use == Mode::ReadRandomAccess))
+        else if ((mode_to_use == Mode::Read) || (mode_to_use == Mode::ReadRandomAccess) ||
+                 (mode_to_use == Mode::ReadFlattenSteps))
         {
             if (adios2sys::SystemTools::FileIsDirectory(name))
             {
@@ -668,10 +677,26 @@ Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm)
         }
     }
 
+    if (mode_to_use == Mode::ReadFlattenSteps)
+    {
+        // older engines don't know about ReadFlattenSteps Mode, throw an exception
+        auto it = ReadFlattenSteps_Supported.find(engineTypeLC);
+        if (it != ReadFlattenSteps_Supported.end())
+        {
+            if (!it->second)
+            {
+                helper::Throw<std::runtime_error>("Core", "IO", "Open",
+                                                  "Engine " + engineTypeLC +
+                                                      " doesn't support ReadFlattenSteps mode");
+            }
+        }
+    }
+
     auto f = FactoryLookup(engineTypeLC);
     if (f != Factory.end())
     {
-        if ((mode_to_use == Mode::Read) || (mode_to_use == Mode::ReadRandomAccess))
+        if ((mode_to_use == Mode::Read) || (mode_to_use == Mode::ReadRandomAccess) ||
+            (mode_to_use == Mode::ReadFlattenSteps))
         {
             engine = f->second.MakeReader(*this, name, mode_to_use, std::move(comm));
         }

source/adios2/core/Stream.cpp

@@ -23,7 +23,8 @@ Stream::Stream(const std::string &name, const Mode mode, helper::Comm comm,
 : m_ADIOS(std::make_shared<ADIOS>(std::move(comm), hostLanguage)), m_IO(&m_ADIOS->DeclareIO(name)),
   m_Name(name), m_Mode(mode), m_EngineType(engineType)
 {
-    if ((mode == adios2::Mode::Read) || (mode == adios2::Mode::ReadRandomAccess))
+    if ((mode == adios2::Mode::Read) || (mode == adios2::Mode::ReadRandomAccess) ||
+        (mode == adios2::Mode::ReadFlattenSteps))
     {
         CheckOpen();
     }
@@ -41,7 +42,8 @@ Stream::Stream(const std::string &name, const Mode mode, helper::Comm comm,
 : m_ADIOS(std::make_shared<ADIOS>(configFile, std::move(comm), hostLanguage)),
   m_IO(&m_ADIOS->DeclareIO(ioInConfigFile)), m_Name(name), m_Mode(mode)
 {
-    if ((mode == adios2::Mode::Read) || (mode == adios2::Mode::ReadRandomAccess))
+    if ((mode == adios2::Mode::Read) || (mode == adios2::Mode::ReadRandomAccess) ||
+        (mode == adios2::Mode::ReadFlattenSteps))
     {
         CheckOpen();
     }

source/adios2/engine/bp5/BP5Reader.cpp

@@ -72,7 +72,7 @@ void BP5Reader::InstallMetadataForTimestep(size_t Step)
         size_t ThisMDSize =
             helper::ReadValue<uint64_t>(m_Metadata.Data(), Position, m_Minifooter.IsLittleEndian);
         char *ThisMD = m_Metadata.Data() + MDPosition;
-        if (m_OpenMode == Mode::ReadRandomAccess)
+        if ((m_OpenMode == Mode::ReadRandomAccess) || (m_OpenMode == Mode::ReadFlattenSteps))
         {
             m_BP5Deserializer->InstallMetaData(ThisMD, ThisMDSize, WriterRank, Step);
         }
@@ -98,7 +98,7 @@ StepStatus BP5Reader::BeginStep(StepMode mode, const float timeoutSeconds)
 {
     PERFSTUBS_SCOPED_TIMER("BP5Reader::BeginStep");
 
-    if (m_OpenMode == Mode::ReadRandomAccess)
+    if (m_OpenMode != Mode::Read)
     {
         helper::Throw<std::logic_error>("Engine", "BP5Reader", "BeginStep",
                                         "BeginStep called in random access mode");
@@ -184,7 +184,7 @@ size_t BP5Reader::CurrentStep() const { return m_CurrentStep; }
 
 void BP5Reader::EndStep()
 {
-    if (m_OpenMode == Mode::ReadRandomAccess)
+    if (m_OpenMode != Mode::Read)
     {
         helper::Throw<std::logic_error>("Engine", "BP5Reader", "EndStep",
                                         "EndStep called in random access mode");
@@ -475,12 +475,14 @@ void BP5Reader::PerformLocalGets()
 // PRIVATE
 void BP5Reader::Init()
 {
-    if ((m_OpenMode != Mode::Read) && (m_OpenMode != Mode::ReadRandomAccess))
+    if ((m_OpenMode != Mode::Read) && (m_OpenMode != Mode::ReadRandomAccess) &&
+        (m_OpenMode != Mode::ReadFlattenSteps))
     {
-        helper::Throw<std::invalid_argument>("Engine", "BP5Reader", "Init",
-                                             "BPFileReader only supports OpenMode::Read or "
-                                             "OpenMode::ReadRandomAccess from" +
-                                                 m_Name);
+        helper::Throw<std::invalid_argument>(
+            "Engine", "BP5Reader", "Init",
+            "BPFileReader only supports OpenMode::Read, "
+            "OpenMode::ReadRandomAccess, or OpenMode::ReadFlattenSteps from" +
+                m_Name);
     }
 
     // if IO was involved in reading before this flag may be true now
@@ -519,7 +521,7 @@ void BP5Reader::InitParameters()
     ParseParams(m_IO, m_Parameters);
     if (m_Parameters.OpenTimeoutSecs < 0.0f)
     {
-        if (m_OpenMode == Mode::ReadRandomAccess)
+        if ((m_OpenMode == Mode::ReadRandomAccess) || (m_OpenMode == Mode::ReadFlattenSteps))
         {
             m_Parameters.OpenTimeoutSecs = 0.0f;
         }
@@ -808,7 +810,8 @@ void BP5Reader::UpdateBuffer(const TimePoint &timeoutInstant, const Seconds &pol
         if (!m_BP5Deserializer)
         {
             m_BP5Deserializer = new format::BP5Deserializer(m_WriterIsRowMajor, m_ReaderIsRowMajor,
-                                                            (m_OpenMode == Mode::ReadRandomAccess));
+                                                            (m_OpenMode != Mode::Read),
+                                                            (m_OpenMode == Mode::ReadFlattenSteps));
             m_BP5Deserializer->m_Engine = this;
         }
     }
@@ -905,7 +908,7 @@ void BP5Reader::UpdateBuffer(const TimePoint &timeoutInstant, const Seconds &pol
 
         m_Comm.Bcast(m_Metadata.Data(), inputSize, 0);
 
-        if (m_OpenMode == Mode::ReadRandomAccess)
+        if ((m_OpenMode == Mode::ReadRandomAccess) || (m_OpenMode == Mode::ReadFlattenSteps))
         {
             for (size_t Step = 0; Step < m_MetadataIndexTable.size(); Step++)
             {
@@ -1250,7 +1253,7 @@ void BP5Reader::DoGetStructDeferred(VariableStruct &variable, void *data)
 void BP5Reader::DoClose(const int transportIndex)
 {
     PERFSTUBS_SCOPED_TIMER("BP5Reader::Close");
-    if (m_OpenMode == Mode::ReadRandomAccess)
+    if ((m_OpenMode == Mode::ReadRandomAccess) || (m_OpenMode == Mode::ReadFlattenSteps))
     {
         PerformGets();
     }
@@ -1311,7 +1314,13 @@ void BP5Reader::FlushProfiler()
     }
 }
 
-size_t BP5Reader::DoSteps() const { return m_StepsCount; }
+size_t BP5Reader::DoSteps() const
+{
+    if (m_OpenMode == Mode::ReadFlattenSteps)
+        return 1;
+    else
+        return m_StepsCount;
+}
 
 void BP5Reader::NotifyEngineNoVarsQuery()
 {