Open file with in-memory metadata

bindings/C/adios2/c/adios2_c_engine.cpp

@@ -199,6 +199,27 @@ adios2_error adios2_engine_openmode(adios2_mode *mode,
     }
 }
 
+adios2_error adios2_engine_get_metadata(adios2_engine *engine, char **md,
+                                        size_t *size)
+{
+    try
+    {
+        adios2::helper::CheckForNullptr(
+            engine, "for const adios2_engine, in call to adios2_get_metadata");
+
+        adios2::core::Engine *engineCpp =
+            reinterpret_cast<adios2::core::Engine *>(engine);
+
+        engineCpp->GetMetadata(md, size);
+        return adios2_error_none;
+    }
+    catch (...)
+    {
+        return static_cast<adios2_error>(
+            adios2::helper::ExceptionToError("adios2_get_metadata"));
+    }
+}
+
 adios2_error adios2_begin_step(adios2_engine *engine,
                                const adios2_step_mode mode,
                                const float timeout_seconds,

bindings/C/adios2/c/adios2_c_engine.h

@@ -56,6 +56,15 @@ adios2_error adios2_engine_get_type(char *type, size_t *size,
 adios2_error adios2_engine_openmode(adios2_mode *mode,
                                     const adios2_engine *engine);
 
+/** Serialize all metadata right after engine is created, which can be
+ * delivered to other processes to open the same file for reading without
+ * opening and reading in metadata again.
+ * @return metadata (pointer to allocated memory) and size of metadata
+ * the pointer must be deallocated by user using free()
+ */
+adios2_error adios2_engine_get_metadata(adios2_engine *engine, char **md,
+                                        size_t *size);
+
 /**
  * @brief Begin a logical adios2 step stream
  * Check each engine documentation for MPI collective/non-collective

bindings/C/adios2/c/adios2_c_io.cpp

@@ -947,6 +947,24 @@ adios2_engine *adios2_open(adios2_io *io, const char *name,
     return engine;
 }
 
+adios2_engine *adios2_open_with_metadata(adios2_io *io, const char *name,
+                                         const char *md, const size_t mdsize)
+{
+    adios2_engine *engine = nullptr;
+    try
+    {
+        adios2::helper::CheckForNullptr(
+            io, "for adios2_io, in call to adios2_open_with_metadata");
+        engine = reinterpret_cast<adios2_engine *>(
+            &reinterpret_cast<adios2::core::IO *>(io)->Open(name, md, mdsize));
+    }
+    catch (...)
+    {
+        adios2::helper::ExceptionToError("adios2_open_with_metadata");
+    }
+    return engine;
+}
+
 adios2_error adios2_flush_all_engines(adios2_io *io)
 {
     try

bindings/C/adios2/c/adios2_c_io.h

@@ -325,6 +325,20 @@ adios2_error adios2_remove_all_attributes(adios2_io *io);
 adios2_engine *adios2_open(adios2_io *io, const char *name,
                            const adios2_mode mode);
 
+/**
+ * Open an Engine to start heavy-weight input/output operations.
+ * This function is for opening a file (not stream) with ReadRandomAccess mode
+ * and supplying the metadata already in memory. The metadata should be
+ * retrieved by another program calling adios2_engine_get_metadata() after
+ * opening the file.
+ * @param io engine owner
+ * @param name unique engine identifier
+ * @param md file metadata residing in memory
+ * @return success: handler, failure: NULL
+ */
+adios2_engine *adios2_open_with_metadata(adios2_io *io, const char *name,
+                                         const char *md, const size_t mdsize);
+
 #if ADIOS2_USE_MPI
 /**
  * Open an Engine to start heavy-weight input/output operations.

bindings/CXX11/adios2/cxx11/Engine.cpp

@@ -54,6 +54,12 @@ Mode Engine::OpenMode() const
     return m_Engine->OpenMode();
 }
 
+void Engine::GetMetadata(char **md, size_t *size) const
+{
+    helper::CheckForNullptr(m_Engine, "in call to Engine::GetMetadata");
+    m_Engine->GetMetadata(md, size);
+}
+
 StepStatus Engine::BeginStep()
 {
     helper::CheckForNullptr(m_Engine, "in call to Engine::BeginStep");

bindings/CXX11/adios2/cxx11/Engine.h

@@ -75,6 +75,14 @@ class Engine
      */
     Mode OpenMode() const;
 
+    /** Serialize all metadata right after engine is created, which can be
+     * delivered to other processes to open the same file for reading without
+     * opening and reading in metadata again.
+     * @return metadata (pointer to allocated memory) and size of metadata
+     * the pointer must be deallocated by user using free()
+     */
+    void GetMetadata(char **md, size_t *size) const;
+
     /**
      * Begin a logical adios2 step, overloaded version with timeoutSeconds = 0
      * and mode = Read

bindings/CXX11/adios2/cxx11/IO.cpp

@@ -109,6 +109,14 @@ Engine IO::Open(const std::string &name, const Mode mode)
                             "for engine " + name + ", in call to IO::Open");
     return Engine(&m_IO->Open(name, mode));
 }
+
+Engine IO::Open(const std::string &name, const char *md, const size_t mdsize)
+{
+    helper::CheckForNullptr(m_IO,
+                            "for engine " + name + ", in call to IO::Open");
+    return Engine(&m_IO->Open(name, md, mdsize));
+}
+
 Group IO::InquireGroup(char delimiter)
 {
     return Group(&m_IO->CreateGroup(delimiter));

bindings/CXX11/adios2/cxx11/IO.h

@@ -307,6 +307,21 @@ class IO
     Engine Open(const std::string &name, const Mode mode, MPI_Comm comm);
 #endif
 
+    /**
+     * Overloaded version that is specifically for a serial program
+     * opening a file (not stream) with ReadRandomAccess mode and
+     * supplying the metadata already in memory. The metadata
+     * should be retrieved by another program calling engine.GetMetadata()
+     * after opening the file.
+     * @param name unique engine identifier within IO object
+     * (file name in case of File transports)
+     * @param md file metadata residing in memory
+     * @return a reference to a derived object of the Engine class
+     * @exception std::invalid_argument if Engine with unique name is already
+     * created with another Open
+     */
+    Engine Open(const std::string &name, const char *md, const size_t mdsize);
+
     /** Flushes all engines created with this IO with the Open function */
     void FlushAll();
 

examples/useCases/CMakeLists.txt

@@ -4,4 +4,5 @@
 #------------------------------------------------------------------------------#
 
 add_subdirectory(insituGlobalArrays)
+add_subdirectory(ensembleRead)
 

examples/useCases/ensembleRead/CMakeLists.txt

@@ -0,0 +1,9 @@
+#------------------------------------------------------------------------------#
+# Distributed under the OSI-approved Apache License, Version 2.0.  See
+# accompanying file Copyright.txt for details.
+#------------------------------------------------------------------------------#
+
+if(ADIOS2_HAVE_MPI)
+  add_executable(ensembleRead ensembleRead.cpp)
+  target_link_libraries(ensembleRead adios2::cxx11_mpi MPI::MPI_C)
+endif()