Merge pull request #3198 from JasonRuonanWang/api

added BlocksInfo API for VariableNT, and implementation for VariableStruct in SSC
ornladios · May 3, 2022 · 445f4ed · 445f4ed
2 parents add7feb + 7a1096c
commit 445f4ed
Show file tree

Hide file tree

Showing 15 changed files with 294 additions and 16 deletions.
diff --git a/bindings/CXX11/adios2/cxx11/Engine.cpp b/bindings/CXX11/adios2/cxx11/Engine.cpp
@@ -223,6 +223,121 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
 ADIOS2_FOREACH_PRIMITIVE_TYPE_1ARG(declare_template_instantiation)
 #undef declare_template_instantiation
 
+std::vector<VariableNT::Info> Engine::BlocksInfo(const VariableNT &variable,
+                                                 const size_t step) const
+{
+    std::vector<VariableNT::Info> ret;
+    if (variable.m_Variable->m_Type == DataType::Struct)
+    {
+        adios2::helper::CheckForNullptr(
+            m_Engine, "for Engine in call to Engine::BlocksInfo");
+        adios2::helper::CheckForNullptr(
+            variable.m_Variable, "for variable in call to Engine::BlocksInfo");
+        auto blocksInfo = m_Engine->BlocksInfoStruct(
+            *reinterpret_cast<core::VariableStruct *>(variable.m_Variable),
+            step);
+        for (const auto &b : blocksInfo)
+        {
+            ret.emplace_back();
+            auto &br = ret.back();
+            br.Start = b.Start;
+            br.Count = b.Count;
+            br.WriterID = b.WriterID;
+            br.Step = b.Step;
+            br.IsReverseDims = b.IsReverseDims;
+            br.IsValue = b.IsValue;
+        }
+    }
+#define declare_type(T)                                                        \
+    else if (variable.m_Variable->m_Type == helper::GetDataType<T>())          \
+    {                                                                          \
+        auto blocksInfoT =                                                     \
+            BlocksInfo(Variable<T>(reinterpret_cast<core::Variable<T> *>(      \
+                           variable.m_Variable)),                              \
+                       step);                                                  \
+        for (const auto &b : blocksInfoT)                                      \
+        {                                                                      \
+            ret.emplace_back();                                                \
+            auto &br = ret.back();                                             \
+            br.Start = b.Start;                                                \
+            br.Count = b.Count;                                                \
+            br.WriterID = b.WriterID;                                          \
+            br.Step = b.Step;                                                  \
+            br.IsReverseDims = b.IsReverseDims;                                \
+            br.IsValue = b.IsValue;                                            \
+        }                                                                      \
+    }
+    ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
+#undef declare_type
+    else
+    {
+        helper::Throw<std::runtime_error>("bindings::CXX11", "Engine",
+                                          "BlocksInfo", "invalid data type");
+    }
+    return ret;
+}
+
+std::map<size_t, std::vector<VariableNT::Info>>
+Engine::AllStepsBlocksInfo(const VariableNT &variable) const
+{
+    std::map<size_t, std::vector<VariableNT::Info>> ret;
+    if (variable.m_Variable->m_Type == DataType::Struct)
+    {
+        adios2::helper::CheckForNullptr(
+            m_Engine, "for Engine in call to Engine::AllStepsBlocksInfo");
+        adios2::helper::CheckForNullptr(
+            variable.m_Variable,
+            "for variable in call to Engine::AllStepsBlocksInfo");
+        auto blocksInfo = m_Engine->AllStepsBlocksInfoStruct(
+            *reinterpret_cast<core::VariableStruct *>(variable.m_Variable));
+        for (const auto &bv : blocksInfo)
+        {
+            auto &bvr = ret[bv.first];
+            for (const auto &b : bv.second)
+            {
+                bvr.emplace_back();
+                auto &br = bvr.back();
+                br.Start = b.Start;
+                br.Count = b.Count;
+                br.WriterID = b.WriterID;
+                br.Step = b.Step;
+                br.IsReverseDims = b.IsReverseDims;
+                br.IsValue = b.IsValue;
+            }
+        }
+    }
+#define declare_type(T)                                                        \
+    else if (variable.m_Variable->m_Type == helper::GetDataType<T>())          \
+    {                                                                          \
+        auto blocksInfoT = AllStepsBlocksInfo(Variable<T>(                     \
+            reinterpret_cast<core::Variable<T> *>(variable.m_Variable)));      \
+        for (const auto &bv : blocksInfoT)                                     \
+        {                                                                      \
+            auto &bvr = ret[bv.first];                                         \
+            for (const auto &b : bv.second)                                    \
+            {                                                                  \
+                bvr.emplace_back();                                            \
+                auto &br = bvr.back();                                         \
+                br.Start = b.Start;                                            \
+                br.Count = b.Count;                                            \
+                br.WriterID = b.WriterID;                                      \
+                br.Step = b.Step;                                              \
+                br.IsReverseDims = b.IsReverseDims;                            \
+                br.IsValue = b.IsValue;                                        \
+            }                                                                  \
+        }                                                                      \
+    }
+    ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
+#undef declare_type
+    else
+    {
+        helper::Throw<std::runtime_error>("bindings::CXX11", "Engine",
+                                          "AllStepsBlocksInfo",
+                                          "invalid data type");
+    }
+    return ret;
+}
+
 #define declare_template_instantiation(T)                                      \
     template void Engine::Put<T>(Variable<T>, const T *, const Mode);          \
     template void Engine::Put<T>(const std::string &, const T *, const Mode);  \

diff --git a/bindings/CXX11/adios2/cxx11/Engine.h b/bindings/CXX11/adios2/cxx11/Engine.h
@@ -430,6 +430,9 @@ class Engine
     std::map<size_t, std::vector<typename Variable<T>::Info>>
     AllStepsBlocksInfo(const Variable<T> variable) const;
 
+    std::map<size_t, std::vector<VariableNT::Info>>
+    AllStepsBlocksInfo(const VariableNT &variable) const;
+
     /**
      * Extracts all available blocks information for a particular
      * variable and step.
@@ -443,6 +446,9 @@ class Engine
     std::vector<typename Variable<T>::Info>
     BlocksInfo(const Variable<T> variable, const size_t step) const;
 
+    std::vector<VariableNT::Info> BlocksInfo(const VariableNT &variable,
+                                             const size_t step) const;
+
     /**
      * Get the absolute steps of a variable in a file. This is for
      * information purposes only, because absolute steps cannot be used

diff --git a/bindings/CXX11/adios2/cxx11/VariableNT.h b/bindings/CXX11/adios2/cxx11/VariableNT.h
@@ -230,6 +230,17 @@ class VariableNT
         std::complex<double> DComplex;
     };
 
+    struct Info
+    {
+        adios2::Dims Start;
+        adios2::Dims Count;
+        int WriterID = 0;
+        size_t BlockID = 0;
+        size_t Step = 0;
+        bool IsReverseDims = false;
+        bool IsValue = false;
+    };
+
     T Min(const size_t step = adios2::DefaultSizeT) const;
     T Max(const size_t step = adios2::DefaultSizeT) const;
     std::pair<T, T> MinMax(const size_t step = adios2::DefaultSizeT) const;

diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp
@@ -237,6 +237,28 @@ void Engine::DoGetStructDeferred(VariableStruct &, void *)
 ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
 #undef declare_type
 
+std::map<size_t, std::vector<VariableStruct::BPInfo>>
+Engine::DoAllStepsBlocksInfoStruct(const VariableStruct &variable) const
+{
+    ThrowUp("DoAllStepsBlocksInfo");
+    return std::map<size_t, std::vector<VariableStruct::BPInfo>>();
+}
+
+std::vector<std::vector<VariableStruct::BPInfo>>
+Engine::DoAllRelativeStepsBlocksInfoStruct(const VariableStruct &variable) const
+{
+    ThrowUp("DoAllRelativeStepsBlocksInfo");
+    return std::vector<std::vector<VariableStruct::BPInfo>>();
+}
+
+std::vector<VariableStruct::BPInfo>
+Engine::DoBlocksInfoStruct(const VariableStruct &variable,
+                           const size_t step) const
+{
+    ThrowUp("DoBlocksInfo");
+    return std::vector<VariableStruct::BPInfo>();
+}
+
 #define declare_type(T, L)                                                     \
     T *Engine::DoBufferData_##L(const int bufferIdx,                           \
                                 const size_t payloadPosition,                  \
@@ -290,6 +312,25 @@ void Engine::CommonChecks(VariableBase &variable, const void *data,
     }
 }
 
+std::map<size_t, std::vector<VariableStruct::BPInfo>>
+Engine::AllStepsBlocksInfoStruct(const VariableStruct &variable) const
+{
+    return DoAllStepsBlocksInfoStruct(variable);
+}
+
+std::vector<std::vector<VariableStruct::BPInfo>>
+Engine::AllRelativeStepsBlocksInfoStruct(const VariableStruct &variable) const
+{
+    return DoAllRelativeStepsBlocksInfoStruct(variable);
+}
+
+std::vector<VariableStruct::BPInfo>
+Engine::BlocksInfoStruct(const VariableStruct &variable,
+                         const size_t step) const
+{
+    return DoBlocksInfoStruct(variable, step);
+}
+
 // PUBLIC TEMPLATE FUNCTIONS EXPANSION WITH SCOPED TYPES
 #define declare_template_instantiation(T)                                      \
                                                                                \

diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h
@@ -30,6 +30,7 @@
 #include "adios2/common/ADIOSTypes.h"
 #include "adios2/core/IO.h"
 #include "adios2/core/Variable.h"
+#include "adios2/core/VariableStruct.h"
 #include "adios2/helper/adiosComm.h"
 
 namespace adios2
@@ -404,6 +405,9 @@ class Engine
     std::map<size_t, std::vector<typename Variable<T>::BPInfo>>
     AllStepsBlocksInfo(const Variable<T> &variable) const;
 
+    std::map<size_t, std::vector<VariableStruct::BPInfo>>
+    AllStepsBlocksInfoStruct(const VariableStruct &variable) const;
+
     /**
      * This function is internal, for public interface use
      * Variable<T>::AllStepsBlocksInfo
@@ -414,6 +418,9 @@ class Engine
     std::vector<std::vector<typename Variable<T>::BPInfo>>
     AllRelativeStepsBlocksInfo(const Variable<T> &variable) const;
 
+    std::vector<std::vector<VariableStruct::BPInfo>>
+    AllRelativeStepsBlocksInfoStruct(const VariableStruct &variable) const;
+
     /**
      * Extracts all available blocks information for a particular
      * variable and step.
@@ -427,6 +434,9 @@ class Engine
     std::vector<typename Variable<T>::BPInfo>
     BlocksInfo(const Variable<T> &variable, const size_t step) const;
 
+    std::vector<VariableStruct::BPInfo>
+    BlocksInfoStruct(const VariableStruct &variable, const size_t step) const;
+
     /**
      * Get the absolute steps of a variable in a file. This is for
      * information purposes only, because absolute steps cannot be used
@@ -572,6 +582,15 @@ class Engine
     ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
 #undef declare_type
 
+    virtual std::map<size_t, std::vector<VariableStruct::BPInfo>>
+    DoAllStepsBlocksInfoStruct(const VariableStruct &variable) const;
+
+    virtual std::vector<std::vector<VariableStruct::BPInfo>>
+    DoAllRelativeStepsBlocksInfoStruct(const VariableStruct &variable) const;
+
+    virtual std::vector<VariableStruct::BPInfo>
+    DoBlocksInfoStruct(const VariableStruct &variable, const size_t step) const;
+
 #define declare_type(T, L)                                                     \
     virtual T *DoBufferData_##L(const int bufferIdx,                           \
                                 const size_t payloadPosition,                  \

diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp
@@ -121,6 +121,13 @@ void SscReader::DoClose(const int transportIndex)
 ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
 #undef declare_type
 
+std::vector<VariableStruct::BPInfo>
+SscReader::DoBlocksInfoStruct(const VariableStruct &variable,
+                              const size_t step) const
+{
+    return m_EngineInstance->BlocksInfo(variable, step);
+}
+
 void SscReader::DoGetStructSync(VariableStruct &variable, void *data)
 {
     PERFSTUBS_SCOPED_TIMER_FUNC();

diff --git a/source/adios2/engine/ssc/SscReader.h b/source/adios2/engine/ssc/SscReader.h
@@ -45,6 +45,9 @@ class SscReader : public Engine
 
     void DoGetStructSync(VariableStruct &, void *) final;
     void DoGetStructDeferred(VariableStruct &, void *) final;
+    std::vector<VariableStruct::BPInfo>
+    DoBlocksInfoStruct(const VariableStruct &variable,
+                       const size_t step) const final;
 
     void DoClose(const int transportIndex = -1) final;
 

diff --git a/source/adios2/engine/ssc/SscReaderBase.h b/source/adios2/engine/ssc/SscReaderBase.h
@@ -46,6 +46,9 @@ class SscReaderBase
     ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
 #undef declare_type
 
+    virtual std::vector<VariableStruct::BPInfo>
+    BlocksInfo(const VariableStruct &variable, const size_t step) const = 0;
+
     virtual void GetDeferred(VariableBase &, void *) = 0;
 
 protected:

diff --git a/source/adios2/engine/ssc/SscReaderGeneric.cpp b/source/adios2/engine/ssc/SscReaderGeneric.cpp
@@ -570,6 +570,38 @@ void SscReaderGeneric::GetDeferred(VariableBase &variable, void *data)
     }
 }
 
+std::vector<VariableStruct::BPInfo>
+SscReaderGeneric::BlocksInfo(const VariableStruct &variable,
+                             const size_t step) const
+{
+    std::vector<VariableStruct::BPInfo> ret;
+    size_t blockID = 0;
+    for (int i = 0; i < static_cast<int>(m_GlobalWritePattern.size()); ++i)
+    {
+        for (auto &v : m_GlobalWritePattern[i])
+        {
+            if (v.name == variable.m_Name)
+            {
+                ret.emplace_back();
+                auto &b = ret.back();
+                b.Start = v.start;
+                b.Count = v.count;
+                b.Shape = v.shape;
+                b.Step = m_CurrentStep;
+                b.StepsStart = m_CurrentStep;
+                b.StepsCount = 1;
+                b.WriterID = i;
+                b.BlockID = blockID;
+                if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor)
+                {
+                    b.IsReverseDims = true;
+                }
+                ++blockID;
+            }
+        }
+    }
+    return ret;
+}
 }
 }
 }

diff --git a/source/adios2/engine/ssc/SscReaderGeneric.h b/source/adios2/engine/ssc/SscReaderGeneric.h
@@ -45,6 +45,9 @@ class SscReaderGeneric : public SscReaderBase
     ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
 #undef declare_type
 
+    std::vector<VariableStruct::BPInfo>
+    BlocksInfo(const VariableStruct &variable, const size_t step) const final;
+
     void GetDeferred(VariableBase &, void *) final;
 
 private:

diff --git a/source/adios2/engine/ssc/SscReaderGeneric.tcc b/source/adios2/engine/ssc/SscReaderGeneric.tcc
@@ -29,10 +29,10 @@ SscReaderGeneric::BlocksInfoCommon(const Variable<T> &variable,
                                    const size_t step) const
 {
     std::vector<typename Variable<T>::BPInfo> ret;
-
-    for (const auto &r : m_GlobalWritePattern)
+    size_t blockID = 0;
+    for (int i = 0; i < static_cast<int>(m_GlobalWritePattern.size()); ++i)
     {
-        for (auto &v : r)
+        for (auto &v : m_GlobalWritePattern[i])
         {
             if (v.name == variable.m_Name)
             {
@@ -44,11 +44,11 @@ SscReaderGeneric::BlocksInfoCommon(const Variable<T> &variable,
                 b.Step = m_CurrentStep;
                 b.StepsStart = m_CurrentStep;
                 b.StepsCount = 1;
+                b.WriterID = i;
+                b.BlockID = blockID;
                 if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor)
                 {
-                    std::reverse(b.Start.begin(), b.Start.end());
-                    std::reverse(b.Count.begin(), b.Count.end());
-                    std::reverse(b.Shape.begin(), b.Shape.end());
+                    b.IsReverseDims = true;
                 }
                 if (v.shapeId == ShapeID::GlobalValue ||
                     v.shapeId == ShapeID::LocalValue)
@@ -68,6 +68,7 @@ SscReaderGeneric::BlocksInfoCommon(const Variable<T> &variable,
                                     v.bufferCount);
                     }
                 }
+                ++blockID;
             }
         }
     }