Merge pull request #2972 from JasonRuonanWang/operator

Unified operator parameters, and centralized all operator logics into the Operator framework

bindings/CXX11/adios2/cxx11/Operator.cpp

@@ -46,9 +46,4 @@ Params Operator::Parameters() const
 // PRIVATE
 Operator::Operator(core::Operator *op) : m_Operator(op) {}
 
-std::string ToString(const Operator &op)
-{
-    return std::string("Operator(Type: \"" + op.Type() + "\")");
-}
-
 } // end namespace adios2

bindings/CXX11/adios2/cxx11/Operator.h

@@ -78,8 +78,6 @@ class Operator
     core::Operator *m_Operator = nullptr;
 };
 
-std::string ToString(const IO &io);
-
 } // end namespace adios2
 
 #endif /* ADIOS2_BINDINGS_CXX11_CXX11_OPERATOR_H_ */

bindings/CXX11/adios2/cxx11/Variable.cpp

@@ -172,17 +172,23 @@ namespace adios2
     }                                                                          \
                                                                                \
     template <>                                                                \
-    std::vector<typename Variable<T>::Operation> Variable<T>::Operations()     \
-        const                                                                  \
+    size_t Variable<T>::AddOperation(const std::string &type,                  \
+                                     const Params &parameters)                 \
+    {                                                                          \
+        return m_Variable->AddOperation(type, parameters);                     \
+    }                                                                          \
+                                                                               \
+    template <>                                                                \
+    std::vector<Operator> Variable<T>::Operations() const                      \
     {                                                                          \
         helper::CheckForNullptr(m_Variable,                                    \
                                 "in call to Variable<T>::Operations");         \
-        std::vector<Operation> operations;                                     \
+        std::vector<Operator> operations;                                      \
         operations.reserve(m_Variable->m_Operations.size());                   \
                                                                                \
         for (const auto &op : m_Variable->m_Operations)                        \
         {                                                                      \
-            operations.push_back(Operation{Operator(op.Op), op.Parameters});   \
+            operations.push_back(Operator(op));                                \
         }                                                                      \
         return operations;                                                     \
     }                                                                          \

bindings/CXX11/adios2/cxx11/Variable.h

@@ -275,20 +275,6 @@ class Variable
      */
     size_t BlockID() const;
 
-    /**
-     * EXPERIMENTAL: carries information about an Operation added with
-     * AddOperation
-     */
-    struct Operation
-    {
-        /** Operator associated with this operation */
-        const adios2::Operator Op;
-        /** Parameters settings for this operation */
-        const adios2::Params Parameters;
-        /** Information associated with this operation */
-        adios2::Params Info;
-    };
-
     /**
      *Adds operation and parameters to current Variable object
      * @param op operator to be added
@@ -299,11 +285,14 @@ class Variable
     size_t AddOperation(const Operator op,
                         const adios2::Params &parameters = adios2::Params());
 
+    size_t AddOperation(const std::string &type,
+                        const adios2::Params &parameters = adios2::Params());
+
     /**
      * Inspects current operators added with AddOperator
      * @return vector of Variable<T>::OperatorInfo
      */
-    std::vector<Operation> Operations() const;
+    std::vector<Operator> Operations() const;
 
     /**
      * Removes all current Operations associated with AddOperation.

bindings/Python/py11Variable.cpp

@@ -83,15 +83,15 @@ size_t Variable::AddOperation(const Operator op, const Params &parameters)
     return m_VariableBase->AddOperation(*op.m_Operator, parameters);
 }
 
-std::vector<Variable::Operation> Variable::Operations() const
+std::vector<Operator> Variable::Operations() const
 {
     helper::CheckForNullptr(m_VariableBase, "in call to Variable::Operations");
-    std::vector<Variable::Operation> operations;
+    std::vector<Operator> operations;
     operations.reserve(m_VariableBase->m_Operations.size());
 
     for (const auto &op : m_VariableBase->m_Operations)
     {
-        operations.push_back(Operation{Operator(op.Op), op.Parameters});
+        operations.push_back(Operator(op));
     }
     return operations;
 }

bindings/Python/py11Variable.h

@@ -95,17 +95,6 @@ class Variable
 
     size_t BlockID() const;
 
-    /**
-     * EXPERIMENTAL: carries information about an Operation added with
-     * AddOperation
-     */
-    struct Operation
-    {
-        const Operator Op;
-        const Params Parameters;
-        const Params Info;
-    };
-
     /**
      * EXPERIMENTAL: Adds operation and parameters to current Variable object
      * @param op operator to be added
@@ -119,7 +108,7 @@ class Variable
      * EXPERIMENTAL: inspects current operators added with AddOperator
      * @return vector of Variable<T>::OperatorInfo
      */
-    std::vector<Operation> Operations() const;
+    std::vector<Operator> Operations() const;
 
     /** Contains sub-block information for a particular Variable<T> */
     struct Info

source/adios2/CMakeLists.txt

@@ -18,9 +18,11 @@ add_library(adios2_core
   core/VariableCompound.cpp core/VariableCompound.tcc
   core/Group.cpp core/Group.tcc
 
-#operator callback
+#operator
   operator/callback/Signature1.cpp
   operator/callback/Signature2.cpp
+  operator/OperatorFactory.cpp
+  operator/compress/CompressNull.cpp
 
 #helper
   helper/adiosComm.h  helper/adiosComm.cpp
@@ -93,9 +95,6 @@ add_library(adios2_core
 
   toolkit/burstbuffer/FileDrainer.cpp
   toolkit/burstbuffer/FileDrainerSingleThread.cpp
-
-  operator/compress/CompressorFactory.cpp
-  operator/compress/CompressNull.cpp
 )
 set_property(TARGET adios2_core PROPERTY EXPORT_NAME core)
 set_property(TARGET adios2_core PROPERTY OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_core)

source/adios2/common/ADIOSMacros.h

@@ -121,36 +121,6 @@
     MACRO(std::complex<float>)                                                 \
     MACRO(std::complex<double>)
 
-#define ADIOS2_FOREACH_ZFP_TYPE_1ARG(MACRO)                                    \
-    MACRO(int32_t)                                                             \
-    MACRO(int64_t)                                                             \
-    MACRO(float)                                                               \
-    MACRO(double)                                                              \
-    MACRO(std::complex<float>)                                                 \
-    MACRO(std::complex<double>)
-
-#define ADIOS2_FOREACH_SZ_TYPE_1ARG(MACRO)                                     \
-    MACRO(float)                                                               \
-    MACRO(double)                                                              \
-    MACRO(std::complex<float>)                                                 \
-    MACRO(std::complex<double>)
-
-#define ADIOS2_FOREACH_SIRIUS_TYPE_1ARG(MACRO) MACRO(float)
-
-#define ADIOS2_FOREACH_LIBPRESSIO_TYPE_1ARG(MACRO)                             \
-    MACRO(uint8_t)                                                             \
-    MACRO(uint16_t)                                                            \
-    MACRO(uint32_t)                                                            \
-    MACRO(uint64_t)                                                            \
-    MACRO(int8_t)                                                              \
-    MACRO(int16_t)                                                             \
-    MACRO(int32_t)                                                             \
-    MACRO(int64_t)                                                             \
-    MACRO(float)                                                               \
-    MACRO(double)
-
-#define ADIOS2_FOREACH_MGARD_TYPE_1ARG(MACRO) MACRO(double)
-
 #define ADIOS2_FOREACH_ATTRIBUTE_TYPE_1ARG(MACRO)                              \
     MACRO(std::string)                                                         \
     MACRO(char)                                                                \

source/adios2/core/ADIOS.cpp

@@ -18,45 +18,11 @@
 #include "adios2/core/IO.h"
 #include "adios2/helper/adiosCommDummy.h"
 #include "adios2/helper/adiosFunctions.h" //InquireKey, BroadcastFile
+#include "adios2/operator/OperatorFactory.h"
 #include <adios2sys/SystemTools.hxx>
 
 #include <adios2-perfstubs-interface.h>
 
-// OPERATORS
-
-// compress
-#ifdef ADIOS2_HAVE_BZIP2
-#include "adios2/operator/compress/CompressBZIP2.h"
-#endif
-
-#ifdef ADIOS2_HAVE_ZFP
-#include "adios2/operator/compress/CompressZFP.h"
-#endif
-
-#ifdef ADIOS2_HAVE_SZ
-#include "adios2/operator/compress/CompressSZ.h"
-#endif
-
-#ifdef ADIOS2_HAVE_MGARD
-#include "adios2/operator/compress/CompressMGARD.h"
-#endif
-
-#ifdef ADIOS2_HAVE_BZIP2
-#include "adios2/operator/compress/CompressBZIP2.h"
-#endif
-
-#ifdef ADIOS2_HAVE_PNG
-#include "adios2/operator/compress/CompressPNG.h"
-#endif
-
-#ifdef ADIOS2_HAVE_BLOSC
-#include "adios2/operator/compress/CompressBlosc.h"
-#endif
-
-#ifdef ADIOS2_HAVE_LIBPRESSIO
-#include "adios2/operator/compress/CompressLibPressio.h"
-#endif
-
 // callbacks
 #include "adios2/operator/callback/Signature1.h"
 #include "adios2/operator/callback/Signature2.h"
@@ -188,102 +154,9 @@ void ADIOS::FlushAll()
 Operator &ADIOS::DefineOperator(const std::string &name, const std::string type,
                                 const Params &parameters)
 {
-    auto lf_ErrorMessage = [](const std::string type) -> std::string {
-        return "ERROR: this version of ADIOS2 didn't compile with the " + type +
-               " library, when parsing config file in ADIOS constructor or in "
-               "call to ADIOS::DefineOperator";
-    };
-
-    std::shared_ptr<Operator> operatorPtr;
-
     CheckOperator(name);
-    const std::string typeLowerCase = helper::LowerCase(type);
-
-    if (typeLowerCase == "bzip2")
-    {
-#ifdef ADIOS2_HAVE_BZIP2
-        auto itPair = m_Operators.emplace(
-            name, std::make_shared<compress::CompressBZIP2>(parameters));
-        operatorPtr = itPair.first->second;
-#else
-        throw std::invalid_argument(lf_ErrorMessage("BZip2"));
-#endif
-    }
-    else if (typeLowerCase == "zfp")
-    {
-#ifdef ADIOS2_HAVE_ZFP
-        auto itPair = m_Operators.emplace(
-            name, std::make_shared<compress::CompressZFP>(parameters));
-        operatorPtr = itPair.first->second;
-#else
-        throw std::invalid_argument(lf_ErrorMessage("ZFP"));
-#endif
-    }
-    else if (typeLowerCase == "sz")
-    {
-#ifdef ADIOS2_HAVE_SZ
-        auto itPair = m_Operators.emplace(
-            name, std::make_shared<compress::CompressSZ>(parameters));
-        operatorPtr = itPair.first->second;
-#else
-        throw std::invalid_argument(lf_ErrorMessage("SZ"));
-#endif
-    }
-    else if (typeLowerCase == "mgard")
-    {
-#ifdef ADIOS2_HAVE_MGARD
-        auto itPair = m_Operators.emplace(
-            name, std::make_shared<compress::CompressMGARD>(parameters));
-        operatorPtr = itPair.first->second;
-#else
-        throw std::invalid_argument(lf_ErrorMessage("MGARD"));
-#endif
-    }
-    else if (typeLowerCase == "png")
-    {
-#ifdef ADIOS2_HAVE_PNG
-        auto itPair = m_Operators.emplace(
-            name, std::make_shared<compress::CompressPNG>(parameters));
-        operatorPtr = itPair.first->second;
-#else
-        throw std::invalid_argument(lf_ErrorMessage("PNG"));
-#endif
-    }
-    else if (typeLowerCase == "blosc")
-    {
-#ifdef ADIOS2_HAVE_BLOSC
-        auto itPair = m_Operators.emplace(
-            name, std::make_shared<compress::CompressBlosc>(parameters));
-        operatorPtr = itPair.first->second;
-#else
-        throw std::invalid_argument(lf_ErrorMessage("Blosc"));
-#endif
-    }
-    else if (typeLowerCase == "libpressio")
-    {
-#ifdef ADIOS2_HAVE_LIBPRESSIO
-        auto itPair = m_Operators.emplace(
-            name, std::make_shared<compress::CompressLibPressio>(parameters));
-        operatorPtr = itPair.first->second;
-#else
-        throw std::invalid_argument(lf_ErrorMessage("LibPressio"));
-#endif
-    }
-    else
-    {
-        throw std::invalid_argument(
-            "ERROR: Operator " + name + " of type " + type +
-            " is not supported by ADIOS2, in call to DefineOperator\n");
-    }
-
-    if (!operatorPtr)
-    {
-        throw std::invalid_argument(
-            "ERROR: Operator " + name + " of type " + type +
-            " couldn't be defined, in call to DefineOperator\n");
-    }
-
-    return *operatorPtr.get();
+    auto itPair = m_Operators.emplace(name, MakeOperator(type, parameters));
+    return *itPair.first->second.get();
 }
 
 Operator *ADIOS::InquireOperator(const std::string &name) noexcept

source/adios2/core/Operator.h

@@ -76,7 +76,7 @@ class Operator
      */
     virtual size_t Operate(const char *dataIn, const Dims &blockStart,
                            const Dims &blockCount, const DataType type,
-                           char *bufferOut, const Params &parameters) = 0;
+                           char *bufferOut) = 0;
 
     /**
      * @param bufferIn

source/adios2/core/Variable.h

@@ -95,7 +95,7 @@ class Variable : public VariableBase
         Dims Count;
         Dims MemoryStart;
         Dims MemoryCount;
-        std::vector<Operation> Operations;
+        std::vector<Operator *> Operations;
         size_t Step = 0;
         size_t StepsStart = 0;
         size_t StepsCount = 0;