added CompressMGARDPlus

source/adios2/CMakeLists.txt

@@ -228,8 +228,8 @@ if(ADIOS2_HAVE_DataSpaces)
   add_subdirectory(toolkit/dataspaces)
   target_sources(adios2_core_mpi PRIVATE
      toolkit/dataspaces/ds_writer.c
-     engine/dataspaces/DataSpacesWriter.cpp engine/dataspaces/DataSpacesWriter.tcc 
-     engine/dataspaces/DataSpacesReader.cpp engine/dataspaces/DataSpacesReader.tcc 
+     engine/dataspaces/DataSpacesWriter.cpp engine/dataspaces/DataSpacesWriter.tcc
+     engine/dataspaces/DataSpacesReader.cpp engine/dataspaces/DataSpacesReader.tcc
   )
   target_link_libraries(adios2_core_mpi PRIVATE DataSpaces::DataSpaces)
 endif()
@@ -260,6 +260,7 @@ if(ADIOS2_HAVE_LIBPRESSIO)
 endif()
 
 if(ADIOS2_HAVE_MGARD)
+  target_sources(adios2_core PRIVATE operator/compress/CompressMGARDPlus.cpp)
   target_sources(adios2_core PRIVATE operator/compress/CompressMGARD.cpp)
   target_link_libraries(adios2_core PRIVATE MGARD::MGARD)
 endif()

source/adios2/core/Operator.h

@@ -36,6 +36,7 @@ class Operator
         COMPRESS_SIRIUS = 5,
         COMPRESS_SZ = 6,
         COMPRESS_ZFP = 7,
+        COMPRESS_MGARDPLUS = 8,
         CALLBACK_SIGNATURE1 = 51,
         CALLBACK_SIGNATURE2 = 52,
         COMPRESS_NULL = 127,

source/adios2/operator/OperatorFactory.cpp

@@ -27,6 +27,7 @@
 
 #ifdef ADIOS2_HAVE_MGARD
 #include "adios2/operator/compress/CompressMGARD.h"
+#include "adios2/operator/compress/CompressMGARDPlus.h"
 #endif
 
 #ifdef ADIOS2_HAVE_PNG
@@ -62,6 +63,8 @@ std::string OperatorTypeToString(const Operator::OperatorType type)
         return "libpressio";
     case Operator::COMPRESS_MGARD:
         return "mgard";
+    case Operator::COMPRESS_MGARDPLUS:
+        return "mgardplus";
     case Operator::COMPRESS_PNG:
         return "png";
     case Operator::COMPRESS_SIRIUS:
@@ -104,6 +107,12 @@ std::shared_ptr<Operator> MakeOperator(const std::string &type,
     {
 #ifdef ADIOS2_HAVE_MGARD
         ret = std::make_shared<compress::CompressMGARD>(parameters);
+#endif
+    }
+    else if (typeLowerCase == "mgardplus")
+    {
+#ifdef ADIOS2_HAVE_MGARD
+        ret = std::make_shared<compress::CompressMGARDPlus>(parameters);
 #endif
     }
     else if (typeLowerCase == "png")

source/adios2/operator/compress/CompressMGARDPlus.cpp

@@ -0,0 +1,144 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * CompressMGARDPlus.cpp :
+ *
+ *  Created on: Feb 23, 2022
+ *      Author: Jason Wang [email protected]
+ */
+
+#include "CompressMGARDPlus.h"
+#include "CompressMGARD.h"
+#include "adios2/helper/adiosFunctions.h"
+
+namespace adios2
+{
+namespace core
+{
+namespace compress
+{
+
+CompressMGARDPlus::CompressMGARDPlus(const Params &parameters)
+: Operator("mgardplus", COMPRESS_MGARDPLUS, "compress", parameters)
+{
+}
+
+size_t CompressMGARDPlus::Operate(const char *dataIn, const Dims &blockStart,
+                                  const Dims &blockCount, const DataType type,
+                                  char *bufferOut)
+{
+    size_t bufferOutOffset = 0;
+    const uint8_t bufferVersion = 1;
+
+    MakeCommonHeader(bufferOut, bufferOutOffset, bufferVersion);
+
+    bufferOutOffset += 32; // TODO: reserve memory space
+
+    CompressMGARD mgard(m_Parameters);
+    size_t mgardBufferSize = mgard.Operate(dataIn, blockStart, blockCount, type,
+                                           bufferOut + bufferOutOffset);
+
+    if (*reinterpret_cast<OperatorType *>(bufferOut + bufferOutOffset) ==
+        COMPRESS_MGARD)
+    {
+        std::vector<char> tmpDecompressBuffer(
+            helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type)));
+
+        mgard.InverseOperate(bufferOut + bufferOutOffset, mgardBufferSize,
+                             tmpDecompressBuffer.data());
+
+        // TODO: now the original data is in dataIn, the compressed and then
+        // decompressed data is in tmpDecompressBuffer.data(). However, these
+        // are char pointers, you will need to convert them into right types as
+        // follows:
+        if (type == DataType::Double || type == DataType::DoubleComplex)
+        {
+            // TODO: use reinterpret_cast<double*>(dataIn) and
+            // reinterpret_cast<double*>(tmpDecompressBuffer.data())
+            // to read original data and decompressed data
+        }
+        else if (type == DataType::Float || type == DataType::FloatComplex)
+        {
+            // TODO: use reinterpret_cast<float*>(dataIn) and
+            // reinterpret_cast<double*>(tmpDecompressBuffer.data())
+            // to read original data and decompressed data
+        }
+        // TODO: after your algorithm is done, put the result into
+        // *reinterpret_cast<double*>(bufferOut+bufferOutOffset) for your first
+        // double number *reinterpret_cast<double*>(bufferOut+bufferOutOffset+8)
+        // for your second double number and so on
+    }
+
+    bufferOutOffset += mgardBufferSize;
+    return bufferOutOffset;
+}
+
+size_t CompressMGARDPlus::DecompressV1(const char *bufferIn,
+                                       const size_t sizeIn, char *dataOut)
+{
+    // Do NOT remove even if the buffer version is updated. Data might be still
+    // in lagacy formats. This function must be kept for backward compatibility.
+    // If a newer buffer format is implemented, create another function, e.g.
+    // DecompressV2 and keep this function for decompressing lagacy data.
+
+    // TODO: read your results here from
+    // *reinterpret_cast<double*>(bufferIn) for your first double number
+    // *reinterpret_cast<double*>(bufferIn+8) for your second double number and
+    // so on
+
+    size_t bufferInOffset = 32; // this number needs to be the same as the
+                                // memory space you reserved in Operate()
+
+    CompressMGARD mgard(m_Parameters);
+    size_t sizeOut = mgard.InverseOperate(bufferIn + bufferInOffset,
+                                          sizeIn - bufferInOffset, dataOut);
+
+    // TODO: the regular decompressed buffer is in dataOut, with the size of
+    // sizeOut. Here you may want to do your magic to change the decompressed
+    // data somehow to improve its accuracy :)
+
+    return sizeOut;
+}
+
+size_t CompressMGARDPlus::InverseOperate(const char *bufferIn,
+                                         const size_t sizeIn, char *dataOut)
+{
+    size_t bufferInOffset = 1; // skip operator type
+    const uint8_t bufferVersion =
+        GetParameter<uint8_t>(bufferIn, bufferInOffset);
+    bufferInOffset += 2; // skip two reserved bytes
+
+    if (bufferVersion == 1)
+    {
+        return DecompressV1(bufferIn + bufferInOffset, sizeIn - bufferInOffset,
+                            dataOut);
+    }
+    else if (bufferVersion == 2)
+    {
+        // TODO: if a Version 2 mgard buffer is being implemented, put it here
+        // and keep the DecompressV1 routine for backward compatibility
+    }
+    else
+    {
+        helper::Throw<std::runtime_error>("Operator", "CompressMGARDPlus",
+                                          "InverseOperate",
+                                          "invalid mgard buffer version");
+    }
+
+    return 0;
+}
+
+bool CompressMGARDPlus::IsDataTypeValid(const DataType type) const
+{
+    if (type == DataType::Double || type == DataType::Float ||
+        type == DataType::DoubleComplex || type == DataType::FloatComplex)
+    {
+        return true;
+    }
+    return false;
+}
+
+} // end namespace compress
+} // end namespace core
+} // end namespace adios2

source/adios2/operator/compress/CompressMGARDPlus.h

@@ -0,0 +1,74 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * CompressMGARDPlus.h :
+ *
+ *  Created on: Feb 23, 2022
+ *      Author: Jason Wang [email protected]
+ */
+
+#ifndef ADIOS2_OPERATOR_COMPRESS_COMPRESSMGARDPLUS_H_
+#define ADIOS2_OPERATOR_COMPRESS_COMPRESSMGARDPLUS_H_
+
+#include "adios2/core/Operator.h"
+
+namespace adios2
+{
+namespace core
+{
+namespace compress
+{
+
+class CompressMGARDPlus : public Operator
+{
+
+public:
+    CompressMGARDPlus(const Params &parameters);
+
+    ~CompressMGARDPlus() = default;
+
+    /**
+     * @param dataIn
+     * @param blockStart
+     * @param blockCount
+     * @param type
+     * @param bufferOut
+     * @return size of compressed buffer
+     */
+    size_t Operate(const char *dataIn, const Dims &blockStart,
+                   const Dims &blockCount, const DataType type,
+                   char *bufferOut) final;
+
+    /**
+     * @param bufferIn
+     * @param sizeIn
+     * @param dataOut
+     * @return size of decompressed buffer
+     */
+    size_t InverseOperate(const char *bufferIn, const size_t sizeIn,
+                          char *dataOut) final;
+
+    bool IsDataTypeValid(const DataType type) const final;
+
+private:
+    /**
+     * Decompress function for V1 buffer. Do NOT remove even if the buffer
+     * version is updated. Data might be still in lagacy formats. This function
+     * must be kept for backward compatibility
+     * @param bufferIn : compressed data buffer (V1 only)
+     * @param sizeIn : number of bytes in bufferIn
+     * @param dataOut : decompressed data buffer
+     * @return : number of bytes in dataOut
+     */
+    size_t DecompressV1(const char *bufferIn, const size_t sizeIn,
+                        char *dataOut);
+
+    std::string m_VersionInfo;
+};
+
+} // end namespace compress
+} // end namespace core
+} // end namespace adios2
+
+#endif /* ADIOS2_OPERATOR_COMPRESS_COMPRESSMGARDPLUS_H_ */

source/adios2/toolkit/format/bp/BPBase.cpp

@@ -444,14 +444,20 @@ std::string BPBase::ReadBPString(const std::vector<char> &buffer,
 // static members
 const std::set<std::string> BPBase::m_TransformTypes = {
     {"unknown", "none", "identity", "bzip2", "sz", "zfp", "mgard", "png",
-     "blosc", "sirius"}};
+     "blosc", "sirius", "mgardplus"}};
 
 const std::map<int, std::string> BPBase::m_TransformTypesToNames = {
-    {transform_unknown, "unknown"},   {transform_none, "none"},
-    {transform_identity, "identity"}, {transform_sz, "sz"},
-    {transform_zfp, "zfp"},           {transform_mgard, "mgard"},
-    {transform_png, "png"},           {transform_bzip2, "bzip2"},
-    {transform_blosc, "blosc"},       {transform_sirius, "sirius"}};
+    {transform_unknown, "unknown"},
+    {transform_none, "none"},
+    {transform_identity, "identity"},
+    {transform_sz, "sz"},
+    {transform_zfp, "zfp"},
+    {transform_mgard, "mgard"},
+    {transform_png, "png"},
+    {transform_bzip2, "bzip2"},
+    {transform_blosc, "blosc"},
+    {transform_sirius, "sirius"},
+    {transform_mgardplus, "mgardplus"}};
 
 BPBase::TransformTypes
 BPBase::TransformTypeEnum(const std::string transformType) const noexcept

source/adios2/toolkit/format/bp/BPBase.h

@@ -452,7 +452,8 @@ class BPBase
         transform_blosc = 11,
         transform_mgard = 12,
         transform_png = 13,
-        transform_sirius = 14
+        transform_sirius = 14,
+        transform_mgardplus = 15
     };
 
     /** Supported transform types */

testing/adios2/engine/bp/operations/CMakeLists.txt

@@ -51,6 +51,7 @@ endif()
 
 if(ADIOS2_HAVE_MGARD)
   bp_gtest_add_tests_helper(WriteReadMGARD MPI_ALLOW)
+  bp_gtest_add_tests_helper(WriteReadMGARDPlus MPI_ALLOW)
 endif()
 
 if(ADIOS2_HAVE_BZip2)