From 5b7feee0cd806d580a89a0ba67531280f2cc901b Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sat, 4 Dec 2021 23:58:07 -0500 Subject: [PATCH 01/10] removed Variable::Operation struct --- bindings/CXX11/adios2/cxx11/Operator.cpp | 5 ----- bindings/CXX11/adios2/cxx11/Operator.h | 2 -- bindings/CXX11/adios2/cxx11/Variable.cpp | 7 +++---- bindings/CXX11/adios2/cxx11/Variable.h | 16 +------------- bindings/Python/py11Variable.cpp | 6 +++--- bindings/Python/py11Variable.h | 13 +----------- source/adios2/core/Variable.h | 2 +- source/adios2/core/VariableBase.cpp | 9 +++++--- source/adios2/core/VariableBase.h | 12 +---------- .../adios2/toolkit/format/bp/BPSerializer.tcc | 9 ++++---- .../toolkit/format/bp/bp3/BP3Deserializer.h | 6 +++--- .../toolkit/format/bp/bp3/BP3Deserializer.tcc | 7 +++---- .../toolkit/format/bp/bp4/BP4Deserializer.h | 6 +++--- .../toolkit/format/bp/bp4/BP4Deserializer.tcc | 7 +++---- .../toolkit/format/bp5/BP5Serializer.cpp | 5 ++--- .../format/dataman/DataManSerializer.cpp | 16 +++++++------- .../format/dataman/DataManSerializer.h | 2 +- .../format/dataman/DataManSerializer.tcc | 21 ++++++++++--------- 18 files changed, 55 insertions(+), 96 deletions(-) diff --git a/bindings/CXX11/adios2/cxx11/Operator.cpp b/bindings/CXX11/adios2/cxx11/Operator.cpp index 08030a61f6..5a6752b83c 100644 --- a/bindings/CXX11/adios2/cxx11/Operator.cpp +++ b/bindings/CXX11/adios2/cxx11/Operator.cpp @@ -46,9 +46,4 @@ Params Operator::Parameters() const // PRIVATE Operator::Operator(core::Operator *op) : m_Operator(op) {} -std::string ToString(const Operator &op) -{ - return std::string("Operator(Type: \"" + op.Type() + "\")"); -} - } // end namespace adios2 diff --git a/bindings/CXX11/adios2/cxx11/Operator.h b/bindings/CXX11/adios2/cxx11/Operator.h index 79ff4832bb..44b5bb8601 100644 --- a/bindings/CXX11/adios2/cxx11/Operator.h +++ b/bindings/CXX11/adios2/cxx11/Operator.h @@ -78,8 +78,6 @@ class Operator core::Operator *m_Operator = nullptr; }; -std::string ToString(const IO &io); - } // end namespace adios2 #endif /* ADIOS2_BINDINGS_CXX11_CXX11_OPERATOR_H_ */ diff --git a/bindings/CXX11/adios2/cxx11/Variable.cpp b/bindings/CXX11/adios2/cxx11/Variable.cpp index bb1fa464c5..dd54511481 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.cpp +++ b/bindings/CXX11/adios2/cxx11/Variable.cpp @@ -172,17 +172,16 @@ namespace adios2 } \ \ template <> \ - std::vector::Operation> Variable::Operations() \ - const \ + std::vector Variable::Operations() const \ { \ helper::CheckForNullptr(m_Variable, \ "in call to Variable::Operations"); \ - std::vector operations; \ + std::vector operations; \ operations.reserve(m_Variable->m_Operations.size()); \ \ for (const auto &op : m_Variable->m_Operations) \ { \ - operations.push_back(Operation{Operator(op.Op), op.Parameters}); \ + operations.push_back(Operator(op)); \ } \ return operations; \ } \ diff --git a/bindings/CXX11/adios2/cxx11/Variable.h b/bindings/CXX11/adios2/cxx11/Variable.h index ef43c21e3c..a2994c86ef 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.h +++ b/bindings/CXX11/adios2/cxx11/Variable.h @@ -275,20 +275,6 @@ class Variable */ size_t BlockID() const; - /** - * EXPERIMENTAL: carries information about an Operation added with - * AddOperation - */ - struct Operation - { - /** Operator associated with this operation */ - const adios2::Operator Op; - /** Parameters settings for this operation */ - const adios2::Params Parameters; - /** Information associated with this operation */ - adios2::Params Info; - }; - /** *Adds operation and parameters to current Variable object * @param op operator to be added @@ -303,7 +289,7 @@ class Variable * Inspects current operators added with AddOperator * @return vector of Variable::OperatorInfo */ - std::vector Operations() const; + std::vector Operations() const; /** * Removes all current Operations associated with AddOperation. diff --git a/bindings/Python/py11Variable.cpp b/bindings/Python/py11Variable.cpp index 60ac8cb1ce..56b20e4c9e 100644 --- a/bindings/Python/py11Variable.cpp +++ b/bindings/Python/py11Variable.cpp @@ -83,15 +83,15 @@ size_t Variable::AddOperation(const Operator op, const Params ¶meters) return m_VariableBase->AddOperation(*op.m_Operator, parameters); } -std::vector Variable::Operations() const +std::vector Variable::Operations() const { helper::CheckForNullptr(m_VariableBase, "in call to Variable::Operations"); - std::vector operations; + std::vector operations; operations.reserve(m_VariableBase->m_Operations.size()); for (const auto &op : m_VariableBase->m_Operations) { - operations.push_back(Operation{Operator(op.Op), op.Parameters}); + operations.push_back(Operator(op)); } return operations; } diff --git a/bindings/Python/py11Variable.h b/bindings/Python/py11Variable.h index c50456169d..d04d1fc898 100644 --- a/bindings/Python/py11Variable.h +++ b/bindings/Python/py11Variable.h @@ -95,17 +95,6 @@ class Variable size_t BlockID() const; - /** - * EXPERIMENTAL: carries information about an Operation added with - * AddOperation - */ - struct Operation - { - const Operator Op; - const Params Parameters; - const Params Info; - }; - /** * EXPERIMENTAL: Adds operation and parameters to current Variable object * @param op operator to be added @@ -119,7 +108,7 @@ class Variable * EXPERIMENTAL: inspects current operators added with AddOperator * @return vector of Variable::OperatorInfo */ - std::vector Operations() const; + std::vector Operations() const; /** Contains sub-block information for a particular Variable */ struct Info diff --git a/source/adios2/core/Variable.h b/source/adios2/core/Variable.h index d49f30afb3..f06de2df64 100644 --- a/source/adios2/core/Variable.h +++ b/source/adios2/core/Variable.h @@ -95,7 +95,7 @@ class Variable : public VariableBase Dims Count; Dims MemoryStart; Dims MemoryCount; - std::vector Operations; + std::vector Operations; size_t Step = 0; size_t StepsStart = 0; size_t StepsCount = 0; diff --git a/source/adios2/core/VariableBase.cpp b/source/adios2/core/VariableBase.cpp index 206c7164b1..3fa2c8a428 100644 --- a/source/adios2/core/VariableBase.cpp +++ b/source/adios2/core/VariableBase.cpp @@ -245,8 +245,11 @@ size_t VariableBase::AddOperation(Operator &op, { if (op.IsDataTypeValid(m_Type)) { - m_Operations.push_back( - Operation{&op, helper::LowerCaseParams(parameters)}); + for (const auto &p : parameters) + { + op.SetParameter(helper::LowerCase(p.first), p.second); + } + m_Operations.push_back(&op); } else { @@ -271,7 +274,7 @@ void VariableBase::SetOperationParameter(const size_t operationID, "SetOperationParameter\n"); } - m_Operations[operationID].Parameters[key] = value; + m_Operations[operationID]->SetParameter(key, value); } void VariableBase::CheckDimensions(const std::string hint) const diff --git a/source/adios2/core/VariableBase.h b/source/adios2/core/VariableBase.h index fbf93b881c..f02865fdac 100644 --- a/source/adios2/core/VariableBase.h +++ b/source/adios2/core/VariableBase.h @@ -73,18 +73,8 @@ class VariableBase * already encountered in previous step */ bool m_FirstStreamingStep = true; - /** Operators metadata info */ - struct Operation - { - /** reference to object derived from Operator class, - * needs a pointer to enable assignment operator (C++ class) */ - core::Operator *Op; - /** Variable specific parameters */ - Params Parameters; - }; - /** Registered transforms */ - std::vector m_Operations; + std::vector m_Operations; size_t m_AvailableStepsStart = 0; size_t m_AvailableStepsCount = 0; diff --git a/source/adios2/toolkit/format/bp/BPSerializer.tcc b/source/adios2/toolkit/format/bp/BPSerializer.tcc index c8b460ac3d..c069aa3b2e 100644 --- a/source/adios2/toolkit/format/bp/BPSerializer.tcc +++ b/source/adios2/toolkit/format/bp/BPSerializer.tcc @@ -366,7 +366,7 @@ void BPSerializer::PutCharacteristicOperation( const typename core::Variable::BPInfo &blockInfo, std::vector &buffer) noexcept { - const std::string type = blockInfo.Operations[0].Op->m_TypeString; + const std::string type = blockInfo.Operations[0]->m_TypeString; const uint8_t typeLength = static_cast(type.size()); helper::InsertToBuffer(buffer, &typeLength); helper::InsertToBuffer(buffer, type.c_str(), type.size()); @@ -402,12 +402,11 @@ void BPSerializer::PutOperationPayloadInBuffer( const core::Variable &variable, const typename core::Variable::BPInfo &blockInfo) { - const Params ¶meters = blockInfo.Operations[0].Parameters; - - const size_t outputSize = blockInfo.Operations[0].Op->Operate( + const size_t outputSize = blockInfo.Operations[0]->Operate( reinterpret_cast(blockInfo.Data), blockInfo.Start, blockInfo.Count, variable.m_Type, - m_Data.m_Buffer.data() + m_Data.m_Position, parameters); + m_Data.m_Buffer.data() + m_Data.m_Position, + blockInfo.Operations[0]->GetParameters()); m_Data.m_Position += outputSize; m_Data.m_AbsolutePosition += outputSize; diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.h b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.h index b567d32422..4135f7d3b7 100644 --- a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.h +++ b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.h @@ -213,9 +213,9 @@ class BP3Deserializer : virtual public BP3Base const std::vector &blocksIndexOffsets) const; template - bool IdentityOperation( - const std::vector::Operation> &operations) - const noexcept; + bool + IdentityOperation(const std::vector &operations) const + noexcept; const helper::BlockOperationInfo &InitPostOperatorBlockData( const std::vector &blockOperationsInfo) diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc index ad0ce63904..4c84ff901d 100644 --- a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc +++ b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc @@ -1117,13 +1117,12 @@ BP3Deserializer::BlocksInfoCommon( template bool BP3Deserializer::IdentityOperation( - const std::vector::Operation> &operations) const - noexcept + const std::vector &operations) const noexcept { bool identity = false; - for (const typename core::Variable::Operation &op : operations) + for (const auto &op : operations) { - if (op.Op->m_TypeString == "identity") + if (op->m_TypeString == "identity") { identity = true; } diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.h b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.h index 9caab8e9a5..ea12dc13bf 100644 --- a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.h +++ b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.h @@ -231,9 +231,9 @@ class BP4Deserializer : virtual public BP4Base const std::vector &blocksIndexOffsets) const; template - bool IdentityOperation( - const std::vector::Operation> &operations) - const noexcept; + bool + IdentityOperation(const std::vector &operations) const + noexcept; const helper::BlockOperationInfo &InitPostOperatorBlockData( const std::vector &blockOperationsInfo) diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc index 26d18cb97e..417a70a9cd 100644 --- a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc +++ b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc @@ -1203,13 +1203,12 @@ BP4Deserializer::BlocksInfoCommon( template bool BP4Deserializer::IdentityOperation( - const std::vector::Operation> &operations) const - noexcept + const std::vector &operations) const noexcept { bool identity = false; - for (const typename core::Variable::Operation &op : operations) + for (const auto &op : operations) { - if (op.Op->m_TypeString == "identity") + if (op->m_TypeString == "identity") { identity = true; } diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp index bfec305d2d..908ac27696 100644 --- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp @@ -440,8 +440,7 @@ BP5Serializer::CreateWriterRec(void *Variable, const char *Name, DataType Type, char *OperatorType = NULL; if (VB->m_Operations.size()) { - OperatorType = - strdup((VB->m_Operations[0]).Op->m_TypeString.data()); + OperatorType = strdup((VB->m_Operations[0])->m_TypeString.data()); } // Array field. To Metadata, add FMFields for DimCount, Shape, Count // and Offsets matching _MetaArrayRec @@ -642,7 +641,7 @@ void BP5Serializer::Marshal(void *Variable, const char *Name, DataOffset = m_PriorDataBufferSizeTotal + pos.globalPos; CompressedSize = core::compress::Compress( (const char *)Data, tmpOffsets, tmpCount, (DataType)Rec->Type, - CompressedData, VB->m_Operations[0].Parameters, + CompressedData, VB->m_Operations[0]->GetParameters(), compressionMethod); // use data size to resize allocated buffer } diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp index 5dadcea0ce..2deaea66cf 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp @@ -621,13 +621,15 @@ void DataManSerializer::Log(const int level, const std::string &message, } template <> -void DataManSerializer::PutData( - const std::string *inputData, const std::string &varName, - const Dims &varShape, const Dims &varStart, const Dims &varCount, - const Dims &varMemStart, const Dims &varMemCount, const std::string &doid, - const size_t step, const int rank, const std::string &address, - const std::vector &ops, VecPtr localBuffer, - JsonPtr metadataJson) +void DataManSerializer::PutData(const std::string *inputData, + const std::string &varName, + const Dims &varShape, const Dims &varStart, + const Dims &varCount, const Dims &varMemStart, + const Dims &varMemCount, + const std::string &doid, const size_t step, + const int rank, const std::string &address, + const std::vector &ops, + VecPtr localBuffer, JsonPtr metadataJson) { PERFSTUBS_SCOPED_TIMER_FUNC(); Log(1, diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.h b/source/adios2/toolkit/format/dataman/DataManSerializer.h index 5a8a7ed34a..a67067d15d 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.h +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.h @@ -99,7 +99,7 @@ class DataManSerializer const Dims &varCount, const Dims &varMemStart, const Dims &varMemCount, const std::string &doid, const size_t step, const int rank, const std::string &address, - const std::vector &ops, + const std::vector &ops, VecPtr localBuffer = nullptr, JsonPtr metadataJson = nullptr); // another wrapper for PutData which accepts adios2::core::Variable diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc index 4d61cea49f..43ffc55ee9 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc @@ -97,13 +97,14 @@ void DataManSerializer::PutData(const core::Variable &variable, } template -void DataManSerializer::PutData( - const T *inputData, const std::string &varName, const Dims &varShape, - const Dims &varStart, const Dims &varCount, const Dims &varMemStart, - const Dims &varMemCount, const std::string &doid, const size_t step, - const int rank, const std::string &address, - const std::vector &ops, VecPtr localBuffer, - JsonPtr metadataJson) +void DataManSerializer::PutData(const T *inputData, const std::string &varName, + const Dims &varShape, const Dims &varStart, + const Dims &varCount, const Dims &varMemStart, + const Dims &varMemCount, + const std::string &doid, const size_t step, + const int rank, const std::string &address, + const std::vector &ops, + VecPtr localBuffer, JsonPtr metadataJson) { PERFSTUBS_SCOPED_TIMER_FUNC(); Log(1, @@ -149,7 +150,7 @@ void DataManSerializer::PutData( bool compressed = false; if (not ops.empty()) { - compressionMethod = ops[0].Op->m_TypeString; + compressionMethod = ops[0]->m_TypeString; std::transform(compressionMethod.begin(), compressionMethod.end(), compressionMethod.begin(), ::tolower); @@ -160,14 +161,14 @@ void DataManSerializer::PutData( datasize = core::compress::Compress( reinterpret_cast(inputData), varStart, varCount, helper::GetDataType(), m_CompressBuffer.data(), - ops[0].Parameters, compressionMethod); + ops[0]->GetParameters(), compressionMethod); compressed = true; } if (compressed) { metaj["Z"] = compressionMethod; - metaj["ZP"] = ops[0].Parameters; + metaj["ZP"] = ops[0]->GetParameters(); } else { From c8532317dfee5020d00fd151fa82383ff961e74b Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 5 Dec 2021 17:13:59 -0500 Subject: [PATCH 02/10] moved MakeOperator logic from ADIOS to CompressorFactory --- source/adios2/core/ADIOS.cpp | 134 +----------------- source/adios2/core/VariableBase.cpp | 12 +- source/adios2/core/VariableBase.h | 5 + .../operator/compress/CompressorFactory.cpp | 68 ++++++++- .../operator/compress/CompressorFactory.h | 4 + 5 files changed, 91 insertions(+), 132 deletions(-) diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp index f4fbce5e6a..ea8f00fbfa 100644 --- a/source/adios2/core/ADIOS.cpp +++ b/source/adios2/core/ADIOS.cpp @@ -18,45 +18,11 @@ #include "adios2/core/IO.h" #include "adios2/helper/adiosCommDummy.h" #include "adios2/helper/adiosFunctions.h" //InquireKey, BroadcastFile +#include "adios2/operator/compress/CompressorFactory.h" #include #include -// OPERATORS - -// compress -#ifdef ADIOS2_HAVE_BZIP2 -#include "adios2/operator/compress/CompressBZIP2.h" -#endif - -#ifdef ADIOS2_HAVE_ZFP -#include "adios2/operator/compress/CompressZFP.h" -#endif - -#ifdef ADIOS2_HAVE_SZ -#include "adios2/operator/compress/CompressSZ.h" -#endif - -#ifdef ADIOS2_HAVE_MGARD -#include "adios2/operator/compress/CompressMGARD.h" -#endif - -#ifdef ADIOS2_HAVE_BZIP2 -#include "adios2/operator/compress/CompressBZIP2.h" -#endif - -#ifdef ADIOS2_HAVE_PNG -#include "adios2/operator/compress/CompressPNG.h" -#endif - -#ifdef ADIOS2_HAVE_BLOSC -#include "adios2/operator/compress/CompressBlosc.h" -#endif - -#ifdef ADIOS2_HAVE_LIBPRESSIO -#include "adios2/operator/compress/CompressLibPressio.h" -#endif - // callbacks #include "adios2/operator/callback/Signature1.h" #include "adios2/operator/callback/Signature2.h" @@ -188,102 +154,10 @@ void ADIOS::FlushAll() Operator &ADIOS::DefineOperator(const std::string &name, const std::string type, const Params ¶meters) { - auto lf_ErrorMessage = [](const std::string type) -> std::string { - return "ERROR: this version of ADIOS2 didn't compile with the " + type + - " library, when parsing config file in ADIOS constructor or in " - "call to ADIOS::DefineOperator"; - }; - - std::shared_ptr operatorPtr; - CheckOperator(name); - const std::string typeLowerCase = helper::LowerCase(type); - - if (typeLowerCase == "bzip2") - { -#ifdef ADIOS2_HAVE_BZIP2 - auto itPair = m_Operators.emplace( - name, std::make_shared(parameters)); - operatorPtr = itPair.first->second; -#else - throw std::invalid_argument(lf_ErrorMessage("BZip2")); -#endif - } - else if (typeLowerCase == "zfp") - { -#ifdef ADIOS2_HAVE_ZFP - auto itPair = m_Operators.emplace( - name, std::make_shared(parameters)); - operatorPtr = itPair.first->second; -#else - throw std::invalid_argument(lf_ErrorMessage("ZFP")); -#endif - } - else if (typeLowerCase == "sz") - { -#ifdef ADIOS2_HAVE_SZ - auto itPair = m_Operators.emplace( - name, std::make_shared(parameters)); - operatorPtr = itPair.first->second; -#else - throw std::invalid_argument(lf_ErrorMessage("SZ")); -#endif - } - else if (typeLowerCase == "mgard") - { -#ifdef ADIOS2_HAVE_MGARD - auto itPair = m_Operators.emplace( - name, std::make_shared(parameters)); - operatorPtr = itPair.first->second; -#else - throw std::invalid_argument(lf_ErrorMessage("MGARD")); -#endif - } - else if (typeLowerCase == "png") - { -#ifdef ADIOS2_HAVE_PNG - auto itPair = m_Operators.emplace( - name, std::make_shared(parameters)); - operatorPtr = itPair.first->second; -#else - throw std::invalid_argument(lf_ErrorMessage("PNG")); -#endif - } - else if (typeLowerCase == "blosc") - { -#ifdef ADIOS2_HAVE_BLOSC - auto itPair = m_Operators.emplace( - name, std::make_shared(parameters)); - operatorPtr = itPair.first->second; -#else - throw std::invalid_argument(lf_ErrorMessage("Blosc")); -#endif - } - else if (typeLowerCase == "libpressio") - { -#ifdef ADIOS2_HAVE_LIBPRESSIO - auto itPair = m_Operators.emplace( - name, std::make_shared(parameters)); - operatorPtr = itPair.first->second; -#else - throw std::invalid_argument(lf_ErrorMessage("LibPressio")); -#endif - } - else - { - throw std::invalid_argument( - "ERROR: Operator " + name + " of type " + type + - " is not supported by ADIOS2, in call to DefineOperator\n"); - } - - if (!operatorPtr) - { - throw std::invalid_argument( - "ERROR: Operator " + name + " of type " + type + - " couldn't be defined, in call to DefineOperator\n"); - } - - return *operatorPtr.get(); + auto itPair = + m_Operators.emplace(name, compress::MakeOperator(type, parameters)); + return *itPair.first->second.get(); } Operator *ADIOS::InquireOperator(const std::string &name) noexcept diff --git a/source/adios2/core/VariableBase.cpp b/source/adios2/core/VariableBase.cpp index 3fa2c8a428..9f402227b2 100644 --- a/source/adios2/core/VariableBase.cpp +++ b/source/adios2/core/VariableBase.cpp @@ -240,6 +240,12 @@ void VariableBase::SetStepSelection(const Box &boxSteps) } } +size_t VariableBase::AddOperation(const std::string &op, + const Params ¶meters) noexcept +{ + return m_Operations.size() - 1; +} + size_t VariableBase::AddOperation(Operator &op, const Params ¶meters) noexcept { @@ -260,7 +266,11 @@ size_t VariableBase::AddOperation(Operator &op, return m_Operations.size() - 1; } -void VariableBase::RemoveOperations() noexcept { m_Operations.clear(); } +void VariableBase::RemoveOperations() noexcept +{ + m_PrivateOperations.clear(); + m_Operations.clear(); +} void VariableBase::SetOperationParameter(const size_t operationID, const std::string key, diff --git a/source/adios2/core/VariableBase.h b/source/adios2/core/VariableBase.h index f02865fdac..17620be8e8 100644 --- a/source/adios2/core/VariableBase.h +++ b/source/adios2/core/VariableBase.h @@ -169,6 +169,9 @@ class VariableBase size_t AddOperation(core::Operator &op, const Params ¶meters = Params()) noexcept; + size_t AddOperation(const std::string &op, + const Params ¶meters = Params()) noexcept; + /** * Removes all current Operations associated with AddOperation. * Provides the posibility to apply or not operators on a step basis. @@ -224,6 +227,8 @@ class VariableBase unsigned int m_DeferredCounter = 0; + std::vector> m_PrivateOperations; + void InitShapeType(); /** Self-check dims according to type, called right after DefineVariable and diff --git a/source/adios2/operator/compress/CompressorFactory.cpp b/source/adios2/operator/compress/CompressorFactory.cpp index 802f8f3b57..37a5903a4a 100644 --- a/source/adios2/operator/compress/CompressorFactory.cpp +++ b/source/adios2/operator/compress/CompressorFactory.cpp @@ -9,7 +9,6 @@ */ #include "CompressorFactory.h" -#include "adios2/core/Operator.h" #include "adios2/helper/adiosFunctions.h" #include "adios2/operator/compress/CompressNull.h" #include @@ -53,6 +52,73 @@ namespace core namespace compress { +std::shared_ptr MakeOperator(const std::string &type, + const Params ¶meters) +{ + std::shared_ptr ret = nullptr; + + const std::string typeLowerCase = helper::LowerCase(type); + + if (typeLowerCase == "blosc") + { +#ifdef ADIOS2_HAVE_BLOSC + ret = std::make_shared(parameters); +#endif + } + else if (typeLowerCase == "bzip2") + { +#ifdef ADIOS2_HAVE_BZIP2 + ret = std::make_shared(parameters); +#endif + } + else if (typeLowerCase == "libpressio") + { +#ifdef ADIOS2_HAVE_LIBPRESSIO + ret = std::make_shared(parameters); +#endif + } + else if (typeLowerCase == "mgard") + { +#ifdef ADIOS2_HAVE_MGARD + ret = std::make_shared(parameters); +#endif + } + else if (typeLowerCase == "png") + { +#ifdef ADIOS2_HAVE_PNG + ret = std::make_shared(parameters); +#endif + } + else if (typeLowerCase == "sz") + { +#ifdef ADIOS2_HAVE_SZ + ret = std::make_shared(parameters); +#endif + } + else if (typeLowerCase == "zfp") + { +#ifdef ADIOS2_HAVE_ZFP + ret = std::make_shared(parameters); +#endif + } + else + { + helper::Log("Operator", "CompressorFactory", "MakeOperator", + "ADIOS2 does not support " + typeLowerCase + " operation", + helper::EXCEPTION); + } + + if (ret == nullptr) + { + helper::Log("Operator", "CompressorFactory", "MakeOperator", + "ADIOS2 didn't compile with " + typeLowerCase + + "library, operator not added", + helper::EXCEPTION); + } + + return ret; +} + size_t Compress(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType dataType, char *bufferOut, const Params ¶meters, diff --git a/source/adios2/operator/compress/CompressorFactory.h b/source/adios2/operator/compress/CompressorFactory.h index 19129027f8..7c6a610a9b 100644 --- a/source/adios2/operator/compress/CompressorFactory.h +++ b/source/adios2/operator/compress/CompressorFactory.h @@ -9,6 +9,7 @@ */ #include "adios2/common/ADIOSTypes.h" +#include "adios2/core/Operator.h" namespace adios2 { @@ -17,6 +18,9 @@ namespace core namespace compress { +std::shared_ptr MakeOperator(const std::string &type, + const Params ¶meters); + size_t Compress(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType type, char *bufferOut, const Params ¶meters, const std::string &compressorType); From 731321e85919ff3d5542d04afd4fd9ba6c8f64f1 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 5 Dec 2021 18:06:35 -0500 Subject: [PATCH 03/10] moved adios2/operator/compress/CompressorFactory to adios2/operator/OperatorFactory --- source/adios2/CMakeLists.txt | 7 +++--- source/adios2/core/ADIOS.cpp | 5 ++-- ...pressorFactory.cpp => OperatorFactory.cpp} | 23 ++++++++---------- .../CompressorFactory.h => OperatorFactory.h} | 5 +--- .../toolkit/format/bp/bp3/BP3Deserializer.tcc | 5 ++-- .../toolkit/format/bp/bp4/BP4Deserializer.tcc | 5 ++-- .../toolkit/format/bp5/BP5Deserializer.cpp | 4 ++-- .../toolkit/format/bp5/BP5Serializer.cpp | 24 ++++--------------- .../format/dataman/DataManSerializer.tcc | 14 +++++------ 9 files changed, 33 insertions(+), 59 deletions(-) rename source/adios2/operator/{compress/CompressorFactory.cpp => OperatorFactory.cpp} (96%) rename source/adios2/operator/{compress/CompressorFactory.h => OperatorFactory.h} (91%) diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 5064d76a44..f71a997031 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -18,9 +18,11 @@ add_library(adios2_core core/VariableCompound.cpp core/VariableCompound.tcc core/Group.cpp core/Group.tcc -#operator callback +#operator operator/callback/Signature1.cpp operator/callback/Signature2.cpp + operator/OperatorFactory.cpp + operator/compress/CompressNull.cpp #helper helper/adiosComm.h helper/adiosComm.cpp @@ -93,9 +95,6 @@ add_library(adios2_core toolkit/burstbuffer/FileDrainer.cpp toolkit/burstbuffer/FileDrainerSingleThread.cpp - - operator/compress/CompressorFactory.cpp - operator/compress/CompressNull.cpp ) set_property(TARGET adios2_core PROPERTY EXPORT_NAME core) set_property(TARGET adios2_core PROPERTY OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_core) diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp index ea8f00fbfa..105b87af52 100644 --- a/source/adios2/core/ADIOS.cpp +++ b/source/adios2/core/ADIOS.cpp @@ -18,7 +18,7 @@ #include "adios2/core/IO.h" #include "adios2/helper/adiosCommDummy.h" #include "adios2/helper/adiosFunctions.h" //InquireKey, BroadcastFile -#include "adios2/operator/compress/CompressorFactory.h" +#include "adios2/operator/OperatorFactory.h" #include #include @@ -155,8 +155,7 @@ Operator &ADIOS::DefineOperator(const std::string &name, const std::string type, const Params ¶meters) { CheckOperator(name); - auto itPair = - m_Operators.emplace(name, compress::MakeOperator(type, parameters)); + auto itPair = m_Operators.emplace(name, MakeOperator(type, parameters)); return *itPair.first->second.get(); } diff --git a/source/adios2/operator/compress/CompressorFactory.cpp b/source/adios2/operator/OperatorFactory.cpp similarity index 96% rename from source/adios2/operator/compress/CompressorFactory.cpp rename to source/adios2/operator/OperatorFactory.cpp index 37a5903a4a..02bdd2339e 100644 --- a/source/adios2/operator/compress/CompressorFactory.cpp +++ b/source/adios2/operator/OperatorFactory.cpp @@ -8,7 +8,7 @@ * Author: Jason Wang jason.ruonan.wang@gmail.com */ -#include "CompressorFactory.h" +#include "OperatorFactory.h" #include "adios2/helper/adiosFunctions.h" #include "adios2/operator/compress/CompressNull.h" #include @@ -49,8 +49,6 @@ namespace adios2 { namespace core { -namespace compress -{ std::shared_ptr MakeOperator(const std::string &type, const Params ¶meters) @@ -130,7 +128,7 @@ size_t Compress(const char *dataIn, const Dims &blockStart, if (compressorType == "blosc") { #ifdef ADIOS2_HAVE_BLOSC - CompressBlosc c({}); + compress::CompressBlosc c({}); ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, parameters); #else @@ -143,7 +141,7 @@ size_t Compress(const char *dataIn, const Dims &blockStart, else if (compressorType == "bzip2") { #ifdef ADIOS2_HAVE_BZIP2 - CompressBZIP2 c({}); + compress::CompressBZIP2 c({}); ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, parameters); #else @@ -156,7 +154,7 @@ size_t Compress(const char *dataIn, const Dims &blockStart, else if (compressorType == "libpressio") { #ifdef ADIOS2_HAVE_LIBPRESSIO - CompressLibPressio c({}); + compress::CompressLibPressio c({}); ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, parameters); #else @@ -169,7 +167,7 @@ size_t Compress(const char *dataIn, const Dims &blockStart, else if (compressorType == "mgard") { #ifdef ADIOS2_HAVE_MGARD - CompressMGARD c({}); + compress::CompressMGARD c({}); ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, parameters); #else @@ -182,7 +180,7 @@ size_t Compress(const char *dataIn, const Dims &blockStart, else if (compressorType == "png") { #ifdef ADIOS2_HAVE_PNG - CompressPNG c({}); + compress::CompressPNG c({}); ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, parameters); #else @@ -194,7 +192,7 @@ size_t Compress(const char *dataIn, const Dims &blockStart, else if (compressorType == "sirius") { #ifdef ADIOS2_HAVE_MHS - CompressSirius c({}); + compress::CompressSirius c({}); ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, parameters); #else @@ -206,7 +204,7 @@ size_t Compress(const char *dataIn, const Dims &blockStart, else if (compressorType == "sz") { #ifdef ADIOS2_HAVE_SZ - CompressSZ c({}); + compress::CompressSZ c({}); ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, parameters); #else @@ -218,7 +216,7 @@ size_t Compress(const char *dataIn, const Dims &blockStart, else if (compressorType == "zfp") { #ifdef ADIOS2_HAVE_ZFP - CompressZFP c({}); + compress::CompressZFP c({}); ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, parameters); #else @@ -249,7 +247,7 @@ size_t Compress(const char *dataIn, const Dims &blockStart, { helper::Log("Operator", "CompressorFactory", "Compress", e.what(), helper::WARNING); - CompressNull c({}); + compress::CompressNull c({}); ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, parameters); } @@ -350,6 +348,5 @@ size_t Decompress(const char *bufferIn, const size_t sizeIn, char *dataOut) return 0; } -} // end namespace compress } // end namespace core } // end namespace adios2 diff --git a/source/adios2/operator/compress/CompressorFactory.h b/source/adios2/operator/OperatorFactory.h similarity index 91% rename from source/adios2/operator/compress/CompressorFactory.h rename to source/adios2/operator/OperatorFactory.h index 7c6a610a9b..72b7e4dc69 100644 --- a/source/adios2/operator/compress/CompressorFactory.h +++ b/source/adios2/operator/OperatorFactory.h @@ -2,7 +2,7 @@ * Distributed under the OSI-approved Apache License, Version 2.0. See * accompanying file Copyright.txt for details. * - * CompressorFactory.h : + * OperatorFactory.h : * * Created on: Sep 29, 2021 * Author: Jason Wang jason.ruonan.wang@gmail.com @@ -15,8 +15,6 @@ namespace adios2 { namespace core { -namespace compress -{ std::shared_ptr MakeOperator(const std::string &type, const Params ¶meters); @@ -27,6 +25,5 @@ size_t Compress(const char *dataIn, const Dims &blockStart, size_t Decompress(const char *bufferIn, const size_t sizeIn, char *dataOut); -} // end namespace compress } // end namespace core } // end namespace adios2 diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc index 4c84ff901d..d1fc504433 100644 --- a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc +++ b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc @@ -17,7 +17,7 @@ #include #include "adios2/helper/adiosFunctions.h" -#include "adios2/operator/compress/CompressorFactory.h" +#include "adios2/operator/OperatorFactory.h" namespace adios2 { @@ -520,8 +520,7 @@ void BP3Deserializer::PostDataRead( char *preOpData = m_ThreadBuffers[threadID][0].data(); const char *postOpData = m_ThreadBuffers[threadID][1].data(); - core::compress::Decompress(postOpData, blockOperationInfo.PayloadSize, - preOpData); + core::Decompress(postOpData, blockOperationInfo.PayloadSize, preOpData); // clip block to match selection helper::ClipVector(m_ThreadBuffers[threadID][0], diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc index 417a70a9cd..a7ef0378d6 100644 --- a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc +++ b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc @@ -20,7 +20,7 @@ #include #include "adios2/helper/adiosFunctions.h" -#include "adios2/operator/compress/CompressorFactory.h" +#include "adios2/operator/OperatorFactory.h" namespace adios2 { @@ -524,8 +524,7 @@ void BP4Deserializer::PostDataRead( char *preOpData = m_ThreadBuffers[threadID][0].data(); const char *postOpData = m_ThreadBuffers[threadID][1].data(); - core::compress::Decompress(postOpData, blockOperationInfo.PayloadSize, - preOpData); + core::Decompress(postOpData, blockOperationInfo.PayloadSize, preOpData); // clip block to match selection helper::ClipVector(m_ThreadBuffers[threadID][0], diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp index 4d32026302..da6dcdc31b 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp @@ -27,7 +27,7 @@ #include "adios2/operator/compress/CompressMGARD.h" #endif -#include "adios2/operator/compress/CompressorFactory.h" +#include "adios2/operator/OperatorFactory.h" #include #include @@ -1095,7 +1095,7 @@ void BP5Deserializer::FinalizeGets(std::vector Requests) ->Count[dim * writer_meta_base->Dims]; } decompressBuffer.reserve(DestSize); - core::compress::Decompress( + core::Decompress( IncomingData, ((MetaArrayRecOperator *)writer_meta_base) ->DataLengths[i], diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp index 908ac27696..384b703973 100644 --- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp @@ -9,26 +9,11 @@ #include "adios2/core/Attribute.h" #include "adios2/core/IO.h" #include "adios2/core/VariableBase.h" +#include "adios2/helper/adiosFunctions.h" #include "adios2/helper/adiosMemory.h" #include "adios2/toolkit/format/buffer/ffs/BufferFFS.h" -#include // max_align_t - -#ifdef ADIOS2_HAVE_ZFP -#include "adios2/operator/compress/CompressZFP.h" -#endif -#ifdef ADIOS2_HAVE_SZ -#include "adios2/operator/compress/CompressSZ.h" -#endif -#ifdef ADIOS2_HAVE_BZIP2 -#include "adios2/operator/compress/CompressBZIP2.h" -#endif -#ifdef ADIOS2_HAVE_MGARD -#include "adios2/operator/compress/CompressMGARD.h" -#endif - -#include "adios2/operator/compress/CompressorFactory.h" -#include "adios2/helper/adiosFunctions.h" +#include // max_align_t #include @@ -639,10 +624,9 @@ void BP5Serializer::Marshal(void *Variable, const char *Name, char *CompressedData = (char *)GetPtr(pos.bufferIdx, pos.posInBuffer); DataOffset = m_PriorDataBufferSizeTotal + pos.globalPos; - CompressedSize = core::compress::Compress( + CompressedSize = VB->m_Operations[0]->Operate( (const char *)Data, tmpOffsets, tmpCount, (DataType)Rec->Type, - CompressedData, VB->m_Operations[0]->GetParameters(), - compressionMethod); + CompressedData, VB->m_Operations[0]->GetParameters()); // use data size to resize allocated buffer } else if (Span == nullptr) diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc index 43ffc55ee9..50da9efd1e 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc @@ -26,7 +26,7 @@ #include "adios2/operator/compress/CompressMGARD.h" #endif -#include "adios2/operator/compress/CompressorFactory.h" +#include "adios2/operator/OperatorFactory.h" #include "adios2/helper/adiosFunctions.h" @@ -158,10 +158,10 @@ void DataManSerializer::PutData(const T *inputData, const std::string &varName, varCount.end(), sizeof(T), std::multiplies())); - datasize = core::compress::Compress( - reinterpret_cast(inputData), varStart, varCount, - helper::GetDataType(), m_CompressBuffer.data(), - ops[0]->GetParameters(), compressionMethod); + datasize = core::Compress(reinterpret_cast(inputData), + varStart, varCount, helper::GetDataType(), + m_CompressBuffer.data(), + ops[0]->GetParameters(), compressionMethod); compressed = true; } @@ -265,8 +265,8 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName, m_OperatorMapMutex.unlock(); decompressBuffer.reserve( helper::GetTotalSize(j.count, sizeof(T))); - core::compress::Decompress(j.buffer->data() + j.position, - j.size, decompressBuffer.data()); + core::Decompress(j.buffer->data() + j.position, j.size, + decompressBuffer.data()); decompressed = true; input_data = decompressBuffer.data(); } From 13edd13c8ecd86ca329ae320a78ef19136013096 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 5 Dec 2021 18:32:16 -0500 Subject: [PATCH 04/10] added Variable::AddOperation(std::string, adios2::Params) --- bindings/CXX11/adios2/cxx11/Variable.cpp | 7 ++++++ bindings/CXX11/adios2/cxx11/Variable.h | 3 +++ source/adios2/core/VariableBase.cpp | 5 +++- source/adios2/core/VariableBase.h | 1 + source/adios2/operator/OperatorFactory.h | 1 + .../bp/operations/TestBPWriteReadPNG.cpp | 24 ++++++++----------- 6 files changed, 26 insertions(+), 15 deletions(-) diff --git a/bindings/CXX11/adios2/cxx11/Variable.cpp b/bindings/CXX11/adios2/cxx11/Variable.cpp index dd54511481..d16e24f08b 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.cpp +++ b/bindings/CXX11/adios2/cxx11/Variable.cpp @@ -172,6 +172,13 @@ namespace adios2 } \ \ template <> \ + size_t Variable::AddOperation(const std::string &type, \ + const Params ¶meters) \ + { \ + return m_Variable->AddOperation(type, parameters); \ + } \ + \ + template <> \ std::vector Variable::Operations() const \ { \ helper::CheckForNullptr(m_Variable, \ diff --git a/bindings/CXX11/adios2/cxx11/Variable.h b/bindings/CXX11/adios2/cxx11/Variable.h index a2994c86ef..ac5888ca65 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.h +++ b/bindings/CXX11/adios2/cxx11/Variable.h @@ -285,6 +285,9 @@ class Variable size_t AddOperation(const Operator op, const adios2::Params ¶meters = adios2::Params()); + size_t AddOperation(const std::string &type, + const adios2::Params ¶meters = adios2::Params()); + /** * Inspects current operators added with AddOperator * @return vector of Variable::OperatorInfo diff --git a/source/adios2/core/VariableBase.cpp b/source/adios2/core/VariableBase.cpp index 9f402227b2..69ca21476b 100644 --- a/source/adios2/core/VariableBase.cpp +++ b/source/adios2/core/VariableBase.cpp @@ -22,6 +22,7 @@ #include "adios2/core/Variable.h" #include "adios2/helper/adiosFunctions.h" //helper::GetTotalSize #include "adios2/helper/adiosString.h" +#include "adios2/operator/OperatorFactory.h" #ifdef ADIOS2_HAVE_CUDA #include @@ -240,9 +241,11 @@ void VariableBase::SetStepSelection(const Box &boxSteps) } } -size_t VariableBase::AddOperation(const std::string &op, +size_t VariableBase::AddOperation(const std::string &type, const Params ¶meters) noexcept { + m_PrivateOperations.emplace_back(MakeOperator(type, parameters)); + m_Operations.push_back(m_PrivateOperations.back().get()); return m_Operations.size() - 1; } diff --git a/source/adios2/core/VariableBase.h b/source/adios2/core/VariableBase.h index 17620be8e8..8d3cceceee 100644 --- a/source/adios2/core/VariableBase.h +++ b/source/adios2/core/VariableBase.h @@ -13,6 +13,7 @@ #define ADIOS2_CORE_VARIABLEBASE_H_ /// \cond EXCLUDE_FROM_DOXYGEN +#include #include #include #include diff --git a/source/adios2/operator/OperatorFactory.h b/source/adios2/operator/OperatorFactory.h index 72b7e4dc69..71af2ad3a0 100644 --- a/source/adios2/operator/OperatorFactory.h +++ b/source/adios2/operator/OperatorFactory.h @@ -10,6 +10,7 @@ #include "adios2/common/ADIOSTypes.h" #include "adios2/core/Operator.h" +#include namespace adios2 { diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadPNG.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadPNG.cpp index 63443cbd39..ce1c225bb8 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadPNG.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadPNG.cpp @@ -101,47 +101,45 @@ void PNGAccuracy2D(const std::string compressionLevel) adios2::ConstantDims); // add operations - adios2::Operator PNGOp = - adios.DefineOperator("PNGCompressor", adios2::ops::LosslessPNG); - var_i8.AddOperation(PNGOp, {{adios2::ops::png::key::color_type, + var_i8.AddOperation("png", {{adios2::ops::png::key::color_type, adios2::ops::png::value::color_type_GRAY}, {adios2::ops::png::key::compression_level, compressionLevel}}); var_i16.AddOperation( - PNGOp, + "png", {{adios2::ops::png::key::color_type, adios2::ops::png::value::color_type_GRAY_ALPHA}, {adios2::ops::png::key::compression_level, compressionLevel}}); var_i32.AddOperation( - PNGOp, + "png", {{adios2::ops::png::key::color_type, adios2::ops::png::value::color_type_RGB_ALPHA}, {adios2::ops::png::key::compression_level, compressionLevel}}); - var_u8.AddOperation(PNGOp, {{adios2::ops::png::key::color_type, + var_u8.AddOperation("png", {{adios2::ops::png::key::color_type, adios2::ops::png::value::color_type_GRAY}, {adios2::ops::png::key::compression_level, compressionLevel}}); var_u16.AddOperation( - PNGOp, + "png", {{adios2::ops::png::key::color_type, adios2::ops::png::value::color_type_GRAY_ALPHA}, {adios2::ops::png::key::compression_level, compressionLevel}}); var_u32.AddOperation( - PNGOp, + "png", {{adios2::ops::png::key::color_type, adios2::ops::png::value::color_type_RGB_ALPHA}, {adios2::ops::png::key::compression_level, compressionLevel}}); - var_u32.AddOperation(PNGOp, {{adios2::ops::png::key::compression_level, + var_u32.AddOperation("png", {{adios2::ops::png::key::compression_level, compressionLevel}}); - var_r32.AddOperation(PNGOp, {{adios2::ops::png::key::compression_level, + var_r32.AddOperation("png", {{adios2::ops::png::key::compression_level, compressionLevel}}); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); @@ -344,13 +342,11 @@ void PNGAccuracy2DSel(const std::string accuracy) adios2::ConstantDims); // add operations - adios2::Operator PNGOp = - adios.DefineOperator("PNGCompressor", adios2::ops::LosslessPNG); var_r32.AddOperation( - PNGOp, {{adios2::ops::png::key::compression_level, accuracy}}); + "png", {{adios2::ops::png::key::compression_level, accuracy}}); var_r64.AddOperation( - PNGOp, {{adios2::ops::png::key::compression_level, accuracy}}); + "png", {{adios2::ops::png::key::compression_level, accuracy}}); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); From 1fa4961fec5e59c38494f2ad7837892e83169521 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 5 Dec 2021 20:04:11 -0500 Subject: [PATCH 05/10] simplified operator factory --- source/adios2/operator/OperatorFactory.cpp | 260 +++--------------- source/adios2/operator/OperatorFactory.h | 4 - .../format/dataman/DataManSerializer.tcc | 27 +- 3 files changed, 37 insertions(+), 254 deletions(-) diff --git a/source/adios2/operator/OperatorFactory.cpp b/source/adios2/operator/OperatorFactory.cpp index 02bdd2339e..7ac7509d5d 100644 --- a/source/adios2/operator/OperatorFactory.cpp +++ b/source/adios2/operator/OperatorFactory.cpp @@ -2,7 +2,7 @@ * Distributed under the OSI-approved Apache License, Version 2.0. See * accompanying file Copyright.txt for details. * - * CompressorFactory.cpp : + * OperatorFactory.cpp : * * Created on: Sep 29, 2021 * Author: Jason Wang jason.ruonan.wang@gmail.com @@ -50,6 +50,29 @@ namespace adios2 namespace core { +std::string OperatorTypeToString(const Operator::OperatorType type) +{ + switch (type) + { + case Operator::COMPRESS_BLOSC: + return "blosc"; + case Operator::COMPRESS_BZIP2: + return "bzip2"; + case Operator::COMPRESS_LIBPRESSIO: + return "libpressio"; + case Operator::COMPRESS_MGARD: + return "mgard"; + case Operator::COMPRESS_PNG: + return "png"; + case Operator::COMPRESS_SZ: + return "sz"; + case Operator::COMPRESS_ZFP: + return "zfp"; + default: + return "null"; + } +} + std::shared_ptr MakeOperator(const std::string &type, const Params ¶meters) { @@ -99,16 +122,20 @@ std::shared_ptr MakeOperator(const std::string &type, ret = std::make_shared(parameters); #endif } + else if (typeLowerCase == "null") + { + ret = std::make_shared(parameters); + } else { - helper::Log("Operator", "CompressorFactory", "MakeOperator", + helper::Log("Operator", "OperatorFactory", "MakeOperator", "ADIOS2 does not support " + typeLowerCase + " operation", helper::EXCEPTION); } if (ret == nullptr) { - helper::Log("Operator", "CompressorFactory", "MakeOperator", + helper::Log("Operator", "OperatorFactory", "MakeOperator", "ADIOS2 didn't compile with " + typeLowerCase + "library, operator not added", helper::EXCEPTION); @@ -117,235 +144,12 @@ std::shared_ptr MakeOperator(const std::string &type, return ret; } -size_t Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, const DataType dataType, - char *bufferOut, const Params ¶meters, - const std::string &compressorType) -{ - size_t ret = 0; - try - { - if (compressorType == "blosc") - { -#ifdef ADIOS2_HAVE_BLOSC - compress::CompressBlosc c({}); - ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, - parameters); -#else - helper::Log( - "Operator", "CompressorFactory", "Compress", - "ADIOS2 didn't compile with BLOSC, compression disabled", - helper::EXCEPTION); -#endif - } - else if (compressorType == "bzip2") - { -#ifdef ADIOS2_HAVE_BZIP2 - compress::CompressBZIP2 c({}); - ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, - parameters); -#else - helper::Log( - "Operator", "CompressorFactory", "Compress", - "ADIOS2 didn't compile with BZIP2, compression disabled", - helper::EXCEPTION); -#endif - } - else if (compressorType == "libpressio") - { -#ifdef ADIOS2_HAVE_LIBPRESSIO - compress::CompressLibPressio c({}); - ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, - parameters); -#else - helper::Log( - "Operator", "CompressorFactory", "Compress", - "ADIOS2 didn't compile with LibPressio, compression disabled", - helper::EXCEPTION); -#endif - } - else if (compressorType == "mgard") - { -#ifdef ADIOS2_HAVE_MGARD - compress::CompressMGARD c({}); - ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, - parameters); -#else - helper::Log( - "Operator", "CompressorFactory", "Compress", - "ADIOS2 didn't compile with MGARD, compression disabled", - helper::EXCEPTION); -#endif - } - else if (compressorType == "png") - { -#ifdef ADIOS2_HAVE_PNG - compress::CompressPNG c({}); - ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, - parameters); -#else - helper::Log("Operator", "CompressorFactory", "Compress", - "ADIOS2 didn't compile with PNG, compression disabled", - helper::EXCEPTION); -#endif - } - else if (compressorType == "sirius") - { -#ifdef ADIOS2_HAVE_MHS - compress::CompressSirius c({}); - ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, - parameters); -#else - helper::Log("Operator", "CompressorFactory", "Compress", - "ADIOS2 didn't enable MHS, compression disabled", - helper::EXCEPTION); -#endif - } - else if (compressorType == "sz") - { -#ifdef ADIOS2_HAVE_SZ - compress::CompressSZ c({}); - ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, - parameters); -#else - helper::Log("Operator", "CompressorFactory", "Compress", - "ADIOS2 didn't compile with SZ, compression disabled", - helper::EXCEPTION); -#endif - } - else if (compressorType == "zfp") - { -#ifdef ADIOS2_HAVE_ZFP - compress::CompressZFP c({}); - ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, - parameters); -#else - helper::Log("Operator", "CompressorFactory", "Compress", - "ADIOS2 didn't compile with ZFP, compression disabled", - helper::EXCEPTION); -#endif - } - if (ret == 0) - { - helper::Log( - "Operator", "CompressorFactory", "Compress", - "compressor " + compressorType + - " failed with returned buffer size 0, compression disabled", - helper::EXCEPTION); - } - else if (ret > helper::GetTotalSize(blockCount, - helper::GetDataTypeSize(dataType))) - { - helper::Log("Operator", "CompressorFactory", "Compress", - "compressor " + compressorType + - " produced buffer larger than uncompressed data, " - "compression disabled", - helper::EXCEPTION); - } - } - catch (std::exception &e) - { - helper::Log("Operator", "CompressorFactory", "Compress", e.what(), - helper::WARNING); - compress::CompressNull c({}); - ret = c.Operate(dataIn, blockStart, blockCount, dataType, bufferOut, - parameters); - } - return ret; -} - size_t Decompress(const char *bufferIn, const size_t sizeIn, char *dataOut) { Operator::OperatorType compressorType; std::memcpy(&compressorType, bufferIn, 1); - - if (compressorType == Operator::COMPRESS_BLOSC) - { -#ifdef ADIOS2_HAVE_BLOSC - compress::CompressBlosc op({}); - return op.InverseOperate(bufferIn, sizeIn, dataOut); -#else - helper::Log("Operator", "CompressorFactory", "Compress", - "ADIOS2 didn't compile with BLOSC", helper::EXCEPTION); -#endif - } - else if (compressorType == Operator::COMPRESS_BZIP2) - { -#ifdef ADIOS2_HAVE_BZIP2 - compress::CompressBZIP2 op({}); - return op.InverseOperate(bufferIn, sizeIn, dataOut); -#else - helper::Log("Operator", "CompressorFactory", "Compress", - "ADIOS2 didn't compile with BZIP2", helper::EXCEPTION); -#endif - } - else if (compressorType == Operator::COMPRESS_LIBPRESSIO) - { -#ifdef ADIOS2_HAVE_LIBPRESSIO - compress::CompressLibPressio op({}); - return op.InverseOperate(bufferIn, sizeIn, dataOut); -#else - helper::Log("Operator", "CompressorFactory", "Compress", - "ADIOS2 didn't compile with LibPressio", helper::EXCEPTION); -#endif - } - else if (compressorType == Operator::COMPRESS_MGARD) - { -#ifdef ADIOS2_HAVE_MGARD - compress::CompressMGARD op({}); - return op.InverseOperate(bufferIn, sizeIn, dataOut); -#else - helper::Log("Operator", "CompressorFactory", "Compress", - "ADIOS2 didn't compile with MGARD", helper::EXCEPTION); -#endif - } - else if (compressorType == Operator::COMPRESS_PNG) - { -#ifdef ADIOS2_HAVE_PNG - compress::CompressPNG op({}); - return op.InverseOperate(bufferIn, sizeIn, dataOut); -#else - helper::Log("Operator", "CompressorFactory", "Compress", - "ADIOS2 didn't compile with PNG", helper::EXCEPTION); -#endif - } - else if (compressorType == Operator::COMPRESS_SIRIUS) - { -#ifdef ADIOS2_HAVE_MHS - compress::CompressSirius op({}); - return op.InverseOperate(bufferIn, sizeIn, dataOut); -#else - helper::Log("Operator", "CompressorFactory", "Compress", - "ADIOS2 didn't enable MHS", helper::EXCEPTION); -#endif - } - else if (compressorType == Operator::COMPRESS_SZ) - { -#ifdef ADIOS2_HAVE_SZ - compress::CompressSZ op({}); - return op.InverseOperate(bufferIn, sizeIn, dataOut); -#else - helper::Log("Operator", "CompressorFactory", "Compress", - "ADIOS2 didn't compile with SZ", helper::EXCEPTION); -#endif - } - else if (compressorType == Operator::COMPRESS_ZFP) - { -#ifdef ADIOS2_HAVE_ZFP - compress::CompressZFP op({}); - return op.InverseOperate(bufferIn, sizeIn, dataOut); -#else - helper::Log("Operator", "CompressorFactory", "Compress", - "ADIOS2 didn't compile with ZFP", helper::EXCEPTION); -#endif - } - else if (compressorType == Operator::COMPRESS_NULL) - { - compress::CompressNull op({}); - return op.InverseOperate(bufferIn, sizeIn, dataOut); - } - - return 0; + auto op = MakeOperator(OperatorTypeToString(compressorType), Params()); + return op->InverseOperate(bufferIn, sizeIn, dataOut); } } // end namespace core diff --git a/source/adios2/operator/OperatorFactory.h b/source/adios2/operator/OperatorFactory.h index 71af2ad3a0..bc3c5dcbb5 100644 --- a/source/adios2/operator/OperatorFactory.h +++ b/source/adios2/operator/OperatorFactory.h @@ -20,10 +20,6 @@ namespace core std::shared_ptr MakeOperator(const std::string &type, const Params ¶meters); -size_t Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, const DataType type, char *bufferOut, - const Params ¶meters, const std::string &compressorType); - size_t Decompress(const char *bufferIn, const size_t sizeIn, char *dataOut); } // end namespace core diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc index 50da9efd1e..2de5c95718 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc @@ -12,26 +12,9 @@ #define ADIOS2_TOOLKIT_FORMAT_DATAMAN_DATAMANSERIALIZER_TCC_ #include "DataManSerializer.h" - -#ifdef ADIOS2_HAVE_ZFP -#include "adios2/operator/compress/CompressZFP.h" -#endif -#ifdef ADIOS2_HAVE_SZ -#include "adios2/operator/compress/CompressSZ.h" -#endif -#ifdef ADIOS2_HAVE_BZIP2 -#include "adios2/operator/compress/CompressBZIP2.h" -#endif -#ifdef ADIOS2_HAVE_MGARD -#include "adios2/operator/compress/CompressMGARD.h" -#endif - -#include "adios2/operator/OperatorFactory.h" - #include "adios2/helper/adiosFunctions.h" - +#include "adios2/operator/OperatorFactory.h" #include - #include namespace adios2 @@ -158,10 +141,10 @@ void DataManSerializer::PutData(const T *inputData, const std::string &varName, varCount.end(), sizeof(T), std::multiplies())); - datasize = core::Compress(reinterpret_cast(inputData), - varStart, varCount, helper::GetDataType(), - m_CompressBuffer.data(), - ops[0]->GetParameters(), compressionMethod); + datasize = + ops[0]->Operate(reinterpret_cast(inputData), varStart, + varCount, helper::GetDataType(), + m_CompressBuffer.data(), ops[0]->GetParameters()); compressed = true; } From 3ca18a082453568dce1f5913a5de4330b444e1cb Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 5 Dec 2021 21:29:06 -0500 Subject: [PATCH 06/10] unified operator parameters --- source/adios2/core/Operator.h | 2 +- source/adios2/helper/adiosLog.cpp | 4 ++-- source/adios2/operator/OperatorFactory.cpp | 6 ++++++ source/adios2/operator/callback/Signature1.cpp | 2 +- source/adios2/operator/callback/Signature1.h | 5 ++--- source/adios2/operator/callback/Signature2.cpp | 2 +- source/adios2/operator/callback/Signature2.h | 5 ++--- source/adios2/operator/compress/CompressBZIP2.cpp | 11 ++++++----- source/adios2/operator/compress/CompressBZIP2.h | 5 ++--- source/adios2/operator/compress/CompressBlosc.cpp | 4 ++-- source/adios2/operator/compress/CompressBlosc.h | 5 ++--- .../operator/compress/CompressLibPressio.cpp | 6 +++--- .../adios2/operator/compress/CompressLibPressio.h | 5 ++--- source/adios2/operator/compress/CompressMGARD.cpp | 14 +++++++------- source/adios2/operator/compress/CompressMGARD.h | 5 ++--- source/adios2/operator/compress/CompressNull.cpp | 2 +- source/adios2/operator/compress/CompressNull.h | 4 ++-- source/adios2/operator/compress/CompressPNG.cpp | 4 ++-- source/adios2/operator/compress/CompressPNG.h | 5 ++--- source/adios2/operator/compress/CompressSZ.cpp | 4 ++-- source/adios2/operator/compress/CompressSZ.h | 5 ++--- source/adios2/operator/compress/CompressSirius.cpp | 2 +- source/adios2/operator/compress/CompressSirius.h | 4 ++-- source/adios2/operator/compress/CompressZFP.cpp | 6 +++--- source/adios2/operator/compress/CompressZFP.h | 5 ++--- source/adios2/toolkit/format/bp/BPSerializer.tcc | 3 +-- source/adios2/toolkit/format/bp5/BP5Serializer.cpp | 2 +- .../toolkit/format/dataman/DataManSerializer.tcc | 7 +++---- .../adios2/interface/TestADIOSInterfaceWrite.cpp | 2 +- 29 files changed, 66 insertions(+), 70 deletions(-) diff --git a/source/adios2/core/Operator.h b/source/adios2/core/Operator.h index 58bf5146a4..b445852269 100644 --- a/source/adios2/core/Operator.h +++ b/source/adios2/core/Operator.h @@ -76,7 +76,7 @@ class Operator */ virtual size_t Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType type, - char *bufferOut, const Params ¶meters) = 0; + char *bufferOut) = 0; /** * @param bufferIn diff --git a/source/adios2/helper/adiosLog.cpp b/source/adios2/helper/adiosLog.cpp index 174fae13db..00dc514a10 100644 --- a/source/adios2/helper/adiosLog.cpp +++ b/source/adios2/helper/adiosLog.cpp @@ -59,7 +59,7 @@ void Log(const std::string &component, const std::string &source, { if (mode == LogMode::EXCEPTION) { - throw(message); + throw message; } else { @@ -118,7 +118,7 @@ void Log(const std::string &component, const std::string &source, else if (mode == EXCEPTION) { std::cerr << m.str(); - throw(m.str()); + throw m.str(); } } diff --git a/source/adios2/operator/OperatorFactory.cpp b/source/adios2/operator/OperatorFactory.cpp index 7ac7509d5d..dec66b5bc8 100644 --- a/source/adios2/operator/OperatorFactory.cpp +++ b/source/adios2/operator/OperatorFactory.cpp @@ -64,6 +64,8 @@ std::string OperatorTypeToString(const Operator::OperatorType type) return "mgard"; case Operator::COMPRESS_PNG: return "png"; + case Operator::COMPRESS_SIRIUS: + return "sirius"; case Operator::COMPRESS_SZ: return "sz"; case Operator::COMPRESS_ZFP: @@ -110,6 +112,10 @@ std::shared_ptr MakeOperator(const std::string &type, ret = std::make_shared(parameters); #endif } + else if (typeLowerCase == "sirius") + { + ret = std::make_shared(parameters); + } else if (typeLowerCase == "sz") { #ifdef ADIOS2_HAVE_SZ diff --git a/source/adios2/operator/callback/Signature1.cpp b/source/adios2/operator/callback/Signature1.cpp index a713354b1a..8067e29f23 100644 --- a/source/adios2/operator/callback/Signature1.cpp +++ b/source/adios2/operator/callback/Signature1.cpp @@ -52,7 +52,7 @@ ADIOS2_FOREACH_STDTYPE_2ARGS(declare_type) size_t Signature1::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType type, - char *bufferOut, const Params ¶meters) + char *bufferOut) { return 0; } diff --git a/source/adios2/operator/callback/Signature1.h b/source/adios2/operator/callback/Signature1.h index 8162018150..8728bf37d2 100644 --- a/source/adios2/operator/callback/Signature1.h +++ b/source/adios2/operator/callback/Signature1.h @@ -50,12 +50,11 @@ class Signature1 : public Operator * @param type * @param bufferOut format will be: 'DataHeader ; (BloscCompressedChunk | * UncompressedData), [ BloscCompressedChunk, ...]' - * @param parameters * @return size of compressed buffer in bytes */ size_t Operate(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, const DataType type, char *bufferOut, - const Params ¶meters) final; + const Dims &blockCount, const DataType type, + char *bufferOut) final; /** * @param bufferIn diff --git a/source/adios2/operator/callback/Signature2.cpp b/source/adios2/operator/callback/Signature2.cpp index a97fd24e99..6683f21bde 100644 --- a/source/adios2/operator/callback/Signature2.cpp +++ b/source/adios2/operator/callback/Signature2.cpp @@ -45,7 +45,7 @@ void Signature2::RunCallback2(void *arg1, const std::string &arg2, size_t Signature2::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType type, - char *bufferOut, const Params ¶meters) + char *bufferOut) { return 0; } diff --git a/source/adios2/operator/callback/Signature2.h b/source/adios2/operator/callback/Signature2.h index 512e28f813..87fe65fc55 100644 --- a/source/adios2/operator/callback/Signature2.h +++ b/source/adios2/operator/callback/Signature2.h @@ -43,12 +43,11 @@ class Signature2 : public Operator * @param type * @param bufferOut format will be: 'DataHeader ; (BloscCompressedChunk | * UncompressedData), [ BloscCompressedChunk, ...]' - * @param parameters * @return size of compressed buffer in bytes */ size_t Operate(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, const DataType type, char *bufferOut, - const Params ¶meters) final; + const Dims &blockCount, const DataType type, + char *bufferOut) final; /** * @param bufferIn diff --git a/source/adios2/operator/compress/CompressBZIP2.cpp b/source/adios2/operator/compress/CompressBZIP2.cpp index abd24fcad0..0e83bb29fc 100644 --- a/source/adios2/operator/compress/CompressBZIP2.cpp +++ b/source/adios2/operator/compress/CompressBZIP2.cpp @@ -34,7 +34,7 @@ CompressBZIP2::CompressBZIP2(const Params ¶meters) size_t CompressBZIP2::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, DataType type, - char *bufferOut, const Params ¶meters) + char *bufferOut) { const uint8_t bufferVersion = 1; @@ -54,14 +54,15 @@ size_t CompressBZIP2::Operate(const char *dataIn, const Dims &blockStart, int blockSize100k = 1; int verbosity = 0; int workFactor = 0; - if (!parameters.empty()) + if (!m_Parameters.empty()) { const std::string hint(" in call to CompressBZIP2 Compress " + ToString(type) + "\n"); - helper::SetParameterValueInt("blockSize100k", parameters, blockSize100k, + helper::SetParameterValueInt("blockSize100k", m_Parameters, + blockSize100k, hint); + helper::SetParameterValueInt("verbosity", m_Parameters, verbosity, hint); - helper::SetParameterValueInt("verbosity", parameters, verbosity, hint); - helper::SetParameterValueInt("workFactor", parameters, workFactor, + helper::SetParameterValueInt("workFactor", m_Parameters, workFactor, hint); if (blockSize100k < 1 || blockSize100k > 9) { diff --git a/source/adios2/operator/compress/CompressBZIP2.h b/source/adios2/operator/compress/CompressBZIP2.h index f2039278c5..28bb744868 100644 --- a/source/adios2/operator/compress/CompressBZIP2.h +++ b/source/adios2/operator/compress/CompressBZIP2.h @@ -37,12 +37,11 @@ class CompressBZIP2 : public Operator * @param blockCount * @param type * @param bufferOut - * @param parameters * @return size of compressed buffer */ size_t Operate(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, const DataType type, char *bufferOut, - const Params ¶meters) final; + const Dims &blockCount, const DataType type, + char *bufferOut) final; /** * @param bufferIn diff --git a/source/adios2/operator/compress/CompressBlosc.cpp b/source/adios2/operator/compress/CompressBlosc.cpp index d8d638360f..fc98dbe3d7 100644 --- a/source/adios2/operator/compress/CompressBlosc.cpp +++ b/source/adios2/operator/compress/CompressBlosc.cpp @@ -44,7 +44,7 @@ CompressBlosc::CompressBlosc(const Params ¶meters) size_t CompressBlosc::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType type, - char *bufferOut, const Params ¶meters) + char *bufferOut) { size_t bufferOutOffset = 0; const uint8_t bufferVersion = 1; @@ -77,7 +77,7 @@ size_t CompressBlosc::Operate(const char *dataIn, const Dims &blockStart, std::string compressor = "blosclz"; size_t blockSize = 0; - for (const auto &itParameter : parameters) + for (const auto &itParameter : m_Parameters) { const std::string key = itParameter.first; const std::string value = itParameter.second; diff --git a/source/adios2/operator/compress/CompressBlosc.h b/source/adios2/operator/compress/CompressBlosc.h index 7e683c5a9f..69f225cc5b 100644 --- a/source/adios2/operator/compress/CompressBlosc.h +++ b/source/adios2/operator/compress/CompressBlosc.h @@ -50,12 +50,11 @@ class CompressBlosc : public Operator * @param type * @param bufferOut format will be: 'DataHeader ; (BloscCompressedChunk | * UncompressedData), [ BloscCompressedChunk, ...]' - * @param parameters * @return size of compressed buffer in bytes */ size_t Operate(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, const DataType type, char *bufferOut, - const Params ¶meters) final; + const Dims &blockCount, const DataType type, + char *bufferOut) final; /** * @param bufferIn diff --git a/source/adios2/operator/compress/CompressLibPressio.cpp b/source/adios2/operator/compress/CompressLibPressio.cpp index 1892695b5a..9f7231b949 100644 --- a/source/adios2/operator/compress/CompressLibPressio.cpp +++ b/source/adios2/operator/compress/CompressLibPressio.cpp @@ -286,7 +286,7 @@ CompressLibPressio::CompressLibPressio(const Params ¶meters) } size_t CompressLibPressio::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType type, - char *bufferOut, const Params ¶meters) + char *bufferOut) { const uint8_t bufferVersion = 1; size_t bufferOutOffset = 0; @@ -308,7 +308,7 @@ size_t CompressLibPressio::Operate(const char *dataIn, const Dims &blockStart, static_cast(pressio_minor_version())); PutParameter(bufferOut, bufferOutOffset, static_cast(pressio_patch_version())); - PutParameters(bufferOut, bufferOutOffset, parameters); + PutParameters(bufferOut, bufferOutOffset, m_Parameters); // zfp V1 metadata end auto inputs_dims = adios_to_libpressio_dims(blockCount); @@ -320,7 +320,7 @@ size_t CompressLibPressio::Operate(const char *dataIn, const Dims &blockStart, pressio_compressor *compressor = nullptr; try { - compressor = adios_to_libpressio_compressor(parameters); + compressor = adios_to_libpressio_compressor(m_Parameters); } catch (std::exception &e) { diff --git a/source/adios2/operator/compress/CompressLibPressio.h b/source/adios2/operator/compress/CompressLibPressio.h index 482c517367..ccff27d491 100644 --- a/source/adios2/operator/compress/CompressLibPressio.h +++ b/source/adios2/operator/compress/CompressLibPressio.h @@ -37,12 +37,11 @@ class CompressLibPressio : public Operator * @param blockCount * @param type * @param bufferOut - * @param parameters * @return size of compressed buffer */ size_t Operate(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, const DataType type, char *bufferOut, - const Params ¶meters) final; + const Dims &blockCount, const DataType type, + char *bufferOut) final; /** * @param bufferIn diff --git a/source/adios2/operator/compress/CompressMGARD.cpp b/source/adios2/operator/compress/CompressMGARD.cpp index c27dfb6c8c..47b93fcd41 100644 --- a/source/adios2/operator/compress/CompressMGARD.cpp +++ b/source/adios2/operator/compress/CompressMGARD.cpp @@ -28,7 +28,7 @@ CompressMGARD::CompressMGARD(const Params ¶meters) size_t CompressMGARD::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType type, - char *bufferOut, const Params ¶meters) + char *bufferOut) { const uint8_t bufferVersion = 1; size_t bufferOutOffset = 0; @@ -84,14 +84,14 @@ size_t CompressMGARD::Operate(const char *dataIn, const Dims &blockStart, // Parameters bool hasTolerance = false; double tolerance, s = 0.0; - auto itAccuracy = parameters.find("accuracy"); - if (itAccuracy != parameters.end()) + auto itAccuracy = m_Parameters.find("accuracy"); + if (itAccuracy != m_Parameters.end()) { tolerance = std::stod(itAccuracy->second); hasTolerance = true; } - auto itTolerance = parameters.find("tolerance"); - if (itTolerance != parameters.end()) + auto itTolerance = m_Parameters.find("tolerance"); + if (itTolerance != m_Parameters.end()) { tolerance = std::stod(itTolerance->second); hasTolerance = true; @@ -102,8 +102,8 @@ size_t CompressMGARD::Operate(const char *dataIn, const Dims &blockStart, "tolerance for MGARD compression " "operator\n"); } - auto itSParameter = parameters.find("s"); - if (itSParameter != parameters.end()) + auto itSParameter = m_Parameters.find("s"); + if (itSParameter != m_Parameters.end()) { s = std::stod(itSParameter->second); } diff --git a/source/adios2/operator/compress/CompressMGARD.h b/source/adios2/operator/compress/CompressMGARD.h index eaa5ba9c33..af0ce1eeaa 100644 --- a/source/adios2/operator/compress/CompressMGARD.h +++ b/source/adios2/operator/compress/CompressMGARD.h @@ -37,12 +37,11 @@ class CompressMGARD : public Operator * @param blockCount * @param type * @param bufferOut - * @param parameters * @return size of compressed buffer */ size_t Operate(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, const DataType type, char *bufferOut, - const Params ¶meters) final; + const Dims &blockCount, const DataType type, + char *bufferOut) final; /** * @param bufferIn diff --git a/source/adios2/operator/compress/CompressNull.cpp b/source/adios2/operator/compress/CompressNull.cpp index 6dd419c4b6..9b3b557441 100644 --- a/source/adios2/operator/compress/CompressNull.cpp +++ b/source/adios2/operator/compress/CompressNull.cpp @@ -25,7 +25,7 @@ CompressNull::CompressNull(const Params ¶meters) size_t CompressNull::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType varType, - char *bufferOut, const Params ¶ms) + char *bufferOut) { const uint8_t bufferVersion = 1; size_t bufferOutOffset = 0; diff --git a/source/adios2/operator/compress/CompressNull.h b/source/adios2/operator/compress/CompressNull.h index 08b86da4af..d772601215 100644 --- a/source/adios2/operator/compress/CompressNull.h +++ b/source/adios2/operator/compress/CompressNull.h @@ -29,8 +29,8 @@ class CompressNull : public Operator ~CompressNull() = default; size_t Operate(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, const DataType type, char *bufferOut, - const Params ¶ms) final; + const Dims &blockCount, const DataType type, + char *bufferOut) final; size_t InverseOperate(const char *bufferIn, const size_t sizeIn, char *dataOut) final; diff --git a/source/adios2/operator/compress/CompressPNG.cpp b/source/adios2/operator/compress/CompressPNG.cpp index 0439e458f9..01fa1de1a7 100644 --- a/source/adios2/operator/compress/CompressPNG.cpp +++ b/source/adios2/operator/compress/CompressPNG.cpp @@ -50,7 +50,7 @@ CompressPNG::CompressPNG(const Params ¶meters) size_t CompressPNG::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType type, - char *bufferOut, const Params ¶meters) + char *bufferOut) { size_t bufferOutOffset = 0; const uint8_t bufferVersion = 1; @@ -84,7 +84,7 @@ size_t CompressPNG::Operate(const char *dataIn, const Dims &blockStart, int bitDepth = 8; std::string colorTypeStr = "PNG_COLOR_TYPE_RGBA"; - for (const auto &itParameter : parameters) + for (const auto &itParameter : m_Parameters) { const std::string key = itParameter.first; const std::string value = itParameter.second; diff --git a/source/adios2/operator/compress/CompressPNG.h b/source/adios2/operator/compress/CompressPNG.h index 0bb5c8a9b6..bbc37ae11a 100644 --- a/source/adios2/operator/compress/CompressPNG.h +++ b/source/adios2/operator/compress/CompressPNG.h @@ -39,12 +39,11 @@ class CompressPNG : public Operator * @param blockCount * @param type * @param bufferOut - * @param parameters * @return size of compressed buffer */ size_t Operate(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, const DataType type, char *bufferOut, - const Params ¶meters) final; + const Dims &blockCount, const DataType type, + char *bufferOut) final; /** * @param bufferIn diff --git a/source/adios2/operator/compress/CompressSZ.cpp b/source/adios2/operator/compress/CompressSZ.cpp index 1ac009501b..ff8bed4774 100644 --- a/source/adios2/operator/compress/CompressSZ.cpp +++ b/source/adios2/operator/compress/CompressSZ.cpp @@ -34,7 +34,7 @@ CompressSZ::CompressSZ(const Params ¶meters) size_t CompressSZ::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType varType, - char *bufferOut, const Params ¶meters) + char *bufferOut) { const uint8_t bufferVersion = 2; size_t bufferOutOffset = 0; @@ -85,7 +85,7 @@ size_t CompressSZ::Operate(const char *dataIn, const Dims &blockStart, std::string sz_configfile = "sz.config"; Params::const_iterator it; - for (it = parameters.begin(); it != parameters.end(); it++) + for (it = m_Parameters.begin(); it != m_Parameters.end(); it++) { if (it->first == "init") { diff --git a/source/adios2/operator/compress/CompressSZ.h b/source/adios2/operator/compress/CompressSZ.h index 0b21086a75..789c0cebc5 100644 --- a/source/adios2/operator/compress/CompressSZ.h +++ b/source/adios2/operator/compress/CompressSZ.h @@ -37,12 +37,11 @@ class CompressSZ : public Operator * @param blockCount * @param type * @param bufferOut - * @param parameters * @return size of compressed buffer */ size_t Operate(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, const DataType type, char *bufferOut, - const Params ¶meters) final; + const Dims &blockCount, const DataType type, + char *bufferOut) final; /** * @param bufferIn diff --git a/source/adios2/operator/compress/CompressSirius.cpp b/source/adios2/operator/compress/CompressSirius.cpp index d03df49a5d..597d7a2aa0 100644 --- a/source/adios2/operator/compress/CompressSirius.cpp +++ b/source/adios2/operator/compress/CompressSirius.cpp @@ -36,7 +36,7 @@ CompressSirius::CompressSirius(const Params ¶meters) size_t CompressSirius::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType varType, - char *bufferOut, const Params ¶ms) + char *bufferOut) { const uint8_t bufferVersion = 1; size_t bufferOutOffset = 0; diff --git a/source/adios2/operator/compress/CompressSirius.h b/source/adios2/operator/compress/CompressSirius.h index a34c99c00b..72d9667ef8 100644 --- a/source/adios2/operator/compress/CompressSirius.h +++ b/source/adios2/operator/compress/CompressSirius.h @@ -30,8 +30,8 @@ class CompressSirius : public Operator ~CompressSirius() = default; size_t Operate(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, const DataType type, char *bufferOut, - const Params ¶ms) final; + const Dims &blockCount, const DataType type, + char *bufferOut) final; size_t InverseOperate(const char *bufferIn, const size_t sizeIn, char *dataOut) final; diff --git a/source/adios2/operator/compress/CompressZFP.cpp b/source/adios2/operator/compress/CompressZFP.cpp index 87cb731b5c..88660d44ba 100644 --- a/source/adios2/operator/compress/CompressZFP.cpp +++ b/source/adios2/operator/compress/CompressZFP.cpp @@ -59,7 +59,7 @@ CompressZFP::CompressZFP(const Params ¶meters) size_t CompressZFP::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType type, - char *bufferOut, const Params ¶meters) + char *bufferOut) { const uint8_t bufferVersion = 1; @@ -82,13 +82,13 @@ size_t CompressZFP::Operate(const char *dataIn, const Dims &blockStart, static_cast(ZFP_VERSION_MINOR)); PutParameter(bufferOut, bufferOutOffset, static_cast(ZFP_VERSION_RELEASE)); - PutParameters(bufferOut, bufferOutOffset, parameters); + PutParameters(bufferOut, bufferOutOffset, m_Parameters); // zfp V1 metadata end Dims convertedDims = ConvertDims(blockCount, type, 3); zfp_field *field = GetZFPField(dataIn, convertedDims, type); - zfp_stream *stream = GetZFPStream(convertedDims, type, parameters); + zfp_stream *stream = GetZFPStream(convertedDims, type, m_Parameters); size_t maxSize = zfp_stream_maximum_size(stream, field); // associate bitstream diff --git a/source/adios2/operator/compress/CompressZFP.h b/source/adios2/operator/compress/CompressZFP.h index 70c6b7f533..0a30160727 100644 --- a/source/adios2/operator/compress/CompressZFP.h +++ b/source/adios2/operator/compress/CompressZFP.h @@ -38,12 +38,11 @@ class CompressZFP : public Operator * @param blockCount * @param type * @param bufferOut - * @param parameters * @return size of compressed buffer */ size_t Operate(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, const DataType type, char *bufferOut, - const Params ¶meters) final; + const Dims &blockCount, const DataType type, + char *bufferOut) final; /** * @param bufferIn diff --git a/source/adios2/toolkit/format/bp/BPSerializer.tcc b/source/adios2/toolkit/format/bp/BPSerializer.tcc index c069aa3b2e..ecd0d88f3b 100644 --- a/source/adios2/toolkit/format/bp/BPSerializer.tcc +++ b/source/adios2/toolkit/format/bp/BPSerializer.tcc @@ -405,8 +405,7 @@ void BPSerializer::PutOperationPayloadInBuffer( const size_t outputSize = blockInfo.Operations[0]->Operate( reinterpret_cast(blockInfo.Data), blockInfo.Start, blockInfo.Count, variable.m_Type, - m_Data.m_Buffer.data() + m_Data.m_Position, - blockInfo.Operations[0]->GetParameters()); + m_Data.m_Buffer.data() + m_Data.m_Position); m_Data.m_Position += outputSize; m_Data.m_AbsolutePosition += outputSize; diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp index 384b703973..a9039dfb29 100644 --- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp @@ -626,7 +626,7 @@ void BP5Serializer::Marshal(void *Variable, const char *Name, DataOffset = m_PriorDataBufferSizeTotal + pos.globalPos; CompressedSize = VB->m_Operations[0]->Operate( (const char *)Data, tmpOffsets, tmpCount, (DataType)Rec->Type, - CompressedData, VB->m_Operations[0]->GetParameters()); + CompressedData); // use data size to resize allocated buffer } else if (Span == nullptr) diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc index 2de5c95718..6cc9a181c7 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc @@ -141,10 +141,9 @@ void DataManSerializer::PutData(const T *inputData, const std::string &varName, varCount.end(), sizeof(T), std::multiplies())); - datasize = - ops[0]->Operate(reinterpret_cast(inputData), varStart, - varCount, helper::GetDataType(), - m_CompressBuffer.data(), ops[0]->GetParameters()); + datasize = ops[0]->Operate(reinterpret_cast(inputData), + varStart, varCount, helper::GetDataType(), + m_CompressBuffer.data()); compressed = true; } diff --git a/testing/adios2/interface/TestADIOSInterfaceWrite.cpp b/testing/adios2/interface/TestADIOSInterfaceWrite.cpp index 20efe62899..db36415a92 100644 --- a/testing/adios2/interface/TestADIOSInterfaceWrite.cpp +++ b/testing/adios2/interface/TestADIOSInterfaceWrite.cpp @@ -592,7 +592,7 @@ TEST_F(ADIOSInterfaceWriteTest, Exceptions) EXPECT_FALSE(invalidOp); EXPECT_THROW(adios.AtIO("IOnull"), std::invalid_argument); EXPECT_THROW(adios.DefineOperator("WrongOp", "UnsupportedType"), - std::invalid_argument); + std::string); #ifdef ADIOS2_HAVE_BZIP2 EXPECT_NO_THROW(adios.DefineOperator("bzip2Op", "bzip2")); From 9375b98981004a9e64e76c0cd351fe4fe0daa7cc Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 5 Dec 2021 22:40:06 -0500 Subject: [PATCH 07/10] fix exception types --- testing/adios2/interface/TestADIOSInterfaceWrite.cpp | 7 +++---- testing/adios2/xml/TestXMLConfig.cpp | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/testing/adios2/interface/TestADIOSInterfaceWrite.cpp b/testing/adios2/interface/TestADIOSInterfaceWrite.cpp index db36415a92..c8138349b6 100644 --- a/testing/adios2/interface/TestADIOSInterfaceWrite.cpp +++ b/testing/adios2/interface/TestADIOSInterfaceWrite.cpp @@ -599,22 +599,21 @@ TEST_F(ADIOSInterfaceWriteTest, Exceptions) EXPECT_THROW(adios.DefineOperator("bzip2Op", "bzip2"), std::invalid_argument); #else - EXPECT_THROW(adios.DefineOperator("bzip2Op", "bzip2"), - std::invalid_argument); + EXPECT_THROW(adios.DefineOperator("bzip2Op", "bzip2"), std::string); #endif #ifdef ADIOS2_HAVE_ZFP EXPECT_NO_THROW(adios.DefineOperator("zfpOp", "zfp")); EXPECT_THROW(adios.DefineOperator("zfpOp", "zfp"), std::invalid_argument); #else - EXPECT_THROW(adios.DefineOperator("zfpOp", "zfp"), std::invalid_argument); + EXPECT_THROW(adios.DefineOperator("zfpOp", "zfp"), std::string); #endif #ifdef ADIOS2_HAVE_SZ EXPECT_NO_THROW(adios.DefineOperator("szOp", "sz")); EXPECT_THROW(adios.DefineOperator("szOp", "sz"), std::invalid_argument); #else - EXPECT_THROW(adios.DefineOperator("szOp", "sz"), std::invalid_argument); + EXPECT_THROW(adios.DefineOperator("szOp", "sz"), std::string); #endif } diff --git a/testing/adios2/xml/TestXMLConfig.cpp b/testing/adios2/xml/TestXMLConfig.cpp index 3667284e46..83d7d3320e 100644 --- a/testing/adios2/xml/TestXMLConfig.cpp +++ b/testing/adios2/xml/TestXMLConfig.cpp @@ -92,7 +92,7 @@ TEST_F(XMLConfigTest, OpTypeException) if (rank == 0) { EXPECT_THROW(adios2::ADIOS adios(configFile, MPI_COMM_SELF), - std::invalid_argument); + std::string); } #else EXPECT_THROW(adios2::ADIOS adios(configFile), std::invalid_argument); @@ -106,10 +106,9 @@ TEST_F(XMLConfigTest, OpNullException) "configOpNullException.xml"); #if ADIOS2_USE_MPI - EXPECT_THROW(adios2::ADIOS adios(configFile, MPI_COMM_WORLD), - std::invalid_argument); + EXPECT_THROW(adios2::ADIOS adios(configFile, MPI_COMM_WORLD), std::string); #else - EXPECT_THROW(adios2::ADIOS adios(configFile), std::invalid_argument); + EXPECT_THROW(adios2::ADIOS adios(configFile), std::string); #endif } From a499d8bf30c83768c3db1897731fe951f6edca16 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 5 Dec 2021 23:08:15 -0500 Subject: [PATCH 08/10] fix more exception types --- source/adios2/operator/OperatorFactory.cpp | 2 +- testing/adios2/xml/TestXMLConfig.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/adios2/operator/OperatorFactory.cpp b/source/adios2/operator/OperatorFactory.cpp index dec66b5bc8..a6f07374ea 100644 --- a/source/adios2/operator/OperatorFactory.cpp +++ b/source/adios2/operator/OperatorFactory.cpp @@ -143,7 +143,7 @@ std::shared_ptr MakeOperator(const std::string &type, { helper::Log("Operator", "OperatorFactory", "MakeOperator", "ADIOS2 didn't compile with " + typeLowerCase + - "library, operator not added", + " library, operator not added", helper::EXCEPTION); } diff --git a/testing/adios2/xml/TestXMLConfig.cpp b/testing/adios2/xml/TestXMLConfig.cpp index 83d7d3320e..4cbf3d2c5b 100644 --- a/testing/adios2/xml/TestXMLConfig.cpp +++ b/testing/adios2/xml/TestXMLConfig.cpp @@ -92,7 +92,7 @@ TEST_F(XMLConfigTest, OpTypeException) if (rank == 0) { EXPECT_THROW(adios2::ADIOS adios(configFile, MPI_COMM_SELF), - std::string); + std::invalid_argument); } #else EXPECT_THROW(adios2::ADIOS adios(configFile), std::invalid_argument); @@ -122,7 +122,7 @@ TEST_F(XMLConfigTest, OpNoneException) EXPECT_THROW(adios2::ADIOS adios(configFile, MPI_COMM_WORLD), std::invalid_argument); #else - EXPECT_THROW(adios2::ADIOS adios(configFile), std::invalid_argument); + EXPECT_THROW(adios2::ADIOS adios(configFile), std::string); #endif } From a9d8ceb5073f499b24d11754edfe85be1df48f2c Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 5 Dec 2021 23:27:55 -0500 Subject: [PATCH 09/10] centralized operator data type check knowledge from global macros to specific operator classes --- source/adios2/common/ADIOSMacros.h | 30 ------------------- .../operator/compress/CompressLibPressio.cpp | 13 ++++---- .../operator/compress/CompressMGARD.cpp | 9 ++---- .../adios2/operator/compress/CompressSZ.cpp | 10 +++---- .../operator/compress/CompressSirius.cpp | 9 ++---- .../adios2/operator/compress/CompressZFP.cpp | 11 ++++--- testing/adios2/xml/TestXMLConfig.cpp | 7 +++-- 7 files changed, 26 insertions(+), 63 deletions(-) diff --git a/source/adios2/common/ADIOSMacros.h b/source/adios2/common/ADIOSMacros.h index 6e05fd968f..8b986901d5 100644 --- a/source/adios2/common/ADIOSMacros.h +++ b/source/adios2/common/ADIOSMacros.h @@ -121,36 +121,6 @@ MACRO(std::complex) \ MACRO(std::complex) -#define ADIOS2_FOREACH_ZFP_TYPE_1ARG(MACRO) \ - MACRO(int32_t) \ - MACRO(int64_t) \ - MACRO(float) \ - MACRO(double) \ - MACRO(std::complex) \ - MACRO(std::complex) - -#define ADIOS2_FOREACH_SZ_TYPE_1ARG(MACRO) \ - MACRO(float) \ - MACRO(double) \ - MACRO(std::complex) \ - MACRO(std::complex) - -#define ADIOS2_FOREACH_SIRIUS_TYPE_1ARG(MACRO) MACRO(float) - -#define ADIOS2_FOREACH_LIBPRESSIO_TYPE_1ARG(MACRO) \ - MACRO(uint8_t) \ - MACRO(uint16_t) \ - MACRO(uint32_t) \ - MACRO(uint64_t) \ - MACRO(int8_t) \ - MACRO(int16_t) \ - MACRO(int32_t) \ - MACRO(int64_t) \ - MACRO(float) \ - MACRO(double) - -#define ADIOS2_FOREACH_MGARD_TYPE_1ARG(MACRO) MACRO(double) - #define ADIOS2_FOREACH_ATTRIBUTE_TYPE_1ARG(MACRO) \ MACRO(std::string) \ MACRO(char) \ diff --git a/source/adios2/operator/compress/CompressLibPressio.cpp b/source/adios2/operator/compress/CompressLibPressio.cpp index 9f7231b949..b56c790408 100644 --- a/source/adios2/operator/compress/CompressLibPressio.cpp +++ b/source/adios2/operator/compress/CompressLibPressio.cpp @@ -376,13 +376,14 @@ size_t CompressLibPressio::InverseOperate(const char *bufferIn, bool CompressLibPressio::IsDataTypeValid(const DataType type) const { -#define declare_type(T) \ - if (helper::GetDataType() == type) \ - { \ - return true; \ + if (type == DataType::Int8 || type == DataType::UInt8 || + type == DataType::Int16 || type == DataType::UInt16 || + type == DataType::Int32 || type == DataType::UInt32 || + type == DataType::Int64 || type == DataType::UInt64 || + type == DataType::Float || type == DataType::Double) + { + return true; } - ADIOS2_FOREACH_LIBPRESSIO_TYPE_1ARG(declare_type) -#undef declare_type return false; } diff --git a/source/adios2/operator/compress/CompressMGARD.cpp b/source/adios2/operator/compress/CompressMGARD.cpp index 47b93fcd41..d66c4fec9a 100644 --- a/source/adios2/operator/compress/CompressMGARD.cpp +++ b/source/adios2/operator/compress/CompressMGARD.cpp @@ -150,13 +150,10 @@ size_t CompressMGARD::InverseOperate(const char *bufferIn, const size_t sizeIn, bool CompressMGARD::IsDataTypeValid(const DataType type) const { -#define declare_type(T) \ - if (helper::GetDataType() == type) \ - { \ - return true; \ + if (type == DataType::Double) + { + return true; } - ADIOS2_FOREACH_MGARD_TYPE_1ARG(declare_type) -#undef declare_type return false; } diff --git a/source/adios2/operator/compress/CompressSZ.cpp b/source/adios2/operator/compress/CompressSZ.cpp index ff8bed4774..0041b82962 100644 --- a/source/adios2/operator/compress/CompressSZ.cpp +++ b/source/adios2/operator/compress/CompressSZ.cpp @@ -321,13 +321,11 @@ size_t CompressSZ::InverseOperate(const char *bufferIn, const size_t sizeIn, bool CompressSZ::IsDataTypeValid(const DataType type) const { -#define declare_type(T) \ - if (helper::GetDataType() == type) \ - { \ - return true; \ + if (type == DataType::Float || type == DataType::Double || + type == DataType::FloatComplex || type == DataType::DoubleComplex) + { + return true; } - ADIOS2_FOREACH_SZ_TYPE_1ARG(declare_type) -#undef declare_type return false; } diff --git a/source/adios2/operator/compress/CompressSirius.cpp b/source/adios2/operator/compress/CompressSirius.cpp index 597d7a2aa0..42db452621 100644 --- a/source/adios2/operator/compress/CompressSirius.cpp +++ b/source/adios2/operator/compress/CompressSirius.cpp @@ -115,13 +115,10 @@ size_t CompressSirius::InverseOperate(const char *bufferIn, const size_t sizeIn, bool CompressSirius::IsDataTypeValid(const DataType type) const { -#define declare_type(T) \ - if (helper::GetDataType() == type) \ - { \ - return true; \ + if (type == DataType::Float) + { + return true; } - ADIOS2_FOREACH_SIRIUS_TYPE_1ARG(declare_type) -#undef declare_type return false; } diff --git a/source/adios2/operator/compress/CompressZFP.cpp b/source/adios2/operator/compress/CompressZFP.cpp index 88660d44ba..2f896ea829 100644 --- a/source/adios2/operator/compress/CompressZFP.cpp +++ b/source/adios2/operator/compress/CompressZFP.cpp @@ -145,13 +145,12 @@ size_t CompressZFP::InverseOperate(const char *bufferIn, const size_t sizeIn, bool CompressZFP::IsDataTypeValid(const DataType type) const { -#define declare_type(T) \ - if (helper::GetDataType() == type) \ - { \ - return true; \ + if (type == DataType::Float || type == DataType::Double || + type == DataType::FloatComplex || type == DataType::DoubleComplex || + type == DataType::Int32 || type == DataType::Int64) + { + return true; } - ADIOS2_FOREACH_ZFP_TYPE_1ARG(declare_type) -#undef declare_type return false; } diff --git a/testing/adios2/xml/TestXMLConfig.cpp b/testing/adios2/xml/TestXMLConfig.cpp index 4cbf3d2c5b..3667284e46 100644 --- a/testing/adios2/xml/TestXMLConfig.cpp +++ b/testing/adios2/xml/TestXMLConfig.cpp @@ -106,9 +106,10 @@ TEST_F(XMLConfigTest, OpNullException) "configOpNullException.xml"); #if ADIOS2_USE_MPI - EXPECT_THROW(adios2::ADIOS adios(configFile, MPI_COMM_WORLD), std::string); + EXPECT_THROW(adios2::ADIOS adios(configFile, MPI_COMM_WORLD), + std::invalid_argument); #else - EXPECT_THROW(adios2::ADIOS adios(configFile), std::string); + EXPECT_THROW(adios2::ADIOS adios(configFile), std::invalid_argument); #endif } @@ -122,7 +123,7 @@ TEST_F(XMLConfigTest, OpNoneException) EXPECT_THROW(adios2::ADIOS adios(configFile, MPI_COMM_WORLD), std::invalid_argument); #else - EXPECT_THROW(adios2::ADIOS adios(configFile), std::string); + EXPECT_THROW(adios2::ADIOS adios(configFile), std::invalid_argument); #endif } From 5e59300b494a885129ca3b11e78a2ee4e67716ed Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Mon, 6 Dec 2021 00:01:11 -0500 Subject: [PATCH 10/10] use adiosLog in adiosXML --- source/adios2/helper/adiosXML.cpp | 44 +++++++++++++++------------- testing/adios2/xml/TestXMLConfig.cpp | 14 ++++----- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/source/adios2/helper/adiosXML.cpp b/source/adios2/helper/adiosXML.cpp index dbbb3056c9..08d714cf36 100644 --- a/source/adios2/helper/adiosXML.cpp +++ b/source/adios2/helper/adiosXML.cpp @@ -19,6 +19,7 @@ #include "adios2/common/ADIOSTypes.h" #include "adios2/core/IO.h" +#include "adios2/helper/adiosLog.h" #include "adios2/helper/adiosString.h" #include "adios2/helper/adiosXMLUtil.h" @@ -44,8 +45,8 @@ void ParseConfigXML( if (fileContents.empty()) { - throw std::invalid_argument("ERROR: config xml file is empty, " + - hint + "\n"); + helper::Log("Helper", "AdiosXML", "ParseConfigXML", + "empty config xml file", helper::EXCEPTION); } return fileContents; }; @@ -82,23 +83,25 @@ void ParseConfigXML( if (*opName && *opType) { - throw std::invalid_argument( - "ERROR: operator (" + std::string(opName->value()) + - ") and type (" + std::string(opType->value()) + - ") attributes can't coexist in element " - "inside element, " + hint + "\n"); + helper::Log( + "Helper", "AdiosXML", "ParseConfigXML", + "operator (" + std::string(opName->value()) + + ") and type (" + std::string(opType->value()) + + ") attributes can't coexist in element " + "inside element", + helper::EXCEPTION); } if (!*opName && !*opType) { - throw std::invalid_argument( - "ERROR: element " - "inside element requires either operator " - "(existing) or type (supported) attribute, " + - hint + "\n"); + helper::Log("Helper", "AdiosXML", "ParseConfigXML", + " element " + "inside element requires either operator " + "(existing) or type (supported) attribute", + helper::EXCEPTION); } core::Operator *op = nullptr; @@ -108,11 +111,12 @@ void ParseConfigXML( auto itOperator = operators.find(std::string(opName->value())); if (itOperator == operators.end()) { - throw std::invalid_argument( - "ERROR: operator " + std::string(opName->value()) + - " not previously defined, from variable " + - variableName + " inside io " + currentIO.m_Name + ", " + - hint + "\n"); + helper::Log("Helper", "AdiosXML", "ParseConfigXML", + "operator " + std::string(opName->value()) + + " not previously defined, from variable " + + variableName + " inside io " + + currentIO.m_Name, + helper::EXCEPTION); } op = itOperator->second.get(); } diff --git a/testing/adios2/xml/TestXMLConfig.cpp b/testing/adios2/xml/TestXMLConfig.cpp index 3667284e46..2748dc2608 100644 --- a/testing/adios2/xml/TestXMLConfig.cpp +++ b/testing/adios2/xml/TestXMLConfig.cpp @@ -92,10 +92,10 @@ TEST_F(XMLConfigTest, OpTypeException) if (rank == 0) { EXPECT_THROW(adios2::ADIOS adios(configFile, MPI_COMM_SELF), - std::invalid_argument); + std::string); } #else - EXPECT_THROW(adios2::ADIOS adios(configFile), std::invalid_argument); + EXPECT_THROW(adios2::ADIOS adios(configFile), std::string); #endif } @@ -106,10 +106,9 @@ TEST_F(XMLConfigTest, OpNullException) "configOpNullException.xml"); #if ADIOS2_USE_MPI - EXPECT_THROW(adios2::ADIOS adios(configFile, MPI_COMM_WORLD), - std::invalid_argument); + EXPECT_THROW(adios2::ADIOS adios(configFile, MPI_COMM_WORLD), std::string); #else - EXPECT_THROW(adios2::ADIOS adios(configFile), std::invalid_argument); + EXPECT_THROW(adios2::ADIOS adios(configFile), std::string); #endif } @@ -120,10 +119,9 @@ TEST_F(XMLConfigTest, OpNoneException) "configOpNoneException.xml"); #if ADIOS2_USE_MPI - EXPECT_THROW(adios2::ADIOS adios(configFile, MPI_COMM_WORLD), - std::invalid_argument); + EXPECT_THROW(adios2::ADIOS adios(configFile, MPI_COMM_WORLD), std::string); #else - EXPECT_THROW(adios2::ADIOS adios(configFile), std::invalid_argument); + EXPECT_THROW(adios2::ADIOS adios(configFile), std::string); #endif }