diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index ec9a6a814b..8e1a9c131a 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -73,7 +73,6 @@ add_library(adios2_core toolkit/format/bp/bpOperation/BPOperation.cpp toolkit/format/bp/bpOperation/BPOperation.tcc toolkit/format/bp/bpOperation/compress/BPZFP.cpp - toolkit/format/bp/bpOperation/compress/BPZFP.tcc toolkit/format/bp/bpOperation/compress/BPSZ.cpp toolkit/format/bp/bpOperation/compress/BPSirius.cpp toolkit/format/bp/bpOperation/compress/BPMGARD.cpp diff --git a/source/adios2/core/Operator.cpp b/source/adios2/core/Operator.cpp index c6b8d80fc8..ec5d8605ad 100644 --- a/source/adios2/core/Operator.cpp +++ b/source/adios2/core/Operator.cpp @@ -50,9 +50,8 @@ void Operator::RunCallback2(void *arg0, const std::string &arg1, } size_t Operator::Compress(const char * /*dataIn*/, const Dims & /*blockStart*/, - const Dims & /*blockCount*/, DataType /*type*/, - char * /*bufferOut*/, const Params & /*params*/, - Params & /*info*/) + const Dims & /*blockCount*/, const DataType /*type*/, + char * /*bufferOut*/, const Params & /*params*/) { throw std::invalid_argument("ERROR: signature (const void*, const " "Dims, const size_t, const std::string, " @@ -62,9 +61,7 @@ size_t Operator::Compress(const char * /*dataIn*/, const Dims & /*blockStart*/, } size_t Operator::Decompress(const char *bufferIn, const size_t sizeIn, - char *dataOut, const DataType type, - const Dims &blockStart, const Dims &blockCount, - const Params ¶meters, Params &info) + char *dataOut) { throw std::invalid_argument( "ERROR: signature (const void*, const size_t, void) not supported " diff --git a/source/adios2/core/Operator.h b/source/adios2/core/Operator.h index e6e4dfd239..46c6cdc7a5 100644 --- a/source/adios2/core/Operator.h +++ b/source/adios2/core/Operator.h @@ -56,21 +56,25 @@ class Operator /** * @param dataIn - * @param dimensions + * @param blockStart + * @param blockCount * @param type * @param bufferOut * @param parameters * @return size of compressed buffer */ virtual size_t Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, DataType type, - char *bufferOut, const Params ¶meters, - Params &info); + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters); + /** + * @param bufferIn + * @param sizeIn + * @param dataOut + * @return size of decompressed buffer + */ virtual size_t Decompress(const char *bufferIn, const size_t sizeIn, - char *dataOut, const DataType type, - const Dims &blockStart, const Dims &blockCount, - const Params ¶meters, Params &info); + char *dataOut); virtual bool IsDataTypeValid(const DataType type) const = 0; diff --git a/source/adios2/operator/compress/CompressBZIP2.cpp b/source/adios2/operator/compress/CompressBZIP2.cpp index 8ea686a761..0ec22a22b8 100644 --- a/source/adios2/operator/compress/CompressBZIP2.cpp +++ b/source/adios2/operator/compress/CompressBZIP2.cpp @@ -34,8 +34,7 @@ CompressBZIP2::CompressBZIP2(const Params ¶meters) size_t CompressBZIP2::Compress(const char *dataIn, const Dims &blockStart, const Dims &blockCount, DataType type, - char *bufferOut, const Params ¶meters, - Params & /*info*/) + char *bufferOut, const Params ¶meters) { const uint8_t bufferVersion = 1; @@ -174,11 +173,7 @@ size_t CompressBZIP2::DecompressV1(const char *bufferIn, const size_t sizeIn, return sizeOut; } size_t CompressBZIP2::Decompress(const char *bufferIn, const size_t sizeIn, - char *dataOut, const DataType /*type*/, - const Dims & /*blockStart*/, - const Dims & /*blockCount*/, - const Params & /*parameters*/, - Params & /*info*/) + char *dataOut) { size_t bufferInOffset = 1; // skip operator type const uint8_t bufferVersion = diff --git a/source/adios2/operator/compress/CompressBZIP2.h b/source/adios2/operator/compress/CompressBZIP2.h index ce11b36a32..6e9388f914 100644 --- a/source/adios2/operator/compress/CompressBZIP2.h +++ b/source/adios2/operator/compress/CompressBZIP2.h @@ -32,31 +32,26 @@ class CompressBZIP2 : public Operator ~CompressBZIP2() = default; /** - * Compression signature for legacy libraries that use char* * @param dataIn - * @param dimensions + * @param blockStart + * @param blockCount * @param type * @param bufferOut * @param parameters - * @return size of compressed buffer in bytes + * @return size of compressed buffer */ size_t Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, DataType type, char *bufferOut, - const Params ¶meters, Params &info) final; + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) final; /** - * Decompression signature for legacy libraries that use char* * @param bufferIn * @param sizeIn * @param dataOut - * @param dimensions - * @param type - * @return size of decompressed buffer in bytes + * @return size of decompressed buffer */ - size_t Decompress(const char *bufferIn, const size_t sizeIn, char *dataOut, - const DataType type, const Dims &blockStart, - const Dims &blockCount, const Params ¶meters, - Params &info) final; + size_t Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) final; bool IsDataTypeValid(const DataType type) const final; diff --git a/source/adios2/operator/compress/CompressBlosc.cpp b/source/adios2/operator/compress/CompressBlosc.cpp index 2d0b91b570..e7b0d2a27d 100644 --- a/source/adios2/operator/compress/CompressBlosc.cpp +++ b/source/adios2/operator/compress/CompressBlosc.cpp @@ -43,9 +43,8 @@ CompressBlosc::CompressBlosc(const Params ¶meters) } size_t CompressBlosc::Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, DataType type, - char *bufferOut, const Params ¶meters, - Params &info) + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) { size_t currentOutputSize = 0; const uint8_t bufferVersion = 1; @@ -264,11 +263,7 @@ size_t CompressBlosc::DecompressV1(const char *bufferIn, const size_t sizeIn, } size_t CompressBlosc::Decompress(const char *bufferIn, const size_t sizeIn, - char *dataOut, const DataType /*type*/, - const Dims & /*blockStart*/, - const Dims & /*blockCount*/, - const Params & /*parameters*/, - Params & /*info*/) + char *dataOut) { size_t bufferInOffset = 1; // skip operator type const uint8_t bufferVersion = diff --git a/source/adios2/operator/compress/CompressBlosc.h b/source/adios2/operator/compress/CompressBlosc.h index 7c99e760db..89a7c8ecaa 100644 --- a/source/adios2/operator/compress/CompressBlosc.h +++ b/source/adios2/operator/compress/CompressBlosc.h @@ -44,9 +44,9 @@ class CompressBlosc : public Operator ~CompressBlosc() = default; /** - * Compression signature for legacy libraries that use void* * @param dataIn - * @param dimensions + * @param blockStart + * @param blockCount * @param type * @param bufferOut format will be: 'DataHeader ; (BloscCompressedChunk | * UncompressedData), [ BloscCompressedChunk, ...]' @@ -54,22 +54,17 @@ class CompressBlosc : public Operator * @return size of compressed buffer in bytes */ size_t Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, DataType type, char *bufferOut, - const Params ¶meters, Params &info) final; + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) final; /** - * Decompression signature for legacy libraries that use char* * @param bufferIn * @param sizeIn * @param dataOut - * @param dimensions - * @param type - * @return size of decompressed buffer in bytes + * @return size of decompressed buffer */ - size_t Decompress(const char *bufferIn, const size_t sizeIn, char *dataOut, - const DataType type, const Dims &blockStart, - const Dims &blockCount, const Params ¶meters, - Params &info) final; + size_t Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) final; bool IsDataTypeValid(const DataType type) const final; diff --git a/source/adios2/operator/compress/CompressLibPressio.cpp b/source/adios2/operator/compress/CompressLibPressio.cpp index 6b4174e41b..069031a3e9 100644 --- a/source/adios2/operator/compress/CompressLibPressio.cpp +++ b/source/adios2/operator/compress/CompressLibPressio.cpp @@ -286,13 +286,33 @@ CompressLibPressio::CompressLibPressio(const Params ¶meters) } size_t CompressLibPressio::Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, DataType varType, - char *bufferOut, const Params ¶meters, - Params &info) + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) { + const uint8_t bufferVersion = 1; + size_t bufferOutOffset = 0; + + // Universal operator metadata + PutParameter(bufferOut, bufferOutOffset, OperatorType::Sz); + PutParameter(bufferOut, bufferOutOffset, bufferVersion); + bufferOutOffset += 2; + // Universal operator metadata end + + const size_t ndims = blockCount.size(); + + // zfp V1 metadata + PutParameter(bufferOut, bufferOutOffset, ndims); + for (const auto &d : blockCount) + { + PutParameter(bufferOut, bufferOutOffset, d); + } + PutParameter(bufferOut, bufferOutOffset, type); + PutParameters(bufferOut, bufferOutOffset, parameters); + // zfp V1 metadata end + auto inputs_dims = adios_to_libpressio_dims(blockCount); pressio_data *input_buf = pressio_data_new_nonowning( - adios_to_libpressio_dtype(varType), const_cast(dataIn), + adios_to_libpressio_dtype(type), const_cast(dataIn), inputs_dims.size(), inputs_dims.data()); pressio_data *output_buf = pressio_data_new_empty(pressio_byte_dtype, 0, nullptr); @@ -318,26 +338,42 @@ size_t CompressLibPressio::Compress(const char *dataIn, const Dims &blockStart, size_t size_in_bytes = 0; void *bytes = pressio_data_ptr(output_buf, &size_in_bytes); - memcpy(bufferOut, bytes, size_in_bytes); + memcpy(bufferOut + bufferOutOffset, bytes, size_in_bytes); + bufferOutOffset += size_in_bytes; pressio_data_free(input_buf); pressio_data_free(output_buf); - return static_cast(size_in_bytes); + return bufferOutOffset; } -size_t CompressLibPressio::Decompress(const char *bufferIn, const size_t sizeIn, - char *dataOut, const DataType type, - const Dims &blockStart, - const Dims &blockCount, - const Params ¶meters, Params &info) +size_t CompressLibPressio::DecompressV1(const char *bufferIn, + const size_t sizeIn, char *dataOut) { + // Do NOT remove even if the buffer version is updated. Data might be still + // in lagacy formats. This function must be kept for backward compatibility. + // If a newer buffer format is implemented, create another function, e.g. + // DecompressV2 and keep this function for decompressing lagacy data. + + size_t bufferInOffset = 0; + + const size_t ndims = GetParameter(bufferIn, bufferInOffset); + Dims blockCount(ndims); + for (size_t i = 0; i < ndims; ++i) + { + blockCount[i] = GetParameter(bufferIn, bufferInOffset); + } + const DataType type = GetParameter(bufferIn, bufferInOffset); + const Params parameters = GetParameters(bufferIn, bufferInOffset); + std::vector dims = adios_to_libpressio_dims(blockCount); pressio_data *output_buf = pressio_data_new_owning( adios_to_libpressio_dtype(type), dims.size(), dims.data()); + size_t newSizeIn = sizeIn - bufferInOffset; pressio_data *input_buf = pressio_data_new_nonowning( - pressio_byte_dtype, const_cast(bufferIn), 1, &sizeIn); + pressio_byte_dtype, const_cast(bufferIn + bufferInOffset), 1, + &newSizeIn); pressio_compressor *compressor = nullptr; try @@ -369,6 +405,32 @@ size_t CompressLibPressio::Decompress(const char *bufferIn, const size_t sizeIn, return size_in_bytes; } +size_t CompressLibPressio::Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) +{ + size_t bufferInOffset = 1; // skip operator type + const uint8_t bufferVersion = + GetParameter(bufferIn, bufferInOffset); + bufferInOffset += 2; // skip two reserved bytes + + if (bufferVersion == 1) + { + return DecompressV1(bufferIn + bufferInOffset, sizeIn - bufferInOffset, + dataOut); + } + else if (bufferVersion == 2) + { + // TODO: if a Version 2 zfp buffer is being implemented, put it here + // and keep the DecompressV1 routine for backward compatibility + } + else + { + throw("unknown zfp buffer version"); + } + + return 0; +} + bool CompressLibPressio::IsDataTypeValid(const DataType type) const { #define declare_type(T) \ diff --git a/source/adios2/operator/compress/CompressLibPressio.h b/source/adios2/operator/compress/CompressLibPressio.h index eb69449614..5ce5c06ac8 100644 --- a/source/adios2/operator/compress/CompressLibPressio.h +++ b/source/adios2/operator/compress/CompressLibPressio.h @@ -32,33 +32,41 @@ class CompressLibPressio : public Operator ~CompressLibPressio() = default; /** - * Compression signature for legacy libraries that use char* * @param dataIn - * @param dimensions + * @param blockStart + * @param blockCount * @param type * @param bufferOut * @param parameters - * @return size of compressed buffer in bytes + * @return size of compressed buffer */ size_t Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, DataType type, char *bufferOut, - const Params ¶meters, Params &info) final; + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) final; /** - * Wrapper around zfp decompression * @param bufferIn * @param sizeIn * @param dataOut - * @param dimensions - * @param type - * @return size of decompressed data in dataOut + * @return size of decompressed buffer */ - size_t Decompress(const char *bufferIn, const size_t sizeIn, char *dataOut, - const DataType type, const Dims &blockStart, - const Dims &blockCount, const Params ¶meters, - Params &info) final; + size_t Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) final; bool IsDataTypeValid(const DataType type) const final; + +private: + /** + * Decompress function for V1 buffer. Do NOT remove even if the buffer + * version is updated. Data might be still in lagacy formats. This function + * must be kept for backward compatibility + * @param bufferIn : compressed data buffer (V1 only) + * @param sizeIn : number of bytes in bufferIn + * @param dataOut : decompressed data buffer + * @return : number of bytes in dataOut + */ + size_t DecompressV1(const char *bufferIn, const size_t sizeIn, + char *dataOut); }; } // end namespace compress diff --git a/source/adios2/operator/compress/CompressMGARD.cpp b/source/adios2/operator/compress/CompressMGARD.cpp index 1abc929139..5bd1e0e097 100644 --- a/source/adios2/operator/compress/CompressMGARD.cpp +++ b/source/adios2/operator/compress/CompressMGARD.cpp @@ -29,9 +29,8 @@ CompressMGARD::CompressMGARD(const Params ¶meters) } size_t CompressMGARD::Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, DataType type, - char *bufferOut, const Params ¶meters, - Params &info) + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) { const uint8_t bufferVersion = 1; size_t bufferOutOffset = 0; @@ -181,11 +180,7 @@ size_t CompressMGARD::DecompressV1(const char *bufferIn, const size_t sizeIn, } size_t CompressMGARD::Decompress(const char *bufferIn, const size_t sizeIn, - char *dataOut, const DataType /*type*/, - const Dims & /*blockStart*/, - const Dims & /*blockCount*/, - const Params & /*parameters*/, - Params & /*info*/) + char *dataOut) { size_t bufferInOffset = 1; // skip operator type const uint8_t bufferVersion = diff --git a/source/adios2/operator/compress/CompressMGARD.h b/source/adios2/operator/compress/CompressMGARD.h index 9b331dbd7f..3c4cf91028 100644 --- a/source/adios2/operator/compress/CompressMGARD.h +++ b/source/adios2/operator/compress/CompressMGARD.h @@ -32,32 +32,26 @@ class CompressMGARD : public Operator ~CompressMGARD() = default; /** - * Compression signature for legacy libraries that use char* * @param dataIn - * @param dimensions + * @param blockStart + * @param blockCount * @param type * @param bufferOut * @param parameters - * @return size of compressed buffer in bytes + * @return size of compressed buffer */ size_t Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, DataType type, char *bufferOut, - const Params ¶meters, Params &info) final; + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) final; /** - * * @param bufferIn * @param sizeIn * @param dataOut - * @param dimensions - * @param varType - * @param - * @return + * @return size of decompressed buffer */ - size_t Decompress(const char *bufferIn, const size_t sizeIn, char *dataOut, - const DataType type, const Dims &blockStart, - const Dims &blockCount, const Params ¶meters, - Params &info) final; + size_t Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) final; bool IsDataTypeValid(const DataType type) const final; diff --git a/source/adios2/operator/compress/CompressPNG.cpp b/source/adios2/operator/compress/CompressPNG.cpp index 88e306c617..d175d00995 100644 --- a/source/adios2/operator/compress/CompressPNG.cpp +++ b/source/adios2/operator/compress/CompressPNG.cpp @@ -48,9 +48,8 @@ CompressPNG::CompressPNG(const Params ¶meters) : Operator("png", parameters) } size_t CompressPNG::Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, DataType type, - char *bufferOut, const Params ¶meters, - Params &info) + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) { size_t bufferOutOffset = 0; const uint8_t bufferVersion = 1; @@ -223,10 +222,7 @@ size_t CompressPNG::DecompressV1(const char *bufferIn, const size_t sizeIn, } size_t CompressPNG::Decompress(const char *bufferIn, const size_t sizeIn, - char *dataOut, const DataType /*type*/, - const Dims & /*blockStart*/, - const Dims & /*blockCount*/, - const Params & /*parameters*/, Params & /*info*/) + char *dataOut) { size_t bufferInOffset = 1; // skip operator type const uint8_t bufferVersion = diff --git a/source/adios2/operator/compress/CompressPNG.h b/source/adios2/operator/compress/CompressPNG.h index c5d2993d32..a76777d52d 100644 --- a/source/adios2/operator/compress/CompressPNG.h +++ b/source/adios2/operator/compress/CompressPNG.h @@ -34,31 +34,26 @@ class CompressPNG : public Operator ~CompressPNG() = default; /** - * Compression signature for legacy libraries that use char* * @param dataIn - * @param dimensions + * @param blockStart + * @param blockCount * @param type * @param bufferOut * @param parameters - * @return size of compressed buffer in bytes + * @return size of compressed buffer */ size_t Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, DataType type, char *bufferOut, - const Params ¶meters, Params &info) final; + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) final; /** - * Decompression signature for legacy libraries that use char* * @param bufferIn * @param sizeIn * @param dataOut - * @param dimensions - * @param type - * @return size of decompressed buffer in bytes + * @return size of decompressed buffer */ - size_t Decompress(const char *bufferIn, const size_t sizeIn, char *dataOut, - const DataType type, const Dims &blockStart, - const Dims &blockCount, const Params ¶meters, - Params &info) final; + size_t Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) final; bool IsDataTypeValid(const DataType type) const final; diff --git a/source/adios2/operator/compress/CompressSZ.cpp b/source/adios2/operator/compress/CompressSZ.cpp index d0ecd02811..fca0d1bffd 100644 --- a/source/adios2/operator/compress/CompressSZ.cpp +++ b/source/adios2/operator/compress/CompressSZ.cpp @@ -30,9 +30,8 @@ namespace compress CompressSZ::CompressSZ(const Params ¶meters) : Operator("sz", parameters) {} size_t CompressSZ::Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, DataType varType, - char *bufferOut, const Params ¶meters, - Params &info) + const Dims &blockCount, const DataType varType, + char *bufferOut, const Params ¶meters) { const uint8_t bufferVersion = 1; size_t bufferOutOffset = 0; @@ -283,9 +282,7 @@ size_t CompressSZ::Compress(const char *dataIn, const Dims &blockStart, } size_t CompressSZ::Decompress(const char *bufferIn, const size_t sizeIn, - char *dataOut, const DataType type, - const Dims &blockStart, const Dims &blockCount, - const Params ¶meters, Params &info) + char *dataOut) { size_t bufferInOffset = 1; // skip operator type const uint8_t bufferVersion = diff --git a/source/adios2/operator/compress/CompressSZ.h b/source/adios2/operator/compress/CompressSZ.h index f31e7bc8dc..b3c178d17f 100644 --- a/source/adios2/operator/compress/CompressSZ.h +++ b/source/adios2/operator/compress/CompressSZ.h @@ -32,31 +32,26 @@ class CompressSZ : public Operator ~CompressSZ() = default; /** - * Compression signature for legacy libraries that use char* * @param dataIn - * @param dimensions + * @param blockStart + * @param blockCount * @param type * @param bufferOut * @param parameters - * @return size of compressed buffer in bytes + * @return size of compressed buffer */ size_t Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, DataType type, char *bufferOut, - const Params ¶meters, Params &info) final; + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) final; /** - * Wrapper around zfp decompression * @param bufferIn * @param sizeIn * @param dataOut - * @param dimensions - * @param type - * @return size of decompressed data in dataOut + * @return size of decompressed buffer */ - size_t Decompress(const char *bufferIn, const size_t sizeIn, char *dataOut, - const DataType type, const Dims &blockStart, - const Dims &blockCount, const Params ¶meters, - Params &info) final; + size_t Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) final; bool IsDataTypeValid(const DataType type) const final; diff --git a/source/adios2/operator/compress/CompressSirius.cpp b/source/adios2/operator/compress/CompressSirius.cpp index 33247d6ec5..bb122232e5 100644 --- a/source/adios2/operator/compress/CompressSirius.cpp +++ b/source/adios2/operator/compress/CompressSirius.cpp @@ -35,9 +35,8 @@ CompressSirius::CompressSirius(const Params ¶meters) } size_t CompressSirius::Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, DataType varType, - char *bufferOut, const Params ¶ms, - Params &info) + const Dims &blockCount, const DataType varType, + char *bufferOut, const Params ¶ms) { const uint8_t bufferVersion = 1; size_t bufferOutOffset = 0; @@ -172,10 +171,7 @@ size_t CompressSirius::DecompressV1(const char *bufferIn, const size_t sizeIn, } size_t CompressSirius::Decompress(const char *bufferIn, const size_t sizeIn, - char *dataOut, const DataType type1, - const Dims &blockStart1, - const Dims &blockCount1, - const Params ¶meters, Params &info) + char *dataOut) { size_t bufferInOffset = 1; // skip operator type const uint8_t bufferVersion = diff --git a/source/adios2/operator/compress/CompressSirius.h b/source/adios2/operator/compress/CompressSirius.h index ac3b176ab2..4c08aafedc 100644 --- a/source/adios2/operator/compress/CompressSirius.h +++ b/source/adios2/operator/compress/CompressSirius.h @@ -30,13 +30,11 @@ class CompressSirius : public Operator ~CompressSirius() = default; size_t Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, DataType type, char *bufferOut, - const Params ¶ms, Params &info) final; + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶ms) final; - size_t Decompress(const char *bufferIn, const size_t sizeIn, char *dataOut, - const DataType type, const Dims &blockStart, - const Dims &blockCount, const Params ¶meters, - Params &info) final; + size_t Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) final; bool IsDataTypeValid(const DataType type) const final; diff --git a/source/adios2/operator/compress/CompressZFP.cpp b/source/adios2/operator/compress/CompressZFP.cpp index 6a7a0a6565..17a53dc0ff 100644 --- a/source/adios2/operator/compress/CompressZFP.cpp +++ b/source/adios2/operator/compress/CompressZFP.cpp @@ -23,9 +23,8 @@ CompressZFP::CompressZFP(const Params ¶meters) : Operator("zfp", parameters) } size_t CompressZFP::Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, DataType type, - char *bufferOut, const Params ¶meters, - Params &info) + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) { const uint8_t bufferVersion = 1; @@ -124,10 +123,7 @@ size_t CompressZFP::DecompressV1(const char *bufferIn, const size_t sizeIn, } size_t CompressZFP::Decompress(const char *bufferIn, const size_t sizeIn, - char *dataOut, const DataType /*type*/, - const Dims & /*blockStart*/, - const Dims & /*blockCount*/, - const Params & /*parameters*/, Params &info) + char *dataOut) { size_t bufferInOffset = 1; // skip operator type const uint8_t bufferVersion = diff --git a/source/adios2/operator/compress/CompressZFP.h b/source/adios2/operator/compress/CompressZFP.h index e5dfd02731..a1cef7e4c4 100644 --- a/source/adios2/operator/compress/CompressZFP.h +++ b/source/adios2/operator/compress/CompressZFP.h @@ -37,31 +37,26 @@ class CompressZFP : public Operator ~CompressZFP() = default; /** - * Wrapper around zfp compression * @param dataIn - * @param dimensions + * @param blockStart + * @param blockCount * @param type * @param bufferOut * @param parameters - * @return size of compressed buffer in bytes + * @return size of compressed buffer */ size_t Compress(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, DataType type, char *bufferOut, - const Params ¶meters, Params &info) final; + const Dims &blockCount, const DataType type, + char *bufferOut, const Params ¶meters) final; /** - * Wrapper around zfp decompression * @param bufferIn * @param sizeIn * @param dataOut - * @param dimensions - * @param type - * @return size of decompressed data in dataOut + * @return size of decompressed buffer */ - size_t Decompress(const char *bufferIn, const size_t sizeIn, char *dataOut, - const DataType type, const Dims &blockStart, - const Dims &blockCount, const Params ¶meters, - Params &info) final; + size_t Decompress(const char *bufferIn, const size_t sizeIn, + char *dataOut) final; bool IsDataTypeValid(const DataType type) const final; diff --git a/source/adios2/toolkit/format/bp/bpOperation/BPOperation.tcc b/source/adios2/toolkit/format/bp/bpOperation/BPOperation.tcc index da312f1ebd..1782256235 100644 --- a/source/adios2/toolkit/format/bp/bpOperation/BPOperation.tcc +++ b/source/adios2/toolkit/format/bp/bpOperation/BPOperation.tcc @@ -34,7 +34,7 @@ void BPOperation::SetDataDefault( const size_t outputSize = op.Compress( reinterpret_cast(blockInfo.Data), blockInfo.Start, blockInfo.Count, variable.m_Type, - bufferSTL.m_Buffer.data() + bufferSTL.m_Position, parameters, info); + bufferSTL.m_Buffer.data() + bufferSTL.m_Position, parameters); info["OutputSize"] = std::to_string(outputSize); diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPBZIP2.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPBZIP2.cpp index 2369256e82..eac3f30ae8 100644 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPBZIP2.cpp +++ b/source/adios2/toolkit/format/bp/bpOperation/compress/BPBZIP2.cpp @@ -67,14 +67,8 @@ void BPBZIP2::GetData(const char *input, char *dataOutput) const { #ifdef ADIOS2_HAVE_BZIP2 - Params params; - core::compress::CompressBZIP2 op(params); - op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput, - helper::GetDataTypeFromString( - blockOperationInfo.Info.at("PreDataType")), - blockOperationInfo.PreStart, blockOperationInfo.PreCount, - params, const_cast(blockOperationInfo.Info)); - + core::compress::CompressBZIP2 op({}); + op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput); #else throw std::runtime_error( "ERROR: current ADIOS2 library didn't compile " diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPBlosc.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPBlosc.cpp index fd2bd1b0a8..584667f4cd 100644 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPBlosc.cpp +++ b/source/adios2/toolkit/format/bp/bpOperation/compress/BPBlosc.cpp @@ -67,14 +67,8 @@ void BPBlosc::GetData(const char *input, char *dataOutput) const { #ifdef ADIOS2_HAVE_BLOSC - Params params; - core::compress::CompressBlosc op(params); - op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput, - helper::GetDataTypeFromString( - blockOperationInfo.Info.at("PreDataType")), - blockOperationInfo.PreStart, blockOperationInfo.PreCount, - blockOperationInfo.Info, params); - + core::compress::CompressBlosc op({}); + op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput); #else throw std::runtime_error( "ERROR: current ADIOS2 library didn't compile " diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPLIBPRESSIO.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPLIBPRESSIO.cpp index 7222e62f65..9d87adfd65 100644 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPLIBPRESSIO.cpp +++ b/source/adios2/toolkit/format/bp/bpOperation/compress/BPLIBPRESSIO.cpp @@ -68,14 +68,8 @@ void BPLIBPRESSIO::GetData(const char *input, char *dataOutput) const { #ifdef ADIOS2_HAVE_LIBPRESSIO - Params params; - core::compress::CompressLibPressio op(params); - op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput, - helper::GetDataTypeFromString( - blockOperationInfo.Info.at("PreDataType")), - blockOperationInfo.PreStart, blockOperationInfo.PreCount, - blockOperationInfo.Info, params); - + core::compress::CompressLibPressio op({}); + op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput); #else throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " "with SZ, can't read SZ compressed data, in call " diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPMGARD.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPMGARD.cpp index ae01ce6f7d..eda59dc5a7 100644 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPMGARD.cpp +++ b/source/adios2/toolkit/format/bp/bpOperation/compress/BPMGARD.cpp @@ -68,14 +68,8 @@ void BPMGARD::GetData(const char *input, char *dataOutput) const { #ifdef ADIOS2_HAVE_MGARD - Params params; - core::compress::CompressMGARD op(params); - op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput, - helper::GetDataTypeFromString( - blockOperationInfo.Info.at("PreDataType")), - blockOperationInfo.PreStart, blockOperationInfo.PreCount, - blockOperationInfo.Info, params); - + core::compress::CompressMGARD op({}); + op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput); #else throw std::runtime_error( "ERROR: current ADIOS2 library didn't compile " diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPPNG.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPPNG.cpp index 48b3876c04..0a5b0033cd 100644 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPPNG.cpp +++ b/source/adios2/toolkit/format/bp/bpOperation/compress/BPPNG.cpp @@ -67,15 +67,8 @@ void BPPNG::GetData(const char *input, char *dataOutput) const { #ifdef ADIOS2_HAVE_PNG - Params params; - core::compress::CompressPNG op(params); - Params &info = const_cast(blockOperationInfo.Info); - op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput, - helper::GetDataTypeFromString( - blockOperationInfo.Info.at("PreDataType")), - blockOperationInfo.PreStart, blockOperationInfo.PreCount, - blockOperationInfo.Info, params); - + core::compress::CompressPNG op({}); + op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput); #else throw std::runtime_error( "ERROR: current ADIOS2 library didn't compile " diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPSZ.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPSZ.cpp index b66d1e4ffe..512c8bba11 100644 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPSZ.cpp +++ b/source/adios2/toolkit/format/bp/bpOperation/compress/BPSZ.cpp @@ -67,14 +67,8 @@ void BPSZ::GetData(const char *input, char *dataOutput) const { #ifdef ADIOS2_HAVE_SZ - Params params; - core::compress::CompressSZ op(params); - op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput, - helper::GetDataTypeFromString( - blockOperationInfo.Info.at("PreDataType")), - blockOperationInfo.PreStart, blockOperationInfo.PreCount, - blockOperationInfo.Info, params); - + core::compress::CompressSZ op({}); + op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput); #else throw std::runtime_error("ERROR: current ADIOS2 library didn't compile " "with SZ, can't read SZ compressed data, in call " diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPSirius.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPSirius.cpp index 45805fc884..4616045ef8 100644 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPSirius.cpp +++ b/source/adios2/toolkit/format/bp/bpOperation/compress/BPSirius.cpp @@ -67,13 +67,8 @@ void BPSirius::GetData(const char *input, char *dataOutput) const { #ifdef ADIOS2_HAVE_MHS - Params params; - core::compress::CompressSirius op(params); - op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput, - helper::GetDataTypeFromString( - blockOperationInfo.Info.at("PreDataType")), - blockOperationInfo.PreStart, blockOperationInfo.PreCount, - blockOperationInfo.Info, params); + core::compress::CompressSirius op({}); + op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput); #else throw std::runtime_error( "ERROR: current ADIOS2 library didn't compile " diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.cpp b/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.cpp index 6526c83be1..bcf9dbcf21 100644 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.cpp +++ b/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.cpp @@ -9,7 +9,6 @@ */ #include "BPZFP.h" -#include "BPZFP.tcc" #include "adios2/helper/adiosFunctions.h" #include "adios2/helper/adiosType.h" @@ -39,7 +38,7 @@ namespace format const typename core::Variable::Operation &operation, \ std::vector &buffer) const noexcept \ { \ - SetMetadataCommon(variable, blockInfo, operation, buffer); \ + SetMetadataDefault(variable, blockInfo, operation, buffer); \ } \ \ void BPZFP::UpdateMetadata( \ @@ -88,13 +87,8 @@ void BPZFP::GetData(const char *input, char *dataOutput) const { #ifdef ADIOS2_HAVE_ZFP - Params params; - core::compress::CompressZFP op(params); - op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput, - helper::GetDataTypeFromString( - blockOperationInfo.Info.at("PreDataType")), - blockOperationInfo.PreStart, blockOperationInfo.PreCount, - blockOperationInfo.Info, params); + core::compress::CompressZFP op({}); + op.Decompress(input, blockOperationInfo.PayloadSize, dataOutput); #else throw std::runtime_error( "ERROR: current ADIOS2 library didn't compile " diff --git a/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.tcc b/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.tcc deleted file mode 100644 index 8b03a70900..0000000000 --- a/source/adios2/toolkit/format/bp/bpOperation/compress/BPZFP.tcc +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * BPZFP.tcc : - * - * Created on: Jul 18, 2018 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPZFP_TCC_ -#define ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPZFP_TCC_ - -#include "BPZFP.h" - -#include "adios2/helper/adiosFunctions.h" - -namespace adios2 -{ -namespace format -{ - -template -void BPZFP::SetMetadataCommon( - const core::Variable &variable, - const typename core::Variable::BPInfo &blockInfo, - const typename core::Variable::Operation &operation, - std::vector &buffer) const noexcept -{ - const uint64_t inputSize = - helper::GetTotalSize(blockInfo.Count) * sizeof(T); - // being naughty here - Params &info = const_cast(operation.Info); - info["InputSize"] = std::to_string(inputSize); - - const uint64_t outputSize = 0; // not known yet - - auto itMode = operation.Parameters.find("accuracy"); - int32_t mode = -1; - - if (itMode != operation.Parameters.end()) - { - mode = static_cast(zfp_mode_accuracy); - } - else - { - itMode = operation.Parameters.find("precision"); - if (itMode != operation.Parameters.end()) - { - mode = static_cast(zfp_mode_precision); - } - else - { - itMode = operation.Parameters.find("rate"); - if (itMode != operation.Parameters.end()) - { - mode = static_cast(zfp_mode_rate); - } - } - } - const std::string modeStr = itMode->second; - - // fixed size - constexpr uint16_t metadataSize = 532; - helper::InsertToBuffer(buffer, &metadataSize); - helper::InsertToBuffer(buffer, &inputSize); - // to be filled out after operation is applied on data - info["OutputSizeMetadataPosition"] = std::to_string(buffer.size()); - helper::InsertToBuffer(buffer, &outputSize); - helper::InsertToBuffer(buffer, &mode); - - const size_t fixedRecordsPosition = buffer.size(); - buffer.resize(fixedRecordsPosition + 512, '\0'); - size_t backPosition = fixedRecordsPosition; - helper::CopyToBuffer(buffer, backPosition, modeStr.data(), modeStr.size()); - backPosition = fixedRecordsPosition + 256; - helper::CopyToBuffer(buffer, backPosition, variable.m_Name.data(), - variable.m_Name.size()); -} - -} // end namespace format -} // end namespace adios2 - -#endif /* ADIOS2_TOOLKIT_FORMAT_BP_BPOPERATION_COMPRESS_BPZFP_TCC_ */ diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc index f37d2aae84..ae485a02ae 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.tcc +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.tcc @@ -336,9 +336,7 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName, try { decompressor.Decompress(j.buffer->data() + j.position, - j.size, decompressBuffer.data(), - j.type, j.start, j.count, j.params, - const_cast(j.params)); + j.size, decompressBuffer.data()); decompressed = true; } catch (std::exception &e) @@ -364,9 +362,7 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName, try { decompressor.Decompress(j.buffer->data() + j.position, - j.size, decompressBuffer.data(), - j.type, j.start, j.count, j.params, - const_cast(j.params)); + j.size, decompressBuffer.data()); decompressed = true; } catch (std::exception &e) @@ -390,11 +386,8 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName, decompressBuffer.reserve(datasize); try { - Params info; decompressor.Decompress(j.buffer->data() + j.position, - j.size, decompressBuffer.data(), - j.type, j.start, j.count, j.params, - const_cast(j.params)); + j.size, decompressBuffer.data()); decompressed = true; } catch (std::exception &e) @@ -419,9 +412,7 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName, try { decompressor.Decompress(j.buffer->data() + j.position, - j.size, decompressBuffer.data(), - j.type, j.start, j.count, j.params, - const_cast(j.params)); + j.size, decompressBuffer.data()); decompressed = true; } catch (std::exception &e) @@ -486,10 +477,9 @@ void DataManSerializer::PutZfp(nlohmann::json &metaj, size_t &datasize, std::multiplies())); try { - Params info; datasize = compressor.Compress( reinterpret_cast(inputData), {}, varCount, - helper::GetDataType(), m_CompressBuffer.data(), params, info); + helper::GetDataType(), m_CompressBuffer.data(), params); } catch (std::exception &e) { @@ -514,10 +504,9 @@ void DataManSerializer::PutSz(nlohmann::json &metaj, size_t &datasize, core::compress::CompressSZ compressor(params); try { - Params info; datasize = compressor.Compress( reinterpret_cast(inputData), {}, varCount, - helper::GetDataType(), m_CompressBuffer.data(), params, info); + helper::GetDataType(), m_CompressBuffer.data(), params); } catch (std::exception &e) { @@ -542,10 +531,9 @@ void DataManSerializer::PutBZip2(nlohmann::json &metaj, size_t &datasize, core::compress::CompressBZIP2 compressor(params); try { - Params info; datasize = compressor.Compress( reinterpret_cast(inputData), {}, varCount, - helper::GetDataType(), m_CompressBuffer.data(), params, info); + helper::GetDataType(), m_CompressBuffer.data(), params); } catch (std::exception &e) { @@ -570,10 +558,9 @@ void DataManSerializer::PutMgard(nlohmann::json &metaj, size_t &datasize, std::multiplies())); try { - Params info; datasize = compressor.Compress( reinterpret_cast(inputData), {}, varCount, - helper::GetDataType(), m_CompressBuffer.data(), params, info); + helper::GetDataType(), m_CompressBuffer.data(), params); } catch (std::exception &e) {