From b6c69251b6439f0cf70e7ff74ba87181017c4c7a Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 28 Sep 2021 18:57:50 -0400 Subject: [PATCH 1/9] added zfp library version info into operator buffer --- .../adios2/operator/compress/CompressZFP.cpp | 31 +++++++++++++------ source/adios2/operator/compress/CompressZFP.h | 9 ++---- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/source/adios2/operator/compress/CompressZFP.cpp b/source/adios2/operator/compress/CompressZFP.cpp index 17a53dc0ff..d0a8c3fb83 100644 --- a/source/adios2/operator/compress/CompressZFP.cpp +++ b/source/adios2/operator/compress/CompressZFP.cpp @@ -45,6 +45,12 @@ size_t CompressZFP::Compress(const char *dataIn, const Dims &blockStart, PutParameter(bufferOut, bufferOutOffset, d); } PutParameter(bufferOut, bufferOutOffset, type); + PutParameter(bufferOut, bufferOutOffset, + static_cast(ZFP_VERSION_MAJOR)); + PutParameter(bufferOut, bufferOutOffset, + static_cast(ZFP_VERSION_MINOR)); + PutParameter(bufferOut, bufferOutOffset, + static_cast(ZFP_VERSION_PATCH)); PutParameters(bufferOut, bufferOutOffset, parameters); // zfp V1 metadata end @@ -90,6 +96,12 @@ size_t CompressZFP::DecompressV1(const char *bufferIn, const size_t sizeIn, blockCount[i] = GetParameter(bufferIn, bufferInOffset); } const DataType type = GetParameter(bufferIn, bufferInOffset); + m_VersionInfo = + " Data is compressed using ZFP Version " + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + + ". Please make sure a compatible version is used for decompression."; const Params parameters = GetParameters(bufferIn, bufferInOffset); Dims convertedDims = ConvertDims(blockCount, type, 3); @@ -107,19 +119,16 @@ size_t CompressZFP::DecompressV1(const char *bufferIn, const size_t sizeIn, if (!status) { - throw std::invalid_argument("ERROR: zfp failed with status " + - std::to_string(status) + - ", in call to CompressZfp Decompress\n"); + throw std::runtime_error( + "ERROR: zfp failed with status " + std::to_string(status) + + ", in call to CompressZfp Decompress." + m_VersionInfo + "\n"); } zfp_field_free(field); zfp_stream_close(stream); stream_close(bitstream); - const size_t dataSizeBytes = - helper::GetTotalSize(convertedDims, helper::GetDataTypeSize(type)); - - return dataSizeBytes; + return helper::GetTotalSize(convertedDims, helper::GetDataTypeSize(type)); } size_t CompressZFP::Decompress(const char *bufferIn, const size_t sizeIn, @@ -142,7 +151,7 @@ size_t CompressZFP::Decompress(const char *bufferIn, const size_t sizeIn, } else { - throw("unknown zfp buffer version"); + throw std::runtime_error("unknown zfp buffer version"); } return 0; @@ -227,7 +236,8 @@ zfp_field *CompressZFP::GetZFPField(const char *data, const Dims &dimensions, throw std::invalid_argument( "ERROR: zfp_field* failed for data of type " + ToString(type) + ", only 1D, 2D and 3D dimensions are supported, from " - "class CompressZfp\n"); + "class CompressZfp." + + m_VersionInfo + "\n"); } if (field == nullptr) @@ -235,7 +245,8 @@ zfp_field *CompressZFP::GetZFPField(const char *data, const Dims &dimensions, throw std::invalid_argument( "ERROR: zfp_field_" + std::to_string(dimensions.size()) + "d failed for data of type " + ToString(type) + - ", data might be corrupted, from class CompressZfp\n"); + ", data might be corrupted, from class CompressZfp." + + m_VersionInfo + "\n"); } return field; diff --git a/source/adios2/operator/compress/CompressZFP.h b/source/adios2/operator/compress/CompressZFP.h index a1cef7e4c4..ade19c7595 100644 --- a/source/adios2/operator/compress/CompressZFP.h +++ b/source/adios2/operator/compress/CompressZFP.h @@ -83,13 +83,6 @@ class CompressZFP : public Operator zfp_stream *GetZFPStream(const Dims &dimensions, DataType type, const Params ¶meters) const; - /** - * check status from BZip compression and decompression functions - * @param status returned by BZip2 library - * @param hint extra exception information - */ - void CheckStatus(const int status, const std::string hint) const; - /** * Decompress function for V1 buffer. Do NOT remove even if the buffer * version is updated. Data might be still in lagacy formats. This function @@ -101,6 +94,8 @@ class CompressZFP : public Operator */ size_t DecompressV1(const char *bufferIn, const size_t sizeIn, char *dataOut); + + std::string m_VersionInfo; }; } // end namespace compress From 5f88cda5b65f1e032d7fa57ddcace64969516738 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 28 Sep 2021 19:53:37 -0400 Subject: [PATCH 2/9] added sz library version info into operator buffer --- source/adios2/operator/compress/CompressSZ.cpp | 16 +++++++++++++++- source/adios2/operator/compress/CompressSZ.h | 2 ++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/source/adios2/operator/compress/CompressSZ.cpp b/source/adios2/operator/compress/CompressSZ.cpp index fca0d1bffd..705ae37986 100644 --- a/source/adios2/operator/compress/CompressSZ.cpp +++ b/source/adios2/operator/compress/CompressSZ.cpp @@ -51,6 +51,11 @@ size_t CompressSZ::Compress(const char *dataIn, const Dims &blockStart, PutParameter(bufferOut, bufferOutOffset, d); } PutParameter(bufferOut, bufferOutOffset, varType); + for (uint8_t i = 0; i < 4; ++i) + { + PutParameter(bufferOut, bufferOutOffset, + static_cast(versionNumber[i])); + } // sz V1 metadata end Dims convertedDims = ConvertDims(blockCount, varType, 4); @@ -324,6 +329,14 @@ size_t CompressSZ::DecompressV1(const char *bufferIn, const size_t sizeIn, } const DataType type = GetParameter(bufferIn, bufferInOffset); + m_VersionInfo = + " Data is compressed using SZ Version " + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + + ". Please make sure a compatible version is used for decompression."; + Dims convertedDims = ConvertDims(blockCount, type, 4, true, 1); // Get type info @@ -359,7 +372,8 @@ size_t CompressSZ::DecompressV1(const char *bufferIn, const size_t sizeIn, if (result == nullptr) { - throw std::runtime_error("ERROR: SZ_decompress failed\n"); + throw std::runtime_error("ERROR: SZ_decompress failed." + + m_VersionInfo + "\n"); } std::memcpy(dataOut, result, dataSizeBytes); free(result); diff --git a/source/adios2/operator/compress/CompressSZ.h b/source/adios2/operator/compress/CompressSZ.h index b3c178d17f..e0e00e48af 100644 --- a/source/adios2/operator/compress/CompressSZ.h +++ b/source/adios2/operator/compress/CompressSZ.h @@ -67,6 +67,8 @@ class CompressSZ : public Operator */ size_t DecompressV1(const char *bufferIn, const size_t sizeIn, char *dataOut); + + std::string m_VersionInfo; }; } // end namespace compress From aeaeae42f0033f63e83e976429fa41eb8c065e05 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 28 Sep 2021 20:10:55 -0400 Subject: [PATCH 3/9] added blosc library version info into operator buffer --- .../operator/compress/CompressBlosc.cpp | 46 ++++++++++++------- .../adios2/operator/compress/CompressBlosc.h | 1 + 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/source/adios2/operator/compress/CompressBlosc.cpp b/source/adios2/operator/compress/CompressBlosc.cpp index e7b0d2a27d..8117bea957 100644 --- a/source/adios2/operator/compress/CompressBlosc.cpp +++ b/source/adios2/operator/compress/CompressBlosc.cpp @@ -46,20 +46,26 @@ size_t CompressBlosc::Compress(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType type, char *bufferOut, const Params ¶meters) { - size_t currentOutputSize = 0; + size_t bufferOutOffset = 0; const uint8_t bufferVersion = 1; // Universal operator metadata - PutParameter(bufferOut, currentOutputSize, OperatorType::BLOSC); - PutParameter(bufferOut, currentOutputSize, bufferVersion); - currentOutputSize += 2; + PutParameter(bufferOut, bufferOutOffset, OperatorType::BLOSC); + PutParameter(bufferOut, bufferOutOffset, bufferVersion); + bufferOutOffset += 2; // Universal operator metadata end const size_t sizeIn = helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type)); // blosc V1 metadata - PutParameter(bufferOut, currentOutputSize, sizeIn); + PutParameter(bufferOut, bufferOutOffset, sizeIn); + PutParameter(bufferOut, bufferOutOffset, + static_cast(BLOSC_VERSION_MAJOR)); + PutParameter(bufferOut, bufferOutOffset, + static_cast(BLOSC_VERSION_MINOR)); + PutParameter(bufferOut, bufferOutOffset, + static_cast(BLOSC_VERSION_RELEASE)); // blosc V1 metadata end bool useMemcpy = false; @@ -145,11 +151,11 @@ size_t CompressBlosc::Compress(const char *dataIn, const Dims &blockStart, // write header to detect new compression format (set first 8 byte to zero) DataHeader *headerPtr = - reinterpret_cast(bufferOut + currentOutputSize); + reinterpret_cast(bufferOut + bufferOutOffset); // set default header *headerPtr = DataHeader{}; - currentOutputSize += sizeof(DataHeader); + bufferOutOffset += sizeof(DataHeader); int32_t typesize = helper::GetDataTypeSize(type); if (typesize > BLOSC_MAX_TYPESIZE) @@ -190,10 +196,10 @@ size_t CompressBlosc::Compress(const char *dataIn, const Dims &blockStart, bloscSize_t compressedChunkSize = blosc_compress(compressionLevel, doShuffle, typesize, maxIntputSize, dataIn + inputOffset, - bufferOut + currentOutputSize, maxChunkSize); + bufferOut + bufferOutOffset, maxChunkSize); if (compressedChunkSize > 0) - currentOutputSize += static_cast(compressedChunkSize); + bufferOutOffset += static_cast(compressedChunkSize); else { // something went wrong with the compression switch to memcopy @@ -214,14 +220,13 @@ size_t CompressBlosc::Compress(const char *dataIn, const Dims &blockStart, if (useMemcpy) { - std::memcpy(bufferOut + currentOutputSize, dataIn + inputOffset, - sizeIn); - currentOutputSize += sizeIn; + std::memcpy(bufferOut + bufferOutOffset, dataIn + inputOffset, sizeIn); + bufferOutOffset += sizeIn; headerPtr->SetNumChunks(0u); } blosc_destroy(); - return currentOutputSize; + return bufferOutOffset; } size_t CompressBlosc::DecompressV1(const char *bufferIn, const size_t sizeIn, @@ -234,9 +239,17 @@ size_t CompressBlosc::DecompressV1(const char *bufferIn, const size_t sizeIn, size_t bufferInOffset = 0; size_t sizeOut = GetParameter(bufferIn, bufferInOffset); + + m_VersionInfo = + " Data is compressed using BLOSC Version " + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + + ". Please make sure a compatible version is used for decompression."; + if (sizeIn - bufferInOffset < sizeof(DataHeader)) { - throw("corrupted blosc buffer header"); + throw("corrupted blosc buffer header." + m_VersionInfo + "\n"); } const bool isChunked = reinterpret_cast(bufferIn + bufferInOffset) @@ -257,7 +270,7 @@ size_t CompressBlosc::DecompressV1(const char *bufferIn, const size_t sizeIn, } if (decompressedSize != sizeOut) { - throw("corrupted blosc buffer"); + throw("corrupted blosc buffer." + m_VersionInfo + "\n"); } return sizeOut; } @@ -352,7 +365,8 @@ size_t CompressBlosc::DecompressChunkedFormat(const char *bufferIn, { throw std::runtime_error( "ERROR: ADIOS2 Blosc Decompress failed. Decompressed chunk " - "results in zero decompressed bytes.\n"); + "results in zero decompressed bytes." + + m_VersionInfo + "\n"); } inputOffset += static_cast(max_inputDataSize); } diff --git a/source/adios2/operator/compress/CompressBlosc.h b/source/adios2/operator/compress/CompressBlosc.h index 89a7c8ecaa..b2eaafa001 100644 --- a/source/adios2/operator/compress/CompressBlosc.h +++ b/source/adios2/operator/compress/CompressBlosc.h @@ -126,6 +126,7 @@ class CompressBlosc : public Operator static const std::map m_Shuffles; static const std::set m_Compressors; + std::string m_VersionInfo; }; } // end namespace compress From 0a3e15e0e75f8582d0fc42e58d567d9b645789c2 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 28 Sep 2021 20:55:06 -0400 Subject: [PATCH 4/9] added mgard and libpressio library version info into operator buffer --- .../adios2/operator/compress/CompressBlosc.h | 1 + .../operator/compress/CompressLibPressio.cpp | 22 ++++++++++++++----- .../operator/compress/CompressLibPressio.h | 2 ++ .../operator/compress/CompressMGARD.cpp | 20 ++++++++++++----- .../adios2/operator/compress/CompressMGARD.h | 2 ++ 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/source/adios2/operator/compress/CompressBlosc.h b/source/adios2/operator/compress/CompressBlosc.h index b2eaafa001..8b05df38be 100644 --- a/source/adios2/operator/compress/CompressBlosc.h +++ b/source/adios2/operator/compress/CompressBlosc.h @@ -126,6 +126,7 @@ class CompressBlosc : public Operator static const std::map m_Shuffles; static const std::set m_Compressors; + std::string m_VersionInfo; }; diff --git a/source/adios2/operator/compress/CompressLibPressio.cpp b/source/adios2/operator/compress/CompressLibPressio.cpp index 069031a3e9..5a35a07c0a 100644 --- a/source/adios2/operator/compress/CompressLibPressio.cpp +++ b/source/adios2/operator/compress/CompressLibPressio.cpp @@ -307,6 +307,12 @@ size_t CompressLibPressio::Compress(const char *dataIn, const Dims &blockStart, PutParameter(bufferOut, bufferOutOffset, d); } PutParameter(bufferOut, bufferOutOffset, type); + PutParameter(bufferOut, bufferOutOffset, + static_cast(pressio_major_version())); + PutParameter(bufferOut, bufferOutOffset, + static_cast(pressio_minor_version())); + PutParameter(bufferOut, bufferOutOffset, + static_cast(pressio_patch_version())); PutParameters(bufferOut, bufferOutOffset, parameters); // zfp V1 metadata end @@ -364,6 +370,12 @@ size_t CompressLibPressio::DecompressV1(const char *bufferIn, blockCount[i] = GetParameter(bufferIn, bufferInOffset); } const DataType type = GetParameter(bufferIn, bufferInOffset); + m_VersionInfo = + " Data is compressed using LibPressio Version " + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + + ". Please make sure a compatible version is used for decompression."; const Params parameters = GetParameters(bufferIn, bufferInOffset); std::vector dims = adios_to_libpressio_dims(blockCount); @@ -384,7 +396,7 @@ size_t CompressLibPressio::DecompressV1(const char *bufferIn, { pressio_data_free(input_buf); pressio_data_free(output_buf); - throw; + throw std::runtime_error(m_VersionInfo + "\n"); } if (pressio_compressor_decompress(compressor, input_buf, output_buf) != 0) @@ -393,7 +405,7 @@ size_t CompressLibPressio::DecompressV1(const char *bufferIn, pressio_data_free(output_buf); throw std::runtime_error( std::string("pressio_compressor_decompress: ") + - pressio_compressor_error_msg(compressor)); + pressio_compressor_error_msg(compressor) + m_VersionInfo + "\n"); } size_t size_in_bytes = 0; @@ -420,12 +432,12 @@ size_t CompressLibPressio::Decompress(const char *bufferIn, const size_t sizeIn, } else if (bufferVersion == 2) { - // TODO: if a Version 2 zfp buffer is being implemented, put it here - // and keep the DecompressV1 routine for backward compatibility + // TODO: if a Version 2 LibPressio buffer is being implemented, put it + // here and keep the DecompressV1 routine for backward compatibility } else { - throw("unknown zfp buffer version"); + throw("unknown LibPressio buffer version"); } return 0; diff --git a/source/adios2/operator/compress/CompressLibPressio.h b/source/adios2/operator/compress/CompressLibPressio.h index 5ce5c06ac8..925a17791a 100644 --- a/source/adios2/operator/compress/CompressLibPressio.h +++ b/source/adios2/operator/compress/CompressLibPressio.h @@ -67,6 +67,8 @@ class CompressLibPressio : public Operator */ size_t DecompressV1(const char *bufferIn, const size_t sizeIn, char *dataOut); + + std::string m_VersionInfo; }; } // end namespace compress diff --git a/source/adios2/operator/compress/CompressMGARD.cpp b/source/adios2/operator/compress/CompressMGARD.cpp index 5bd1e0e097..556787dc70 100644 --- a/source/adios2/operator/compress/CompressMGARD.cpp +++ b/source/adios2/operator/compress/CompressMGARD.cpp @@ -9,12 +9,10 @@ */ #include "CompressMGARD.h" - -#include //std::memcpy - -#include - #include "adios2/helper/adiosFunctions.h" +#include +#include +#include namespace adios2 { @@ -50,6 +48,12 @@ size_t CompressMGARD::Compress(const char *dataIn, const Dims &blockStart, PutParameter(bufferOut, bufferOutOffset, d); } PutParameter(bufferOut, bufferOutOffset, type); + PutParameter(bufferOut, bufferOutOffset, + static_cast(MGARD_VERSION_MAJOR)); + PutParameter(bufferOut, bufferOutOffset, + static_cast(MGARD_VERSION_MINOR)); + PutParameter(bufferOut, bufferOutOffset, + static_cast(MGARD_VERSION_PATCH)); // mgard V1 metadata end if (ndims > 3) @@ -139,6 +143,12 @@ size_t CompressMGARD::DecompressV1(const char *bufferIn, const size_t sizeIn, blockCount[i] = GetParameter(bufferIn, bufferInOffset); } const DataType type = GetParameter(bufferIn, bufferInOffset); + m_VersionInfo = + " Data is compressed using MGARD Version " + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + + ". Please make sure a compatible version is used for decompression."; int mgardType = -1; diff --git a/source/adios2/operator/compress/CompressMGARD.h b/source/adios2/operator/compress/CompressMGARD.h index 3c4cf91028..a7147d68ca 100644 --- a/source/adios2/operator/compress/CompressMGARD.h +++ b/source/adios2/operator/compress/CompressMGARD.h @@ -67,6 +67,8 @@ class CompressMGARD : public Operator */ size_t DecompressV1(const char *bufferIn, const size_t sizeIn, char *dataOut); + + std::string m_VersionInfo; }; } // end namespace compress From e3cb6c73a64b0094455e61b535e160e348f1b858 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 28 Sep 2021 21:03:42 -0400 Subject: [PATCH 5/9] added png library version info into operator buffer --- .../adios2/operator/compress/CompressPNG.cpp | 19 +++++++++++++++++-- source/adios2/operator/compress/CompressPNG.h | 2 ++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/source/adios2/operator/compress/CompressPNG.cpp b/source/adios2/operator/compress/CompressPNG.cpp index d175d00995..1a0cba7f6e 100644 --- a/source/adios2/operator/compress/CompressPNG.cpp +++ b/source/adios2/operator/compress/CompressPNG.cpp @@ -181,6 +181,12 @@ size_t CompressPNG::Compress(const char *dataIn, const Dims &blockStart, png_destroy_write_struct(&pngWrite, &pngInfo); PutParameter(bufferOut, paramOffset, destInfo.Offset); + PutParameter(bufferOut, bufferOutOffset, + static_cast(PNG_LIBPNG_VER_MAJOR)); + PutParameter(bufferOut, bufferOutOffset, + static_cast(PNG_LIBPNG_VER_MINOR)); + PutParameter(bufferOut, bufferOutOffset, + static_cast(PNG_LIBPNG_VER_RELEASE)); return destInfo.Offset; } @@ -196,6 +202,13 @@ size_t CompressPNG::DecompressV1(const char *bufferIn, const size_t sizeIn, size_t bufferInOffset = 0; const size_t outSize = GetParameter(bufferIn, bufferInOffset); + m_VersionInfo = + " Data is compressed using PNG Version " + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + + std::to_string(GetParameter(bufferIn, bufferInOffset)) + + ". Please make sure a compatible version is used for decompression."; + png_image image; std::memset(&image, 0, sizeof(image)); image.version = PNG_IMAGE_VERSION; @@ -207,7 +220,8 @@ size_t CompressPNG::DecompressV1(const char *bufferIn, const size_t sizeIn, { throw std::runtime_error( "ERROR: png_image_begin_read_from_memory failed in call " - "to ADIOS2 PNG Decompress\n"); + "to ADIOS2 PNG Decompress." + + m_VersionInfo + "\n"); } // TODO might be needed from parameters? @@ -216,7 +230,8 @@ size_t CompressPNG::DecompressV1(const char *bufferIn, const size_t sizeIn, { throw std::runtime_error( "ERROR: png_image_finish_read_from_memory failed in call " - "to ADIOS2 PNG Decompress\n"); + "to ADIOS2 PNG Decompress." + + m_VersionInfo + "\n"); } return outSize; } diff --git a/source/adios2/operator/compress/CompressPNG.h b/source/adios2/operator/compress/CompressPNG.h index a76777d52d..003f8d9a79 100644 --- a/source/adios2/operator/compress/CompressPNG.h +++ b/source/adios2/operator/compress/CompressPNG.h @@ -86,6 +86,8 @@ class CompressPNG : public Operator char *BufferOut = nullptr; size_t Offset = 0; }; + + std::string m_VersionInfo; }; } // end namespace compress From 4f5ead1b296b7068ed8a7fc0958ac7f8b3cad8ad Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 28 Sep 2021 21:45:24 -0400 Subject: [PATCH 6/9] fix memory alignment for blosc --- source/adios2/operator/compress/CompressBlosc.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/adios2/operator/compress/CompressBlosc.cpp b/source/adios2/operator/compress/CompressBlosc.cpp index 8117bea957..03a5131eca 100644 --- a/source/adios2/operator/compress/CompressBlosc.cpp +++ b/source/adios2/operator/compress/CompressBlosc.cpp @@ -345,8 +345,8 @@ size_t CompressBlosc::DecompressChunkedFormat(const char *bufferIn, * * we need only the compressed size ( source address + 12 byte) */ - bloscSize_t max_inputDataSize = - *reinterpret_cast(in_ptr + 12u); + bloscSize_t max_inputDataSize; + std::memcpy(&max_inputDataSize, in_ptr + 12, sizeof(bloscSize_t)); char *out_ptr = dataOut + currentOutputSize; From 414af5e0ffb3aec0de5c3ca2bfd66bc8e9d6d08a Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 28 Sep 2021 22:01:43 -0400 Subject: [PATCH 7/9] fixed png bug --- source/adios2/operator/compress/CompressPNG.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/operator/compress/CompressPNG.cpp b/source/adios2/operator/compress/CompressPNG.cpp index 1a0cba7f6e..23e0fc9c3a 100644 --- a/source/adios2/operator/compress/CompressPNG.cpp +++ b/source/adios2/operator/compress/CompressPNG.cpp @@ -61,7 +61,7 @@ size_t CompressPNG::Compress(const char *dataIn, const Dims &blockStart, // Universal operator metadata end size_t paramOffset = bufferOutOffset; - bufferOutOffset += sizeof(size_t); + bufferOutOffset += sizeof(size_t) + 3; auto lf_Write = [](png_structp png_ptr, png_bytep data, png_size_t length) { DestInfo *pDestInfo = From 22f64e97b7e395a3fb9392da007f15bf83ac1d15 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 28 Sep 2021 22:08:15 -0400 Subject: [PATCH 8/9] assign zeros to reserved buffer bytes --- source/adios2/operator/compress/CompressBZIP2.cpp | 2 +- source/adios2/operator/compress/CompressBlosc.cpp | 2 +- source/adios2/operator/compress/CompressLibPressio.cpp | 2 +- source/adios2/operator/compress/CompressMGARD.cpp | 2 +- source/adios2/operator/compress/CompressPNG.cpp | 2 +- source/adios2/operator/compress/CompressSZ.cpp | 2 +- source/adios2/operator/compress/CompressSirius.cpp | 2 +- source/adios2/operator/compress/CompressZFP.cpp | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/adios2/operator/compress/CompressBZIP2.cpp b/source/adios2/operator/compress/CompressBZIP2.cpp index 0ec22a22b8..5c85719b1a 100644 --- a/source/adios2/operator/compress/CompressBZIP2.cpp +++ b/source/adios2/operator/compress/CompressBZIP2.cpp @@ -43,7 +43,7 @@ size_t CompressBZIP2::Compress(const char *dataIn, const Dims &blockStart, // Universal operator metadata PutParameter(bufferOut, destOffset, OperatorType::BZIP2); PutParameter(bufferOut, destOffset, bufferVersion); - destOffset += 2; + PutParameter(bufferOut, destOffset, static_cast(0)); // Universal operator metadata end const size_t sizeIn = diff --git a/source/adios2/operator/compress/CompressBlosc.cpp b/source/adios2/operator/compress/CompressBlosc.cpp index 03a5131eca..eee5b9282e 100644 --- a/source/adios2/operator/compress/CompressBlosc.cpp +++ b/source/adios2/operator/compress/CompressBlosc.cpp @@ -52,7 +52,7 @@ size_t CompressBlosc::Compress(const char *dataIn, const Dims &blockStart, // Universal operator metadata PutParameter(bufferOut, bufferOutOffset, OperatorType::BLOSC); PutParameter(bufferOut, bufferOutOffset, bufferVersion); - bufferOutOffset += 2; + PutParameter(bufferOut, bufferOutOffset, static_cast(0)); // Universal operator metadata end const size_t sizeIn = diff --git a/source/adios2/operator/compress/CompressLibPressio.cpp b/source/adios2/operator/compress/CompressLibPressio.cpp index 5a35a07c0a..52cc0930e4 100644 --- a/source/adios2/operator/compress/CompressLibPressio.cpp +++ b/source/adios2/operator/compress/CompressLibPressio.cpp @@ -295,7 +295,7 @@ size_t CompressLibPressio::Compress(const char *dataIn, const Dims &blockStart, // Universal operator metadata PutParameter(bufferOut, bufferOutOffset, OperatorType::Sz); PutParameter(bufferOut, bufferOutOffset, bufferVersion); - bufferOutOffset += 2; + PutParameter(bufferOut, bufferOutOffset, static_cast(0)); // Universal operator metadata end const size_t ndims = blockCount.size(); diff --git a/source/adios2/operator/compress/CompressMGARD.cpp b/source/adios2/operator/compress/CompressMGARD.cpp index 556787dc70..885239ffb8 100644 --- a/source/adios2/operator/compress/CompressMGARD.cpp +++ b/source/adios2/operator/compress/CompressMGARD.cpp @@ -36,7 +36,7 @@ size_t CompressMGARD::Compress(const char *dataIn, const Dims &blockStart, // Universal operator metadata PutParameter(bufferOut, bufferOutOffset, OperatorType::MGARD); PutParameter(bufferOut, bufferOutOffset, bufferVersion); - bufferOutOffset += 2; + PutParameter(bufferOut, bufferOutOffset, static_cast(0)); // Universal operator metadata end const size_t ndims = blockCount.size(); diff --git a/source/adios2/operator/compress/CompressPNG.cpp b/source/adios2/operator/compress/CompressPNG.cpp index 23e0fc9c3a..e37edb72e8 100644 --- a/source/adios2/operator/compress/CompressPNG.cpp +++ b/source/adios2/operator/compress/CompressPNG.cpp @@ -57,7 +57,7 @@ size_t CompressPNG::Compress(const char *dataIn, const Dims &blockStart, // Universal operator metadata PutParameter(bufferOut, bufferOutOffset, OperatorType::BLOSC); PutParameter(bufferOut, bufferOutOffset, bufferVersion); - bufferOutOffset += 2; + PutParameter(bufferOut, bufferOutOffset, static_cast(0)); // Universal operator metadata end size_t paramOffset = bufferOutOffset; diff --git a/source/adios2/operator/compress/CompressSZ.cpp b/source/adios2/operator/compress/CompressSZ.cpp index 705ae37986..82d8f51b80 100644 --- a/source/adios2/operator/compress/CompressSZ.cpp +++ b/source/adios2/operator/compress/CompressSZ.cpp @@ -39,7 +39,7 @@ size_t CompressSZ::Compress(const char *dataIn, const Dims &blockStart, // Universal operator metadata PutParameter(bufferOut, bufferOutOffset, OperatorType::Sz); PutParameter(bufferOut, bufferOutOffset, bufferVersion); - bufferOutOffset += 2; + PutParameter(bufferOut, bufferOutOffset, static_cast(0)); // Universal operator metadata end const size_t ndims = blockCount.size(); diff --git a/source/adios2/operator/compress/CompressSirius.cpp b/source/adios2/operator/compress/CompressSirius.cpp index bb122232e5..2047f005e8 100644 --- a/source/adios2/operator/compress/CompressSirius.cpp +++ b/source/adios2/operator/compress/CompressSirius.cpp @@ -44,7 +44,7 @@ size_t CompressSirius::Compress(const char *dataIn, const Dims &blockStart, // Universal operator metadata PutParameter(bufferOut, bufferOutOffset, OperatorType::SIRIUS); PutParameter(bufferOut, bufferOutOffset, bufferVersion); - bufferOutOffset += 2; + PutParameter(bufferOut, bufferOutOffset, static_cast(0)); // Universal operator metadata end const size_t ndims = blockCount.size(); diff --git a/source/adios2/operator/compress/CompressZFP.cpp b/source/adios2/operator/compress/CompressZFP.cpp index d0a8c3fb83..2a61e2d87c 100644 --- a/source/adios2/operator/compress/CompressZFP.cpp +++ b/source/adios2/operator/compress/CompressZFP.cpp @@ -33,7 +33,7 @@ size_t CompressZFP::Compress(const char *dataIn, const Dims &blockStart, // Universal operator metadata PutParameter(bufferOut, bufferOutOffset, OperatorType::Sz); PutParameter(bufferOut, bufferOutOffset, bufferVersion); - bufferOutOffset += 2; + PutParameter(bufferOut, bufferOutOffset, static_cast(0)); // Universal operator metadata end const size_t ndims = blockCount.size(); From 62b94a74f34ebdb83a80c534bc0a380b1cd034ff Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 28 Sep 2021 22:30:41 -0400 Subject: [PATCH 9/9] fix multiple bugs --- source/adios2/operator/compress/CompressLibPressio.cpp | 2 +- source/adios2/operator/compress/CompressPNG.cpp | 8 ++++---- source/adios2/operator/compress/CompressZFP.cpp | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/adios2/operator/compress/CompressLibPressio.cpp b/source/adios2/operator/compress/CompressLibPressio.cpp index 52cc0930e4..7e696ad095 100644 --- a/source/adios2/operator/compress/CompressLibPressio.cpp +++ b/source/adios2/operator/compress/CompressLibPressio.cpp @@ -293,7 +293,7 @@ size_t CompressLibPressio::Compress(const char *dataIn, const Dims &blockStart, size_t bufferOutOffset = 0; // Universal operator metadata - PutParameter(bufferOut, bufferOutOffset, OperatorType::Sz); + PutParameter(bufferOut, bufferOutOffset, OperatorType::LIBPRESSIO); PutParameter(bufferOut, bufferOutOffset, bufferVersion); PutParameter(bufferOut, bufferOutOffset, static_cast(0)); // Universal operator metadata end diff --git a/source/adios2/operator/compress/CompressPNG.cpp b/source/adios2/operator/compress/CompressPNG.cpp index e37edb72e8..565a9d69d4 100644 --- a/source/adios2/operator/compress/CompressPNG.cpp +++ b/source/adios2/operator/compress/CompressPNG.cpp @@ -55,7 +55,7 @@ size_t CompressPNG::Compress(const char *dataIn, const Dims &blockStart, const uint8_t bufferVersion = 1; // Universal operator metadata - PutParameter(bufferOut, bufferOutOffset, OperatorType::BLOSC); + PutParameter(bufferOut, bufferOutOffset, OperatorType::PNG); PutParameter(bufferOut, bufferOutOffset, bufferVersion); PutParameter(bufferOut, bufferOutOffset, static_cast(0)); // Universal operator metadata end @@ -181,11 +181,11 @@ size_t CompressPNG::Compress(const char *dataIn, const Dims &blockStart, png_destroy_write_struct(&pngWrite, &pngInfo); PutParameter(bufferOut, paramOffset, destInfo.Offset); - PutParameter(bufferOut, bufferOutOffset, + PutParameter(bufferOut, paramOffset, static_cast(PNG_LIBPNG_VER_MAJOR)); - PutParameter(bufferOut, bufferOutOffset, + PutParameter(bufferOut, paramOffset, static_cast(PNG_LIBPNG_VER_MINOR)); - PutParameter(bufferOut, bufferOutOffset, + PutParameter(bufferOut, paramOffset, static_cast(PNG_LIBPNG_VER_RELEASE)); return destInfo.Offset; diff --git a/source/adios2/operator/compress/CompressZFP.cpp b/source/adios2/operator/compress/CompressZFP.cpp index 2a61e2d87c..1ed8d5c312 100644 --- a/source/adios2/operator/compress/CompressZFP.cpp +++ b/source/adios2/operator/compress/CompressZFP.cpp @@ -31,7 +31,7 @@ size_t CompressZFP::Compress(const char *dataIn, const Dims &blockStart, size_t bufferOutOffset = 0; // Universal operator metadata - PutParameter(bufferOut, bufferOutOffset, OperatorType::Sz); + PutParameter(bufferOut, bufferOutOffset, OperatorType::ZFP); PutParameter(bufferOut, bufferOutOffset, bufferVersion); PutParameter(bufferOut, bufferOutOffset, static_cast(0)); // Universal operator metadata end