Skip to content

Commit

Permalink
Merge pull request #2886 from JasonRuonanWang/operator
Browse files Browse the repository at this point in the history 
make libpressio buffer self-contained and backward-compatible, and cleaned up Operator classes
  • Loading branch information
@JasonRuonanWang
JasonRuonanWang authored Sep 28, 2021
2 parents b3d5fab + 58a5996 commit 8bbd685
Show file tree
Hide file tree
Showing 30 changed files with 206 additions and 344 deletions.
1 change: 0 additions & 1 deletion source/adios2/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@ add_library(adios2_core
toolkit/format/bp/bpOperation/BPOperation.cpp
toolkit/format/bp/bpOperation/BPOperation.tcc
toolkit/format/bp/bpOperation/compress/BPZFP.cpp
toolkit/format/bp/bpOperation/compress/BPZFP.tcc
toolkit/format/bp/bpOperation/compress/BPSZ.cpp
toolkit/format/bp/bpOperation/compress/BPSirius.cpp
toolkit/format/bp/bpOperation/compress/BPMGARD.cpp
Expand Down
9 changes: 3 additions & 6 deletions source/adios2/core/Operator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,8 @@ void Operator::RunCallback2(void *arg0, const std::string &arg1,
}

size_t Operator::Compress(const char * /*dataIn*/, const Dims & /*blockStart*/,
const Dims & /*blockCount*/, DataType /*type*/,
char * /*bufferOut*/, const Params & /*params*/,
Params & /*info*/)
const Dims & /*blockCount*/, const DataType /*type*/,
char * /*bufferOut*/, const Params & /*params*/)
{
throw std::invalid_argument("ERROR: signature (const void*, const "
"Dims, const size_t, const std::string, "
Expand All @@ -62,9 +61,7 @@ size_t Operator::Compress(const char * /*dataIn*/, const Dims & /*blockStart*/,
}

size_t Operator::Decompress(const char *bufferIn, const size_t sizeIn,
char *dataOut, const DataType type,
const Dims &blockStart, const Dims &blockCount,
const Params &parameters, Params &info)
char *dataOut)
{
throw std::invalid_argument(
"ERROR: signature (const void*, const size_t, void) not supported "
Expand Down
18 changes: 11 additions & 7 deletions source/adios2/core/Operator.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,21 +56,25 @@ class Operator

/**
* @param dataIn
* @param dimensions
* @param blockStart
* @param blockCount
* @param type
* @param bufferOut
* @param parameters
* @return size of compressed buffer
*/
virtual size_t Compress(const char *dataIn, const Dims &blockStart,
const Dims &blockCount, DataType type,
char *bufferOut, const Params &parameters,
Params &info);
const Dims &blockCount, const DataType type,
char *bufferOut, const Params &parameters);

/**
* @param bufferIn
* @param sizeIn
* @param dataOut
* @return size of decompressed buffer
*/
virtual size_t Decompress(const char *bufferIn, const size_t sizeIn,
char *dataOut, const DataType type,
const Dims &blockStart, const Dims &blockCount,
const Params &parameters, Params &info);
char *dataOut);

virtual bool IsDataTypeValid(const DataType type) const = 0;

Expand Down
9 changes: 2 additions & 7 deletions source/adios2/operator/compress/CompressBZIP2.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,7 @@ CompressBZIP2::CompressBZIP2(const Params &parameters)

size_t CompressBZIP2::Compress(const char *dataIn, const Dims &blockStart,
const Dims &blockCount, DataType type,
char *bufferOut, const Params &parameters,
Params & /*info*/)
char *bufferOut, const Params &parameters)
{

const uint8_t bufferVersion = 1;
Expand Down Expand Up @@ -174,11 +173,7 @@ size_t CompressBZIP2::DecompressV1(const char *bufferIn, const size_t sizeIn,
return sizeOut;
}
size_t CompressBZIP2::Decompress(const char *bufferIn, const size_t sizeIn,
char *dataOut, const DataType /*type*/,
const Dims & /*blockStart*/,
const Dims & /*blockCount*/,
const Params & /*parameters*/,
Params & /*info*/)
char *dataOut)
{
size_t bufferInOffset = 1; // skip operator type
const uint8_t bufferVersion =
Expand Down
21 changes: 8 additions & 13 deletions source/adios2/operator/compress/CompressBZIP2.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,31 +32,26 @@ class CompressBZIP2 : public Operator
~CompressBZIP2() = default;

/**
* Compression signature for legacy libraries that use char*
* @param dataIn
* @param dimensions
* @param blockStart
* @param blockCount
* @param type
* @param bufferOut
* @param parameters
* @return size of compressed buffer in bytes
* @return size of compressed buffer
*/
size_t Compress(const char *dataIn, const Dims &blockStart,
const Dims &blockCount, DataType type, char *bufferOut,
const Params &parameters, Params &info) final;
const Dims &blockCount, const DataType type,
char *bufferOut, const Params &parameters) final;

/**
* Decompression signature for legacy libraries that use char*
* @param bufferIn
* @param sizeIn
* @param dataOut
* @param dimensions
* @param type
* @return size of decompressed buffer in bytes
* @return size of decompressed buffer
*/
size_t Decompress(const char *bufferIn, const size_t sizeIn, char *dataOut,
const DataType type, const Dims &blockStart,
const Dims &blockCount, const Params &parameters,
Params &info) final;
size_t Decompress(const char *bufferIn, const size_t sizeIn,
char *dataOut) final;

bool IsDataTypeValid(const DataType type) const final;

Expand Down
11 changes: 3 additions & 8 deletions source/adios2/operator/compress/CompressBlosc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,8 @@ CompressBlosc::CompressBlosc(const Params &parameters)
}

size_t CompressBlosc::Compress(const char *dataIn, const Dims &blockStart,
const Dims &blockCount, DataType type,
char *bufferOut, const Params &parameters,
Params &info)
const Dims &blockCount, const DataType type,
char *bufferOut, const Params &parameters)
{
size_t currentOutputSize = 0;
const uint8_t bufferVersion = 1;
Expand Down Expand Up @@ -264,11 +263,7 @@ size_t CompressBlosc::DecompressV1(const char *bufferIn, const size_t sizeIn,
}

size_t CompressBlosc::Decompress(const char *bufferIn, const size_t sizeIn,
char *dataOut, const DataType /*type*/,
const Dims & /*blockStart*/,
const Dims & /*blockCount*/,
const Params & /*parameters*/,
Params & /*info*/)
char *dataOut)
{
size_t bufferInOffset = 1; // skip operator type
const uint8_t bufferVersion =
Expand Down
19 changes: 7 additions & 12 deletions source/adios2/operator/compress/CompressBlosc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,32 +44,27 @@ class CompressBlosc : public Operator
~CompressBlosc() = default;

/**
* Compression signature for legacy libraries that use void*
* @param dataIn
* @param dimensions
* @param blockStart
* @param blockCount
* @param type
* @param bufferOut format will be: 'DataHeader ; (BloscCompressedChunk |
* UncompressedData), [ BloscCompressedChunk, ...]'
* @param parameters
* @return size of compressed buffer in bytes
*/
size_t Compress(const char *dataIn, const Dims &blockStart,
const Dims &blockCount, DataType type, char *bufferOut,
const Params &parameters, Params &info) final;
const Dims &blockCount, const DataType type,
char *bufferOut, const Params &parameters) final;

/**
* Decompression signature for legacy libraries that use char*
* @param bufferIn
* @param sizeIn
* @param dataOut
* @param dimensions
* @param type
* @return size of decompressed buffer in bytes
* @return size of decompressed buffer
*/
size_t Decompress(const char *bufferIn, const size_t sizeIn, char *dataOut,
const DataType type, const Dims &blockStart,
const Dims &blockCount, const Params &parameters,
Params &info) final;
size_t Decompress(const char *bufferIn, const size_t sizeIn,
char *dataOut) final;

bool IsDataTypeValid(const DataType type) const final;

Expand Down
86 changes: 74 additions & 12 deletions source/adios2/operator/compress/CompressLibPressio.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,13 +286,33 @@ CompressLibPressio::CompressLibPressio(const Params &parameters)
}

size_t CompressLibPressio::Compress(const char *dataIn, const Dims &blockStart,
const Dims &blockCount, DataType varType,
char *bufferOut, const Params &parameters,
Params &info)
const Dims &blockCount, const DataType type,
char *bufferOut, const Params &parameters)
{
const uint8_t bufferVersion = 1;
size_t bufferOutOffset = 0;

// Universal operator metadata
PutParameter(bufferOut, bufferOutOffset, OperatorType::Sz);
PutParameter(bufferOut, bufferOutOffset, bufferVersion);
bufferOutOffset += 2;
// Universal operator metadata end

const size_t ndims = blockCount.size();

// zfp V1 metadata
PutParameter(bufferOut, bufferOutOffset, ndims);
for (const auto &d : blockCount)
{
PutParameter(bufferOut, bufferOutOffset, d);
}
PutParameter(bufferOut, bufferOutOffset, type);
PutParameters(bufferOut, bufferOutOffset, parameters);
// zfp V1 metadata end

auto inputs_dims = adios_to_libpressio_dims(blockCount);
pressio_data *input_buf = pressio_data_new_nonowning(
adios_to_libpressio_dtype(varType), const_cast<char *>(dataIn),
adios_to_libpressio_dtype(type), const_cast<char *>(dataIn),
inputs_dims.size(), inputs_dims.data());
pressio_data *output_buf =
pressio_data_new_empty(pressio_byte_dtype, 0, nullptr);
Expand All @@ -318,26 +338,42 @@ size_t CompressLibPressio::Compress(const char *dataIn, const Dims &blockStart,

size_t size_in_bytes = 0;
void *bytes = pressio_data_ptr(output_buf, &size_in_bytes);
memcpy(bufferOut, bytes, size_in_bytes);
memcpy(bufferOut + bufferOutOffset, bytes, size_in_bytes);
bufferOutOffset += size_in_bytes;

pressio_data_free(input_buf);
pressio_data_free(output_buf);

return static_cast<size_t>(size_in_bytes);
return bufferOutOffset;
}

size_t CompressLibPressio::Decompress(const char *bufferIn, const size_t sizeIn,
char *dataOut, const DataType type,
const Dims &blockStart,
const Dims &blockCount,
const Params &parameters, Params &info)
size_t CompressLibPressio::DecompressV1(const char *bufferIn,
const size_t sizeIn, char *dataOut)
{
// Do NOT remove even if the buffer version is updated. Data might be still
// in lagacy formats. This function must be kept for backward compatibility.
// If a newer buffer format is implemented, create another function, e.g.
// DecompressV2 and keep this function for decompressing lagacy data.

size_t bufferInOffset = 0;

const size_t ndims = GetParameter<size_t, size_t>(bufferIn, bufferInOffset);
Dims blockCount(ndims);
for (size_t i = 0; i < ndims; ++i)
{
blockCount[i] = GetParameter<size_t, size_t>(bufferIn, bufferInOffset);
}
const DataType type = GetParameter<DataType>(bufferIn, bufferInOffset);
const Params parameters = GetParameters(bufferIn, bufferInOffset);

std::vector<size_t> dims = adios_to_libpressio_dims(blockCount);
pressio_data *output_buf = pressio_data_new_owning(
adios_to_libpressio_dtype(type), dims.size(), dims.data());

size_t newSizeIn = sizeIn - bufferInOffset;
pressio_data *input_buf = pressio_data_new_nonowning(
pressio_byte_dtype, const_cast<char *>(bufferIn), 1, &sizeIn);
pressio_byte_dtype, const_cast<char *>(bufferIn + bufferInOffset), 1,
&newSizeIn);

pressio_compressor *compressor = nullptr;
try
Expand Down Expand Up @@ -369,6 +405,32 @@ size_t CompressLibPressio::Decompress(const char *bufferIn, const size_t sizeIn,
return size_in_bytes;
}

size_t CompressLibPressio::Decompress(const char *bufferIn, const size_t sizeIn,
char *dataOut)
{
size_t bufferInOffset = 1; // skip operator type
const uint8_t bufferVersion =
GetParameter<uint8_t>(bufferIn, bufferInOffset);
bufferInOffset += 2; // skip two reserved bytes

if (bufferVersion == 1)
{
return DecompressV1(bufferIn + bufferInOffset, sizeIn - bufferInOffset,
dataOut);
}
else if (bufferVersion == 2)
{
// TODO: if a Version 2 zfp buffer is being implemented, put it here
// and keep the DecompressV1 routine for backward compatibility
}
else
{
throw("unknown zfp buffer version");
}

return 0;
}

bool CompressLibPressio::IsDataTypeValid(const DataType type) const
{
#define declare_type(T) \
Expand Down
34 changes: 21 additions & 13 deletions source/adios2/operator/compress/CompressLibPressio.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,33 +32,41 @@ class CompressLibPressio : public Operator
~CompressLibPressio() = default;

/**
* Compression signature for legacy libraries that use char*
* @param dataIn
* @param dimensions
* @param blockStart
* @param blockCount
* @param type
* @param bufferOut
* @param parameters
* @return size of compressed buffer in bytes
* @return size of compressed buffer
*/
size_t Compress(const char *dataIn, const Dims &blockStart,
const Dims &blockCount, DataType type, char *bufferOut,
const Params &parameters, Params &info) final;
const Dims &blockCount, const DataType type,
char *bufferOut, const Params &parameters) final;

/**
* Wrapper around zfp decompression
* @param bufferIn
* @param sizeIn
* @param dataOut
* @param dimensions
* @param type
* @return size of decompressed data in dataOut
* @return size of decompressed buffer
*/
size_t Decompress(const char *bufferIn, const size_t sizeIn, char *dataOut,
const DataType type, const Dims &blockStart,
const Dims &blockCount, const Params &parameters,
Params &info) final;
size_t Decompress(const char *bufferIn, const size_t sizeIn,
char *dataOut) final;

bool IsDataTypeValid(const DataType type) const final;

private:
/**
* Decompress function for V1 buffer. Do NOT remove even if the buffer
* version is updated. Data might be still in lagacy formats. This function
* must be kept for backward compatibility
* @param bufferIn : compressed data buffer (V1 only)
* @param sizeIn : number of bytes in bufferIn
* @param dataOut : decompressed data buffer
* @return : number of bytes in dataOut
*/
size_t DecompressV1(const char *bufferIn, const size_t sizeIn,
char *dataOut);
};

} // end namespace compress
Expand Down
11 changes: 3 additions & 8 deletions source/adios2/operator/compress/CompressMGARD.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,8 @@ CompressMGARD::CompressMGARD(const Params &parameters)
}

size_t CompressMGARD::Compress(const char *dataIn, const Dims &blockStart,
const Dims &blockCount, DataType type,
char *bufferOut, const Params &parameters,
Params &info)
const Dims &blockCount, const DataType type,
char *bufferOut, const Params &parameters)
{
const uint8_t bufferVersion = 1;
size_t bufferOutOffset = 0;
Expand Down Expand Up @@ -181,11 +180,7 @@ size_t CompressMGARD::DecompressV1(const char *bufferIn, const size_t sizeIn,
}

size_t CompressMGARD::Decompress(const char *bufferIn, const size_t sizeIn,
char *dataOut, const DataType /*type*/,
const Dims & /*blockStart*/,
const Dims & /*blockCount*/,
const Params & /*parameters*/,
Params & /*info*/)
char *dataOut)
{
size_t bufferInOffset = 1; // skip operator type
const uint8_t bufferVersion =
Expand Down
Loading

0 comments on commit 8bbd685

Please sign in to comment.