From 081005fddbf03ba5600536016d30648c1afb0cc8 Mon Sep 17 00:00:00 2001 From: SchrodingerZhu Date: Wed, 27 Apr 2022 16:49:18 +0800 Subject: [PATCH 1/2] fix: ut and more idx error Signed-off-by: SchrodingerZhu --- dbms/src/Server/DTTool/DTToolMigrate.cpp | 2 ++ dbms/src/Server/tests/gtest_dttool.cpp | 22 +++++++++++++------- dbms/src/Storages/DeltaMerge/File/DMFile.cpp | 5 +++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/dbms/src/Server/DTTool/DTToolMigrate.cpp b/dbms/src/Server/DTTool/DTToolMigrate.cpp index 1be0a68f715..d54fc1ed9ed 100644 --- a/dbms/src/Server/DTTool/DTToolMigrate.cpp +++ b/dbms/src/Server/DTTool/DTToolMigrate.cpp @@ -190,6 +190,8 @@ int migrateServiceMain(DB::Context & context, const MigrateArgs & args) LOG_FMT_INFO(logger, "source version: {}", (src_file->getConfiguration() ? 2 : 1)); LOG_FMT_INFO(logger, "source bytes: {}", src_file->getBytesOnDisk()); LOG_FMT_INFO(logger, "migration temporary directory: {}", keeper.migration_temp_dir.path().c_str()); + LOG_FMT_INFO(logger, "target version: {}", args.version); + LOG_FMT_INFO(logger, "target frame size: {}", args.frame); DB::DM::DMConfigurationOpt option{}; // if new format is the target, we construct a config file. diff --git a/dbms/src/Server/tests/gtest_dttool.cpp b/dbms/src/Server/tests/gtest_dttool.cpp index 4ea163f0507..e1bc8a25c00 100644 --- a/dbms/src/Server/tests/gtest_dttool.cpp +++ b/dbms/src/Server/tests/gtest_dttool.cpp @@ -233,20 +233,23 @@ TEST_F(DTToolTest, BlockwiseInvariant) stream->readSuffix(); } - std::vector> test_cases{ - {2, DB::ChecksumAlgo::XXH3, DB::CompressionMethod::LZ4, -1}, - {1, DB::ChecksumAlgo::XXH3, DB::CompressionMethod::ZSTD, 1}, - {2, DB::ChecksumAlgo::City128, DB::CompressionMethod::LZ4HC, 0}, - {2, DB::ChecksumAlgo::CRC64, DB::CompressionMethod::ZSTD, 22}, - {1, DB::ChecksumAlgo::XXH3, DB::CompressionMethod::NONE, -1}}; - for (auto [version, algo, comp, level] : test_cases) + std::vector> test_cases{ + {2, DBMS_DEFAULT_BUFFER_SIZE, DB::ChecksumAlgo::XXH3, DB::CompressionMethod::LZ4, -1}, + {1, 64, DB::ChecksumAlgo::XXH3, DB::CompressionMethod::ZSTD, 1}, + {2, DBMS_DEFAULT_BUFFER_SIZE * 2, DB::ChecksumAlgo::City128, DB::CompressionMethod::LZ4HC, 0}, + {2, DBMS_DEFAULT_BUFFER_SIZE * 4, DB::ChecksumAlgo::City128, DB::CompressionMethod::LZ4HC, 0}, + {2, 4, DB::ChecksumAlgo::CRC64, DB::CompressionMethod::ZSTD, 22}, + {2, 13, DB::ChecksumAlgo::CRC64, DB::CompressionMethod::ZSTD, 22}, + {2, 5261, DB::ChecksumAlgo::CRC64, DB::CompressionMethod::ZSTD, 22}, + {1, DBMS_DEFAULT_BUFFER_SIZE, DB::ChecksumAlgo::XXH3, DB::CompressionMethod::NONE, -1}}; + for (auto [version, frame_size, algo, comp, level] : test_cases) { auto a = DTTool::Migrate::MigrateArgs{ .no_keep = false, .dry_mode = false, .file_id = 1, .version = version, - .frame = DBMS_DEFAULT_BUFFER_SIZE, + .frame = frame_size, .algorithm = algo, .workdir = getTemporaryPath(), .compression_method = comp, @@ -260,6 +263,9 @@ TEST_F(DTToolTest, BlockwiseInvariant) 0, getTemporaryPath(), DB::DM::DMFile::ReadMetaMode::all()); + if (version == 2) { + EXPECT_EQ(refreshed_file->getConfiguration()->getChecksumFrameLength(), frame_size); + } auto stream = DB::DM::createSimpleBlockInputStream(*db_context, refreshed_file); auto size_iter = size_info.begin(); auto prop_iter = dmfile->getPackProperties().property().begin(); diff --git a/dbms/src/Storages/DeltaMerge/File/DMFile.cpp b/dbms/src/Storages/DeltaMerge/File/DMFile.cpp index 928bf8a6415..d40a6bf5bab 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFile.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFile.cpp @@ -579,8 +579,9 @@ void DMFile::readMetadata(const FileProviderPtr & file_provider, const ReadMetaM auto recheck = [&](size_t size) { if (this->configuration) { - auto frame_count = size / this->configuration->getChecksumFrameLength() - + (0 != size % this->configuration->getChecksumFrameLength()); + auto total_size = this->configuration->getChecksumFrameLength() + this->configuration->getChecksumHeaderLength(); + auto frame_count = size / total_size + + (0 != size % total_size); size -= frame_count * this->configuration->getChecksumHeaderLength(); } return size; From 2b3417efb2ba71dac2e41958490668de2055fccf Mon Sep 17 00:00:00 2001 From: SchrodingerZhu Date: Wed, 27 Apr 2022 17:06:57 +0800 Subject: [PATCH 2/2] chore: format Signed-off-by: SchrodingerZhu --- dbms/src/Server/tests/gtest_dttool.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Server/tests/gtest_dttool.cpp b/dbms/src/Server/tests/gtest_dttool.cpp index e1bc8a25c00..6b0d6e3d5c2 100644 --- a/dbms/src/Server/tests/gtest_dttool.cpp +++ b/dbms/src/Server/tests/gtest_dttool.cpp @@ -263,7 +263,8 @@ TEST_F(DTToolTest, BlockwiseInvariant) 0, getTemporaryPath(), DB::DM::DMFile::ReadMetaMode::all()); - if (version == 2) { + if (version == 2) + { EXPECT_EQ(refreshed_file->getConfiguration()->getChecksumFrameLength(), frame_size); } auto stream = DB::DM::createSimpleBlockInputStream(*db_context, refreshed_file);