From b29ee6b278a03521e8e3602c3c53377f98c95806 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 5 Jul 2022 16:42:57 +0800 Subject: [PATCH 01/26] add fast mode logic --- dbms/src/Interpreters/Settings.h | 1 - dbms/src/Server/tests/gtest_server_config.cpp | 2 - .../Storages/DeltaMerge/DMDecoratorStreams.h | 101 +++++++++++++-- .../DeltaMerge/DMSegmentThreadInputStream.h | 5 +- .../DeltaMerge/Delta/DeltaValueSpace.h | 21 ++-- .../Storages/DeltaMerge/DeltaMergeStore.cpp | 5 +- .../src/Storages/DeltaMerge/DeltaMergeStore.h | 1 + dbms/src/Storages/DeltaMerge/Segment.cpp | 66 +++++----- dbms/src/Storages/DeltaMerge/Segment.h | 3 +- .../DeltaMerge/tests/gtest_column_filter.cpp | 117 ++++++++++++++++++ dbms/src/Storages/StorageDeltaMerge.cpp | 1 + dbms/src/Storages/Transaction/TiDB.h | 7 ++ 12 files changed, 272 insertions(+), 58 deletions(-) create mode 100644 dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index 9361e0525d2..7daa22974ff 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -279,7 +279,6 @@ struct Settings M(SettingFloat, dt_bg_gc_delta_delete_ratio_to_trigger_gc, 0.3, "Trigger segment's gc when the ratio of delta delete range to stable exceeds this ratio.") \ M(SettingUInt64, dt_insert_max_rows, 0, "Max rows of insert blocks when write into DeltaTree Engine. By default 0 means no limit.") \ M(SettingBool, dt_enable_rough_set_filter, true, "Whether to parse where expression as Rough Set Index filter or not.") \ - M(SettingBool, dt_raw_filter_range, true, "Do range filter or not when read data in raw mode in DeltaTree Engine.") \ M(SettingBool, dt_read_delta_only, false, "Only read delta data in DeltaTree Engine.") \ M(SettingBool, dt_read_stable_only, false, "Only read stable data in DeltaTree Engine.") \ M(SettingBool, dt_enable_logical_split, false, "Enable logical split or not in DeltaTree Engine.") \ diff --git a/dbms/src/Server/tests/gtest_server_config.cpp b/dbms/src/Server/tests/gtest_server_config.cpp index cf53a8d6c18..2e20d39ab58 100644 --- a/dbms/src/Server/tests/gtest_server_config.cpp +++ b/dbms/src/Server/tests/gtest_server_config.cpp @@ -453,7 +453,6 @@ try [profiles] [profiles.default] dt_enable_rough_set_filter = false -dt_raw_filter_range = 0 dt_read_delta_only = 1 dt_read_stable_only = true )"}; @@ -468,7 +467,6 @@ dt_read_stable_only = true global_ctx.reloadDeltaTreeConfig(*config); ASSERT_EQ(global_ctx.getSettingsRef().dt_enable_rough_set_filter, false); - ASSERT_EQ(global_ctx.getSettingsRef().dt_raw_filter_range, false); ASSERT_EQ(global_ctx.getSettingsRef().dt_read_delta_only, true); ASSERT_EQ(global_ctx.getSettingsRef().dt_read_stable_only, true); } diff --git a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h index 49eac0b25a2..f37283793f9 100644 --- a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h +++ b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h @@ -14,8 +14,10 @@ #pragma once +#include #include #include +#include #include @@ -24,14 +26,31 @@ namespace DB { namespace DM { + +static constexpr size_t UNROLL_BATCH = 64; + +/// DMColumnFilterBlockInputStream is used to filter the column and filter out the rows whose del_mark is true class DMColumnFilterBlockInputStream : public IBlockInputStream { public: DMColumnFilterBlockInputStream(const BlockInputStreamPtr & input, const ColumnDefines & columns_to_read_) : columns_to_read(columns_to_read_) , header(toEmptyBlock(columns_to_read)) + , log(&Poco::Logger::get("DMColumnFilterBlockInputStream")) { children.emplace_back(input); + delete_col_pos = input->getHeader().getPositionByName(TAG_COLUMN_NAME); + } + + ~DMColumnFilterBlockInputStream() + { + LOG_FMT_DEBUG(log, + "Total rows: {}, pass: {:.2f}%" + ", complete pass: {:.2f}%, complete not pass: {:.2f}%", + total_rows, + passed_rows * 100.0 / total_rows, + complete_passed * 100.0 / total_blocks, + complete_not_passed * 100.0 / total_blocks); } String getName() const override { return "DMColumnFilter"; } @@ -40,20 +59,88 @@ class DMColumnFilterBlockInputStream : public IBlockInputStream Block read() override { - Block block = children.back()->read(); - if (!block) - return {}; - Block res; - for (auto & cd : columns_to_read) + while (true) { - res.insert(block.getByName(cd.name)); + Block block = children.back()->read(); + if (!block) + return {}; + if (!block.rows()) + continue; + + delete_col_data = getColumnVectorDataPtr(block, delete_col_pos); + + size_t rows = block.rows(); + delete_filter.resize(rows); + + const size_t batch_rows = (rows - 1) / UNROLL_BATCH * UNROLL_BATCH; + + // The following is trying to unroll the filtering operations, + // so that optimizer could use vectorized optimization. + // The original logic can be seen in #checkWithNextIndex(). + { + UInt8 * filter_pos = delete_filter.data(); + auto * delete_pos = const_cast(delete_col_data->data()); + for (size_t i = 0; i < batch_rows; ++i) + { + (*filter_pos) = (*delete_pos) == 0; + ++filter_pos; + ++delete_pos; + } + } + + for (size_t i = batch_rows; i < rows; ++i) + { + delete_filter[i] = !(*delete_col_data)[i]; + } + + const size_t passed_count = countBytesInFilter(delete_filter); + + ++total_blocks; + total_rows += rows; + passed_rows += passed_count; + + // This block is empty after filter, continue to process next block + if (passed_count == 0) + { + ++complete_not_passed; + continue; + } + + + if (passed_count == rows) + { + ++complete_passed; + return getNewBlockByHeader(header, block); + } + + Block res; + for (auto & cd : columns_to_read) + { + auto & column = block.getByName(cd.name); + column.column = column.column->filter(delete_filter, passed_count); + res.insert(std::move(column)); + } + return res; } - return res; } private: ColumnDefines columns_to_read; Block header; + + size_t delete_col_pos; + + IColumn::Filter delete_filter{}; + + PaddedPODArray const * delete_col_data = nullptr; + + size_t total_blocks = 0; + size_t total_rows = 0; + size_t passed_rows = 0; + size_t complete_passed = 0; + size_t complete_not_passed = 0; + + Poco::Logger * log; }; class DMHandleConvertBlockInputStream : public IBlockInputStream diff --git a/dbms/src/Storages/DeltaMerge/DMSegmentThreadInputStream.h b/dbms/src/Storages/DeltaMerge/DMSegmentThreadInputStream.h index d47c3b105cc..ea9074165d2 100644 --- a/dbms/src/Storages/DeltaMerge/DMSegmentThreadInputStream.h +++ b/dbms/src/Storages/DeltaMerge/DMSegmentThreadInputStream.h @@ -48,7 +48,6 @@ class DMSegmentThreadInputStream : public IProfilingBlockInputStream UInt64 max_version_, size_t expected_block_size_, bool is_raw_, - bool do_range_filter_for_raw_, const int extra_table_id_index, const TableID physical_table_id, const String & req_id) @@ -61,7 +60,6 @@ class DMSegmentThreadInputStream : public IProfilingBlockInputStream , max_version(max_version_) , expected_block_size(expected_block_size_) , is_raw(is_raw_) - , do_range_filter_for_raw(do_range_filter_for_raw_) , extra_table_id_index(extra_table_id_index) , physical_table_id(physical_table_id) , log(Logger::get(NAME, req_id)) @@ -104,7 +102,7 @@ class DMSegmentThreadInputStream : public IProfilingBlockInputStream cur_segment = task->segment; if (is_raw) { - cur_stream = cur_segment->getInputStreamRaw(*dm_context, columns_to_read, task->read_snapshot, do_range_filter_for_raw); + cur_stream = cur_segment->getInputStreamRaw(*dm_context, columns_to_read, task->read_snapshot, task->ranges, filter); } else { @@ -167,7 +165,6 @@ class DMSegmentThreadInputStream : public IProfilingBlockInputStream const UInt64 max_version; const size_t expected_block_size; const bool is_raw; - const bool do_range_filter_for_raw; // position of the ExtraPhysTblID column in column_names parameter in the StorageDeltaMerge::read function. const int extra_table_id_index; diff --git a/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.h b/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.h index 8f14682caa8..4ccddacdbb9 100644 --- a/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.h +++ b/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.h @@ -376,8 +376,10 @@ class DeltaValueReader class DeltaValueInputStream : public IBlockInputStream { private: - ColumnFileSetInputStream mem_table_input_stream; - ColumnFileSetInputStream persisted_files_input_stream; + // ColumnFileSetInputStream mem_table_input_stream; + // ColumnFileSetInputStream persisted_files_input_stream; + BlockInputStreamPtr mem_table_input_stream; + BlockInputStreamPtr persisted_files_input_stream; bool persisted_files_done = false; @@ -386,25 +388,28 @@ class DeltaValueInputStream : public IBlockInputStream const DeltaSnapshotPtr & delta_snap_, const ColumnDefinesPtr & col_defs_, const RowKeyRange & segment_range_) - : mem_table_input_stream(context_, delta_snap_->getMemTableSetSnapshot(), col_defs_, segment_range_) - , persisted_files_input_stream(context_, delta_snap_->getPersistedFileSetSnapshot(), col_defs_, segment_range_) + : mem_table_input_stream{std::make_shared(context_, delta_snap_->getMemTableSetSnapshot(), col_defs_, segment_range_)} + , persisted_files_input_stream{std::make_shared(context_, delta_snap_->getPersistedFileSetSnapshot(), col_defs_, segment_range_)} {} String getName() const override { return "DeltaValue"; } - Block getHeader() const override { return persisted_files_input_stream.getHeader(); } + Block getHeader() const override { return persisted_files_input_stream->getHeader(); } + + BlockInputStreamPtr getPersistedFilesInputStream() { return persisted_files_input_stream; } + BlockInputStreamPtr getMemTableInputStream() { return mem_table_input_stream; } Block read() override { if (persisted_files_done) - return mem_table_input_stream.read(); + return mem_table_input_stream->read(); - Block block = persisted_files_input_stream.read(); + Block block = persisted_files_input_stream->read(); if (block) return block; else { persisted_files_done = true; - return mem_table_input_stream.read(); + return mem_table_input_stream->read(); } } }; diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index 195ed5c53c2..15665f39170 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -1168,7 +1168,6 @@ BlockInputStreams DeltaMergeStore::readRaw(const Context & db_context, std::numeric_limits::max(), DEFAULT_BLOCK_SIZE, true, - db_settings.dt_raw_filter_range, extra_table_id_index, physical_table_id, req_info); @@ -1185,6 +1184,7 @@ BlockInputStreams DeltaMergeStore::read(const Context & db_context, UInt64 max_version, const RSOperatorPtr & filter, const String & tracing_id, + bool is_raw_read, size_t expected_block_size, const SegmentIdSet & read_segments, size_t extra_table_id_index) @@ -1220,8 +1220,7 @@ BlockInputStreams DeltaMergeStore::read(const Context & db_context, filter, max_version, expected_block_size, - false, - db_settings.dt_raw_filter_range, + is_raw_read, extra_table_id_index, physical_table_id, req_info); diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h index 705481ca107..5f035bf1c5b 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.h @@ -363,6 +363,7 @@ class DeltaMergeStore : private boost::noncopyable UInt64 max_version, const RSOperatorPtr & filter, const String & tracing_id, + bool is_raw_read = false, // set true when read in fast mode size_t expected_block_size = DEFAULT_BLOCK_SIZE, const SegmentIdSet & read_segments = {}, size_t extra_table_id_index = InvalidColumnID); diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index 8398fdcee40..1c03e2a639e 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -500,56 +500,56 @@ BlockInputStreamPtr Segment::getInputStreamForDataExport(const DMContext & dm_co BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, const ColumnDefines & columns_to_read, const SegmentSnapshotPtr & segment_snap, - bool do_range_filter, + const RowKeyRanges & data_ranges, + const RSOperatorPtr & filter, size_t expected_block_size) { auto new_columns_to_read = std::make_shared(); - if (!do_range_filter) - { - (*new_columns_to_read) = columns_to_read; - } - else - { - new_columns_to_read->push_back(getExtraHandleColumnDefine(is_common_handle)); - - for (const auto & c : columns_to_read) - { - if (c.id != EXTRA_HANDLE_COLUMN_ID) - new_columns_to_read->push_back(c); - } - } + new_columns_to_read->push_back(getExtraHandleColumnDefine(is_common_handle)); - BlockInputStreamPtr delta_stream = std::make_shared(dm_context, // - segment_snap->delta, - new_columns_to_read, - this->rowkey_range); + for (const auto & c : columns_to_read) + { + if (c.id != EXTRA_HANDLE_COLUMN_ID) + new_columns_to_read->push_back(c); + } - RowKeyRanges rowkey_ranges{rowkey_range}; BlockInputStreamPtr stable_stream = segment_snap->stable->getInputStream( dm_context, *new_columns_to_read, - rowkey_ranges, - EMPTY_FILTER, + data_ranges, + filter, std::numeric_limits::max(), expected_block_size, false); - if (do_range_filter) - { - delta_stream = std::make_shared>(delta_stream, rowkey_ranges, 0); - delta_stream = std::make_shared(delta_stream, columns_to_read); + //BlockInputStreamPtr delta_stream = std::make_shared(dm_context, segment_snap->delta, new_columns_to_read, this->rowkey_range); + + //delta_stream = std::make_shared>(delta_stream, data_ranges, 0); + //delta_stream = std::make_shared(delta_stream, columns_to_read); + + DeltaValueInputStream delta_stream(dm_context, segment_snap->delta, new_columns_to_read, this->rowkey_range); + auto memtable_stream = delta_stream.getMemTableInputStream(); + auto persisted_files_stream = delta_stream.getPersistedFilesInputStream(); + + memtable_stream = std::make_shared>(memtable_stream, data_ranges, 0); + memtable_stream = std::make_shared(memtable_stream, columns_to_read); + + persisted_files_stream = std::make_shared>(persisted_files_stream, data_ranges, 0); // related to minor compaction sorted + persisted_files_stream = std::make_shared(persisted_files_stream, columns_to_read); + + stable_stream = std::make_shared>(stable_stream, data_ranges, 0); + stable_stream = std::make_shared(stable_stream, columns_to_read); - stable_stream = std::make_shared>(stable_stream, rowkey_ranges, 0); - stable_stream = std::make_shared(stable_stream, columns_to_read); - } BlockInputStreams streams; if (dm_context.read_delta_only) { - streams.push_back(delta_stream); + //streams.push_back(delta_stream); + streams.push_back(memtable_stream); + streams.push_back(persisted_files_stream); } else if (dm_context.read_stable_only) { @@ -557,7 +557,9 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, } else { - streams.push_back(delta_stream); + //streams.push_back(delta_stream); + streams.push_back(memtable_stream); + streams.push_back(persisted_files_stream); streams.push_back(stable_stream); } return std::make_shared(streams, /*req_id=*/""); @@ -568,7 +570,7 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, con auto segment_snap = createSnapshot(dm_context, false, CurrentMetrics::DT_SnapshotOfReadRaw); if (!segment_snap) return {}; - return getInputStreamRaw(dm_context, columns_to_read, segment_snap, true); + return getInputStreamRaw(dm_context, columns_to_read, segment_snap, {rowkey_range}, EMPTY_FILTER); } SegmentPtr Segment::mergeDelta(DMContext & dm_context, const ColumnDefinesPtr & schema_snap) const diff --git a/dbms/src/Storages/DeltaMerge/Segment.h b/dbms/src/Storages/DeltaMerge/Segment.h index cccfc5091b9..10b16b9e823 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.h +++ b/dbms/src/Storages/DeltaMerge/Segment.h @@ -182,7 +182,8 @@ class Segment : private boost::noncopyable const DMContext & dm_context, const ColumnDefines & columns_to_read, const SegmentSnapshotPtr & segment_snap, - bool do_range_filter, + const RowKeyRanges & data_ranges, + const RSOperatorPtr & filter, size_t expected_block_size = DEFAULT_BLOCK_SIZE); BlockInputStreamPtr getInputStreamRaw( diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp new file mode 100644 index 00000000000..e59e4e9bcc2 --- /dev/null +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp @@ -0,0 +1,117 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include +#include + +namespace DB +{ +namespace DM +{ +namespace tests +{ +namespace +{ +constexpr const char * str_col_name = "col_a"; + +class DebugBlockInputStream : public IProfilingBlockInputStream +{ +public: + DebugBlockInputStream(const BlocksList & blocks, bool is_common_handle_) + : begin(blocks.begin()) + , end(blocks.end()) + , it(blocks.begin()) + , is_common_handle(is_common_handle_) + { + } + String getName() const override { return "Debug"; } + Block getHeader() const override + { + auto cds = DMTestEnv::getDefaultColumns(is_common_handle ? DMTestEnv::PkType::CommonHandle : DMTestEnv::PkType::HiddenTiDBRowID); + cds->push_back(ColumnDefine(100, str_col_name, DataTypeFactory::instance().get("String"))); + return toEmptyBlock(*cds); + } + +protected: + Block readImpl() override + { + if (it == end) + return Block(); + else + return *(it++); + } + +private: + BlocksList::const_iterator begin; + BlocksList::const_iterator end; + BlocksList::const_iterator it; + bool is_common_handle; +}; + +BlockInputStreamPtr genInputStream(const BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle) +{ + ColumnDefine handle_define( + TiDBPkColumnID, + DMTestEnv::pk_name, + is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE); + return std::make_shared( + std::make_shared(blocks, is_common_handle), + columns); +} +} + +TEST(ColumnFilterTest, NormalCase) +{ + BlocksList blocks; + + { + Int64 pk_value = 4; + blocks.push_back(DMTestEnv::prepareOneRowBlock(pk_value, 10, 0, str_col_name, "hello", false, 1)); + blocks.push_back(DMTestEnv::prepareOneRowBlock(pk_value, 20, 0, str_col_name, "world", false, 1)); + blocks.push_back(DMTestEnv::prepareOneRowBlock(pk_value, 30, 1, str_col_name, "", false, 1)); + blocks.push_back(DMTestEnv::prepareOneRowBlock(pk_value, 40, 0, str_col_name, "TiFlash", false, 1)); + blocks.push_back(DMTestEnv::prepareOneRowBlock(pk_value, 40, 1, str_col_name, "Storage", false, 1)); + } + + ColumnDefines columns = getColumnDefinesFromBlock(blocks.back()); + + { + auto in = genInputStream(blocks, columns, false); + in->readPrefix(); + Block block = in->read(); + ASSERT_EQ(block.rows(), 1); + auto col = block.getByName(str_col_name); + auto val = col.column->getDataAt(0); + ASSERT_EQ(val, "hello"); + + block = in->read(); + ASSERT_EQ(block.rows(), 1); + col = block.getByName(str_col_name); + val = col.column->getDataAt(0); + ASSERT_EQ(val, "world"); + + block = in->read(); + ASSERT_EQ(block.rows(), 1); + col = block.getByName(str_col_name); + val = col.column->getDataAt(0); + ASSERT_EQ(val, "TiFlash"); + + block = in->read(); + ASSERT_EQ(block.rows(), 0); + in->readSuffix(); + } +} +} +} +} \ No newline at end of file diff --git a/dbms/src/Storages/StorageDeltaMerge.cpp b/dbms/src/Storages/StorageDeltaMerge.cpp index 67d32c73a05..e2c06f5193f 100644 --- a/dbms/src/Storages/StorageDeltaMerge.cpp +++ b/dbms/src/Storages/StorageDeltaMerge.cpp @@ -756,6 +756,7 @@ BlockInputStreams StorageDeltaMerge::read( /*max_version=*/mvcc_query_info.read_tso, rs_operator, query_info.req_id, + /* is_raw_read */ tidb_table_info.tiflash_mode == TiDB::TiFlashMode::Fast, max_block_size, parseSegmentSet(select_query.segment_expression_list), extra_table_id_index); diff --git a/dbms/src/Storages/Transaction/TiDB.h b/dbms/src/Storages/Transaction/TiDB.h index 4c28a614857..b1d2a26dbdc 100644 --- a/dbms/src/Storages/Transaction/TiDB.h +++ b/dbms/src/Storages/Transaction/TiDB.h @@ -333,6 +333,11 @@ struct IndexInfo bool is_global; }; +enum class TiFlashMode +{ + Normal, + Fast, +}; struct TableInfo { TableInfo() = default; @@ -382,6 +387,8 @@ struct TableInfo // The TiFlash replica info persisted by TiDB TiFlashReplicaInfo replica_info; + TiFlashMode tiflash_mode = TiFlashMode::Normal; + ::TiDB::StorageEngine engine_type = ::TiDB::StorageEngine::UNSPECIFIED; ColumnID getColumnID(const String & name) const; From c60185ee7bdebf6df5c1e461713805e567e91967 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 5 Jul 2022 17:48:43 +0800 Subject: [PATCH 02/26] fix code format --- .../Storages/DeltaMerge/tests/gtest_column_filter.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp index e59e4e9bcc2..5c4f6ee4a97 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp @@ -11,9 +11,9 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include -#include namespace DB { @@ -69,7 +69,7 @@ BlockInputStreamPtr genInputStream(const BlocksList & blocks, const ColumnDefine std::make_shared(blocks, is_common_handle), columns); } -} +} // namespace TEST(ColumnFilterTest, NormalCase) { @@ -112,6 +112,6 @@ TEST(ColumnFilterTest, NormalCase) in->readSuffix(); } } -} -} -} \ No newline at end of file +} // namespace tests +} // namespace DM +} // namespace DB \ No newline at end of file From 0dff68b06c034d12bccd052ec960b7a2dc8170d8 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Thu, 7 Jul 2022 17:22:04 +0800 Subject: [PATCH 03/26] compatible old code --- .../Storages/DeltaMerge/DMDecoratorStreams.h | 106 ++++++++++-------- .../DeltaMerge/DMSegmentThreadInputStream.h | 5 +- .../Storages/DeltaMerge/DeltaMergeStore.cpp | 2 + dbms/src/Storages/DeltaMerge/Segment.cpp | 11 +- dbms/src/Storages/DeltaMerge/Segment.h | 4 +- .../DeltaMerge/tests/gtest_column_filter.cpp | 61 +++++++++- .../tests/gtest_dm_delta_merge_store.cpp | 57 +++++++++- 7 files changed, 187 insertions(+), 59 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h index f37283793f9..9dfab115b63 100644 --- a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h +++ b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h @@ -33,10 +33,11 @@ static constexpr size_t UNROLL_BATCH = 64; class DMColumnFilterBlockInputStream : public IBlockInputStream { public: - DMColumnFilterBlockInputStream(const BlockInputStreamPtr & input, const ColumnDefines & columns_to_read_) + DMColumnFilterBlockInputStream(const BlockInputStreamPtr & input, const ColumnDefines & columns_to_read_, const bool filter_delete_mark = true) : columns_to_read(columns_to_read_) , header(toEmptyBlock(columns_to_read)) , log(&Poco::Logger::get("DMColumnFilterBlockInputStream")) + , filter_delete_mark(filter_delete_mark) { children.emplace_back(input); delete_col_pos = input->getHeader().getPositionByName(TAG_COLUMN_NAME); @@ -59,66 +60,81 @@ class DMColumnFilterBlockInputStream : public IBlockInputStream Block read() override { - while (true) + if (filter_delete_mark) { - Block block = children.back()->read(); - if (!block) - return {}; - if (!block.rows()) - continue; + while (true) + { + Block block = children.back()->read(); + if (!block) + return {}; + if (!block.rows()) + continue; - delete_col_data = getColumnVectorDataPtr(block, delete_col_pos); + delete_col_data = getColumnVectorDataPtr(block, delete_col_pos); - size_t rows = block.rows(); - delete_filter.resize(rows); + size_t rows = block.rows(); + delete_filter.resize(rows); - const size_t batch_rows = (rows - 1) / UNROLL_BATCH * UNROLL_BATCH; + const size_t batch_rows = (rows - 1) / UNROLL_BATCH * UNROLL_BATCH; - // The following is trying to unroll the filtering operations, - // so that optimizer could use vectorized optimization. - // The original logic can be seen in #checkWithNextIndex(). - { - UInt8 * filter_pos = delete_filter.data(); - auto * delete_pos = const_cast(delete_col_data->data()); - for (size_t i = 0; i < batch_rows; ++i) + // The following is trying to unroll the filtering operations, + // so that optimizer could use vectorized optimization. + // The original logic can be seen in #checkWithNextIndex(). { - (*filter_pos) = (*delete_pos) == 0; - ++filter_pos; - ++delete_pos; + UInt8 * filter_pos = delete_filter.data(); + auto * delete_pos = const_cast(delete_col_data->data()); + for (size_t i = 0; i < batch_rows; ++i) + { + (*filter_pos) = (*delete_pos) == 0; + ++filter_pos; + ++delete_pos; + } } - } - for (size_t i = batch_rows; i < rows; ++i) - { - delete_filter[i] = !(*delete_col_data)[i]; - } + for (size_t i = batch_rows; i < rows; ++i) + { + delete_filter[i] = !(*delete_col_data)[i]; + } - const size_t passed_count = countBytesInFilter(delete_filter); + const size_t passed_count = countBytesInFilter(delete_filter); - ++total_blocks; - total_rows += rows; - passed_rows += passed_count; + ++total_blocks; + total_rows += rows; + passed_rows += passed_count; - // This block is empty after filter, continue to process next block - if (passed_count == 0) - { - ++complete_not_passed; - continue; - } + // This block is empty after filter, continue to process next block + if (passed_count == 0) + { + ++complete_not_passed; + continue; + } - if (passed_count == rows) - { - ++complete_passed; - return getNewBlockByHeader(header, block); - } + if (passed_count == rows) + { + ++complete_passed; + return getNewBlockByHeader(header, block); + } + Block res; + for (auto & cd : columns_to_read) + { + auto & column = block.getByName(cd.name); + column.column = column.column->filter(delete_filter, passed_count); + res.insert(std::move(column)); + } + return res; + } + } + else + { + Block block = children.back()->read(); + if (!block) + return {}; Block res; for (auto & cd : columns_to_read) { - auto & column = block.getByName(cd.name); - column.column = column.column->filter(delete_filter, passed_count); - res.insert(std::move(column)); + res.insert(block.getByName(cd.name)); } return res; } @@ -141,6 +157,8 @@ class DMColumnFilterBlockInputStream : public IBlockInputStream size_t complete_not_passed = 0; Poco::Logger * log; + + bool filter_delete_mark; }; class DMHandleConvertBlockInputStream : public IBlockInputStream diff --git a/dbms/src/Storages/DeltaMerge/DMSegmentThreadInputStream.h b/dbms/src/Storages/DeltaMerge/DMSegmentThreadInputStream.h index ea9074165d2..33433da10f8 100644 --- a/dbms/src/Storages/DeltaMerge/DMSegmentThreadInputStream.h +++ b/dbms/src/Storages/DeltaMerge/DMSegmentThreadInputStream.h @@ -48,6 +48,7 @@ class DMSegmentThreadInputStream : public IProfilingBlockInputStream UInt64 max_version_, size_t expected_block_size_, bool is_raw_, + bool do_delete_mark_filter_for_raw_, const int extra_table_id_index, const TableID physical_table_id, const String & req_id) @@ -60,6 +61,7 @@ class DMSegmentThreadInputStream : public IProfilingBlockInputStream , max_version(max_version_) , expected_block_size(expected_block_size_) , is_raw(is_raw_) + , do_delete_mark_filter_for_raw(do_delete_mark_filter_for_raw_) , extra_table_id_index(extra_table_id_index) , physical_table_id(physical_table_id) , log(Logger::get(NAME, req_id)) @@ -102,7 +104,7 @@ class DMSegmentThreadInputStream : public IProfilingBlockInputStream cur_segment = task->segment; if (is_raw) { - cur_stream = cur_segment->getInputStreamRaw(*dm_context, columns_to_read, task->read_snapshot, task->ranges, filter); + cur_stream = cur_segment->getInputStreamRaw(*dm_context, columns_to_read, task->read_snapshot, task->ranges, filter, do_delete_mark_filter_for_raw); } else { @@ -165,6 +167,7 @@ class DMSegmentThreadInputStream : public IProfilingBlockInputStream const UInt64 max_version; const size_t expected_block_size; const bool is_raw; + const bool do_delete_mark_filter_for_raw; // position of the ExtraPhysTblID column in column_names parameter in the StorageDeltaMerge::read function. const int extra_table_id_index; diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index 8638fdafa99..f2aa49f2b7d 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -1180,6 +1180,7 @@ BlockInputStreams DeltaMergeStore::readRaw(const Context & db_context, std::numeric_limits::max(), DEFAULT_BLOCK_SIZE, true, + false, extra_table_id_index, physical_table_id, req_info); @@ -1233,6 +1234,7 @@ BlockInputStreams DeltaMergeStore::read(const Context & db_context, max_version, expected_block_size, is_raw_read, + true, extra_table_id_index, physical_table_id, req_info); diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index 1c03e2a639e..05969ac36bf 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -502,6 +502,7 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, const SegmentSnapshotPtr & segment_snap, const RowKeyRanges & data_ranges, const RSOperatorPtr & filter, + bool filter_delete_mark, size_t expected_block_size) { auto new_columns_to_read = std::make_shared(); @@ -534,13 +535,13 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, auto persisted_files_stream = delta_stream.getPersistedFilesInputStream(); memtable_stream = std::make_shared>(memtable_stream, data_ranges, 0); - memtable_stream = std::make_shared(memtable_stream, columns_to_read); + memtable_stream = std::make_shared(memtable_stream, columns_to_read, filter_delete_mark); persisted_files_stream = std::make_shared>(persisted_files_stream, data_ranges, 0); // related to minor compaction sorted - persisted_files_stream = std::make_shared(persisted_files_stream, columns_to_read); + persisted_files_stream = std::make_shared(persisted_files_stream, columns_to_read, filter_delete_mark); stable_stream = std::make_shared>(stable_stream, data_ranges, 0); - stable_stream = std::make_shared(stable_stream, columns_to_read); + stable_stream = std::make_shared(stable_stream, columns_to_read, filter_delete_mark); BlockInputStreams streams; @@ -565,12 +566,12 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, return std::make_shared(streams, /*req_id=*/""); } -BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, const ColumnDefines & columns_to_read) +BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, const ColumnDefines & columns_to_read, bool filter_delete_mark) { auto segment_snap = createSnapshot(dm_context, false, CurrentMetrics::DT_SnapshotOfReadRaw); if (!segment_snap) return {}; - return getInputStreamRaw(dm_context, columns_to_read, segment_snap, {rowkey_range}, EMPTY_FILTER); + return getInputStreamRaw(dm_context, columns_to_read, segment_snap, {rowkey_range}, EMPTY_FILTER, filter_delete_mark); } SegmentPtr Segment::mergeDelta(DMContext & dm_context, const ColumnDefinesPtr & schema_snap) const diff --git a/dbms/src/Storages/DeltaMerge/Segment.h b/dbms/src/Storages/DeltaMerge/Segment.h index 24df72709c1..69792ed7239 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.h +++ b/dbms/src/Storages/DeltaMerge/Segment.h @@ -184,11 +184,13 @@ class Segment : private boost::noncopyable const SegmentSnapshotPtr & segment_snap, const RowKeyRanges & data_ranges, const RSOperatorPtr & filter, + bool filter_delete_mark = true, size_t expected_block_size = DEFAULT_BLOCK_SIZE); BlockInputStreamPtr getInputStreamRaw( const DMContext & dm_context, - const ColumnDefines & columns_to_read); + const ColumnDefines & columns_to_read, + bool filter_delete_mark = false); /// For those split, merge and mergeDelta methods, we should use prepareXXX/applyXXX combo in real production. /// split(), merge() and mergeDelta() are only used in test cases. diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp index 5c4f6ee4a97..12f683d15b4 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp @@ -59,7 +59,7 @@ class DebugBlockInputStream : public IProfilingBlockInputStream bool is_common_handle; }; -BlockInputStreamPtr genInputStream(const BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle) +BlockInputStreamPtr genInputStream(const BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle, bool filter_delete_mark = true) { ColumnDefine handle_define( TiDBPkColumnID, @@ -67,11 +67,12 @@ BlockInputStreamPtr genInputStream(const BlocksList & blocks, const ColumnDefine is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE); return std::make_shared( std::make_shared(blocks, is_common_handle), - columns); + columns, + filter_delete_mark); } } // namespace -TEST(ColumnFilterTest, NormalCase) +TEST(ColumnFilterTest, NormalCaseFilterDeleteMark) { BlocksList blocks; @@ -112,6 +113,60 @@ TEST(ColumnFilterTest, NormalCase) in->readSuffix(); } } + +TEST(ColumnFilterTest, WithoutFilterDeleteMark) +{ + BlocksList blocks; + + { + Int64 pk_value = 4; + blocks.push_back(DMTestEnv::prepareOneRowBlock(pk_value, 10, 0, str_col_name, "hello", false, 1)); + blocks.push_back(DMTestEnv::prepareOneRowBlock(pk_value, 20, 0, str_col_name, "world", false, 1)); + blocks.push_back(DMTestEnv::prepareOneRowBlock(pk_value, 30, 1, str_col_name, "", false, 1)); + blocks.push_back(DMTestEnv::prepareOneRowBlock(pk_value, 40, 0, str_col_name, "TiFlash", false, 1)); + blocks.push_back(DMTestEnv::prepareOneRowBlock(pk_value, 40, 1, str_col_name, "Storage", false, 1)); + } + + ColumnDefines columns = getColumnDefinesFromBlock(blocks.back()); + + { + auto in = genInputStream(blocks, columns, false); + in->readPrefix(); + Block block = in->read(); + ASSERT_EQ(block.rows(), 1); + auto col = block.getByName(str_col_name); + auto val = col.column->getDataAt(0); + ASSERT_EQ(val, "hello"); + + block = in->read(); + ASSERT_EQ(block.rows(), 1); + col = block.getByName(str_col_name); + val = col.column->getDataAt(0); + ASSERT_EQ(val, "world"); + + block = in->read(); + ASSERT_EQ(block.rows(), 1); + col = block.getByName(str_col_name); + val = col.column->getDataAt(0); + ASSERT_EQ(val, ""); + + block = in->read(); + ASSERT_EQ(block.rows(), 1); + col = block.getByName(str_col_name); + val = col.column->getDataAt(0); + ASSERT_EQ(val, "TiFlash"); + + block = in->read(); + ASSERT_EQ(block.rows(), 1); + col = block.getByName(str_col_name); + val = col.column->getDataAt(0); + ASSERT_EQ(val, "Storage"); + + block = in->read(); + ASSERT_EQ(block.rows(), 0); + in->readSuffix(); + } +} } // namespace tests } // namespace DM } // namespace DB \ No newline at end of file diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp index b7913c44a2c..1b199577ba1 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp @@ -202,7 +202,7 @@ class DeltaMergeStoreRWTest store->preIngestFile(store_path, file_id, dmfile->getBytesOnDisk()); - auto & pk_column = block.getByPosition(0).column; + const auto & pk_column = block.getByPosition(0).column; auto min_pk = pk_column->getInt(0); auto max_pk = pk_column->getInt(block.rows() - 1); HandleRange range(min_pk, max_pk + 1); @@ -423,6 +423,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; @@ -565,6 +566,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; @@ -650,6 +652,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; @@ -721,6 +724,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; while (Block block = in->read()) @@ -758,6 +762,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; while (Block block = in->read()) @@ -843,6 +848,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; while (Block block = in->read()) @@ -928,6 +934,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; while (Block block = in->read()) @@ -959,6 +966,7 @@ try /* max_version= */ UInt64(1), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; while (Block block = in->read()) @@ -1014,6 +1022,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; while (Block block = in->read()) @@ -1053,6 +1062,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; // block_num represents index of current segment @@ -1112,6 +1122,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; @@ -1135,6 +1146,7 @@ try /* max_version= */ tso2, EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; @@ -1158,6 +1170,7 @@ try /* max_version= */ tso1, EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; @@ -1181,6 +1194,7 @@ try /* max_version= */ tso1 - 1, EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; @@ -1242,6 +1256,7 @@ try /* max_version= */ tso1, EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; @@ -1279,6 +1294,7 @@ try /* max_version= */ tso2 - 1, EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; @@ -1317,6 +1333,7 @@ try /* max_version= */ tso3 - 1, EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; @@ -1342,6 +1359,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; @@ -1367,6 +1385,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; @@ -1428,6 +1447,7 @@ try /* max_version= */ tso1, EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1); BlockInputStreamPtr in = ins[0]; @@ -1465,6 +1485,7 @@ try /* max_version= */ tso2 - 1, EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1); BlockInputStreamPtr in = ins[0]; @@ -1503,6 +1524,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1); BlockInputStreamPtr in = ins[0]; @@ -1559,6 +1581,7 @@ try /* max_version= */ tso1, EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1); BlockInputStreamPtr in = ins[0]; @@ -1596,6 +1619,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1); BlockInputStreamPtr in = ins[0]; @@ -1682,6 +1706,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; @@ -1790,6 +1815,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr & in = ins[0]; @@ -1897,6 +1923,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr & in = ins[0]; @@ -1985,6 +2012,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr & in = ins[0]; @@ -2088,6 +2116,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; in->readPrefix(); @@ -2164,6 +2193,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; in->readPrefix(); @@ -2240,6 +2270,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; in->readPrefix(); @@ -2316,6 +2347,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; in->readPrefix(); @@ -2392,6 +2424,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; in->readPrefix(); @@ -2466,6 +2499,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; @@ -2539,6 +2573,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; in->readPrefix(); @@ -2629,6 +2664,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr & in = ins[0]; @@ -2763,6 +2799,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr & in = ins[0]; @@ -2824,6 +2861,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr & in = ins[0]; @@ -2923,6 +2961,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; in->readPrefix(); @@ -2976,6 +3015,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; in->readPrefix(); @@ -3005,14 +3045,14 @@ try store = reload(table_column_defines, true, 2); { // check handle column of store - auto & h = store->getHandle(); + const auto & h = store->getHandle(); ASSERT_EQ(h.name, EXTRA_HANDLE_COLUMN_NAME); ASSERT_EQ(h.id, EXTRA_HANDLE_COLUMN_ID); ASSERT_TRUE(h.type->equals(*EXTRA_HANDLE_COLUMN_STRING_TYPE)); } { // check column structure of store - auto & cols = store->getTableColumns(); + const auto & cols = store->getTableColumns(); // version & tag column added ASSERT_EQ(cols.size(), 3UL); } @@ -3092,6 +3132,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; @@ -3221,6 +3262,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; while (Block block = in->read()) @@ -3295,6 +3337,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; while (Block block = in->read()) @@ -3326,6 +3369,7 @@ try /* max_version= */ UInt64(1), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; while (Block block = in->read()) @@ -3384,6 +3428,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; while (Block block = in->read()) @@ -3434,6 +3479,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; while (Block block = in->read()) @@ -3507,6 +3553,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, + /* is_raw_read= */ false, /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; @@ -3600,11 +3647,11 @@ class DeltaMergeStoreMergeDeltaBySegmentTest } protected: - std::unique_ptr helper; + std::unique_ptr helper{}; DeltaMergeStorePtr store; DMContextPtr dm_context; - UInt64 ps_ver; + UInt64 ps_ver{}; DMTestEnv::PkType pk_type; }; From 3039aab0fd0aefd22191a4dc43532b6390bcf608 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Thu, 7 Jul 2022 20:50:55 +0800 Subject: [PATCH 04/26] fix bug --- dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h index 9dfab115b63..5c0b96cd4d8 100644 --- a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h +++ b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h @@ -40,7 +40,10 @@ class DMColumnFilterBlockInputStream : public IBlockInputStream , filter_delete_mark(filter_delete_mark) { children.emplace_back(input); - delete_col_pos = input->getHeader().getPositionByName(TAG_COLUMN_NAME); + if (filter_delete_mark) + { + delete_col_pos = input->getHeader().getPositionByName(TAG_COLUMN_NAME); + } } ~DMColumnFilterBlockInputStream() From 7898ae8bd3228f87cd239e7d2c437a7a28874342 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Thu, 7 Jul 2022 21:51:35 +0800 Subject: [PATCH 05/26] fix test --- dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp index 12f683d15b4..fe56bbd8d4b 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp @@ -130,7 +130,7 @@ TEST(ColumnFilterTest, WithoutFilterDeleteMark) ColumnDefines columns = getColumnDefinesFromBlock(blocks.back()); { - auto in = genInputStream(blocks, columns, false); + auto in = genInputStream(blocks, columns, false, false); in->readPrefix(); Block block = in->read(); ASSERT_EQ(block.rows(), 1); @@ -150,6 +150,7 @@ TEST(ColumnFilterTest, WithoutFilterDeleteMark) val = col.column->getDataAt(0); ASSERT_EQ(val, ""); + block = in->read(); ASSERT_EQ(block.rows(), 1); col = block.getByName(str_col_name); From e8c52beb72aee251704d4bc94324f38f3612a215 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Sun, 10 Jul 2022 17:31:49 +0800 Subject: [PATCH 06/26] add test for fast mode --- dbms/src/Interpreters/Settings.h | 2 + .../Storages/DeltaMerge/DMDecoratorStreams.h | 150 +- dbms/src/Storages/DeltaMerge/Segment.cpp | 18 +- .../Storages/DeltaMerge/tests/CMakeLists.txt | 3 + .../DeltaMerge/tests/gtest_column_filter.cpp | 26 +- ...est_dm_delta_merge_store_for_fast_mode.cpp | 1448 +++++++++++++++++ 6 files changed, 1565 insertions(+), 82 deletions(-) create mode 100644 dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index 7daa22974ff..192a016bf78 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -279,6 +279,8 @@ struct Settings M(SettingFloat, dt_bg_gc_delta_delete_ratio_to_trigger_gc, 0.3, "Trigger segment's gc when the ratio of delta delete range to stable exceeds this ratio.") \ M(SettingUInt64, dt_insert_max_rows, 0, "Max rows of insert blocks when write into DeltaTree Engine. By default 0 means no limit.") \ M(SettingBool, dt_enable_rough_set_filter, true, "Whether to parse where expression as Rough Set Index filter or not.") \ + /* unused */ \ + M(SettingBool, dt_raw_filter_range, true, "Do range filter or not when read data in raw mode in DeltaTree Engine.") \ M(SettingBool, dt_read_delta_only, false, "Only read delta data in DeltaTree Engine.") \ M(SettingBool, dt_read_stable_only, false, "Only read stable data in DeltaTree Engine.") \ M(SettingBool, dt_enable_logical_split, false, "Enable logical split or not in DeltaTree Engine.") \ diff --git a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h index 5c0b96cd4d8..63fed02a2f9 100644 --- a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h +++ b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h @@ -29,24 +29,19 @@ namespace DM static constexpr size_t UNROLL_BATCH = 64; -/// DMColumnFilterBlockInputStream is used to filter the column and filter out the rows whose del_mark is true -class DMColumnFilterBlockInputStream : public IBlockInputStream +/// DMDeleteFilterBlockInputStream is used to filter the column and filter out the rows whose del_mark is true +class DMDeleteFilterBlockInputStream : public IBlockInputStream { public: - DMColumnFilterBlockInputStream(const BlockInputStreamPtr & input, const ColumnDefines & columns_to_read_, const bool filter_delete_mark = true) + DMDeleteFilterBlockInputStream(const BlockInputStreamPtr & input, const ColumnDefines & columns_to_read_) : columns_to_read(columns_to_read_) , header(toEmptyBlock(columns_to_read)) - , log(&Poco::Logger::get("DMColumnFilterBlockInputStream")) - , filter_delete_mark(filter_delete_mark) + , log(&Poco::Logger::get("DMDeleteFilterBlockInputStream")) { children.emplace_back(input); - if (filter_delete_mark) - { - delete_col_pos = input->getHeader().getPositionByName(TAG_COLUMN_NAME); - } + delete_col_pos = input->getHeader().getPositionByName(TAG_COLUMN_NAME); } - - ~DMColumnFilterBlockInputStream() + ~DMDeleteFilterBlockInputStream() { LOG_FMT_DEBUG(log, "Total rows: {}, pass: {:.2f}%" @@ -63,81 +58,66 @@ class DMColumnFilterBlockInputStream : public IBlockInputStream Block read() override { - if (filter_delete_mark) + while (true) { - while (true) - { - Block block = children.back()->read(); - if (!block) - return {}; - if (!block.rows()) - continue; - - delete_col_data = getColumnVectorDataPtr(block, delete_col_pos); + Block block = children.back()->read(); + if (!block) + return {}; + if (!block.rows()) + continue; - size_t rows = block.rows(); - delete_filter.resize(rows); + delete_col_data = getColumnVectorDataPtr(block, delete_col_pos); - const size_t batch_rows = (rows - 1) / UNROLL_BATCH * UNROLL_BATCH; + size_t rows = block.rows(); + delete_filter.resize(rows); - // The following is trying to unroll the filtering operations, - // so that optimizer could use vectorized optimization. - // The original logic can be seen in #checkWithNextIndex(). - { - UInt8 * filter_pos = delete_filter.data(); - auto * delete_pos = const_cast(delete_col_data->data()); - for (size_t i = 0; i < batch_rows; ++i) - { - (*filter_pos) = (*delete_pos) == 0; - ++filter_pos; - ++delete_pos; - } - } + const size_t batch_rows = (rows - 1) / UNROLL_BATCH * UNROLL_BATCH; - for (size_t i = batch_rows; i < rows; ++i) + // The following is trying to unroll the filtering operations, + // so that optimizer could use vectorized optimization. + // The original logic can be seen in #checkWithNextIndex(). + { + UInt8 * filter_pos = delete_filter.data(); + auto * delete_pos = const_cast(delete_col_data->data()); + for (size_t i = 0; i < batch_rows; ++i) { - delete_filter[i] = !(*delete_col_data)[i]; + (*filter_pos) = (*delete_pos) == 0; + ++filter_pos; + ++delete_pos; } + } - const size_t passed_count = countBytesInFilter(delete_filter); + for (size_t i = batch_rows; i < rows; ++i) + { + delete_filter[i] = !(*delete_col_data)[i]; + } - ++total_blocks; - total_rows += rows; - passed_rows += passed_count; + const size_t passed_count = countBytesInFilter(delete_filter); - // This block is empty after filter, continue to process next block - if (passed_count == 0) - { - ++complete_not_passed; - continue; - } + ++total_blocks; + total_rows += rows; + passed_rows += passed_count; + // This block is empty after filter, continue to process next block + if (passed_count == 0) + { + ++complete_not_passed; + continue; + } - if (passed_count == rows) - { - ++complete_passed; - return getNewBlockByHeader(header, block); - } - Block res; - for (auto & cd : columns_to_read) - { - auto & column = block.getByName(cd.name); - column.column = column.column->filter(delete_filter, passed_count); - res.insert(std::move(column)); - } - return res; + if (passed_count == rows) + { + ++complete_passed; + return getNewBlockByHeader(header, block); } - } - else - { - Block block = children.back()->read(); - if (!block) - return {}; + Block res; for (auto & cd : columns_to_read) { - res.insert(block.getByName(cd.name)); + auto & column = block.getByName(cd.name); + column.column = column.column->filter(delete_filter, passed_count); + res.insert(std::move(column)); } return res; } @@ -160,8 +140,38 @@ class DMColumnFilterBlockInputStream : public IBlockInputStream size_t complete_not_passed = 0; Poco::Logger * log; +}; + +class DMColumnFilterBlockInputStream : public IBlockInputStream +{ +public: + DMColumnFilterBlockInputStream(const BlockInputStreamPtr & input, const ColumnDefines & columns_to_read_) + : columns_to_read(columns_to_read_) + , header(toEmptyBlock(columns_to_read)) + { + children.emplace_back(input); + } + + String getName() const override { return "DMColumnFilter"; } - bool filter_delete_mark; + Block getHeader() const override { return header; } + + Block read() override + { + Block block = children.back()->read(); + if (!block) + return {}; + Block res; + for (auto & cd : columns_to_read) + { + res.insert(block.getByName(cd.name)); + } + return res; + } + +private: + ColumnDefines columns_to_read; + Block header; }; class DMHandleConvertBlockInputStream : public IBlockInputStream diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index 05969ac36bf..f262c83ee6c 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -535,15 +535,23 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, auto persisted_files_stream = delta_stream.getPersistedFilesInputStream(); memtable_stream = std::make_shared>(memtable_stream, data_ranges, 0); - memtable_stream = std::make_shared(memtable_stream, columns_to_read, filter_delete_mark); - persisted_files_stream = std::make_shared>(persisted_files_stream, data_ranges, 0); // related to minor compaction sorted - persisted_files_stream = std::make_shared(persisted_files_stream, columns_to_read, filter_delete_mark); - stable_stream = std::make_shared>(stable_stream, data_ranges, 0); - stable_stream = std::make_shared(stable_stream, columns_to_read, filter_delete_mark); + if (filter_delete_mark) + { + memtable_stream = std::make_shared(memtable_stream, columns_to_read); + persisted_files_stream = std::make_shared(persisted_files_stream, columns_to_read); + stable_stream = std::make_shared(stable_stream, columns_to_read); + } + else + { + memtable_stream = std::make_shared(memtable_stream, columns_to_read); + persisted_files_stream = std::make_shared(persisted_files_stream, columns_to_read); + stable_stream = std::make_shared(stable_stream, columns_to_read); + } + BlockInputStreams streams; if (dm_context.read_delta_only) diff --git a/dbms/src/Storages/DeltaMerge/tests/CMakeLists.txt b/dbms/src/Storages/DeltaMerge/tests/CMakeLists.txt index fd02bcebd2f..14ecdfc5e79 100644 --- a/dbms/src/Storages/DeltaMerge/tests/CMakeLists.txt +++ b/dbms/src/Storages/DeltaMerge/tests/CMakeLists.txt @@ -42,6 +42,9 @@ target_link_libraries(dm_test_storage_delta_merge add_executable(dm_test_delta_merge_store EXCLUDE_FROM_ALL gtest_dm_delta_merge_store.cpp) target_link_libraries(dm_test_delta_merge_store dbms gtest_main clickhouse_functions) +add_executable(dm_test_delta_merge_store_for_fast_mode EXCLUDE_FROM_ALL gtest_dm_delta_merge_store_for_fast_mode.cpp) +target_link_libraries(dm_test_delta_merge_store_for_fast_mode dbms gtest_main clickhouse_functions) + add_executable(dm_test_segment EXCLUDE_FROM_ALL gtest_dm_segment.cpp) target_link_libraries(dm_test_segment dbms gtest_main clickhouse_functions) diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp index fe56bbd8d4b..bc7d7dfbef5 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp @@ -59,20 +59,32 @@ class DebugBlockInputStream : public IProfilingBlockInputStream bool is_common_handle; }; -BlockInputStreamPtr genInputStream(const BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle, bool filter_delete_mark = true) +BlockInputStreamPtr genColumnFilterInputStream(const BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle) { ColumnDefine handle_define( TiDBPkColumnID, DMTestEnv::pk_name, is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE); + return std::make_shared( std::make_shared(blocks, is_common_handle), - columns, - filter_delete_mark); + columns); +} + +BlockInputStreamPtr genDeleteFilterInputStream(const BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle) +{ + ColumnDefine handle_define( + TiDBPkColumnID, + DMTestEnv::pk_name, + is_common_handle ? EXTRA_HANDLE_COLUMN_STRING_TYPE : EXTRA_HANDLE_COLUMN_INT_TYPE); + + return std::make_shared( + std::make_shared(blocks, is_common_handle), + columns); } } // namespace -TEST(ColumnFilterTest, NormalCaseFilterDeleteMark) +TEST(DeleteFilterTest, NormalCase) { BlocksList blocks; @@ -88,7 +100,7 @@ TEST(ColumnFilterTest, NormalCaseFilterDeleteMark) ColumnDefines columns = getColumnDefinesFromBlock(blocks.back()); { - auto in = genInputStream(blocks, columns, false); + auto in = genDeleteFilterInputStream(blocks, columns, false); in->readPrefix(); Block block = in->read(); ASSERT_EQ(block.rows(), 1); @@ -114,7 +126,7 @@ TEST(ColumnFilterTest, NormalCaseFilterDeleteMark) } } -TEST(ColumnFilterTest, WithoutFilterDeleteMark) +TEST(ColumnFilterTest, NormalCase) { BlocksList blocks; @@ -130,7 +142,7 @@ TEST(ColumnFilterTest, WithoutFilterDeleteMark) ColumnDefines columns = getColumnDefinesFromBlock(blocks.back()); { - auto in = genInputStream(blocks, columns, false, false); + auto in = genColumnFilterInputStream(blocks, columns, false); in->readPrefix(); Block block = in->read(); ASSERT_EQ(block.rows(), 1); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp new file mode 100644 index 00000000000..e9e5fac4ab8 --- /dev/null +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp @@ -0,0 +1,1448 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/// This test file is mainly test on the correctness of read in fast mode. +/// Because the basic functions are tested in gtest_dm_delta_merge_storage.cpp, we will not cover it here. + +namespace DB { +namespace FailPoints { +} // namespace FailPoints + +namespace DM +{ +extern DMFilePtr writeIntoNewDMFile(DMContext & dm_context, // + const ColumnDefinesPtr & schema_snap, + const BlockInputStreamPtr & input_stream, + UInt64 file_id, + const String & parent_path, + DMFileBlockOutputStream::Flags flags); +namespace tests +{ +class DeltaMergeStoreTest : public DB::base::TiFlashStorageTestBasic +{ +public: + void SetUp() override + { + TiFlashStorageTestBasic::SetUp(); + store = reload(); + } + + DeltaMergeStorePtr + reload(const ColumnDefinesPtr & pre_define_columns = {}, bool is_common_handle = false, size_t rowkey_column_size = 1) + { + TiFlashStorageTestBasic::reload(); + ColumnDefinesPtr cols; + if (!pre_define_columns) + cols = DMTestEnv::getDefaultColumns(is_common_handle ? DMTestEnv::PkType::CommonHandle : DMTestEnv::PkType::HiddenTiDBRowID); + else + cols = pre_define_columns; + + ColumnDefine handle_column_define = (*cols)[0]; + + DeltaMergeStorePtr s = std::make_shared(*db_context, + false, + "test", + "DeltaMergeStoreTest", + 100, + *cols, + handle_column_define, + is_common_handle, + rowkey_column_size, + DeltaMergeStore::Settings()); + return s; + } + +protected: + DeltaMergeStorePtr store; +}; + +enum TestMode +{ + V1_BlockOnly, + V2_BlockOnly, + V2_FileOnly, + V2_Mix, +}; + +class DeltaMergeStoreRWTest + : public DB::base::TiFlashStorageTestBasic + , public testing::WithParamInterface +{ +public: + DeltaMergeStoreRWTest() + { + mode = GetParam(); + + switch (mode) + { + case TestMode::V1_BlockOnly: + setStorageFormat(1); + break; + case TestMode::V2_BlockOnly: + case TestMode::V2_FileOnly: + case TestMode::V2_Mix: + setStorageFormat(2); + break; + } + } + + void SetUp() override + { + TiFlashStorageTestBasic::SetUp(); + store = reload(); + } + + DeltaMergeStorePtr + reload(const ColumnDefinesPtr & pre_define_columns = {}, bool is_common_handle = false, size_t rowkey_column_size = 1) + { + TiFlashStorageTestBasic::reload(); + ColumnDefinesPtr cols; + if (!pre_define_columns) + cols = DMTestEnv::getDefaultColumns(is_common_handle ? DMTestEnv::PkType::CommonHandle : DMTestEnv::PkType::HiddenTiDBRowID); + else + cols = pre_define_columns; + + ColumnDefine handle_column_define = (*cols)[0]; + + DeltaMergeStorePtr s = std::make_shared(*db_context, + false, + "test", + "DeltaMergeStoreRWTest", + 101, + *cols, + handle_column_define, + is_common_handle, + rowkey_column_size, + DeltaMergeStore::Settings()); + return s; + } + + std::pair genDMFile(DMContext & context, const Block & block) + { + auto input_stream = std::make_shared(block); + auto [store_path, file_id] = store->preAllocateIngestFile(); + + DMFileBlockOutputStream::Flags flags; + flags.setSingleFile(DMTestEnv::getPseudoRandomNumber() % 2); + + auto dmfile = writeIntoNewDMFile( + context, + std::make_shared(store->getTableColumns()), + input_stream, + file_id, + store_path, + flags); + + + store->preIngestFile(store_path, file_id, dmfile->getBytesOnDisk()); + + const auto & pk_column = block.getByPosition(0).column; + auto min_pk = pk_column->getInt(0); + auto max_pk = pk_column->getInt(block.rows() - 1); + HandleRange range(min_pk, max_pk + 1); + + return {RowKeyRange::fromHandleRange(range), {file_id}}; + } + +protected: + TestMode mode; + DeltaMergeStorePtr store; + + constexpr static const char * TRACING_NAME = "DeltaMergeStoreRWTest"; +}; + +TEST_P(DeltaMergeStoreRWTest, TestFastModeWithOnlyInsertWithoutRangeFilter) +{ + /// test under only insert data (no update, no delete) with all range + + const ColumnDefine col_str_define(2, "col2", std::make_shared()); + const ColumnDefine col_i8_define(3, "i8", std::make_shared()); + { + auto table_column_defines = DMTestEnv::getDefaultColumns(); + table_column_defines->emplace_back(col_str_define); + table_column_defines->emplace_back(col_i8_define); + + store = reload(table_column_defines); + } + + const size_t num_rows_write = 128; + { + // write to store + Block block; + { + block = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false); + // Add a column of col2:String for test + block.insert(DB::tests::createColumn( + createNumberStrings(0, num_rows_write), + col_str_define.name, + col_str_define.id)); + // Add a column of i8:Int8 for test + block.insert(DB::tests::createColumn( + createSignedNumbers(0, num_rows_write), + col_i8_define.name, + col_i8_define.id)); + } + + switch (mode) + { + case TestMode::V1_BlockOnly: + case TestMode::V2_BlockOnly: + store->write(*db_context, db_context->getSettingsRef(), block); + break; + default: + { + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range, file_ids] = genDMFile(*dm_context, block); + store->ingestFiles(dm_context, range, file_ids, false); + break; + } + } + } + + { + // read all columns from store with all range in fast mode + const auto & columns = store->getTableColumns(); + BlockInputStreamPtr in = store->read(*db_context, + db_context->getSettingsRef(), + columns, + {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ std::numeric_limits::max(), + EMPTY_FILTER, + TRACING_NAME, + /* is_raw_read= */ true, + /* expected_block_size= */ 1024)[0]; + + size_t num_rows_read = 0; + in->readPrefix(); + while (Block block = in->read()) + { + num_rows_read += block.rows(); + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + EXPECT_EQ(c->getInt(i), i); + } + else if (iter.name == col_str_define.name) + { + EXPECT_EQ(c->getDataAt(i), DB::toString(i)); + } + else if (iter.name == col_i8_define.name) + { + Int64 num = i * (i % 2 == 0 ? -1 : 1); + EXPECT_EQ(c->getInt(i), num); + } + } + } + } + in->readSuffix(); + ASSERT_EQ(num_rows_read, num_rows_write); + } + +} + +TEST_P(DeltaMergeStoreRWTest, TestFastModeWithOnlyInsertWithRangeFilter) +{ + /// test under only insert data (no update, no delete) with range filter + + const ColumnDefine col_str_define(2, "col2", std::make_shared()); + const ColumnDefine col_i8_define(3, "i8", std::make_shared()); + { + auto table_column_defines = DMTestEnv::getDefaultColumns(); + table_column_defines->emplace_back(col_str_define); + table_column_defines->emplace_back(col_i8_define); + + store = reload(table_column_defines); + } + + const size_t num_rows_write = 128; + { + // write to store + Block block; + { + block = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false); + // Add a column of col2:String for test + block.insert(DB::tests::createColumn( + createNumberStrings(0, num_rows_write), + col_str_define.name, + col_str_define.id)); + // Add a column of i8:Int8 for test + block.insert(DB::tests::createColumn( + createSignedNumbers(0, num_rows_write), + col_i8_define.name, + col_i8_define.id)); + } + + switch (mode) + { + case TestMode::V1_BlockOnly: + case TestMode::V2_BlockOnly: + store->write(*db_context, db_context->getSettingsRef(), block); + break; + default: + { + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range, file_ids] = genDMFile(*dm_context, block); + store->ingestFiles(dm_context, range, file_ids, false); + break; + } + } + } + { + // read all columns from store with row key range in fast mode + WriteBufferFromOwnString start_key_ss; + DB::EncodeInt64(0, start_key_ss); + + WriteBufferFromOwnString end_key_ss; + DB::EncodeInt64(64, end_key_ss); + + + const auto & columns = store->getTableColumns(); + BlockInputStreamPtr in = store->read(*db_context, + db_context->getSettingsRef(), + columns, + {RowKeyRange( + RowKeyValue(false, std::make_shared(start_key_ss.releaseStr()), 0), + RowKeyValue(false, std::make_shared(end_key_ss.releaseStr()), 64), + false, + store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ std::numeric_limits::max(), + EMPTY_FILTER, + TRACING_NAME, + /* is_raw_read= */ true, + /* expected_block_size= */ 1024)[0]; + + size_t num_rows_read = 0; + in->readPrefix(); + while (Block block = in->read()) + { + num_rows_read += block.rows(); + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + EXPECT_EQ(c->getInt(i), i); + } + else if (iter.name == col_str_define.name) + { + EXPECT_EQ(c->getDataAt(i), DB::toString(i)); + } + else if (iter.name == col_i8_define.name) + { + Int64 num = i * (i % 2 == 0 ? -1 : 1); + EXPECT_EQ(c->getInt(i), num); + } + } + } + } + in->readSuffix(); + ASSERT_EQ(num_rows_read, 64); + } +} + +TEST_P(DeltaMergeStoreRWTest, TestFastModeWithOnlyInsertWithMultipleBlockWithoutFlushCache) +try +{ + const size_t num_write_rows = 32; + auto table_column_defines = DMTestEnv::getDefaultColumns(); + store = reload(table_column_defines); + + //Test write multi blocks without overlap and do not compact + { + Block block1 = DMTestEnv::prepareSimpleWriteBlock(0, 1 * num_write_rows, false); + Block block2 = DMTestEnv::prepareSimpleWriteBlock(1 * num_write_rows, 2 * num_write_rows, false); + Block block3 = DMTestEnv::prepareSimpleWriteBlock(2 * num_write_rows, 3 * num_write_rows, false); + switch (mode) + { + case TestMode::V1_BlockOnly: + case TestMode::V2_BlockOnly: + { + store->write(*db_context, db_context->getSettingsRef(), block1); + store->write(*db_context, db_context->getSettingsRef(), block2); + store->write(*db_context, db_context->getSettingsRef(), block3); + break; + } + case TestMode::V2_FileOnly: + { + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range1, file_ids1] = genDMFile(*dm_context, block1); + auto [range2, file_ids2] = genDMFile(*dm_context, block2); + auto [range3, file_ids3] = genDMFile(*dm_context, block3); + auto range = range1.merge(range2).merge(range3); + auto file_ids = file_ids1; + file_ids.insert(file_ids.cend(), file_ids2.begin(), file_ids2.end()); + file_ids.insert(file_ids.cend(), file_ids3.begin(), file_ids3.end()); + store->ingestFiles(dm_context, range, file_ids, false); + break; + } + case TestMode::V2_Mix: // disk + memory + { + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range1, file_ids1] = genDMFile(*dm_context, block1); + auto [range3, file_ids3] = genDMFile(*dm_context, block3); + auto range = range1.merge(range3); + auto file_ids = file_ids1; + file_ids.insert(file_ids.cend(), file_ids3.begin(), file_ids3.end()); + store->ingestFiles(dm_context, range, file_ids, false); // in disk + store->write(*db_context, db_context->getSettingsRef(), block2); + + break; + } + } + } + + { + const auto & columns = store->getTableColumns(); + BlockInputStreamPtr in = store->read(*db_context, + db_context->getSettingsRef(), + columns, + {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ std::numeric_limits::max(), + EMPTY_FILTER, + TRACING_NAME, + /* is_raw_read= */ true, + /* expected_block_size= */ 1024)[0]; + size_t num_rows_read = 0; + + switch (mode) + { + case TestMode::V1_BlockOnly: + case TestMode::V2_BlockOnly: + { + while (Block block = in->read()) + { + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + ASSERT_EQ(c->getInt(i), i + num_rows_read); + } + } + } + num_rows_read += block.rows(); + } + break; + } + case TestMode::V2_FileOnly: + { + while (Block block = in->read()) + { + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + ASSERT_EQ(c->getInt(i), i + num_rows_read); + } + } + } + num_rows_read += block.rows(); + } + break; + } + case TestMode::V2_Mix: + { + int block_index = 0; + int begin_value = num_write_rows; // memory first, then persist, finally stable + while (Block block = in->read()) + { + if (block_index == 1) { + begin_value = 0; + } else if (block_index == 2) { + begin_value = num_write_rows * 2; + } + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + ASSERT_EQ(c->getInt(i), i + begin_value); + } + } + } + num_rows_read += block.rows(); + block_index += 1; + } + break; + } + } + + + ASSERT_EQ(num_rows_read, 3 * num_write_rows); + } +} +CATCH + +TEST_P(DeltaMergeStoreRWTest, TestFastModeWithOnlyInsertWithMultipleBlockWithoutCompact) +try +{ + const size_t num_write_rows = 32; + auto table_column_defines = DMTestEnv::getDefaultColumns(); + store = reload(table_column_defines); + + //Test write multi blocks without overlap and do not compact + { + Block block1 = DMTestEnv::prepareSimpleWriteBlock(0, 1 * num_write_rows, false); + Block block2 = DMTestEnv::prepareSimpleWriteBlock(1 * num_write_rows, 2 * num_write_rows, false); + Block block3 = DMTestEnv::prepareSimpleWriteBlock(2 * num_write_rows, 3 * num_write_rows, false); + switch (mode) + { + case TestMode::V1_BlockOnly: + case TestMode::V2_BlockOnly: + { + store->write(*db_context, db_context->getSettingsRef(), block1); + store->write(*db_context, db_context->getSettingsRef(), block2); + store->write(*db_context, db_context->getSettingsRef(), block3); + break; + } + case TestMode::V2_FileOnly: + { + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range1, file_ids1] = genDMFile(*dm_context, block1); + auto [range2, file_ids2] = genDMFile(*dm_context, block2); + auto [range3, file_ids3] = genDMFile(*dm_context, block3); + auto range = range1.merge(range2).merge(range3); + auto file_ids = file_ids1; + file_ids.insert(file_ids.cend(), file_ids2.begin(), file_ids2.end()); + file_ids.insert(file_ids.cend(), file_ids3.begin(), file_ids3.end()); + store->ingestFiles(dm_context, range, file_ids, false); + break; + } + case TestMode::V2_Mix: + { + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range1, file_ids1] = genDMFile(*dm_context, block1); + auto [range3, file_ids3] = genDMFile(*dm_context, block3); + auto range = range1.merge(range3); + auto file_ids = file_ids1; + file_ids.insert(file_ids.cend(), file_ids3.begin(), file_ids3.end()); + store->ingestFiles(dm_context, range, file_ids, false); + store->write(*db_context, db_context->getSettingsRef(), block2); + + break; + } + } + + store->flushCache(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); + } + + { + const auto & columns = store->getTableColumns(); + BlockInputStreamPtr in = store->read(*db_context, + db_context->getSettingsRef(), + columns, + {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ std::numeric_limits::max(), + EMPTY_FILTER, + TRACING_NAME, + /* is_raw_read= */ true, + /* expected_block_size= */ 1024)[0]; + size_t num_rows_read = 0; + + switch (mode) + { + case TestMode::V1_BlockOnly: + case TestMode::V2_BlockOnly: + { + while (Block block = in->read()) + { + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + ASSERT_EQ(c->getInt(i), i); + } + } + } + num_rows_read += block.rows(); + } + break; + } + case TestMode::V2_FileOnly: + { + while (Block block = in->read()) + { + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + ASSERT_EQ(c->getInt(i), i + num_rows_read); + } + } + } + num_rows_read += block.rows(); + } + break; + } + case TestMode::V2_Mix: + { + int block_index = 0; + int begin_value = 0; + while (Block block = in->read()) + { + if (block_index == 1) { + begin_value = num_write_rows * 2; + } else if (block_index == 2) { + begin_value = num_write_rows; + } + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + ASSERT_EQ(c->getInt(i), i + begin_value); + } + } + } + num_rows_read += block.rows(); + block_index += 1; + } + break; + } + } + + + ASSERT_EQ(num_rows_read, 3 * num_write_rows); + } +} +CATCH + +TEST_P(DeltaMergeStoreRWTest, TestFastModeWithOnlyInsertWithMultipleBlockWithCompact) +try +{ + const size_t num_write_rows = 32; + auto table_column_defines = DMTestEnv::getDefaultColumns(); + store = reload(table_column_defines); + + //Test write multi blocks without overlap and do not compact + { + Block block1 = DMTestEnv::prepareSimpleWriteBlock(0, 1 * num_write_rows, false); + Block block2 = DMTestEnv::prepareSimpleWriteBlock(1 * num_write_rows, 2 * num_write_rows, false); + Block block3 = DMTestEnv::prepareSimpleWriteBlock(2 * num_write_rows, 3 * num_write_rows, false); + switch (mode) + { + case TestMode::V1_BlockOnly: + case TestMode::V2_BlockOnly: + { + store->write(*db_context, db_context->getSettingsRef(), block1); + store->write(*db_context, db_context->getSettingsRef(), block2); + store->write(*db_context, db_context->getSettingsRef(), block3); + break; + } + case TestMode::V2_FileOnly: + { + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range1, file_ids1] = genDMFile(*dm_context, block1); + auto [range2, file_ids2] = genDMFile(*dm_context, block2); + auto [range3, file_ids3] = genDMFile(*dm_context, block3); + auto range = range1.merge(range2).merge(range3); + auto file_ids = file_ids1; + file_ids.insert(file_ids.cend(), file_ids2.begin(), file_ids2.end()); + file_ids.insert(file_ids.cend(), file_ids3.begin(), file_ids3.end()); + store->ingestFiles(dm_context, range, file_ids, false); + break; + } + case TestMode::V2_Mix: + { + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range1, file_ids1] = genDMFile(*dm_context, block1); + auto [range3, file_ids3] = genDMFile(*dm_context, block3); + auto range = range1.merge(range3); + auto file_ids = file_ids1; + file_ids.insert(file_ids.cend(), file_ids3.begin(), file_ids3.end()); + store->ingestFiles(dm_context, range, file_ids, false); + store->write(*db_context, db_context->getSettingsRef(), block2); + + break; + } + } + + store->flushCache(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); + } + + store->compact(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); + + { + const auto & columns = store->getTableColumns(); + BlockInputStreamPtr in = store->read(*db_context, + db_context->getSettingsRef(), + columns, + {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ std::numeric_limits::max(), + EMPTY_FILTER, + TRACING_NAME, + /* is_raw_read= */ true, + /* expected_block_size= */ 1024)[0]; + size_t num_rows_read = 0; + + switch (mode) + { + case TestMode::V1_BlockOnly: + case TestMode::V2_BlockOnly: + { + while (Block block = in->read()) + { + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + ASSERT_EQ(c->getInt(i), i); + } + } + } + num_rows_read += block.rows(); + } + break; + } + case TestMode::V2_FileOnly: + { + while (Block block = in->read()) + { + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + ASSERT_EQ(c->getInt(i), i + num_rows_read); + } + } + } + num_rows_read += block.rows(); + } + break; + } + case TestMode::V2_Mix: + { + int block_index = 0; + int begin_value = 0; + while (Block block = in->read()) + { + if (block_index == 1) { + begin_value = num_write_rows * 2; + } else if (block_index == 2) { + begin_value = num_write_rows; + } + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + ASSERT_EQ(c->getInt(i), i + begin_value); + } + } + } + num_rows_read += block.rows(); + block_index += 1; + } + break; + } + } + + + ASSERT_EQ(num_rows_read, 3 * num_write_rows); + } +} +CATCH + +TEST_P(DeltaMergeStoreRWTest, TestFastModeWithOnlyInsertWithMultipleBlockWithCompactAndMergeDelta) +try +{ + const size_t num_write_rows = 32; + auto table_column_defines = DMTestEnv::getDefaultColumns(); + store = reload(table_column_defines); + + //Test write multi blocks without overlap and do not compact + { + Block block1 = DMTestEnv::prepareSimpleWriteBlock(0, 1 * num_write_rows, false); + Block block2 = DMTestEnv::prepareSimpleWriteBlock(1 * num_write_rows, 2 * num_write_rows, false); + Block block3 = DMTestEnv::prepareSimpleWriteBlock(2 * num_write_rows, 3 * num_write_rows, false); + switch (mode) + { + case TestMode::V1_BlockOnly: + case TestMode::V2_BlockOnly: + { + store->write(*db_context, db_context->getSettingsRef(), block1); + store->write(*db_context, db_context->getSettingsRef(), block2); + store->write(*db_context, db_context->getSettingsRef(), block3); + break; + } + case TestMode::V2_FileOnly: + { + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range1, file_ids1] = genDMFile(*dm_context, block1); + auto [range2, file_ids2] = genDMFile(*dm_context, block2); + auto [range3, file_ids3] = genDMFile(*dm_context, block3); + auto range = range1.merge(range2).merge(range3); + auto file_ids = file_ids1; + file_ids.insert(file_ids.cend(), file_ids2.begin(), file_ids2.end()); + file_ids.insert(file_ids.cend(), file_ids3.begin(), file_ids3.end()); + store->ingestFiles(dm_context, range, file_ids, false); + break; + } + case TestMode::V2_Mix: + { + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range1, file_ids1] = genDMFile(*dm_context, block1); + auto [range3, file_ids3] = genDMFile(*dm_context, block3); + auto range = range1.merge(range3); + auto file_ids = file_ids1; + file_ids.insert(file_ids.cend(), file_ids3.begin(), file_ids3.end()); + store->ingestFiles(dm_context, range, file_ids, false); + store->write(*db_context, db_context->getSettingsRef(), block2); + + break; + } + } + + store->flushCache(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); + } + + store->compact(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); + + store->mergeDeltaAll(*db_context); + + { + const auto & columns = store->getTableColumns(); + BlockInputStreamPtr in = store->read(*db_context, + db_context->getSettingsRef(), + columns, + {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ std::numeric_limits::max(), + EMPTY_FILTER, + TRACING_NAME, + /* is_raw_read= */ true, + /* expected_block_size= */ 1024)[0]; + size_t num_rows_read = 0; + + while (Block block = in->read()) + { + num_rows_read += block.rows(); + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + ASSERT_EQ(c->getInt(i), i); + } + } + } + } + + + + ASSERT_EQ(num_rows_read, 3 * num_write_rows); + } +} +CATCH + +// Insert + Update +TEST_P(DeltaMergeStoreRWTest, TestFastModeWithMultipleBlockWithOverlap) +try +{ + const size_t num_write_rows = 32; + + // Test write multi blocks with overlap and do compact + { + UInt64 tso1 = 1; + UInt64 tso2 = 100; + Block block1 = DMTestEnv::prepareSimpleWriteBlock(0, 1 * num_write_rows, false, tso1); + Block block2 = DMTestEnv::prepareSimpleWriteBlock(1 * num_write_rows, 2 * num_write_rows, false, tso1); + Block block3 = DMTestEnv::prepareSimpleWriteBlock(num_write_rows / 2, num_write_rows / 2 + num_write_rows, false, tso2); + + switch (mode) + { + case TestMode::V1_BlockOnly: + case TestMode::V2_BlockOnly: + { + store->write(*db_context, db_context->getSettingsRef(), block1); + store->write(*db_context, db_context->getSettingsRef(), block2); + store->write(*db_context, db_context->getSettingsRef(), block3); + break; + } + case TestMode::V2_FileOnly: + { + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range1, file_ids1] = genDMFile(*dm_context, block1); + auto [range2, file_ids2] = genDMFile(*dm_context, block2); + auto [range3, file_ids3] = genDMFile(*dm_context, block3); + auto range = range1.merge(range2).merge(range3); + auto file_ids = file_ids1; + file_ids.insert(file_ids.cend(), file_ids2.begin(), file_ids2.end()); + file_ids.insert(file_ids.cend(), file_ids3.begin(), file_ids3.end()); + store->ingestFiles(dm_context, range, file_ids, false); + break; + } + case TestMode::V2_Mix: + { + store->write(*db_context, db_context->getSettingsRef(), block2); + + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range1, file_ids1] = genDMFile(*dm_context, block1); + auto [range3, file_ids3] = genDMFile(*dm_context, block3); + auto range = range1.merge(range3); + auto file_ids = file_ids1; + file_ids.insert(file_ids.cend(), file_ids3.begin(), file_ids3.end()); + store->ingestFiles(dm_context, range, file_ids, false); + break; + } + } + + // in V1_BlockOnly and V2_BlockOnly mode, flush cache will make sort + store->flushCache(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); + } + + store->compact(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); + + { + const auto & columns = store->getTableColumns(); + BlockInputStreamPtr in = store->read(*db_context, + db_context->getSettingsRef(), + columns, + {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ std::numeric_limits::max(), + EMPTY_FILTER, + TRACING_NAME, + /* is_raw_read= */ true, + /* expected_block_size= */ 1024)[0]; + size_t num_rows_read = 0; + + switch (mode) + { + case TestMode::V1_BlockOnly: + case TestMode::V2_BlockOnly: + { + while (Block block = in->read()) + { + for (auto && iter : block) + { + if (iter.name == DMTestEnv::pk_name) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (i < Int64(num_write_rows / 2)) { + ASSERT_EQ(c->getInt(i), i); + } else if (i < Int64(2.5 * num_write_rows)) { + ASSERT_EQ(c->getInt(i), (i - num_write_rows / 2) / 2 + num_write_rows / 2); + } else { + ASSERT_EQ(c->getInt(i), (i - num_write_rows * 2) + num_write_rows); + } + } + } + } + num_rows_read += block.rows(); + } + break; + } + case TestMode::V2_FileOnly: + { + auto block_index = 0; + auto begin_value = 0; + while (Block block = in->read()) + { + if (block_index == 1) { + begin_value = num_write_rows; + } else if (block_index == 2) { + begin_value = num_write_rows / 2; + } + for (auto && iter : block) + { + if (iter.name == DMTestEnv::pk_name) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + ASSERT_EQ(c->getInt(i), i + begin_value); + } + } + } + num_rows_read += block.rows(); + block_index += 1; + } + break; + } + case TestMode::V2_Mix: + { + auto block_index = 0; + auto begin_value = num_write_rows; + while (Block block = in->read()) + { + if (block_index == 1) { + begin_value = 0; + } else if (block_index == 2) { + begin_value = num_write_rows / 2; + } + for (auto && iter : block) + { + if (iter.name == DMTestEnv::pk_name) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + ASSERT_EQ(c->getInt(i), i + begin_value); + } + } + } + num_rows_read += block.rows(); + block_index += 1; + } + break; + } + } + + + ASSERT_EQ(num_rows_read, 3 * num_write_rows); + } +} +CATCH + +// Insert + Delete +TEST_P(DeltaMergeStoreRWTest, TestFastModeWithDelete) +try +{ + const size_t num_rows_write = 128; + { + // Create a block with sequential Int64 handle in range [0, 128) + Block block = DMTestEnv::prepareSimpleWriteBlock(0, 128, false); + + switch (mode) + { + case TestMode::V1_BlockOnly: + case TestMode::V2_BlockOnly: + store->write(*db_context, db_context->getSettingsRef(), block); + break; + default: + { + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range, file_ids] = genDMFile(*dm_context, block); + store->ingestFiles(dm_context, range, file_ids, false); + break; + } + } + } + // Test Reading first + { + const auto & columns = store->getTableColumns(); + BlockInputStreamPtr in = store->read(*db_context, + db_context->getSettingsRef(), + columns, + {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ std::numeric_limits::max(), + EMPTY_FILTER, + TRACING_NAME, + /* is_raw_read= */ true, + /* expected_block_size= */ 1024)[0]; + size_t num_rows_read = 0; + while (Block block = in->read()) + { + num_rows_read += block.rows(); + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + ASSERT_EQ(c->getInt(i), i); + } + } + } + } + + ASSERT_EQ(num_rows_read, num_rows_write); + } + // Delete range [0, 64) + const size_t num_deleted_rows = 64; + { + HandleRange range(0, num_deleted_rows); + store->deleteRange(*db_context, db_context->getSettingsRef(), RowKeyRange::fromHandleRange(range)); + } + // Read after deletion + { + const auto & columns = store->getTableColumns(); + BlockInputStreamPtr in = store->read(*db_context, + db_context->getSettingsRef(), + columns, + {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ std::numeric_limits::max(), + EMPTY_FILTER, + TRACING_NAME, + /* is_raw_read= */ true, + /* expected_block_size= */ 1024)[0]; + size_t num_rows_read = 0; + + // filter del mark = 1， thus just read the insert data before delete + while (Block block = in->read()) + { + num_rows_read += block.rows(); + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + ASSERT_EQ(c->getInt(i), i); + } + } + } + } + + ASSERT_EQ(num_rows_read, num_rows_write); + } +} +CATCH + +TEST_P(DeltaMergeStoreRWTest, TestFastModeWithDeleteWithMergeDelta) +try +{ + const size_t num_rows_write = 128; + { + // Create a block with sequential Int64 handle in range [0, 128) + Block block = DMTestEnv::prepareSimpleWriteBlock(0, 128, false); + + switch (mode) + { + case TestMode::V1_BlockOnly: + case TestMode::V2_BlockOnly: + store->write(*db_context, db_context->getSettingsRef(), block); + break; + default: + { + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range, file_ids] = genDMFile(*dm_context, block); + store->ingestFiles(dm_context, range, file_ids, false); + break; + } + } + } + + // Delete range [0, 64) + const size_t num_deleted_rows = 64; + { + HandleRange range(0, num_deleted_rows); + store->deleteRange(*db_context, db_context->getSettingsRef(), RowKeyRange::fromHandleRange(range)); + } + + store->flushCache(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); + + store->compact(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); + + store->mergeDeltaAll(*db_context); + + // Read after merge delta + { + const auto & columns = store->getTableColumns(); + BlockInputStreamPtr in = store->read(*db_context, + db_context->getSettingsRef(), + columns, + {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ std::numeric_limits::max(), + EMPTY_FILTER, + TRACING_NAME, + /* is_raw_read= */ true, + /* expected_block_size= */ 1024)[0]; + size_t num_rows_read = 0; + + while (Block block = in->read()) + { + num_rows_read += block.rows(); + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + ASSERT_EQ(c->getInt(i), i + num_deleted_rows); + } + } + } + } + + ASSERT_EQ(num_rows_read, num_rows_write - num_deleted_rows); + } +} +CATCH + +// insert + update + delete and fast mode first and then normal mode, to check the mode conversion is ok +TEST_P(DeltaMergeStoreRWTest, TestFastModeComplexWithModeConversion) +try +{ + const size_t num_write_rows = 128; + + { + UInt64 tso1 = 1; + UInt64 tso2 = 100; + Block block1 = DMTestEnv::prepareSimpleWriteBlock(0, 1 * num_write_rows, false, tso1); + Block block2 = DMTestEnv::prepareSimpleWriteBlock(1 * num_write_rows, 2 * num_write_rows, false, tso1); + Block block3 = DMTestEnv::prepareSimpleWriteBlock(num_write_rows / 2, num_write_rows / 2 + num_write_rows, false, tso2); + + switch (mode) + { + case TestMode::V1_BlockOnly: + case TestMode::V2_BlockOnly: + { + store->write(*db_context, db_context->getSettingsRef(), block1); + store->write(*db_context, db_context->getSettingsRef(), block2); + store->write(*db_context, db_context->getSettingsRef(), block3); + break; + } + case TestMode::V2_FileOnly: + { + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range1, file_ids1] = genDMFile(*dm_context, block1); + auto [range2, file_ids2] = genDMFile(*dm_context, block2); + auto [range3, file_ids3] = genDMFile(*dm_context, block3); + auto range = range1.merge(range2).merge(range3); + auto file_ids = file_ids1; + file_ids.insert(file_ids.cend(), file_ids2.begin(), file_ids2.end()); + file_ids.insert(file_ids.cend(), file_ids3.begin(), file_ids3.end()); + store->ingestFiles(dm_context, range, file_ids, false); + break; + } + case TestMode::V2_Mix: + { + store->write(*db_context, db_context->getSettingsRef(), block2); + + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range1, file_ids1] = genDMFile(*dm_context, block1); + auto [range3, file_ids3] = genDMFile(*dm_context, block3); + auto range = range1.merge(range3); + auto file_ids = file_ids1; + file_ids.insert(file_ids.cend(), file_ids3.begin(), file_ids3.end()); + store->ingestFiles(dm_context, range, file_ids, false); + break; + } + } + + // in V1_BlockOnly and V2_BlockOnly mode, flush cache will make sort + store->flushCache(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); + } + + store->compact(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); + + // Delete range [0, 64) + const size_t num_deleted_rows = 64; + { + HandleRange range(0, num_deleted_rows); + store->deleteRange(*db_context, db_context->getSettingsRef(), RowKeyRange::fromHandleRange(range)); + } + + // Read in fast mode + { + const auto & columns = store->getTableColumns(); + BlockInputStreamPtr in = store->read(*db_context, + db_context->getSettingsRef(), + columns, + {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ std::numeric_limits::max(), + EMPTY_FILTER, + TRACING_NAME, + /* is_raw_read= */ true, + /* expected_block_size= */ 1024)[0]; + size_t num_rows_read = 0; + + switch (mode) + { + case TestMode::V1_BlockOnly: + case TestMode::V2_BlockOnly: + { + while (Block block = in->read()) + { + for (auto && iter : block) + { + if (iter.name == DMTestEnv::pk_name) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (i < Int64(num_write_rows / 2)) { + ASSERT_EQ(c->getInt(i), i); + } else if (i < Int64(2.5 * num_write_rows)) { + ASSERT_EQ(c->getInt(i), (i - num_write_rows / 2) / 2 + num_write_rows / 2); + } else { + ASSERT_EQ(c->getInt(i), (i - num_write_rows * 2) + num_write_rows); + } + } + } + } + num_rows_read += block.rows(); + } + break; + } + case TestMode::V2_FileOnly: + { + auto block_index = 0; + auto begin_value = 0; + while (Block block = in->read()) + { + if (block_index == 1) { + begin_value = num_write_rows; + } else if (block_index == 2) { + begin_value = num_write_rows / 2; + } + for (auto && iter : block) + { + if (iter.name == DMTestEnv::pk_name) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + ASSERT_EQ(c->getInt(i), i + begin_value); + } + } + } + num_rows_read += block.rows(); + block_index += 1; + } + break; + } + case TestMode::V2_Mix: + { + auto block_index = 0; + auto begin_value = num_write_rows; + while (Block block = in->read()) + { + if (block_index == 1) { + begin_value = 0; + } else if (block_index == 2) { + begin_value = num_write_rows / 2; + } + for (auto && iter : block) + { + if (iter.name == DMTestEnv::pk_name) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + ASSERT_EQ(c->getInt(i), i + begin_value); + } + } + } + num_rows_read += block.rows(); + block_index += 1; + } + break; + } + } + + ASSERT_EQ(num_rows_read, 3 * num_write_rows); + } + + // Read with version in normal case + { + const auto & columns = store->getTableColumns(); + BlockInputStreamPtr in = store->read(*db_context, + db_context->getSettingsRef(), + columns, + {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ UInt64(1), + EMPTY_FILTER, + TRACING_NAME, + /* is_raw_read= */ false, + /* expected_block_size= */ 1024)[0]; + size_t num_rows_read = 0; + while (Block block = in->read()) + { + num_rows_read += block.rows(); + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + ASSERT_EQ(c->getInt(i), i + num_write_rows / 2); + } + } + } + } + + ASSERT_EQ(num_rows_read, 1.5 * num_write_rows); + } +} +CATCH + +} +} +} \ No newline at end of file From e01ad7c6e28bd8d51ce34469e99614204e510bc6 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Sun, 10 Jul 2022 17:52:53 +0800 Subject: [PATCH 07/26] fix format --- ...est_dm_delta_merge_store_for_fast_mode.cpp | 143 +++++++++++------- 1 file changed, 87 insertions(+), 56 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp index e9e5fac4ab8..3e33d32b8f7 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp @@ -41,18 +41,20 @@ /// This test file is mainly test on the correctness of read in fast mode. /// Because the basic functions are tested in gtest_dm_delta_merge_storage.cpp, we will not cover it here. -namespace DB { -namespace FailPoints { +namespace DB +{ +namespace FailPoints +{ } // namespace FailPoints namespace DM { extern DMFilePtr writeIntoNewDMFile(DMContext & dm_context, // - const ColumnDefinesPtr & schema_snap, - const BlockInputStreamPtr & input_stream, - UInt64 file_id, - const String & parent_path, - DMFileBlockOutputStream::Flags flags); + const ColumnDefinesPtr & schema_snap, + const BlockInputStreamPtr & input_stream, + UInt64 file_id, + const String & parent_path, + DMFileBlockOutputStream::Flags flags); namespace tests { class DeltaMergeStoreTest : public DB::base::TiFlashStorageTestBasic @@ -279,7 +281,6 @@ TEST_P(DeltaMergeStoreRWTest, TestFastModeWithOnlyInsertWithoutRangeFilter) in->readSuffix(); ASSERT_EQ(num_rows_read, num_rows_write); } - } TEST_P(DeltaMergeStoreRWTest, TestFastModeWithOnlyInsertWithRangeFilter) @@ -330,23 +331,23 @@ TEST_P(DeltaMergeStoreRWTest, TestFastModeWithOnlyInsertWithRangeFilter) } } { - // read all columns from store with row key range in fast mode + // read all columns from store with row key range in fast mode WriteBufferFromOwnString start_key_ss; DB::EncodeInt64(0, start_key_ss); WriteBufferFromOwnString end_key_ss; DB::EncodeInt64(64, end_key_ss); - + const auto & columns = store->getTableColumns(); BlockInputStreamPtr in = store->read(*db_context, db_context->getSettingsRef(), columns, {RowKeyRange( - RowKeyValue(false, std::make_shared(start_key_ss.releaseStr()), 0), - RowKeyValue(false, std::make_shared(end_key_ss.releaseStr()), 64), - false, - store->getRowKeyColumnSize())}, + RowKeyValue(false, std::make_shared(start_key_ss.releaseStr()), 0), + RowKeyValue(false, std::make_shared(end_key_ss.releaseStr()), 64), + false, + store->getRowKeyColumnSize())}, /* num_streams= */ 1, /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, @@ -430,7 +431,7 @@ try file_ids.insert(file_ids.cend(), file_ids3.begin(), file_ids3.end()); store->ingestFiles(dm_context, range, file_ids, false); // in disk store->write(*db_context, db_context->getSettingsRef(), block2); - + break; } } @@ -497,9 +498,12 @@ try int begin_value = num_write_rows; // memory first, then persist, finally stable while (Block block = in->read()) { - if (block_index == 1) { + if (block_index == 1) + { begin_value = 0; - } else if (block_index == 2) { + } + else if (block_index == 2) + { begin_value = num_write_rows * 2; } for (auto && iter : block) @@ -519,7 +523,7 @@ try break; } } - + ASSERT_EQ(num_rows_read, 3 * num_write_rows); } @@ -571,7 +575,7 @@ try file_ids.insert(file_ids.cend(), file_ids3.begin(), file_ids3.end()); store->ingestFiles(dm_context, range, file_ids, false); store->write(*db_context, db_context->getSettingsRef(), block2); - + break; } } @@ -640,9 +644,12 @@ try int begin_value = 0; while (Block block = in->read()) { - if (block_index == 1) { + if (block_index == 1) + { begin_value = num_write_rows * 2; - } else if (block_index == 2) { + } + else if (block_index == 2) + { begin_value = num_write_rows; } for (auto && iter : block) @@ -662,7 +669,7 @@ try break; } } - + ASSERT_EQ(num_rows_read, 3 * num_write_rows); } @@ -714,7 +721,7 @@ try file_ids.insert(file_ids.cend(), file_ids3.begin(), file_ids3.end()); store->ingestFiles(dm_context, range, file_ids, false); store->write(*db_context, db_context->getSettingsRef(), block2); - + break; } } @@ -785,9 +792,12 @@ try int begin_value = 0; while (Block block = in->read()) { - if (block_index == 1) { + if (block_index == 1) + { begin_value = num_write_rows * 2; - } else if (block_index == 2) { + } + else if (block_index == 2) + { begin_value = num_write_rows; } for (auto && iter : block) @@ -807,7 +817,7 @@ try break; } } - + ASSERT_EQ(num_rows_read, 3 * num_write_rows); } @@ -859,7 +869,7 @@ try file_ids.insert(file_ids.cend(), file_ids3.begin(), file_ids3.end()); store->ingestFiles(dm_context, range, file_ids, false); store->write(*db_context, db_context->getSettingsRef(), block2); - + break; } } @@ -868,7 +878,7 @@ try } store->compact(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); - + store->mergeDeltaAll(*db_context); { @@ -901,7 +911,6 @@ try } } - ASSERT_EQ(num_rows_read, 3 * num_write_rows); } @@ -912,7 +921,7 @@ CATCH TEST_P(DeltaMergeStoreRWTest, TestFastModeWithMultipleBlockWithOverlap) try { - const size_t num_write_rows = 32; + const size_t num_write_rows = 32; // Test write multi blocks with overlap and do compact { @@ -994,14 +1003,19 @@ try auto c = iter.column; for (Int64 i = 0; i < Int64(c->size()); ++i) { - if (i < Int64(num_write_rows / 2)) { + if (i < Int64(num_write_rows / 2)) + { ASSERT_EQ(c->getInt(i), i); - } else if (i < Int64(2.5 * num_write_rows)) { + } + else if (i < Int64(2.5 * num_write_rows)) + { ASSERT_EQ(c->getInt(i), (i - num_write_rows / 2) / 2 + num_write_rows / 2); - } else { + } + else + { ASSERT_EQ(c->getInt(i), (i - num_write_rows * 2) + num_write_rows); } - } + } } } num_rows_read += block.rows(); @@ -1014,9 +1028,12 @@ try auto begin_value = 0; while (Block block = in->read()) { - if (block_index == 1) { + if (block_index == 1) + { begin_value = num_write_rows; - } else if (block_index == 2) { + } + else if (block_index == 2) + { begin_value = num_write_rows / 2; } for (auto && iter : block) @@ -1027,7 +1044,7 @@ try for (Int64 i = 0; i < Int64(c->size()); ++i) { ASSERT_EQ(c->getInt(i), i + begin_value); - } + } } } num_rows_read += block.rows(); @@ -1041,9 +1058,12 @@ try auto begin_value = num_write_rows; while (Block block = in->read()) { - if (block_index == 1) { + if (block_index == 1) + { begin_value = 0; - } else if (block_index == 2) { + } + else if (block_index == 2) + { begin_value = num_write_rows / 2; } for (auto && iter : block) @@ -1054,7 +1074,7 @@ try for (Int64 i = 0; i < Int64(c->size()); ++i) { ASSERT_EQ(c->getInt(i), i + begin_value); - } + } } } num_rows_read += block.rows(); @@ -1063,7 +1083,7 @@ try break; } } - + ASSERT_EQ(num_rows_read, 3 * num_write_rows); } @@ -1203,9 +1223,9 @@ try store->flushCache(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); store->compact(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); - + store->mergeDeltaAll(*db_context); - + // Read after merge delta { const auto & columns = store->getTableColumns(); @@ -1335,14 +1355,19 @@ try auto c = iter.column; for (Int64 i = 0; i < Int64(c->size()); ++i) { - if (i < Int64(num_write_rows / 2)) { + if (i < Int64(num_write_rows / 2)) + { ASSERT_EQ(c->getInt(i), i); - } else if (i < Int64(2.5 * num_write_rows)) { + } + else if (i < Int64(2.5 * num_write_rows)) + { ASSERT_EQ(c->getInt(i), (i - num_write_rows / 2) / 2 + num_write_rows / 2); - } else { + } + else + { ASSERT_EQ(c->getInt(i), (i - num_write_rows * 2) + num_write_rows); } - } + } } } num_rows_read += block.rows(); @@ -1355,9 +1380,12 @@ try auto begin_value = 0; while (Block block = in->read()) { - if (block_index == 1) { + if (block_index == 1) + { begin_value = num_write_rows; - } else if (block_index == 2) { + } + else if (block_index == 2) + { begin_value = num_write_rows / 2; } for (auto && iter : block) @@ -1368,7 +1396,7 @@ try for (Int64 i = 0; i < Int64(c->size()); ++i) { ASSERT_EQ(c->getInt(i), i + begin_value); - } + } } } num_rows_read += block.rows(); @@ -1382,9 +1410,12 @@ try auto begin_value = num_write_rows; while (Block block = in->read()) { - if (block_index == 1) { + if (block_index == 1) + { begin_value = 0; - } else if (block_index == 2) { + } + else if (block_index == 2) + { begin_value = num_write_rows / 2; } for (auto && iter : block) @@ -1395,7 +1426,7 @@ try for (Int64 i = 0; i < Int64(c->size()); ++i) { ASSERT_EQ(c->getInt(i), i + begin_value); - } + } } } num_rows_read += block.rows(); @@ -1443,6 +1474,6 @@ try } CATCH -} -} -} \ No newline at end of file +} // namespace tests +} // namespace DM +} // namespace DB \ No newline at end of file From 377734f4b5f56f911f7a33865bea4f12ed63d8e1 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Sun, 10 Jul 2022 18:25:43 +0800 Subject: [PATCH 08/26] support dtworkload in fast mode --- dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp | 2 +- dbms/src/Storages/DeltaMerge/workload/Options.cpp | 5 ++++- dbms/src/Storages/DeltaMerge/workload/Options.h | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp index a53a1b9ebbd..2282186e234 100644 --- a/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp @@ -191,7 +191,7 @@ void DTWorkload::read(const ColumnDefines & columns, int stream_count, T func) auto filter = EMPTY_FILTER; int excepted_block_size = 1024; uint64_t read_ts = ts_gen->get(); - auto streams = store->read(*context, context->getSettingsRef(), columns, ranges, stream_count, read_ts, filter, "DTWorkload", excepted_block_size); + auto streams = store->read(*context, context->getSettingsRef(), columns, ranges, stream_count, read_ts, filter, "DTWorkload", opts->is_fast_mode, excepted_block_size); std::vector threads; threads.reserve(streams.size()); for (auto & stream : streams) diff --git a/dbms/src/Storages/DeltaMerge/workload/Options.cpp b/dbms/src/Storages/DeltaMerge/workload/Options.cpp index 8545d22ca8d..f9c14af7c64 100644 --- a/dbms/src/Storages/DeltaMerge/workload/Options.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/Options.cpp @@ -47,7 +47,8 @@ std::string WorkloadOptions::toString(std::string seperator) const fmt::format("ps_run_mode {}{}", ps_run_mode, seperator) + // fmt::format("bg_thread_count {}{}", bg_thread_count, seperator) + // fmt::format("table_id {}{}", table_id, seperator) + // - fmt::format("table_name {}{}", table_name, seperator); + fmt::format("table_name {}{}", table_name, seperator) + // + fmt::format("is_fast_mode {}{}", is_fast_mode, seperator); } std::pair WorkloadOptions::parseOptions(int argc, char * argv[]) @@ -94,6 +95,7 @@ std::pair WorkloadOptions::parseOptions(int argc, char * argv // ("table_name", value()->default_value(""), "") // ("table_id", value()->default_value(-1), "") // + ("is_fast_mode", value()->default_value(false), "default is false, means normal mode. When we in fast mode, we should set verification as false") // ; boost::program_options::variables_map vm; @@ -174,6 +176,7 @@ std::pair WorkloadOptions::parseOptions(int argc, char * argv table_id = vm["table_id"].as(); table_name = vm["table_name"].as(); + is_fast_mode = vm["is_fast_mode"].as(); return {true, toString()}; } diff --git a/dbms/src/Storages/DeltaMerge/workload/Options.h b/dbms/src/Storages/DeltaMerge/workload/Options.h index f017daf2d8a..5617470a0f7 100644 --- a/dbms/src/Storages/DeltaMerge/workload/Options.h +++ b/dbms/src/Storages/DeltaMerge/workload/Options.h @@ -62,6 +62,8 @@ struct WorkloadOptions int64_t table_id; std::string table_name; + bool is_fast_mode; + std::string toString(std::string seperator = "\n") const; std::pair parseOptions(int argc, char * argv[]); void initFailpoints() const; From acb508017aeef6e8fc21306257c0e2aca227fe68 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Mon, 11 Jul 2022 11:45:45 +0800 Subject: [PATCH 09/26] remove useless code --- dbms/src/Storages/DeltaMerge/Segment.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index f262c83ee6c..3c50b2e2bce 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -525,11 +525,6 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, expected_block_size, false); - //BlockInputStreamPtr delta_stream = std::make_shared(dm_context, segment_snap->delta, new_columns_to_read, this->rowkey_range); - - //delta_stream = std::make_shared>(delta_stream, data_ranges, 0); - //delta_stream = std::make_shared(delta_stream, columns_to_read); - DeltaValueInputStream delta_stream(dm_context, segment_snap->delta, new_columns_to_read, this->rowkey_range); auto memtable_stream = delta_stream.getMemTableInputStream(); auto persisted_files_stream = delta_stream.getPersistedFilesInputStream(); @@ -556,7 +551,6 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, if (dm_context.read_delta_only) { - //streams.push_back(delta_stream); streams.push_back(memtable_stream); streams.push_back(persisted_files_stream); } @@ -566,7 +560,6 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, } else { - //streams.push_back(delta_stream); streams.push_back(memtable_stream); streams.push_back(persisted_files_stream); streams.push_back(stable_stream); From 94a3bc6949662ed5a2b319ef2b3d0cd21a9f01d6 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Mon, 11 Jul 2022 15:43:34 +0800 Subject: [PATCH 10/26] fix delete column error --- dbms/src/Storages/DeltaMerge/Segment.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index 3c50b2e2bce..246d4198d80 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -509,10 +509,14 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, new_columns_to_read->push_back(getExtraHandleColumnDefine(is_common_handle)); + if (filter_delete_mark) + { + new_columns_to_read->push_back(getTagColumnDefine()); + } for (const auto & c : columns_to_read) { - if (c.id != EXTRA_HANDLE_COLUMN_ID) + if (c.id != EXTRA_HANDLE_COLUMN_ID && (!(filter_delete_mark && c.id == TAG_COLUMN_ID))) new_columns_to_read->push_back(c); } From 506bc153cad6af7e1ced124f15a73c36688a4fae Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 12 Jul 2022 09:08:05 +0800 Subject: [PATCH 11/26] support clean read in fast mode --- .../File/DMFileBlockInputStream.cpp | 1 + .../DeltaMerge/File/DMFileBlockInputStream.h | 8 +++-- .../Storages/DeltaMerge/File/DMFileReader.cpp | 7 ++-- .../Storages/DeltaMerge/File/DMFileReader.h | 5 ++- dbms/src/Storages/DeltaMerge/Segment.cpp | 33 +++++++++++++++++-- .../Storages/DeltaMerge/StableValueSpace.cpp | 5 +-- .../Storages/DeltaMerge/StableValueSpace.h | 3 +- .../DeltaMerge/workload/DTWorkload.cpp | 14 ++++++-- 8 files changed, 62 insertions(+), 14 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp index c9212c4b81e..d25378c54cf 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp @@ -61,6 +61,7 @@ DMFileBlockInputStreamPtr DMFileBlockInputStreamBuilder::build(const DMFilePtr & read_columns, is_common_handle, enable_clean_read, + is_raw_read, max_data_version, std::move(pack_filter), mark_cache, diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h b/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h index a36bf50a937..d96339e0f54 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h @@ -74,9 +74,10 @@ class DMFileBlockInputStreamBuilder // 2. You don't need pk, version and delete_tag columns // If you have no idea what it means, then simply set it to false. // `max_data_version_` is the MVCC filter version for reading. Used by clean read check - DMFileBlockInputStreamBuilder & enableCleanRead(bool enable, UInt64 max_data_version_) + DMFileBlockInputStreamBuilder & enableCleanRead(bool enable, bool is_raw_read_, UInt64 max_data_version_) { enable_clean_read = enable; + is_raw_read = is_raw_read_; max_data_version = max_data_version_; return *this; } @@ -139,6 +140,7 @@ class DMFileBlockInputStreamBuilder // clean read bool enable_clean_read = false; + bool is_raw_read = false; UInt64 max_data_version = std::numeric_limits::max(); // Rough set filter RSOperatorPtr rs_filter; @@ -150,8 +152,8 @@ class DMFileBlockInputStreamBuilder bool enable_column_cache = false; ColumnCachePtr column_cache; ReadLimiterPtr read_limiter; - size_t aio_threshold; - size_t max_read_buffer_size; + size_t aio_threshold{}; + size_t max_read_buffer_size{}; size_t rows_threshold_per_read = DMFILE_READ_ROWS_THRESHOLD; bool read_one_pack_every_time = false; diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp index 423d8d4d031..6f8efe62a18 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp @@ -210,6 +210,7 @@ DMFileReader::DMFileReader( bool is_common_handle_, // clean read bool enable_clean_read_, + bool is_raw_read_, UInt64 max_read_version_, // filters DMFilePackFilter && pack_filter_, @@ -230,6 +231,7 @@ DMFileReader::DMFileReader( , read_one_pack_every_time(read_one_pack_every_time_) , single_file_mode(dmfile_->isSingleFileMode()) , enable_clean_read(enable_clean_read_) + , is_raw_read(is_raw_read_) , max_read_version(max_read_version_) , pack_filter(std::move(pack_filter_)) , skip_packs_by_column(read_columns.size(), 0) @@ -338,8 +340,9 @@ Block DMFileReader::read() } // TODO: this will need better algorithm: we should separate those packs which can and can not do clean read. - bool do_clean_read = enable_clean_read && expected_handle_res == All && not_clean_rows == 0; - if (do_clean_read) + bool do_clean_read = enable_clean_read && expected_handle_res == All && (not_clean_rows == 0 || is_raw_read); + + if (do_clean_read && !is_raw_read) { UInt64 max_version = 0; for (size_t pack_id = start_pack_id; pack_id < next_pack_id; ++pack_id) diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileReader.h b/dbms/src/Storages/DeltaMerge/File/DMFileReader.h index 9211918c2d0..e1fbafc0562 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileReader.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFileReader.h @@ -76,6 +76,7 @@ class DMFileReader // 2. You don't need pk, version and delete_tag columns // If you have no idea what it means, then simply set it to false. bool enable_clean_read_, + bool is_raw_read, // The the MVCC filter version. Used by clean read check. UInt64 max_read_version_, // filters @@ -122,8 +123,10 @@ class DMFileReader const bool single_file_mode; /// Clean read optimize - // If there is no delta for some packs in stable, we can try to do clean read. + // In normal mode, if there is no delta for some packs in stable, we can try to do clean read. + // In fast mode, we always try to do clean read. const bool enable_clean_read; + const bool is_raw_read; const UInt64 max_read_version; /// Filters diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index 3c50b2e2bce..8c9dd80cedc 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -39,6 +39,7 @@ #include #include +#include #include namespace ProfileEvents @@ -505,17 +506,41 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, bool filter_delete_mark, size_t expected_block_size) { + /// Now, we use filter_delete_mark to determine whether it is in fast mode or just from `selraw * xxxx` + /// But this way seems not to be robustness enough, maybe we need another flag? auto new_columns_to_read = std::make_shared(); + bool enable_clean_read = true; - new_columns_to_read->push_back(getExtraHandleColumnDefine(is_common_handle)); + new_columns_to_read->push_back(getExtraHandleColumnDefine(is_common_handle)); //所以这条不是必要的？ + if (filter_delete_mark) + { + new_columns_to_read->push_back(getTagColumnDefine()); + } for (const auto & c : columns_to_read) { if (c.id != EXTRA_HANDLE_COLUMN_ID) - new_columns_to_read->push_back(c); + { + if (!(filter_delete_mark && c.id == TAG_COLUMN_ID)) + new_columns_to_read->push_back(c); + } + else + { + enable_clean_read = false; + } } + /// Whether fast mode or raw read(selraw), if columns_to_read does not include EXTRA_HANDLE_COLUMN_ID, + /// we can try to use clean read to make optimization in stable part. + + /// Especially for fast mode, + /// when the pack is under totally data_ranges and has no rows whose del_mark = 1 --> we don't need read handle_column/tag_column/version_column + /// when the pack is under totally data_ranges and has rows whose del_mark = 1 --> we don't need read handle_column/version_column + /// others --> we don't need read version_column + + /// Actually, we first ignore the tag_column optimization + BlockInputStreamPtr stable_stream = segment_snap->stable->getInputStream( dm_context, *new_columns_to_read, @@ -523,7 +548,8 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, filter, std::numeric_limits::max(), expected_block_size, - false); + enable_clean_read, + true); DeltaValueInputStream delta_stream(dm_context, segment_snap->delta, new_columns_to_read, this->rowkey_range); auto memtable_stream = delta_stream.getMemTableInputStream(); @@ -564,6 +590,7 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, streams.push_back(persisted_files_stream); streams.push_back(stable_stream); } + return std::make_shared(streams, /*req_id=*/""); } diff --git a/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp b/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp index ed97bd8f421..7bce8a88156 100644 --- a/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp +++ b/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp @@ -328,7 +328,8 @@ StableValueSpace::Snapshot::getInputStream( const RSOperatorPtr & filter, UInt64 max_data_version, size_t expected_block_size, - bool enable_clean_read) + bool enable_clean_read, + bool is_raw_read) { LOG_FMT_DEBUG(log, "max_data_version: {}, enable_clean_read: {}", max_data_version, enable_clean_read); SkippableBlockInputStreams streams; @@ -337,7 +338,7 @@ StableValueSpace::Snapshot::getInputStream( { DMFileBlockInputStreamBuilder builder(context.db_context); builder - .enableCleanRead(enable_clean_read, max_data_version) + .enableCleanRead(enable_clean_read, is_raw_read, max_data_version) .setRSOperator(filter) .setColumnCache(column_caches[i]) .setTracingID(context.tracing_id) diff --git a/dbms/src/Storages/DeltaMerge/StableValueSpace.h b/dbms/src/Storages/DeltaMerge/StableValueSpace.h index 24384674d80..194d61dc27c 100644 --- a/dbms/src/Storages/DeltaMerge/StableValueSpace.h +++ b/dbms/src/Storages/DeltaMerge/StableValueSpace.h @@ -150,7 +150,8 @@ class StableValueSpace : public std::enable_shared_from_this const RSOperatorPtr & filter, UInt64 max_data_version, size_t expected_block_size, - bool enable_clean_read); + bool enable_clean_read, + bool is_raw_read = false); RowsAndBytes getApproxRowsAndBytes(const DMContext & context, const RowKeyRange & range) const; diff --git a/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp index 2282186e234..38643e2c588 100644 --- a/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp @@ -274,6 +274,16 @@ void DTWorkload::scanAll(ThreadStat & read_stat) while (writing_threads.load(std::memory_order_relaxed) > 0) { const auto & columns = store->getTableColumns(); + ColumnDefines real_columns; + + for (const auto& col : columns) + { + if (col.name != EXTRA_HANDLE_COLUMN_NAME && col.name != VERSION_COLUMN_NAME && col.name != TAG_COLUMN_NAME) + { + real_columns.emplace_back(col); + } + } + int stream_count = opts->read_stream_count; std::atomic read_count = 0; auto count_row = [&read_count](BlockInputStreamPtr in, [[maybe_unused]] uint64_t read_ts) { @@ -283,10 +293,10 @@ void DTWorkload::scanAll(ThreadStat & read_stat) } }; Stopwatch sw; - read(columns, stream_count, count_row); + read(real_columns, stream_count, count_row); read_stat.ms = sw.elapsedMilliseconds(); read_stat.count = read_count; - LOG_FMT_INFO(log, "scanAll: columns {} streams {} read_stat {}", columns.size(), stream_count, read_stat.toString()); + LOG_FMT_INFO(log, "scanAll: columns {} streams {} read_stat {}", real_columns.size(), stream_count, read_stat.toString()); } } catch (...) From 49b6cf96a08dd03e6aba18ad7f412971df2c8fcd Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 12 Jul 2022 11:26:42 +0800 Subject: [PATCH 12/26] fix comments --- dbms/src/Interpreters/Settings.h | 5 ++-- .../Storages/DeltaMerge/DMDecoratorStreams.h | 10 ++++--- .../DeltaMerge/Delta/DeltaValueSpace.h | 21 ++++++--------- dbms/src/Storages/DeltaMerge/Segment.cpp | 24 +++++++---------- .../DeltaMerge/tests/gtest_column_filter.cpp | 26 +++++-------------- ...est_dm_delta_merge_store_for_fast_mode.cpp | 6 ++--- 6 files changed, 35 insertions(+), 57 deletions(-) diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index 192a016bf78..56666640ef2 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -272,15 +272,14 @@ struct Settings M(SettingUInt64, dt_segment_delta_small_column_file_size, 8388608, "Determine whether a column file in delta is small or not. 8MB by default.") \ M(SettingUInt64, dt_segment_stable_pack_rows, DEFAULT_MERGE_BLOCK_SIZE, "Expected stable pack rows in DeltaTree Engine.") \ M(SettingFloat, dt_segment_wait_duration_factor, 1, "The factor of wait duration in a write stall.") \ - M(SettingUInt64, dt_bg_gc_check_interval, 60, "Background gc thread check interval, the unit is second.") \ + M(SettingUInt64, dt_bg_gc_check_interval, 60, "Background gc thread check interval, the unit is second.") \ M(SettingInt64, dt_bg_gc_max_segments_to_check_every_round, 100, "Max segments to check in every gc round, value less than or equal to 0 means gc no segments.") \ M(SettingFloat, dt_bg_gc_ratio_threhold_to_trigger_gc, 1.2, "Trigger segment's gc when the ratio of invalid version exceed this threhold. Values smaller than or equal to 1.0 means gc all " \ "segments") \ M(SettingFloat, dt_bg_gc_delta_delete_ratio_to_trigger_gc, 0.3, "Trigger segment's gc when the ratio of delta delete range to stable exceeds this ratio.") \ M(SettingUInt64, dt_insert_max_rows, 0, "Max rows of insert blocks when write into DeltaTree Engine. By default 0 means no limit.") \ M(SettingBool, dt_enable_rough_set_filter, true, "Whether to parse where expression as Rough Set Index filter or not.") \ - /* unused */ \ - M(SettingBool, dt_raw_filter_range, true, "Do range filter or not when read data in raw mode in DeltaTree Engine.") \ + M(SettingBool, dt_raw_filter_range, true, "[unused] Do range filter or not when read data in raw mode in DeltaTree Engine.") \ M(SettingBool, dt_read_delta_only, false, "Only read delta data in DeltaTree Engine.") \ M(SettingBool, dt_read_stable_only, false, "Only read stable data in DeltaTree Engine.") \ M(SettingBool, dt_enable_logical_split, false, "Enable logical split or not in DeltaTree Engine.") \ diff --git a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h index 63fed02a2f9..8b2879c713b 100644 --- a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h +++ b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h @@ -16,11 +16,13 @@ #include #include +#include #include -#include #include +#include "common/logger_useful.h" + namespace DB { @@ -36,14 +38,14 @@ class DMDeleteFilterBlockInputStream : public IBlockInputStream DMDeleteFilterBlockInputStream(const BlockInputStreamPtr & input, const ColumnDefines & columns_to_read_) : columns_to_read(columns_to_read_) , header(toEmptyBlock(columns_to_read)) - , log(&Poco::Logger::get("DMDeleteFilterBlockInputStream")) + , log(Logger::get("DMDeleteFilterBlockInputStream", /*req_id=*/"")) { children.emplace_back(input); delete_col_pos = input->getHeader().getPositionByName(TAG_COLUMN_NAME); } ~DMDeleteFilterBlockInputStream() { - LOG_FMT_DEBUG(log, + LOG_FMT_TRACE(log, "Total rows: {}, pass: {:.2f}%" ", complete pass: {:.2f}%, complete not pass: {:.2f}%", total_rows, @@ -139,7 +141,7 @@ class DMDeleteFilterBlockInputStream : public IBlockInputStream size_t complete_passed = 0; size_t complete_not_passed = 0; - Poco::Logger * log; + LoggerPtr log; }; class DMColumnFilterBlockInputStream : public IBlockInputStream diff --git a/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.h b/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.h index bab05d9e889..04fb97b3004 100644 --- a/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.h +++ b/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.h @@ -383,10 +383,8 @@ class DeltaValueReader class DeltaValueInputStream : public IBlockInputStream { private: - // ColumnFileSetInputStream mem_table_input_stream; - // ColumnFileSetInputStream persisted_files_input_stream; - BlockInputStreamPtr mem_table_input_stream; - BlockInputStreamPtr persisted_files_input_stream; + ColumnFileSetInputStream mem_table_input_stream; + ColumnFileSetInputStream persisted_files_input_stream; bool persisted_files_done = false; @@ -395,28 +393,25 @@ class DeltaValueInputStream : public IBlockInputStream const DeltaSnapshotPtr & delta_snap_, const ColumnDefinesPtr & col_defs_, const RowKeyRange & segment_range_) - : mem_table_input_stream{std::make_shared(context_, delta_snap_->getMemTableSetSnapshot(), col_defs_, segment_range_)} - , persisted_files_input_stream{std::make_shared(context_, delta_snap_->getPersistedFileSetSnapshot(), col_defs_, segment_range_)} + : mem_table_input_stream(context_, delta_snap_->getMemTableSetSnapshot(), col_defs_, segment_range_) + , persisted_files_input_stream(context_, delta_snap_->getPersistedFileSetSnapshot(), col_defs_, segment_range_) {} String getName() const override { return "DeltaValue"; } - Block getHeader() const override { return persisted_files_input_stream->getHeader(); } - - BlockInputStreamPtr getPersistedFilesInputStream() { return persisted_files_input_stream; } - BlockInputStreamPtr getMemTableInputStream() { return mem_table_input_stream; } + Block getHeader() const override { return persisted_files_input_stream.getHeader(); } Block read() override { if (persisted_files_done) - return mem_table_input_stream->read(); + return mem_table_input_stream.read(); - Block block = persisted_files_input_stream->read(); + Block block = persisted_files_input_stream.read(); if (block) return block; else { persisted_files_done = true; - return mem_table_input_stream->read(); + return mem_table_input_stream.read(); } } }; diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index 246d4198d80..09180690254 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -39,6 +39,7 @@ #include #include +#include #include namespace ProfileEvents @@ -529,34 +530,30 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, expected_block_size, false); - DeltaValueInputStream delta_stream(dm_context, segment_snap->delta, new_columns_to_read, this->rowkey_range); - auto memtable_stream = delta_stream.getMemTableInputStream(); - auto persisted_files_stream = delta_stream.getPersistedFilesInputStream(); + BlockInputStreamPtr delta_stream = std::make_shared(dm_context, segment_snap->delta, new_columns_to_read, this->rowkey_range); - memtable_stream = std::make_shared>(memtable_stream, data_ranges, 0); - persisted_files_stream = std::make_shared>(persisted_files_stream, data_ranges, 0); // related to minor compaction sorted - stable_stream = std::make_shared>(stable_stream, data_ranges, 0); + //delta_stream = std::make_shared(delta_stream, 16384, 0, /*req_id=*/""); // For test + delta_stream = std::make_shared>(delta_stream, data_ranges, 0); + stable_stream = std::make_shared>(stable_stream, data_ranges, 0); if (filter_delete_mark) { - memtable_stream = std::make_shared(memtable_stream, columns_to_read); - persisted_files_stream = std::make_shared(persisted_files_stream, columns_to_read); + delta_stream = std::make_shared(delta_stream, columns_to_read); stable_stream = std::make_shared(stable_stream, columns_to_read); } else { - memtable_stream = std::make_shared(memtable_stream, columns_to_read); - persisted_files_stream = std::make_shared(persisted_files_stream, columns_to_read); + delta_stream = std::make_shared(delta_stream, columns_to_read); stable_stream = std::make_shared(stable_stream, columns_to_read); } + BlockInputStreams streams; if (dm_context.read_delta_only) { - streams.push_back(memtable_stream); - streams.push_back(persisted_files_stream); + streams.push_back(delta_stream); } else if (dm_context.read_stable_only) { @@ -564,8 +561,7 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, } else { - streams.push_back(memtable_stream); - streams.push_back(persisted_files_stream); + streams.push_back(delta_stream); streams.push_back(stable_stream); } return std::make_shared(streams, /*req_id=*/""); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp index bc7d7dfbef5..894f3dc529e 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include #include @@ -25,13 +25,11 @@ namespace { constexpr const char * str_col_name = "col_a"; -class DebugBlockInputStream : public IProfilingBlockInputStream +class DebugBlockInputStream : public BlocksListBlockInputStream { public: - DebugBlockInputStream(const BlocksList & blocks, bool is_common_handle_) - : begin(blocks.begin()) - , end(blocks.end()) - , it(blocks.begin()) + DebugBlockInputStream(BlocksList & blocks, bool is_common_handle_) + : BlocksListBlockInputStream(std::move(blocks)) , is_common_handle(is_common_handle_) { } @@ -43,23 +41,11 @@ class DebugBlockInputStream : public IProfilingBlockInputStream return toEmptyBlock(*cds); } -protected: - Block readImpl() override - { - if (it == end) - return Block(); - else - return *(it++); - } - private: - BlocksList::const_iterator begin; - BlocksList::const_iterator end; - BlocksList::const_iterator it; bool is_common_handle; }; -BlockInputStreamPtr genColumnFilterInputStream(const BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle) +BlockInputStreamPtr genColumnFilterInputStream(BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle) { ColumnDefine handle_define( TiDBPkColumnID, @@ -71,7 +57,7 @@ BlockInputStreamPtr genColumnFilterInputStream(const BlocksList & blocks, const columns); } -BlockInputStreamPtr genDeleteFilterInputStream(const BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle) +BlockInputStreamPtr genDeleteFilterInputStream(BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle) { ColumnDefine handle_define( TiDBPkColumnID, diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp index 3e33d32b8f7..6c61079eefb 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp @@ -495,16 +495,16 @@ try case TestMode::V2_Mix: { int block_index = 0; - int begin_value = num_write_rows; // memory first, then persist, finally stable + int begin_value = 0; // persist first, then memory, finally stable while (Block block = in->read()) { if (block_index == 1) { - begin_value = 0; + begin_value = num_write_rows * 2; } else if (block_index == 2) { - begin_value = num_write_rows * 2; + begin_value = num_write_rows; } for (auto && iter : block) { From 5a8fd0da70dd57b9eb5b8ffbd60294d6532b6c3e Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 12 Jul 2022 13:18:28 +0800 Subject: [PATCH 13/26] update fast mode --- dbms/src/Interpreters/Settings.h | 3 +-- .../Storages/DeltaMerge/DMDecoratorStreams.h | 10 ++++--- .../DeltaMerge/Delta/DeltaValueSpace.h | 21 ++++++--------- .../Storages/DeltaMerge/File/DMFileReader.cpp | 1 + .../Storages/DeltaMerge/File/DMFileReader.h | 2 +- dbms/src/Storages/DeltaMerge/Segment.cpp | 25 +++++++----------- .../DeltaMerge/tests/gtest_column_filter.cpp | 26 +++++-------------- ...est_dm_delta_merge_store_for_fast_mode.cpp | 6 ++--- 8 files changed, 36 insertions(+), 58 deletions(-) diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index 192a016bf78..215b94dda0e 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -279,8 +279,7 @@ struct Settings M(SettingFloat, dt_bg_gc_delta_delete_ratio_to_trigger_gc, 0.3, "Trigger segment's gc when the ratio of delta delete range to stable exceeds this ratio.") \ M(SettingUInt64, dt_insert_max_rows, 0, "Max rows of insert blocks when write into DeltaTree Engine. By default 0 means no limit.") \ M(SettingBool, dt_enable_rough_set_filter, true, "Whether to parse where expression as Rough Set Index filter or not.") \ - /* unused */ \ - M(SettingBool, dt_raw_filter_range, true, "Do range filter or not when read data in raw mode in DeltaTree Engine.") \ + M(SettingBool, dt_raw_filter_range, true, "[unused] Do range filter or not when read data in raw mode in DeltaTree Engine.") \ M(SettingBool, dt_read_delta_only, false, "Only read delta data in DeltaTree Engine.") \ M(SettingBool, dt_read_stable_only, false, "Only read stable data in DeltaTree Engine.") \ M(SettingBool, dt_enable_logical_split, false, "Enable logical split or not in DeltaTree Engine.") \ diff --git a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h index 63fed02a2f9..c7f5bfd162f 100644 --- a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h +++ b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h @@ -17,10 +17,12 @@ #include #include #include -#include +#include #include +#include + namespace DB { @@ -36,14 +38,14 @@ class DMDeleteFilterBlockInputStream : public IBlockInputStream DMDeleteFilterBlockInputStream(const BlockInputStreamPtr & input, const ColumnDefines & columns_to_read_) : columns_to_read(columns_to_read_) , header(toEmptyBlock(columns_to_read)) - , log(&Poco::Logger::get("DMDeleteFilterBlockInputStream")) + , log(Logger::get("DMDeleteFilterBlockInputStream", /*req_id=*/"")) { children.emplace_back(input); delete_col_pos = input->getHeader().getPositionByName(TAG_COLUMN_NAME); } ~DMDeleteFilterBlockInputStream() { - LOG_FMT_DEBUG(log, + LOG_FMT_TRACE(log, "Total rows: {}, pass: {:.2f}%" ", complete pass: {:.2f}%, complete not pass: {:.2f}%", total_rows, @@ -139,7 +141,7 @@ class DMDeleteFilterBlockInputStream : public IBlockInputStream size_t complete_passed = 0; size_t complete_not_passed = 0; - Poco::Logger * log; + LoggerPtr log; }; class DMColumnFilterBlockInputStream : public IBlockInputStream diff --git a/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.h b/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.h index bab05d9e889..04fb97b3004 100644 --- a/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.h +++ b/dbms/src/Storages/DeltaMerge/Delta/DeltaValueSpace.h @@ -383,10 +383,8 @@ class DeltaValueReader class DeltaValueInputStream : public IBlockInputStream { private: - // ColumnFileSetInputStream mem_table_input_stream; - // ColumnFileSetInputStream persisted_files_input_stream; - BlockInputStreamPtr mem_table_input_stream; - BlockInputStreamPtr persisted_files_input_stream; + ColumnFileSetInputStream mem_table_input_stream; + ColumnFileSetInputStream persisted_files_input_stream; bool persisted_files_done = false; @@ -395,28 +393,25 @@ class DeltaValueInputStream : public IBlockInputStream const DeltaSnapshotPtr & delta_snap_, const ColumnDefinesPtr & col_defs_, const RowKeyRange & segment_range_) - : mem_table_input_stream{std::make_shared(context_, delta_snap_->getMemTableSetSnapshot(), col_defs_, segment_range_)} - , persisted_files_input_stream{std::make_shared(context_, delta_snap_->getPersistedFileSetSnapshot(), col_defs_, segment_range_)} + : mem_table_input_stream(context_, delta_snap_->getMemTableSetSnapshot(), col_defs_, segment_range_) + , persisted_files_input_stream(context_, delta_snap_->getPersistedFileSetSnapshot(), col_defs_, segment_range_) {} String getName() const override { return "DeltaValue"; } - Block getHeader() const override { return persisted_files_input_stream->getHeader(); } - - BlockInputStreamPtr getPersistedFilesInputStream() { return persisted_files_input_stream; } - BlockInputStreamPtr getMemTableInputStream() { return mem_table_input_stream; } + Block getHeader() const override { return persisted_files_input_stream.getHeader(); } Block read() override { if (persisted_files_done) - return mem_table_input_stream->read(); + return mem_table_input_stream.read(); - Block block = persisted_files_input_stream->read(); + Block block = persisted_files_input_stream.read(); if (block) return block; else { persisted_files_done = true; - return mem_table_input_stream->read(); + return mem_table_input_stream.read(); } } }; diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp index 6f8efe62a18..82c3dd70289 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp @@ -340,6 +340,7 @@ Block DMFileReader::read() } // TODO: this will need better algorithm: we should separate those packs which can and can not do clean read. + // bool do_clean_read = enable_clean_read && expected_handle_res == All && not_clean_rows == 0 ; bool do_clean_read = enable_clean_read && expected_handle_res == All && (not_clean_rows == 0 || is_raw_read); if (do_clean_read && !is_raw_read) diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileReader.h b/dbms/src/Storages/DeltaMerge/File/DMFileReader.h index e1fbafc0562..0b60ba78d78 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileReader.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFileReader.h @@ -76,7 +76,7 @@ class DMFileReader // 2. You don't need pk, version and delete_tag columns // If you have no idea what it means, then simply set it to false. bool enable_clean_read_, - bool is_raw_read, + bool is_raw_read_, // The the MVCC filter version. Used by clean read check. UInt64 max_read_version_, // filters diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index 8c9dd80cedc..21167e1385c 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -539,7 +539,7 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, /// when the pack is under totally data_ranges and has rows whose del_mark = 1 --> we don't need read handle_column/version_column /// others --> we don't need read version_column - /// Actually, we first ignore the tag_column optimization + /// Actually, we first ignore whether we need to read tag_column, just deal it the same with handle column BlockInputStreamPtr stable_stream = segment_snap->stable->getInputStream( dm_context, @@ -551,25 +551,22 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, enable_clean_read, true); - DeltaValueInputStream delta_stream(dm_context, segment_snap->delta, new_columns_to_read, this->rowkey_range); - auto memtable_stream = delta_stream.getMemTableInputStream(); - auto persisted_files_stream = delta_stream.getPersistedFilesInputStream(); + BlockInputStreamPtr delta_stream = std::make_shared(dm_context, segment_snap->delta, new_columns_to_read, this->rowkey_range); - memtable_stream = std::make_shared>(memtable_stream, data_ranges, 0); - persisted_files_stream = std::make_shared>(persisted_files_stream, data_ranges, 0); // related to minor compaction sorted + //delta_stream = std::make_shared(delta_stream, 16384, 0, /*req_id=*/""); // For test + + delta_stream = std::make_shared>(delta_stream, data_ranges, 0); stable_stream = std::make_shared>(stable_stream, data_ranges, 0); if (filter_delete_mark) { - memtable_stream = std::make_shared(memtable_stream, columns_to_read); - persisted_files_stream = std::make_shared(persisted_files_stream, columns_to_read); + delta_stream = std::make_shared(delta_stream, columns_to_read); stable_stream = std::make_shared(stable_stream, columns_to_read); } else { - memtable_stream = std::make_shared(memtable_stream, columns_to_read); - persisted_files_stream = std::make_shared(persisted_files_stream, columns_to_read); + delta_stream = std::make_shared(delta_stream, columns_to_read); stable_stream = std::make_shared(stable_stream, columns_to_read); } @@ -577,8 +574,7 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, if (dm_context.read_delta_only) { - streams.push_back(memtable_stream); - streams.push_back(persisted_files_stream); + streams.push_back(delta_stream); } else if (dm_context.read_stable_only) { @@ -586,8 +582,7 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, } else { - streams.push_back(memtable_stream); - streams.push_back(persisted_files_stream); + streams.push_back(delta_stream); streams.push_back(stable_stream); } @@ -1484,7 +1479,7 @@ SkippableBlockInputStreamPtr Segment::getPlacedStream(const DMContext & dm_conte { if (unlikely(rowkey_ranges.empty())) throw Exception("rowkey ranges shouldn't be empty", ErrorCodes::LOGICAL_ERROR); - + SkippableBlockInputStreamPtr stable_input_stream = stable_snap->getInputStream(dm_context, read_columns, rowkey_ranges, filter, max_version, expected_block_size, false); RowKeyRange rowkey_range = rowkey_ranges.size() == 1 ? rowkey_ranges[0] : mergeRanges(rowkey_ranges, rowkey_ranges[0].is_common_handle, rowkey_ranges[0].rowkey_column_size); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp index bc7d7dfbef5..894f3dc529e 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include #include @@ -25,13 +25,11 @@ namespace { constexpr const char * str_col_name = "col_a"; -class DebugBlockInputStream : public IProfilingBlockInputStream +class DebugBlockInputStream : public BlocksListBlockInputStream { public: - DebugBlockInputStream(const BlocksList & blocks, bool is_common_handle_) - : begin(blocks.begin()) - , end(blocks.end()) - , it(blocks.begin()) + DebugBlockInputStream(BlocksList & blocks, bool is_common_handle_) + : BlocksListBlockInputStream(std::move(blocks)) , is_common_handle(is_common_handle_) { } @@ -43,23 +41,11 @@ class DebugBlockInputStream : public IProfilingBlockInputStream return toEmptyBlock(*cds); } -protected: - Block readImpl() override - { - if (it == end) - return Block(); - else - return *(it++); - } - private: - BlocksList::const_iterator begin; - BlocksList::const_iterator end; - BlocksList::const_iterator it; bool is_common_handle; }; -BlockInputStreamPtr genColumnFilterInputStream(const BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle) +BlockInputStreamPtr genColumnFilterInputStream(BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle) { ColumnDefine handle_define( TiDBPkColumnID, @@ -71,7 +57,7 @@ BlockInputStreamPtr genColumnFilterInputStream(const BlocksList & blocks, const columns); } -BlockInputStreamPtr genDeleteFilterInputStream(const BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle) +BlockInputStreamPtr genDeleteFilterInputStream(BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle) { ColumnDefine handle_define( TiDBPkColumnID, diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp index 3e33d32b8f7..6c61079eefb 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp @@ -495,16 +495,16 @@ try case TestMode::V2_Mix: { int block_index = 0; - int begin_value = num_write_rows; // memory first, then persist, finally stable + int begin_value = 0; // persist first, then memory, finally stable while (Block block = in->read()) { if (block_index == 1) { - begin_value = 0; + begin_value = num_write_rows * 2; } else if (block_index == 2) { - begin_value = num_write_rows * 2; + begin_value = num_write_rows; } for (auto && iter : block) { From d23e0e8cca8afce81737d32ad142425a1e9272b9 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Tue, 12 Jul 2022 17:53:16 +0800 Subject: [PATCH 14/26] for test --- .../Storages/DeltaMerge/DMDecoratorStreams.h | 55 +++++++++++++------ .../Storages/DeltaMerge/File/DMFileReader.cpp | 4 ++ 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h index c7f5bfd162f..290981aacf4 100644 --- a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h +++ b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h @@ -20,6 +20,7 @@ #include #include +#include "common/types.h" #include @@ -52,6 +53,7 @@ class DMDeleteFilterBlockInputStream : public IBlockInputStream passed_rows * 100.0 / total_rows, complete_passed * 100.0 / total_blocks, complete_not_passed * 100.0 / total_blocks); + std::cout << "Total rows: " << total_rows << ", pass: " << passed_rows * 100.0 / total_rows << ", complete pass: " << complete_passed * 100.0 / total_blocks << "complete not pass: " << complete_not_passed * 100.0 / total_blocks << std::endl; } String getName() const override { return "DMColumnFilter"; } @@ -68,34 +70,51 @@ class DMDeleteFilterBlockInputStream : public IBlockInputStream if (!block.rows()) continue; - delete_col_data = getColumnVectorDataPtr(block, delete_col_pos); + std::cout << "block.dumpStructure()" << block.dumpStructure() << std::endl; + + delete_col_data = toColumnVectorDataPtr(block.getByName(TAG_COLUMN_NAME).column); + //delete_col_data = getColumnVectorDataPtr(block, delete_col_pos); + size_t rows = block.rows(); delete_filter.resize(rows); - const size_t batch_rows = (rows - 1) / UNROLL_BATCH * UNROLL_BATCH; - - // The following is trying to unroll the filtering operations, - // so that optimizer could use vectorized optimization. - // The original logic can be seen in #checkWithNextIndex(). + // const size_t batch_rows = (rows - 1) / UNROLL_BATCH * UNROLL_BATCH; + + // // The following is trying to unroll the filtering operations, + // // so that optimizer could use vectorized optimization. + // // The original logic can be seen in #checkWithNextIndex(). + // { + // UInt8 * filter_pos = delete_filter.data(); + // auto * delete_pos = const_cast(delete_col_data->data()); + // for (size_t i = 0; i < batch_rows; ++i) + // { + // (*filter_pos) = (*delete_pos) == 0; + + // //std::cout << "(*filter_pos) is " << (*filter_pos) << " (*delete_pos) is " << (*delete_pos) << std::endl; + // ++filter_pos; + // ++delete_pos; + // } + // } + + for (size_t i = 0; i < rows; ++i) { - UInt8 * filter_pos = delete_filter.data(); - auto * delete_pos = const_cast(delete_col_data->data()); - for (size_t i = 0; i < batch_rows; ++i) - { - (*filter_pos) = (*delete_pos) == 0; - ++filter_pos; - ++delete_pos; + if ((*delete_col_data)[i] == 0) { + delete_filter[i] = 1; + } else { + delete_filter[i] = 0; + std::cout << " (*delete_col_data)[i] is not 0 " << (*delete_col_data)[i] << std::endl; } - } - - for (size_t i = batch_rows; i < rows; ++i) - { - delete_filter[i] = !(*delete_col_data)[i]; + //delete_filter[i] = ((*delete_col_data)[i] == 0); } const size_t passed_count = countBytesInFilter(delete_filter); + if (passed_count != rows){ + for (size_t index = 0; index < delete_col_data->size(); index++){ + std::cout << "index is " << index << " (*delete_col_data)[index] is " << (*delete_col_data)[index] << " delete_filter[index] is " << delete_filter[index] << " passed_count is " << passed_count << " rows is " << rows << " " << delete_col_data->size() << std::endl; + } + } ++total_blocks; total_rows += rows; passed_rows += passed_count; diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp index 82c3dd70289..2941b0339a3 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp @@ -381,6 +381,9 @@ Block DMFileReader::read() else if (cd.id == TAG_COLUMN_ID) { column = cd.type->createColumnConst(read_rows, Field(static_cast(pack_stats[start_pack_id].first_tag))); + // for (size_t index = 0; index < column->size(); index++){ + // std::cout << "sub1 index is " << index << " (*column)[index] is " << (*column)[index].toString() << std::endl; + // } } res.insert(ColumnWithTypeAndName{column, cd.type, cd.name, cd.id}); @@ -460,6 +463,7 @@ Block DMFileReader::read() dmfile->path()); // New column after ddl is not exist in this DMFile, fill with default value ColumnPtr column = createColumnWithDefaultValue(cd, read_rows); + res.insert(ColumnWithTypeAndName{std::move(column), cd.type, cd.name, cd.id}); skip_packs_by_column[i] = 0; } From 191ac029a16d119d4fd210f858304798f2d0c632 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 13 Jul 2022 13:02:51 +0800 Subject: [PATCH 15/26] include delele handle --- .../Storages/DeltaMerge/DMDecoratorStreams.h | 60 +++++----- .../DeltaMerge/File/DMFilePackFilter.h | 8 ++ .../Storages/DeltaMerge/File/DMFileReader.cpp | 46 +++++-- dbms/src/Storages/DeltaMerge/Segment.cpp | 12 +- ...est_dm_delta_merge_store_for_fast_mode.cpp | 113 ++++++++++++++++++ 5 files changed, 188 insertions(+), 51 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h index a5b84f060f6..24a531f36db 100644 --- a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h +++ b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h @@ -55,7 +55,6 @@ class DMDeleteFilterBlockInputStream : public IBlockInputStream passed_rows * 100.0 / total_rows, complete_passed * 100.0 / total_blocks, complete_not_passed * 100.0 / total_blocks); - std::cout << "Total rows: " << total_rows << ", pass: " << passed_rows * 100.0 / total_rows << ", complete pass: " << complete_passed * 100.0 / total_blocks << "complete not pass: " << complete_not_passed * 100.0 / total_blocks << std::endl; } String getName() const override { return "DMColumnFilter"; } @@ -71,52 +70,49 @@ class DMDeleteFilterBlockInputStream : public IBlockInputStream return {}; if (!block.rows()) continue; + + /// if the pack is in clean read, it's del column return a const column, with size 1. + /// In this case, all the del_mark must be 0. Thus we don't need extra filter. + if (block.getByPosition(delete_col_pos).column->isColumnConst()) { + //std::cout << "del optimization" << std::endl; + ++total_blocks; + ++complete_passed; + total_rows += block.rows(); + passed_rows += block.rows(); - std::cout << "block.dumpStructure()" << block.dumpStructure() << std::endl; + return getNewBlockByHeader(header, block); + } - delete_col_data = toColumnVectorDataPtr(block.getByName(TAG_COLUMN_NAME).column); - //delete_col_data = getColumnVectorDataPtr(block, delete_col_pos); + delete_col_data = getColumnVectorDataPtr(block, delete_col_pos); size_t rows = block.rows(); delete_filter.resize(rows); - // const size_t batch_rows = (rows - 1) / UNROLL_BATCH * UNROLL_BATCH; - - // // The following is trying to unroll the filtering operations, - // // so that optimizer could use vectorized optimization. - // // The original logic can be seen in #checkWithNextIndex(). - // { - // UInt8 * filter_pos = delete_filter.data(); - // auto * delete_pos = const_cast(delete_col_data->data()); - // for (size_t i = 0; i < batch_rows; ++i) - // { - // (*filter_pos) = (*delete_pos) == 0; + const size_t batch_rows = (rows - 1) / UNROLL_BATCH * UNROLL_BATCH; - // //std::cout << "(*filter_pos) is " << (*filter_pos) << " (*delete_pos) is " << (*delete_pos) << std::endl; - // ++filter_pos; - // ++delete_pos; - // } - // } + // The following is trying to unroll the filtering operations, + // so that optimizer could use vectorized optimization. + // The original logic can be seen in #checkWithNextIndex(). + { + UInt8 * filter_pos = delete_filter.data(); + auto * delete_pos = const_cast(delete_col_data->data()); + for (size_t i = 0; i < batch_rows; ++i) + { + (*filter_pos) = (*delete_pos) == 0; + + ++filter_pos; + ++delete_pos; + } + } for (size_t i = 0; i < rows; ++i) { - if ((*delete_col_data)[i] == 0) { - delete_filter[i] = 1; - } else { - delete_filter[i] = 0; - std::cout << " (*delete_col_data)[i] is not 0 " << (*delete_col_data)[i] << std::endl; - } - //delete_filter[i] = ((*delete_col_data)[i] == 0); + delete_filter[i] = ((*delete_col_data)[i] == 0); } const size_t passed_count = countBytesInFilter(delete_filter); - if (passed_count != rows){ - for (size_t index = 0; index < delete_col_data->size(); index++){ - std::cout << "index is " << index << " (*delete_col_data)[index] is " << (*delete_col_data)[index] << " delete_filter[index] is " << delete_filter[index] << " passed_count is " << passed_count << " rows is " << rows << " " << delete_col_data->size() << std::endl; - } - } ++total_blocks; total_rows += rows; passed_rows += passed_count; diff --git a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.h b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.h index 554ecc34b01..8d2806bae00 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.h @@ -84,6 +84,14 @@ class DMFilePackFilter return minmax_index->getUInt64MinMax(pack_id).second; } + UInt64 getMaxDel(size_t pack_id) + { + if (!param.indexes.count(TAG_COLUMN_ID)) + tryLoadIndex(TAG_COLUMN_ID); + auto & minmax_index = param.indexes.find(TAG_COLUMN_ID)->second.minmax; + return minmax_index->getUInt64MinMax(pack_id).second; + } + // Get valid rows and bytes after filter invalid packs by handle_range and filter std::pair validRowsAndBytes() { diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp index 2941b0339a3..72786da3835 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp @@ -24,6 +24,7 @@ #include #include #include +#include namespace CurrentMetrics { @@ -320,8 +321,10 @@ Block DMFileReader::read() { if (read_pack_limit != 0 && next_pack_id - start_pack_id >= read_pack_limit) break; - if (enable_clean_read && handle_res[next_pack_id] != expected_handle_res) - break; + // if (enable_clean_read && handle_res[next_pack_id] != expected_handle_res) + // break; + if (enable_clean_read && (handle_res[next_pack_id] != expected_handle_res || (is_raw_read && (pack_filter.getMaxDel(next_pack_id) != 0 || (next_pack_id > 0 && pack_filter.getMaxDel(next_pack_id - 1) != 0))))) + break; read_rows += pack_stats[next_pack_id].rows; not_clean_rows += pack_stats[next_pack_id].not_clean; @@ -340,15 +343,19 @@ Block DMFileReader::read() } // TODO: this will need better algorithm: we should separate those packs which can and can not do clean read. - // bool do_clean_read = enable_clean_read && expected_handle_res == All && not_clean_rows == 0 ; - bool do_clean_read = enable_clean_read && expected_handle_res == All && (not_clean_rows == 0 || is_raw_read); + bool do_clean_read_on_normal_mode = enable_clean_read && expected_handle_res == All && not_clean_rows == 0 && (!is_raw_read); + + // when is in fast mode and all pack's max del index is 0, then we don't need to read del column + bool do_clean_read_on_handle = is_raw_read && expected_handle_res == All; + bool do_clean_read_on_del = is_raw_read && pack_filter.getMaxDel(start_pack_id) == 0; + //bool do_clean_read_on_del = false; - if (do_clean_read && !is_raw_read) + if (do_clean_read_on_normal_mode) { UInt64 max_version = 0; for (size_t pack_id = start_pack_id; pack_id < next_pack_id; ++pack_id) max_version = std::max(pack_filter.getMaxVersion(pack_id), max_version); - do_clean_read = max_version <= max_read_version; + do_clean_read_on_normal_mode = max_version <= max_read_version; } for (size_t i = 0; i < read_columns.size(); ++i) @@ -357,7 +364,27 @@ Block DMFileReader::read() { // For clean read of column pk, version, tag, instead of loading data from disk, just create placeholder column is OK. auto & cd = read_columns[i]; - if (do_clean_read && isExtraColumn(cd)) + if (cd.id == EXTRA_HANDLE_COLUMN_ID && do_clean_read_on_handle) { + // Return the first row's handle + ColumnPtr column; + if (is_common_handle) + { + StringRef min_handle = pack_filter.getMinStringHandle(start_pack_id); + column = cd.type->createColumnConst(read_rows, Field(min_handle.data, min_handle.size)); + } + else + { + Handle min_handle = pack_filter.getMinHandle(start_pack_id); + column = cd.type->createColumnConst(read_rows, Field(min_handle)); + } + res.insert(ColumnWithTypeAndName{column, cd.type, cd.name, cd.id}); + skip_packs_by_column[i] = read_packs; + } else if (cd.id == TAG_COLUMN_ID && do_clean_read_on_del) { + ColumnPtr column = cd.type->createColumnConst(read_rows, Field(static_cast(pack_stats[start_pack_id].first_tag))); + res.insert(ColumnWithTypeAndName{column, cd.type, cd.name, cd.id}); + + skip_packs_by_column[i] = read_packs; + } else if (do_clean_read_on_normal_mode && isExtraColumn(cd)) { ColumnPtr column; if (cd.id == EXTRA_HANDLE_COLUMN_ID) @@ -381,9 +408,6 @@ Block DMFileReader::read() else if (cd.id == TAG_COLUMN_ID) { column = cd.type->createColumnConst(read_rows, Field(static_cast(pack_stats[start_pack_id].first_tag))); - // for (size_t index = 0; index < column->size(); index++){ - // std::cout << "sub1 index is " << index << " (*column)[index] is " << (*column)[index].toString() << std::endl; - // } } res.insert(ColumnWithTypeAndName{column, cd.type, cd.name, cd.id}); @@ -440,6 +464,7 @@ Block DMFileReader::read() } // Cast column's data from DataType in disk to what we need now auto converted_column = convertColumnByColumnDefineIfNeed(data_type, std::move(result_column), cd); + res.insert(ColumnWithTypeAndName{converted_column, cd.type, cd.name, cd.id}); } else @@ -448,6 +473,7 @@ Block DMFileReader::read() auto column = data_type->createColumn(); readFromDisk(cd, column, start_pack_id, read_rows, skip_packs_by_column[i], single_file_mode); auto converted_column = convertColumnByColumnDefineIfNeed(data_type, std::move(column), cd); + res.insert(ColumnWithTypeAndName{std::move(converted_column), cd.type, cd.name, cd.id}); skip_packs_by_column[i] = 0; } diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index 0f2e25cd2b9..74b5f39ddc9 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -510,8 +510,6 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, /// But this way seems not to be robustness enough, maybe we need another flag? auto new_columns_to_read = std::make_shared(); - bool enable_clean_read = true; - new_columns_to_read->push_back(getExtraHandleColumnDefine(is_common_handle)); if (filter_delete_mark) @@ -526,10 +524,6 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, if (!(filter_delete_mark && c.id == TAG_COLUMN_ID)) new_columns_to_read->push_back(c); } - else - { - enable_clean_read = false; - } } /// Whether fast mode or raw read(selraw), if columns_to_read does not include EXTRA_HANDLE_COLUMN_ID, @@ -549,8 +543,8 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, filter, std::numeric_limits::max(), expected_block_size, - enable_clean_read, - true); + /* enable_clean_read */ filter_delete_mark, + /* is_raw_read */ filter_delete_mark); BlockInputStreamPtr delta_stream = std::make_shared(dm_context, segment_snap->delta, new_columns_to_read, this->rowkey_range); @@ -1480,7 +1474,7 @@ SkippableBlockInputStreamPtr Segment::getPlacedStream(const DMContext & dm_conte { if (unlikely(rowkey_ranges.empty())) throw Exception("rowkey ranges shouldn't be empty", ErrorCodes::LOGICAL_ERROR); - + SkippableBlockInputStreamPtr stable_input_stream = stable_snap->getInputStream(dm_context, read_columns, rowkey_ranges, filter, max_version, expected_block_size, false); RowKeyRange rowkey_range = rowkey_ranges.size() == 1 ? rowkey_ranges[0] : mergeRanges(rowkey_ranges, rowkey_ranges[0].is_common_handle, rowkey_ranges[0].rowkey_column_size); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp index 6c61079eefb..fa332ddb971 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp @@ -1474,6 +1474,119 @@ try } CATCH +TEST_P(DeltaMergeStoreRWTest, TestFastModeForCleanRead) +try +{ + const size_t num_rows_write = 128; + { + // Create a block with sequential Int64 handle in range [0, 128) + Block block = DMTestEnv::prepareSimpleWriteBlock(0, 128, false); + + switch (mode) + { + case TestMode::V1_BlockOnly: + case TestMode::V2_BlockOnly: + store->write(*db_context, db_context->getSettingsRef(), block); + break; + default: + { + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range, file_ids] = genDMFile(*dm_context, block); + store->ingestFiles(dm_context, range, file_ids, false); + break; + } + } + } + + store->flushCache(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); + + store->compact(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); + + store->mergeDeltaAll(*db_context); + + // could do clean read with del optimization + { + const auto & columns = store->getTableColumns(); + BlockInputStreamPtr in = store->read(*db_context, + db_context->getSettingsRef(), + columns, + {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ std::numeric_limits::max(), + EMPTY_FILTER, + TRACING_NAME, + /* is_raw_read= */ true, + /* expected_block_size= */ 1024)[0]; + size_t num_rows_read = 0; + + while (Block block = in->read()) + { + num_rows_read += block.rows(); + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + ASSERT_EQ(c->getInt(i), i); + } + } + } + } + + ASSERT_EQ(num_rows_read, num_rows_write); + } + + // Delete range [0, 64) + const size_t num_deleted_rows = 64; + { + HandleRange range(0, num_deleted_rows); + store->deleteRange(*db_context, db_context->getSettingsRef(), RowKeyRange::fromHandleRange(range)); + } + + store->flushCache(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); + + store->compact(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); + + store->mergeDeltaAll(*db_context); + + // could do clean read with handle and del optimization + { + const auto & columns = store->getTableColumns(); + ColumnDefines real_columns ; + for (auto & col : columns){ + if (col.name != EXTRA_HANDLE_COLUMN_NAME) + { + real_columns.emplace_back(col); + } + } + + BlockInputStreamPtr in = store->read(*db_context, + db_context->getSettingsRef(), + real_columns, + {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ std::numeric_limits::max(), + EMPTY_FILTER, + TRACING_NAME, + /* is_raw_read= */ true, + /* expected_block_size= */ 1024)[0]; + size_t num_rows_read = 0; + + while (Block block = in->read()) + { + num_rows_read += block.rows(); + } + + ASSERT_EQ(num_rows_read, num_rows_write - num_deleted_rows); + } + + + + +} +CATCH } // namespace tests } // namespace DM } // namespace DB \ No newline at end of file From 47a053baf57e84ea7275a46e1f397445104e9c0b Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 13 Jul 2022 13:16:45 +0800 Subject: [PATCH 16/26] support handle optimization --- .../Storages/DeltaMerge/DMDecoratorStreams.h | 19 +++------------- .../DeltaMerge/File/DMFilePackFilter.h | 8 ------- .../Storages/DeltaMerge/File/DMFileReader.cpp | 22 +++++++------------ dbms/src/Storages/DeltaMerge/Segment.cpp | 2 +- ...est_dm_delta_merge_store_for_fast_mode.cpp | 13 +++++------ .../DeltaMerge/workload/DTWorkload.cpp | 2 +- 6 files changed, 18 insertions(+), 48 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h index 24a531f36db..2fac61c769f 100644 --- a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h +++ b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h @@ -18,13 +18,11 @@ #include #include #include -#include +#include #include -#include "common/types.h" - -#include +#include "common/types.h" namespace DB @@ -70,21 +68,10 @@ class DMDeleteFilterBlockInputStream : public IBlockInputStream return {}; if (!block.rows()) continue; - - /// if the pack is in clean read, it's del column return a const column, with size 1. - /// In this case, all the del_mark must be 0. Thus we don't need extra filter. - if (block.getByPosition(delete_col_pos).column->isColumnConst()) { - //std::cout << "del optimization" << std::endl; - ++total_blocks; - ++complete_passed; - total_rows += block.rows(); - passed_rows += block.rows(); - return getNewBlockByHeader(header, block); - } delete_col_data = getColumnVectorDataPtr(block, delete_col_pos); - + size_t rows = block.rows(); delete_filter.resize(rows); diff --git a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.h b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.h index 8d2806bae00..554ecc34b01 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFilePackFilter.h @@ -84,14 +84,6 @@ class DMFilePackFilter return minmax_index->getUInt64MinMax(pack_id).second; } - UInt64 getMaxDel(size_t pack_id) - { - if (!param.indexes.count(TAG_COLUMN_ID)) - tryLoadIndex(TAG_COLUMN_ID); - auto & minmax_index = param.indexes.find(TAG_COLUMN_ID)->second.minmax; - return minmax_index->getUInt64MinMax(pack_id).second; - } - // Get valid rows and bytes after filter invalid packs by handle_range and filter std::pair validRowsAndBytes() { diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp index 72786da3835..21a091b5072 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp @@ -24,6 +24,7 @@ #include #include #include + #include namespace CurrentMetrics @@ -321,10 +322,8 @@ Block DMFileReader::read() { if (read_pack_limit != 0 && next_pack_id - start_pack_id >= read_pack_limit) break; - // if (enable_clean_read && handle_res[next_pack_id] != expected_handle_res) - // break; - if (enable_clean_read && (handle_res[next_pack_id] != expected_handle_res || (is_raw_read && (pack_filter.getMaxDel(next_pack_id) != 0 || (next_pack_id > 0 && pack_filter.getMaxDel(next_pack_id - 1) != 0))))) - break; + if (enable_clean_read && handle_res[next_pack_id] != expected_handle_res) + break; read_rows += pack_stats[next_pack_id].rows; not_clean_rows += pack_stats[next_pack_id].not_clean; @@ -347,8 +346,6 @@ Block DMFileReader::read() // when is in fast mode and all pack's max del index is 0, then we don't need to read del column bool do_clean_read_on_handle = is_raw_read && expected_handle_res == All; - bool do_clean_read_on_del = is_raw_read && pack_filter.getMaxDel(start_pack_id) == 0; - //bool do_clean_read_on_del = false; if (do_clean_read_on_normal_mode) { @@ -364,7 +361,8 @@ Block DMFileReader::read() { // For clean read of column pk, version, tag, instead of loading data from disk, just create placeholder column is OK. auto & cd = read_columns[i]; - if (cd.id == EXTRA_HANDLE_COLUMN_ID && do_clean_read_on_handle) { + if (cd.id == EXTRA_HANDLE_COLUMN_ID && do_clean_read_on_handle) + { // Return the first row's handle ColumnPtr column; if (is_common_handle) @@ -379,12 +377,8 @@ Block DMFileReader::read() } res.insert(ColumnWithTypeAndName{column, cd.type, cd.name, cd.id}); skip_packs_by_column[i] = read_packs; - } else if (cd.id == TAG_COLUMN_ID && do_clean_read_on_del) { - ColumnPtr column = cd.type->createColumnConst(read_rows, Field(static_cast(pack_stats[start_pack_id].first_tag))); - res.insert(ColumnWithTypeAndName{column, cd.type, cd.name, cd.id}); - - skip_packs_by_column[i] = read_packs; - } else if (do_clean_read_on_normal_mode && isExtraColumn(cd)) + } + else if (do_clean_read_on_normal_mode && isExtraColumn(cd)) { ColumnPtr column; if (cd.id == EXTRA_HANDLE_COLUMN_ID) @@ -464,7 +458,7 @@ Block DMFileReader::read() } // Cast column's data from DataType in disk to what we need now auto converted_column = convertColumnByColumnDefineIfNeed(data_type, std::move(result_column), cd); - + res.insert(ColumnWithTypeAndName{converted_column, cd.type, cd.name, cd.id}); } else diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index 74b5f39ddc9..57d189d7bab 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -511,7 +511,7 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, auto new_columns_to_read = std::make_shared(); new_columns_to_read->push_back(getExtraHandleColumnDefine(is_common_handle)); - + if (filter_delete_mark) { new_columns_to_read->push_back(getTagColumnDefine()); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp index fa332ddb971..fdea238d277 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp @@ -1504,7 +1504,7 @@ try store->mergeDeltaAll(*db_context); - // could do clean read with del optimization + // could do clean read with no optimization { const auto & columns = store->getTableColumns(); BlockInputStreamPtr in = store->read(*db_context, @@ -1551,11 +1551,12 @@ try store->mergeDeltaAll(*db_context); - // could do clean read with handle and del optimization + // could do clean read with handle optimization { const auto & columns = store->getTableColumns(); - ColumnDefines real_columns ; - for (auto & col : columns){ + ColumnDefines real_columns; + for (auto & coconst l : columns) + { if (col.name != EXTRA_HANDLE_COLUMN_NAME) { real_columns.emplace_back(col); @@ -1581,10 +1582,6 @@ try ASSERT_EQ(num_rows_read, num_rows_write - num_deleted_rows); } - - - - } CATCH } // namespace tests diff --git a/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp index 38643e2c588..43b9d37079f 100644 --- a/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp @@ -276,7 +276,7 @@ void DTWorkload::scanAll(ThreadStat & read_stat) const auto & columns = store->getTableColumns(); ColumnDefines real_columns; - for (const auto& col : columns) + for (const auto & col : columns) { if (col.name != EXTRA_HANDLE_COLUMN_NAME && col.name != VERSION_COLUMN_NAME && col.name != TAG_COLUMN_NAME) { From f6415308aecae0e724191467e914d2d86e35d62f Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 13 Jul 2022 13:17:45 +0800 Subject: [PATCH 17/26] hongyunyan_clean_read_base_on_fast_mode --- dbms/src/Storages/DeltaMerge/Segment.cpp | 2 +- .../tests/gtest_dm_delta_merge_store_for_fast_mode.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index 57d189d7bab..ca17d35f3d2 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -534,7 +534,7 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, /// when the pack is under totally data_ranges and has rows whose del_mark = 1 --> we don't need read handle_column/version_column /// others --> we don't need read version_column - /// Actually, we first ignore whether we need to read tag_column, just deal it the same with handle column + /// Considering the del min max index has some problem now, we first only handle with handle column. BlockInputStreamPtr stable_stream = segment_snap->stable->getInputStream( dm_context, diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp index fdea238d277..d0a438789c6 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp @@ -1555,7 +1555,7 @@ try { const auto & columns = store->getTableColumns(); ColumnDefines real_columns; - for (auto & coconst l : columns) + for (auto & col : columns) { if (col.name != EXTRA_HANDLE_COLUMN_NAME) { From 6648b6831729fa1a5195a70a04ff9e705bafbc21 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 13 Jul 2022 13:28:43 +0800 Subject: [PATCH 18/26] fix comment --- dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h | 3 +-- dbms/src/Storages/DeltaMerge/workload/Options.cpp | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h index 8b2879c713b..42784c29b8b 100644 --- a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h +++ b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h @@ -18,11 +18,10 @@ #include #include #include +#include #include -#include "common/logger_useful.h" - namespace DB { diff --git a/dbms/src/Storages/DeltaMerge/workload/Options.cpp b/dbms/src/Storages/DeltaMerge/workload/Options.cpp index f9c14af7c64..63734c5bec1 100644 --- a/dbms/src/Storages/DeltaMerge/workload/Options.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/Options.cpp @@ -178,6 +178,11 @@ std::pair WorkloadOptions::parseOptions(int argc, char * argv table_name = vm["table_name"].as(); is_fast_mode = vm["is_fast_mode"].as(); + if (is_fast_mode && verification) + { + return {false, fmt::format("When in_fast_mode, we should set verification as false")}; + } + return {true, toString()}; } From 1187896f6fe86f25668c4dcb15824f294d7ce47c Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 13 Jul 2022 14:12:40 +0800 Subject: [PATCH 19/26] fix bug --- dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h index 2fac61c769f..b6443462277 100644 --- a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h +++ b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h @@ -93,7 +93,7 @@ class DMDeleteFilterBlockInputStream : public IBlockInputStream } } - for (size_t i = 0; i < rows; ++i) + for (size_t i = batch_rows; i < rows; ++i) { delete_filter[i] = ((*delete_col_data)[i] == 0); } From 97e7191bff2a1ba773aa846ff5cf00873086f4cb Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 13 Jul 2022 14:24:05 +0800 Subject: [PATCH 20/26] fix bug --- dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp | 2 +- dbms/src/Storages/DeltaMerge/Segment.cpp | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp index 21a091b5072..b9603d232e4 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp @@ -345,7 +345,7 @@ Block DMFileReader::read() bool do_clean_read_on_normal_mode = enable_clean_read && expected_handle_res == All && not_clean_rows == 0 && (!is_raw_read); // when is in fast mode and all pack's max del index is 0, then we don't need to read del column - bool do_clean_read_on_handle = is_raw_read && expected_handle_res == All; + bool do_clean_read_on_handle = enable_clean_read && is_raw_read && expected_handle_res == All; if (do_clean_read_on_normal_mode) { diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index ca17d35f3d2..2fb091b930f 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -517,12 +517,16 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, new_columns_to_read->push_back(getTagColumnDefine()); } + bool enable_clean_read = filter_delete_mark; + for (const auto & c : columns_to_read) { if (c.id != EXTRA_HANDLE_COLUMN_ID) { if (!(filter_delete_mark && c.id == TAG_COLUMN_ID)) new_columns_to_read->push_back(c); + } else { + enable_clean_read = false; } } @@ -543,7 +547,7 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, filter, std::numeric_limits::max(), expected_block_size, - /* enable_clean_read */ filter_delete_mark, + /* enable_clean_read */ enable_clean_read, /* is_raw_read */ filter_delete_mark); BlockInputStreamPtr delta_stream = std::make_shared(dm_context, segment_snap->delta, new_columns_to_read, this->rowkey_range); From dd56b02bba7ccbb232a0cd9b4eb97e2a73881ced Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Thu, 14 Jul 2022 20:01:00 +0800 Subject: [PATCH 21/26] merge master --- .../Storages/DeltaMerge/DMDecoratorStreams.h | 7 +- .../Storages/DeltaMerge/DeltaMergeStore.cpp | 14 -- .../tests/gtest_dm_delta_merge_store.cpp | 10 +- ...est_dm_delta_merge_store_for_fast_mode.cpp | 149 ------------------ .../DeltaMerge/workload/DTWorkload.cpp | 14 +- 5 files changed, 9 insertions(+), 185 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h index 162d6f8cfb0..7c1feb6cfaf 100644 --- a/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h +++ b/dbms/src/Storages/DeltaMerge/DMDecoratorStreams.h @@ -22,8 +22,6 @@ #include -#include "common/types.h" - namespace DB { namespace DM @@ -81,8 +79,11 @@ class DMDeleteFilterBlockInputStream : public IBlockInputStream auto * delete_pos = const_cast(delete_col_data->data()); for (size_t i = 0; i < batch_rows; ++i) { - (*filter_pos) = (*delete_pos) == 0; + (*filter_pos) = !(*delete_pos); + ++filter_pos; + ++delete_pos; } + } for (size_t i = batch_rows; i < rows; ++i) { diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index ba112870af9..d34856ec32f 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -1180,13 +1180,8 @@ BlockInputStreams DeltaMergeStore::readRaw(const Context & db_context, EMPTY_FILTER, std::numeric_limits::max(), DEFAULT_BLOCK_SIZE, -<<<<<<< HEAD - true, - false, -======= /* is_raw_ */ true, /* do_delete_mark_filter_for_raw_ */ false, // don't do filter based on del_mark = 1 ->>>>>>> 2bbe02bb41d557cd5cf297cc99a145071c17ccf8 extra_table_id_index, physical_table_id, req_info); @@ -1203,11 +1198,7 @@ BlockInputStreams DeltaMergeStore::read(const Context & db_context, UInt64 max_version, const RSOperatorPtr & filter, const String & tracing_id, -<<<<<<< HEAD - bool is_raw_read, -======= bool is_fast_mode, ->>>>>>> 2bbe02bb41d557cd5cf297cc99a145071c17ccf8 size_t expected_block_size, const SegmentIdSet & read_segments, size_t extra_table_id_index) @@ -1243,13 +1234,8 @@ BlockInputStreams DeltaMergeStore::read(const Context & db_context, filter, max_version, expected_block_size, -<<<<<<< HEAD - is_raw_read, - true, -======= /* is_raw_ */ is_fast_mode, /* do_delete_mark_filter_for_raw_ */ is_fast_mode, ->>>>>>> 2bbe02bb41d557cd5cf297cc99a145071c17ccf8 extra_table_id_index, physical_table_id, req_info); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp index ac1fb115c24..79038e1a349 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp @@ -3318,11 +3318,11 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, -<<<<<<< HEAD - /* is_raw_read= */ false, -======= /* is_fast_mode= */ false, + /* expected_block_size= */ 1024)[0]; + size_t num_rows_read = 0; while (Block block = in->read()) + { num_rows_read += block.rows(); for (auto && iter : block) { @@ -3392,11 +3392,7 @@ try /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, TRACING_NAME, -<<<<<<< HEAD - /* is_raw_read= */ false, -======= /* is_fast_mode= */ false, ->>>>>>> 2bbe02bb41d557cd5cf297cc99a145071c17ccf8 /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp index 205e069a0c4..3708d307ae8 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp @@ -930,154 +930,8 @@ try } CATCH -<<<<<<< HEAD // Insert + Delete TEST_P(DeltaMergeStoreRWTest, TestFastModeWithDelete) -======= - -TEST_P(DeltaMergeStoreRWTest, TestFastModeWithDeleteRow) -try -{ - const ColumnDefine col_str_define(2, "col2", std::make_shared()); - const ColumnDefine col_i8_define(3, "i8", std::make_shared()); - { - auto table_column_defines = DMTestEnv::getDefaultColumns(); - table_column_defines->emplace_back(col_str_define); - table_column_defines->emplace_back(col_i8_define); - - store = reload(table_column_defines); - } - - const size_t num_rows_write = 128; - { - Block block1; - { - block1 = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false); - // Add a column of col2:String for test - block1.insert(DB::tests::createColumn( - createNumberStrings(0, num_rows_write), - col_str_define.name, - col_str_define.id)); - // Add a column of i8:Int8 for test - block1.insert(DB::tests::createColumn( - createSignedNumbers(0, num_rows_write), - col_i8_define.name, - col_i8_define.id)); - } - - Block block2; - { - block2 = DMTestEnv::prepareSimpleWriteBlock(num_rows_write, 1.5 * num_rows_write, false, 3, DMTestEnv::pk_name, EXTRA_HANDLE_COLUMN_ID, EXTRA_HANDLE_COLUMN_INT_TYPE, false, 1, true, true); - // Add a column of col2:String for test - block2.insert(DB::tests::createColumn( - createNumberStrings(0.5 * num_rows_write, num_rows_write), - col_str_define.name, - col_str_define.id)); - // Add a column of i8:Int8 for test - block2.insert(DB::tests::createColumn( - createSignedNumbers(0.5 * num_rows_write, num_rows_write), - col_i8_define.name, - col_i8_define.id)); - } - - switch (mode) - { - case TestMode::V1_BlockOnly: - case TestMode::V2_BlockOnly: - store->write(*db_context, db_context->getSettingsRef(), block1); - store->write(*db_context, db_context->getSettingsRef(), block2); - break; - default: - auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); - auto [range1, file_ids1] = genDMFile(*dm_context, block1); - auto [range2, file_ids2] = genDMFile(*dm_context, block2); - auto range = range1.merge(range2); - auto file_ids = file_ids1; - file_ids.insert(file_ids.cend(), file_ids2.begin(), file_ids2.end()); - store->ingestFiles(dm_context, range, file_ids, false); - break; - } - } - - // Read after deletion - { - const auto & columns = store->getTableColumns(); - BlockInputStreamPtr in = store->read(*db_context, - db_context->getSettingsRef(), - columns, - {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, - /* num_streams= */ 1, - /* max_version= */ std::numeric_limits::max(), - EMPTY_FILTER, - TRACING_NAME, - /* is_raw_read= */ true, - /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - - // filter del mark = 1， thus just read the insert data before delete - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i); - } - } - } - } - - ASSERT_EQ(num_rows_read, num_rows_write); - } - - store->flushCache(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); - - store->compact(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); - - store->mergeDeltaAll(*db_context); - - { - const auto & columns = store->getTableColumns(); - BlockInputStreamPtr in = store->read(*db_context, - db_context->getSettingsRef(), - columns, - {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, - /* num_streams= */ 1, - /* max_version= */ std::numeric_limits::max(), - EMPTY_FILTER, - TRACING_NAME, - /* is_raw_read= */ true, - /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i); - } - } - } - } - - ASSERT_EQ(num_rows_read, num_rows_write); - } -} -CATCH - -// Insert + Delete Range -TEST_P(DeltaMergeStoreRWTest, TestFastModeWithDeleteRange) ->>>>>>> 2bbe02bb41d557cd5cf297cc99a145071c17ccf8 try { const size_t num_rows_write = 128; @@ -1460,7 +1314,6 @@ try } CATCH -<<<<<<< HEAD TEST_P(DeltaMergeStoreRWTest, TestFastModeForCleanRead) try { @@ -1571,8 +1424,6 @@ try } } CATCH -======= ->>>>>>> 2bbe02bb41d557cd5cf297cc99a145071c17ccf8 } // namespace tests } // namespace DM } // namespace DB \ No newline at end of file diff --git a/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp index 43b9d37079f..2282186e234 100644 --- a/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp @@ -274,16 +274,6 @@ void DTWorkload::scanAll(ThreadStat & read_stat) while (writing_threads.load(std::memory_order_relaxed) > 0) { const auto & columns = store->getTableColumns(); - ColumnDefines real_columns; - - for (const auto & col : columns) - { - if (col.name != EXTRA_HANDLE_COLUMN_NAME && col.name != VERSION_COLUMN_NAME && col.name != TAG_COLUMN_NAME) - { - real_columns.emplace_back(col); - } - } - int stream_count = opts->read_stream_count; std::atomic read_count = 0; auto count_row = [&read_count](BlockInputStreamPtr in, [[maybe_unused]] uint64_t read_ts) { @@ -293,10 +283,10 @@ void DTWorkload::scanAll(ThreadStat & read_stat) } }; Stopwatch sw; - read(real_columns, stream_count, count_row); + read(columns, stream_count, count_row); read_stat.ms = sw.elapsedMilliseconds(); read_stat.count = read_count; - LOG_FMT_INFO(log, "scanAll: columns {} streams {} read_stat {}", real_columns.size(), stream_count, read_stat.toString()); + LOG_FMT_INFO(log, "scanAll: columns {} streams {} read_stat {}", columns.size(), stream_count, read_stat.toString()); } } catch (...) From 15ea2f4142bc7e54348ea17e9408ad96b8dbd876 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Thu, 14 Jul 2022 20:03:34 +0800 Subject: [PATCH 22/26] add test --- ...est_dm_delta_merge_store_for_fast_mode.cpp | 145 +++++++++++++++++- 1 file changed, 143 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp index 3708d307ae8..6da9800699b 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp @@ -930,8 +930,149 @@ try } CATCH -// Insert + Delete -TEST_P(DeltaMergeStoreRWTest, TestFastModeWithDelete) +// Insert + Delete row +TEST_P(DeltaMergeStoreRWTest, TestFastModeWithDeleteRow) +try +{ + const ColumnDefine col_str_define(2, "col2", std::make_shared()); + const ColumnDefine col_i8_define(3, "i8", std::make_shared()); + { + auto table_column_defines = DMTestEnv::getDefaultColumns(); + table_column_defines->emplace_back(col_str_define); + table_column_defines->emplace_back(col_i8_define); + + store = reload(table_column_defines); + } + + const size_t num_rows_write = 128; + { + Block block1; + { + block1 = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false); + // Add a column of col2:String for test + block1.insert(DB::tests::createColumn( + createNumberStrings(0, num_rows_write), + col_str_define.name, + col_str_define.id)); + // Add a column of i8:Int8 for test + block1.insert(DB::tests::createColumn( + createSignedNumbers(0, num_rows_write), + col_i8_define.name, + col_i8_define.id)); + } + + Block block2; + { + block2 = DMTestEnv::prepareSimpleWriteBlock(num_rows_write, 1.5 * num_rows_write, false, 3, DMTestEnv::pk_name, EXTRA_HANDLE_COLUMN_ID, EXTRA_HANDLE_COLUMN_INT_TYPE, false, 1, true, true); + // Add a column of col2:String for test + block2.insert(DB::tests::createColumn( + createNumberStrings(0.5 * num_rows_write, num_rows_write), + col_str_define.name, + col_str_define.id)); + // Add a column of i8:Int8 for test + block2.insert(DB::tests::createColumn( + createSignedNumbers(0.5 * num_rows_write, num_rows_write), + col_i8_define.name, + col_i8_define.id)); + } + + switch (mode) + { + case TestMode::V1_BlockOnly: + case TestMode::V2_BlockOnly: + store->write(*db_context, db_context->getSettingsRef(), block1); + store->write(*db_context, db_context->getSettingsRef(), block2); + break; + default: + auto dm_context = store->newDMContext(*db_context, db_context->getSettingsRef()); + auto [range1, file_ids1] = genDMFile(*dm_context, block1); + auto [range2, file_ids2] = genDMFile(*dm_context, block2); + auto range = range1.merge(range2); + auto file_ids = file_ids1; + file_ids.insert(file_ids.cend(), file_ids2.begin(), file_ids2.end()); + store->ingestFiles(dm_context, range, file_ids, false); + break; + } + } + + // Read after deletion + { + const auto & columns = store->getTableColumns(); + BlockInputStreamPtr in = store->read(*db_context, + db_context->getSettingsRef(), + columns, + {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ std::numeric_limits::max(), + EMPTY_FILTER, + TRACING_NAME, + /* is_raw_read= */ true, + /* expected_block_size= */ 1024)[0]; + size_t num_rows_read = 0; + + // filter del mark = 1， thus just read the insert data before delete + while (Block block = in->read()) + { + num_rows_read += block.rows(); + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + ASSERT_EQ(c->getInt(i), i); + } + } + } + } + + ASSERT_EQ(num_rows_read, num_rows_write); + } + + store->flushCache(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); + + store->compact(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); + + store->mergeDeltaAll(*db_context); + + { + const auto & columns = store->getTableColumns(); + BlockInputStreamPtr in = store->read(*db_context, + db_context->getSettingsRef(), + columns, + {RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())}, + /* num_streams= */ 1, + /* max_version= */ std::numeric_limits::max(), + EMPTY_FILTER, + TRACING_NAME, + /* is_raw_read= */ true, + /* expected_block_size= */ 1024)[0]; + size_t num_rows_read = 0; + + while (Block block = in->read()) + { + num_rows_read += block.rows(); + for (auto && iter : block) + { + auto c = iter.column; + for (Int64 i = 0; i < Int64(c->size()); ++i) + { + if (iter.name == DMTestEnv::pk_name) + { + ASSERT_EQ(c->getInt(i), i); + } + } + } + } + + ASSERT_EQ(num_rows_read, num_rows_write); + } +} +CATCH + +// Insert + Delete Range +TEST_P(DeltaMergeStoreRWTest, TestFastModeWithDeleteRange) try { const size_t num_rows_write = 128; From 620060f14e2076103ca7b6870957447d811d33a8 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Thu, 14 Jul 2022 20:06:38 +0800 Subject: [PATCH 23/26] fix format --- dbms/src/Storages/DeltaMerge/Segment.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index 74d0803e68e..0bb7c643db8 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -533,7 +533,9 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, { if (!(filter_delete_mark && c.id == TAG_COLUMN_ID)) new_columns_to_read->push_back(c); - } else { + } + else + { enable_clean_read = false; } } @@ -575,7 +577,6 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, } - BlockInputStreams streams; if (dm_context.read_delta_only) From d48eed7e73f1cf57444e3157d2567a14a00afe80 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Fri, 15 Jul 2022 08:58:42 +0800 Subject: [PATCH 24/26] change name --- .../DeltaMerge/File/DMFileBlockInputStream.cpp | 2 +- .../DeltaMerge/File/DMFileBlockInputStream.h | 15 +++++++++------ .../src/Storages/DeltaMerge/File/DMFileReader.cpp | 9 ++++----- dbms/src/Storages/DeltaMerge/File/DMFileReader.h | 4 ++-- dbms/src/Storages/DeltaMerge/Segment.cpp | 9 +++------ dbms/src/Storages/DeltaMerge/StableValueSpace.cpp | 4 ++-- dbms/src/Storages/DeltaMerge/StableValueSpace.h | 2 +- 7 files changed, 22 insertions(+), 23 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp index d25378c54cf..56b11b975a1 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp @@ -61,7 +61,7 @@ DMFileBlockInputStreamPtr DMFileBlockInputStreamBuilder::build(const DMFilePtr & read_columns, is_common_handle, enable_clean_read, - is_raw_read, + is_fast_mode, max_data_version, std::move(pack_filter), mark_cache, diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h b/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h index d96339e0f54..a7f2fe9d556 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h @@ -69,15 +69,18 @@ class DMFileBlockInputStreamBuilder // **** filters **** // - // Only set this param to true when - // 1. There is no delta. - // 2. You don't need pk, version and delete_tag columns + // Only set enable param to true when + // in normal mode: + // 1. There is no delta. + // 2. You don't need pk, version and delete_tag columns + // in fast mode: + // 1. You don't need pk columns // If you have no idea what it means, then simply set it to false. // `max_data_version_` is the MVCC filter version for reading. Used by clean read check - DMFileBlockInputStreamBuilder & enableCleanRead(bool enable, bool is_raw_read_, UInt64 max_data_version_) + DMFileBlockInputStreamBuilder & enableCleanRead(bool enable, bool is_fast_mode_, UInt64 max_data_version_) { enable_clean_read = enable; - is_raw_read = is_raw_read_; + is_fast_mode = is_fast_mode_; max_data_version = max_data_version_; return *this; } @@ -140,7 +143,7 @@ class DMFileBlockInputStreamBuilder // clean read bool enable_clean_read = false; - bool is_raw_read = false; + bool is_fast_mode = false; UInt64 max_data_version = std::numeric_limits::max(); // Rough set filter RSOperatorPtr rs_filter; diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp index b9603d232e4..9424a5063a4 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp @@ -212,7 +212,7 @@ DMFileReader::DMFileReader( bool is_common_handle_, // clean read bool enable_clean_read_, - bool is_raw_read_, + bool is_fast_mode_, UInt64 max_read_version_, // filters DMFilePackFilter && pack_filter_, @@ -233,7 +233,7 @@ DMFileReader::DMFileReader( , read_one_pack_every_time(read_one_pack_every_time_) , single_file_mode(dmfile_->isSingleFileMode()) , enable_clean_read(enable_clean_read_) - , is_raw_read(is_raw_read_) + , is_fast_mode(is_fast_mode_) , max_read_version(max_read_version_) , pack_filter(std::move(pack_filter_)) , skip_packs_by_column(read_columns.size(), 0) @@ -342,10 +342,9 @@ Block DMFileReader::read() } // TODO: this will need better algorithm: we should separate those packs which can and can not do clean read. - bool do_clean_read_on_normal_mode = enable_clean_read && expected_handle_res == All && not_clean_rows == 0 && (!is_raw_read); + bool do_clean_read_on_normal_mode = enable_clean_read && expected_handle_res == All && not_clean_rows == 0 && (!is_fast_mode); - // when is in fast mode and all pack's max del index is 0, then we don't need to read del column - bool do_clean_read_on_handle = enable_clean_read && is_raw_read && expected_handle_res == All; + bool do_clean_read_on_handle = enable_clean_read && is_fast_mode && expected_handle_res == All; if (do_clean_read_on_normal_mode) { diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileReader.h b/dbms/src/Storages/DeltaMerge/File/DMFileReader.h index 0b60ba78d78..c04c93871a7 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileReader.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFileReader.h @@ -76,7 +76,7 @@ class DMFileReader // 2. You don't need pk, version and delete_tag columns // If you have no idea what it means, then simply set it to false. bool enable_clean_read_, - bool is_raw_read_, + bool is_fast_mode_, // The the MVCC filter version. Used by clean read check. UInt64 max_read_version_, // filters @@ -126,7 +126,7 @@ class DMFileReader // In normal mode, if there is no delta for some packs in stable, we can try to do clean read. // In fast mode, we always try to do clean read. const bool enable_clean_read; - const bool is_raw_read; + const bool is_fast_mode; const UInt64 max_read_version; /// Filters diff --git a/dbms/src/Storages/DeltaMerge/Segment.cpp b/dbms/src/Storages/DeltaMerge/Segment.cpp index 0bb7c643db8..91d65b7fee2 100644 --- a/dbms/src/Storages/DeltaMerge/Segment.cpp +++ b/dbms/src/Storages/DeltaMerge/Segment.cpp @@ -540,15 +540,12 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, } } - /// Whether fast mode or raw read(selraw), if columns_to_read does not include EXTRA_HANDLE_COLUMN_ID, + /// when we read in fast mode, if columns_to_read does not include EXTRA_HANDLE_COLUMN_ID, /// we can try to use clean read to make optimization in stable part. - - /// Especially for fast mode, /// when the pack is under totally data_ranges and has no rows whose del_mark = 1 --> we don't need read handle_column/tag_column/version_column /// when the pack is under totally data_ranges and has rows whose del_mark = 1 --> we don't need read handle_column/version_column /// others --> we don't need read version_column - - /// Considering the del min max index has some problem now, we first only handle with handle column. + /// Considering the del min max index has some problem now, we first only optimize with handle column. BlockInputStreamPtr stable_stream = segment_snap->stable->getInputStream( dm_context, @@ -558,7 +555,7 @@ BlockInputStreamPtr Segment::getInputStreamRaw(const DMContext & dm_context, std::numeric_limits::max(), expected_block_size, /* enable_clean_read */ enable_clean_read, - /* is_raw_read */ filter_delete_mark); + /* is_fast_mode */ filter_delete_mark); BlockInputStreamPtr delta_stream = std::make_shared(dm_context, segment_snap->delta, new_columns_to_read, this->rowkey_range); diff --git a/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp b/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp index 7bce8a88156..4a6b653f3af 100644 --- a/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp +++ b/dbms/src/Storages/DeltaMerge/StableValueSpace.cpp @@ -329,7 +329,7 @@ StableValueSpace::Snapshot::getInputStream( UInt64 max_data_version, size_t expected_block_size, bool enable_clean_read, - bool is_raw_read) + bool is_fast_mode) { LOG_FMT_DEBUG(log, "max_data_version: {}, enable_clean_read: {}", max_data_version, enable_clean_read); SkippableBlockInputStreams streams; @@ -338,7 +338,7 @@ StableValueSpace::Snapshot::getInputStream( { DMFileBlockInputStreamBuilder builder(context.db_context); builder - .enableCleanRead(enable_clean_read, is_raw_read, max_data_version) + .enableCleanRead(enable_clean_read, is_fast_mode, max_data_version) .setRSOperator(filter) .setColumnCache(column_caches[i]) .setTracingID(context.tracing_id) diff --git a/dbms/src/Storages/DeltaMerge/StableValueSpace.h b/dbms/src/Storages/DeltaMerge/StableValueSpace.h index 194d61dc27c..542b8ea7a62 100644 --- a/dbms/src/Storages/DeltaMerge/StableValueSpace.h +++ b/dbms/src/Storages/DeltaMerge/StableValueSpace.h @@ -151,7 +151,7 @@ class StableValueSpace : public std::enable_shared_from_this UInt64 max_data_version, size_t expected_block_size, bool enable_clean_read, - bool is_raw_read = false); + bool is_fast_mode = false); RowsAndBytes getApproxRowsAndBytes(const DMContext & context, const RowKeyRange & range) const; From 242a4f8803397e58d40b407e852f54f0943bf380 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Fri, 15 Jul 2022 17:15:38 +0800 Subject: [PATCH 25/26] fix comments --- .../Storages/DeltaMerge/File/DMFileReader.cpp | 1 - ...est_dm_delta_merge_store_for_fast_mode.cpp | 45 ++++++++++++++----- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp index 9424a5063a4..779de36da63 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp +++ b/dbms/src/Storages/DeltaMerge/File/DMFileReader.cpp @@ -457,7 +457,6 @@ Block DMFileReader::read() } // Cast column's data from DataType in disk to what we need now auto converted_column = convertColumnByColumnDefineIfNeed(data_type, std::move(result_column), cd); - res.insert(ColumnWithTypeAndName{converted_column, cd.type, cd.name, cd.id}); } else diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp index 6da9800699b..479b6973101 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp @@ -290,7 +290,7 @@ try /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; - + in->readSuffix(); switch (mode) { case TestMode::V1_BlockOnly: @@ -364,7 +364,7 @@ try } } - + in->readSuffix(); ASSERT_EQ(num_rows_read, 3 * num_write_rows); } } @@ -436,7 +436,7 @@ try /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; - + in->readPrefix(); switch (mode) { case TestMode::V1_BlockOnly: @@ -509,8 +509,7 @@ try break; } } - - + in->readSuffix(); ASSERT_EQ(num_rows_read, 3 * num_write_rows); } } @@ -585,6 +584,7 @@ try /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; + in->readPrefix(); switch (mode) { case TestMode::V1_BlockOnly: @@ -630,6 +630,7 @@ try { int block_index = 0; int begin_value = 0; + while (Block block = in->read()) { if (block_index == 1) @@ -657,8 +658,7 @@ try break; } } - - + in->readSuffix(); ASSERT_EQ(num_rows_read, 3 * num_write_rows); } } @@ -735,6 +735,7 @@ try /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; + in->readPrefix(); while (Block block = in->read()) { num_rows_read += block.rows(); @@ -750,7 +751,7 @@ try } } } - + in->readSuffix(); ASSERT_EQ(num_rows_read, 3 * num_write_rows); } @@ -829,6 +830,7 @@ try /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; + in->readPrefix(); switch (mode) { case TestMode::V1_BlockOnly: @@ -866,6 +868,7 @@ try { auto block_index = 0; auto begin_value = 0; + while (Block block = in->read()) { if (block_index == 1) @@ -896,6 +899,7 @@ try { auto block_index = 0; auto begin_value = num_write_rows; + while (Block block = in->read()) { if (block_index == 1) @@ -920,11 +924,12 @@ try num_rows_read += block.rows(); block_index += 1; } + break; } } - + in->readSuffix(); ASSERT_EQ(num_rows_read, 3 * num_write_rows); } } @@ -1010,6 +1015,7 @@ try /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; + in->readPrefix(); // filter del mark = 1， thus just read the insert data before delete while (Block block = in->read()) { @@ -1026,7 +1032,7 @@ try } } } - + in->readSuffix(); ASSERT_EQ(num_rows_read, num_rows_write); } @@ -1050,6 +1056,7 @@ try /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; + in->readPrefix(); while (Block block = in->read()) { num_rows_read += block.rows(); @@ -1065,6 +1072,7 @@ try } } } + in->readSuffix(); ASSERT_EQ(num_rows_read, num_rows_write); } @@ -1109,6 +1117,7 @@ try /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; + in->readPrefix(); while (Block block = in->read()) { num_rows_read += block.rows(); @@ -1124,7 +1133,7 @@ try } } } - + in->readSuffix(); ASSERT_EQ(num_rows_read, num_rows_write); } // Delete range [0, 64) @@ -1149,6 +1158,7 @@ try size_t num_rows_read = 0; // filter del mark = 1， thus just read the insert data before delete + in->readPrefix(); while (Block block = in->read()) { num_rows_read += block.rows(); @@ -1164,6 +1174,7 @@ try } } } + in->readSuffix(); ASSERT_EQ(num_rows_read, num_rows_write); } @@ -1222,6 +1233,7 @@ try /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; + in->readPrefix(); while (Block block = in->read()) { num_rows_read += block.rows(); @@ -1237,6 +1249,7 @@ try } } } + in->readSuffix(); ASSERT_EQ(num_rows_read, num_rows_write - num_deleted_rows); } @@ -1322,6 +1335,7 @@ try /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; + in->readPrefix(); switch (mode) { case TestMode::V1_BlockOnly: @@ -1416,7 +1430,7 @@ try break; } } - + in->readSuffix(); ASSERT_EQ(num_rows_read, 3 * num_write_rows); } @@ -1434,6 +1448,8 @@ try /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; + + in->readPrefix(); while (Block block = in->read()) { num_rows_read += block.rows(); @@ -1449,6 +1465,7 @@ try } } } + in->readSuffix(); ASSERT_EQ(num_rows_read, 1.5 * num_write_rows); } @@ -1500,6 +1517,7 @@ try /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; + in->readPrefix(); while (Block block = in->read()) { num_rows_read += block.rows(); @@ -1515,6 +1533,7 @@ try } } } + in->readSuffix(); ASSERT_EQ(num_rows_read, num_rows_write); } @@ -1556,10 +1575,12 @@ try /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; + in->readPrefix(); while (Block block = in->read()) { num_rows_read += block.rows(); } + in->readSuffix(); ASSERT_EQ(num_rows_read, num_rows_write - num_deleted_rows); } From 974685426e306139b43981be465171e96d33f603 Mon Sep 17 00:00:00 2001 From: JaySon Date: Fri, 15 Jul 2022 18:29:02 +0800 Subject: [PATCH 26/26] Update dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp --- .../tests/gtest_dm_delta_merge_store_for_fast_mode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp index 479b6973101..360f3e96315 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp @@ -290,7 +290,7 @@ try /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; size_t num_rows_read = 0; - in->readSuffix(); + in->readPrefix(); switch (mode) { case TestMode::V1_BlockOnly: