From 2126ee1fccf69691acd43cb7d6db3b0e2b437584 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Tue, 12 Apr 2022 16:24:01 +0800 Subject: [PATCH 01/16] update. --- dbms/src/TestUtils/mockExecutor.cpp | 237 ++++++++++++++++++ dbms/src/TestUtils/mockExecutor.h | 84 +++++++ .../TestUtils/tests/gtest_mock_executors.cpp | 13 + 3 files changed, 334 insertions(+) create mode 100644 dbms/src/TestUtils/mockExecutor.cpp create mode 100644 dbms/src/TestUtils/mockExecutor.h create mode 100644 dbms/src/TestUtils/tests/gtest_mock_executors.cpp diff --git a/dbms/src/TestUtils/mockExecutor.cpp b/dbms/src/TestUtils/mockExecutor.cpp new file mode 100644 index 00000000000..33c5bee30bd --- /dev/null +++ b/dbms/src/TestUtils/mockExecutor.cpp @@ -0,0 +1,237 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace DB +{ +std::shared_ptr DAGRequestBuilder::build(Context & context) +{ + DAGProperties properties; + MPPInfo mpp_info(properties.start_ts, -1, -1, {}, {}); + std::shared_ptr dag_request_ptr = std::make_shared(); + tipb::DAGRequest & dag_request = *dag_request_ptr; + dag_request.set_time_zone_name(properties.tz_name); + dag_request.set_time_zone_offset(properties.tz_offset); + dag_request.set_flags(dag_request.flags() | (1u << 1u /* TRUNCATE_AS_WARNING */) | (1u << 6u /* OVERFLOW_AS_WARNING */)); + + if (properties.encode_type == "chunk") + dag_request.set_encode_type(tipb::EncodeType::TypeChunk); + else if (properties.encode_type == "chblock") + dag_request.set_encode_type(tipb::EncodeType::TypeCHBlock); + else + dag_request.set_encode_type(tipb::EncodeType::TypeDefault); + + for (size_t i = 0; i < root->output_schema.size(); ++i) + dag_request.add_output_offsets(i); + auto * root_tipb_executor = dag_request.mutable_root_executor(); + + root->toTiPBExecutor(root_tipb_executor, properties.collator, mpp_info, context); + return dag_request_ptr; +} + +DAGRequestBuilder & DAGRequestBuilder::mockTable(String db, String table, const std::vector> & columns) +{ + assert(!columns.empty()); + TableInfo table_info; + table_info.name = db + "." + table; + for (const auto & column : columns) + { + TiDB::ColumnInfo ret; + ret.tp = column.second; + ret.name = column.first; + table_info.columns.push_back(std::move(ret)); + } + String empty_alias; + root = compileTableScan(getExecutorIndex(), table_info, empty_alias, false); + return *this; +} + +DAGRequestBuilder & DAGRequestBuilder::mockTable(MockTableName name, const std::vector> & columns) +{ + return mockTable(name.first, name.second, columns); +} + +DAGRequestBuilder & DAGRequestBuilder::filter(ASTPtr filter_expr) +{ + assert(root); + root = compileSelection(root, getExecutorIndex(), filter_expr); + return *this; +} + +DAGRequestBuilder & DAGRequestBuilder::filter(AstExprBuilder filter_expr) +{ + assert(root); + root = compileSelection(root, getExecutorIndex(), filter_expr.build()); + return *this; +} + +DAGRequestBuilder & DAGRequestBuilder::limit(int limit) +{ + assert(root); + root = compileLimit(root, getExecutorIndex(), AstExprBuilder().appendLiteral(Field(static_cast(limit))).build()); + return *this; +} + + +DAGRequestBuilder & DAGRequestBuilder::topN(ASTPtr order_exprs, ASTPtr limit_expr) +{ + assert(root); + root = compileTopN(root, getExecutorIndex(), order_exprs, limit_expr); + return *this; +} + +DAGRequestBuilder & DAGRequestBuilder::topN(String col_name, bool desc, int limit) +{ + assert(root); + root = compileTopN(root, + getExecutorIndex(), + AstExprBuilder().appendOrderByItem(col_name, desc).appendList().build(), + AstExprBuilder().appendLiteral(Field(static_cast(limit))).build()); + return *this; +} + +DAGRequestBuilder & DAGRequestBuilder::topN(std::initializer_list cols, bool desc, int limit) +{ + assert(root); + auto exp_list = std::make_shared(); + AstExprBuilder expr_builder; + for (const auto & name : cols) + expr_builder.appendOrderByItem(name, desc); + + root = compileTopN(root, + getExecutorIndex(), + expr_builder.appendList().build(), + AstExprBuilder().appendLiteral(Field(static_cast(limit))).build()); + return *this; +} + + +DAGRequestBuilder & DAGRequestBuilder::project(String col_name) +{ + assert(root); + root = compileProject(root, getExecutorIndex(), AstExprBuilder().appendColumnRef(col_name).appendList().build()); + return *this; +} + +DAGRequestBuilder & DAGRequestBuilder::project(std::initializer_list cols) +{ + assert(root); + auto exp_list = std::make_shared(); + for (const auto & col : cols) + { + exp_list->children.push_back(col); + } + + root = compileProject(root, getExecutorIndex(), exp_list); + return *this; +} + +DAGRequestBuilder & DAGRequestBuilder::project(std::initializer_list cols) +{ + assert(root); + auto exp_list = std::make_shared(); + for (const auto & name : cols) + exp_list->children.push_back(COL(name)); + + root = compileProject(root, getExecutorIndex(), exp_list); + return *this; +} + +DAGRequestBuilder & DAGRequestBuilder::join(const DAGRequestBuilder & right, ASTPtr using_expr_list) +{ + assert(root); + assert(right.root); + auto join_ast = std::make_shared(); + join_ast->using_expression_list = using_expr_list; + join_ast->strictness = ASTTableJoin::Strictness::All; + root = compileJoin(getExecutorIndex(), root, right.root, join_ast); + return *this; +} + +AstExprBuilder & AstExprBuilder::appendColumnRef(const String & column_name) +{ + vec.push_back(std::make_shared(column_name)); + return *this; +} + +AstExprBuilder & AstExprBuilder::appendLiteral(const Field & field) +{ + vec.push_back(std::make_shared(field)); + return *this; +} + +AstExprBuilder & AstExprBuilder::appendOrderByItem(const String & column_name, bool asc) +{ + int direction = asc ? 1 : -1; + ASTPtr locale_node; + auto order_by_item = std::make_shared(direction, direction, false, locale_node); + order_by_item->children.push_back(std::make_shared(column_name)); + vec.push_back(order_by_item); + return *this; +} + +AstExprBuilder & AstExprBuilder::appendList() +{ + auto exp_list = std::make_shared(); + for (const auto & ast_ptr : vec) + exp_list->children.push_back(ast_ptr); + vec.clear(); + vec.push_back(exp_list); + return *this; +} + +AstExprBuilder & AstExprBuilder::appendAlias(String alias) +{ + assert(vec.size() == 1); + auto i = std::make_shared(alias); + i->children.push_back(vec.back()); + vec.clear(); + return *this; +} + +AstExprBuilder & AstExprBuilder::appendFunction(const String & func_name) +{ + appendList(); + auto func = std::make_shared(); + func->name = func_name; + ASTPtr exp_list = vec.back(); + func->arguments = exp_list; + func->children.push_back(func->arguments); + vec.clear(); + vec.push_back(func); + return *this; +} + +ASTPtr AstExprBuilder::build() +{ + assert(vec.size() == 1); + ASTPtr ret = vec.back(); + vec.clear(); + return ret; +} + +ASTPtr buildFunction(std::initializer_list exprs, String name) +{ + auto func = std::make_shared(); + func->name = name; + auto expr_list = std::make_shared(); + for (const auto & expr : exprs) + expr_list->children.push_back(expr); + func->arguments = expr_list; + func->children.push_back(func->arguments); + return func; +} + +} // namespace DB \ No newline at end of file diff --git a/dbms/src/TestUtils/mockExecutor.h b/dbms/src/TestUtils/mockExecutor.h new file mode 100644 index 00000000000..277b4d7620f --- /dev/null +++ b/dbms/src/TestUtils/mockExecutor.h @@ -0,0 +1,84 @@ +#pragma once + +#include +#include + +#include +#include + +namespace DB +{ +// +using MockColumnInfo = std::pair; +using MockTableName = std::pair; +class AstExprBuilder; + + +class DAGRequestBuilder +{ +public: + static size_t & getExecutorIndex() + { + thread_local size_t executor_index = 0; + return executor_index; + } + + std::shared_ptr build(Context & context); + + DAGRequestBuilder & mockTable(String db, String table, const std::vector & columns); + DAGRequestBuilder & mockTable(MockTableName name, const std::vector & columns); + + DAGRequestBuilder & filter(AstExprBuilder filter_expr); + DAGRequestBuilder & filter(ASTPtr filter_expr); + + DAGRequestBuilder & limit(int limit); + + DAGRequestBuilder & topN(ASTPtr order_exprs, ASTPtr limit_expr); + DAGRequestBuilder & topN(String col_name, bool desc, int limit); + DAGRequestBuilder & topN(std::initializer_list cols, bool desc, int limit); + + DAGRequestBuilder & project(String col_name); + DAGRequestBuilder & project(std::initializer_list cols); + DAGRequestBuilder & project(std::initializer_list cols); + + // only support inner join + // TODO support more joins + DAGRequestBuilder & join(const DAGRequestBuilder & right, ASTPtr using_expr_list); + +private: + ExecutorPtr root; +}; + +class AstExprBuilder +{ +public: + AstExprBuilder & appendColumnRef(const String & column_name); + + AstExprBuilder & appendLiteral(const Field & field); + + AstExprBuilder & appendOrderByItem(const String & column_name, bool asc); + + AstExprBuilder & appendList(); + + AstExprBuilder & appendAlias(String alias); + + AstExprBuilder & appendFunction(const String & func_name); + + ASTPtr build(); + +private: + std::vector vec; +}; + +ASTPtr buildFunction(std::initializer_list exprs, String name); + +#define COL(name) AstExprBuilder().appendColumnRef((name)).build() +#define EQ(expr1, expr2) buildFunction({(expr1), (expr2)}, "equals") +#define NOT_EQ(expr1, expr2) buildFunction({(expr1), (expr2)}, "notEquals") +#define lt(expr1, expr2) buildFunction({(expr1), (expr2)}, "less") +#define gt(expr1, expr2) buildFunction({(expr1), (expr2)}, "greater") +#define And(expr1, expr2) buildFunction({(expr1), (expr2)}, "and") +#define Or(expr1, expr2) buildFunction({(expr1), (expr2)}, "or") +#define NOT(expr) buildFunction({expr1}, "not") + +} // namespace DB \ No newline at end of file diff --git a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp new file mode 100644 index 00000000000..da46cbb91ee --- /dev/null +++ b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp @@ -0,0 +1,13 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. From a152bae37233d1ac7c2468561eaa7069f89dc13e Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Tue, 12 Apr 2022 18:04:26 +0800 Subject: [PATCH 02/16] update. --- dbms/src/TestUtils/mockExecutor.cpp | 175 +++++++----------- dbms/src/TestUtils/mockExecutor.h | 49 ++--- .../TestUtils/tests/gtest_mock_executors.cpp | 160 ++++++++++++++++ 3 files changed, 245 insertions(+), 139 deletions(-) diff --git a/dbms/src/TestUtils/mockExecutor.cpp b/dbms/src/TestUtils/mockExecutor.cpp index 33c5bee30bd..55b3787a14b 100644 --- a/dbms/src/TestUtils/mockExecutor.cpp +++ b/dbms/src/TestUtils/mockExecutor.cpp @@ -16,12 +16,47 @@ namespace DB { -std::shared_ptr DAGRequestBuilder::build(Context & context) +ASTPtr buildColumn(const String & column_name) +{ + return std::make_shared(column_name); +} + +ASTPtr buildLiteral(const Field & field) +{ + return std::make_shared(field); +} + +ASTPtr buildFunction(std::initializer_list exprs, const String & name) +{ + auto func = std::make_shared(); + func->name = name; + auto expr_list = std::make_shared(); + for (const auto & expr : exprs) + expr_list->children.push_back(expr); + func->arguments = expr_list; + func->children.push_back(func->arguments); + return func; +} + +ASTPtr buildOrderByItemList(std::initializer_list> order_by_items) +{ + std::vector vec; + for (auto item: order_by_items) + { + int direction = item.second ? 1 : -1; + ASTPtr locale_node; + auto order_by_item = std::make_shared(direction, direction, false, locale_node); + order_by_item->children.push_back(std::make_shared(item.first)); + vec.push_back(order_by_item); + } + auto exp_list = std::make_shared(); + for (const auto & ast_ptr : vec) + exp_list->children.push_back(ast_ptr); + return exp_list; +} + +void DAGRequestBuilder::initDAGRequest(tipb::DAGRequest & dag_request) { - DAGProperties properties; - MPPInfo mpp_info(properties.start_ts, -1, -1, {}, {}); - std::shared_ptr dag_request_ptr = std::make_shared(); - tipb::DAGRequest & dag_request = *dag_request_ptr; dag_request.set_time_zone_name(properties.tz_name); dag_request.set_time_zone_offset(properties.tz_offset); dag_request.set_flags(dag_request.flags() | (1u << 1u /* TRUNCATE_AS_WARNING */) | (1u << 6u /* OVERFLOW_AS_WARNING */)); @@ -35,9 +70,16 @@ std::shared_ptr DAGRequestBuilder::build(Context & context) for (size_t i = 0; i < root->output_schema.size(); ++i) dag_request.add_output_offsets(i); - auto * root_tipb_executor = dag_request.mutable_root_executor(); +} + +std::shared_ptr DAGRequestBuilder::build(Context & context) +{ + MPPInfo mpp_info(properties.start_ts, -1, -1, {}, {}); + std::shared_ptr dag_request_ptr = std::make_shared(); + tipb::DAGRequest & dag_request = *dag_request_ptr; + initDAGRequest(dag_request); + root->toTiPBExecutor(dag_request.mutable_root_executor(), properties.collator, mpp_info, context); - root->toTiPBExecutor(root_tipb_executor, properties.collator, mpp_info, context); return dag_request_ptr; } @@ -70,21 +112,20 @@ DAGRequestBuilder & DAGRequestBuilder::filter(ASTPtr filter_expr) return *this; } -DAGRequestBuilder & DAGRequestBuilder::filter(AstExprBuilder filter_expr) +DAGRequestBuilder & DAGRequestBuilder::limit(int limit) { assert(root); - root = compileSelection(root, getExecutorIndex(), filter_expr.build()); + root = compileLimit(root, getExecutorIndex(), buildLiteral(Field(static_cast(limit)))); return *this; } -DAGRequestBuilder & DAGRequestBuilder::limit(int limit) +DAGRequestBuilder & DAGRequestBuilder::limit(ASTPtr limit_expr) { assert(root); - root = compileLimit(root, getExecutorIndex(), AstExprBuilder().appendLiteral(Field(static_cast(limit))).build()); + root = compileLimit(root, getExecutorIndex(), limit_expr); return *this; } - DAGRequestBuilder & DAGRequestBuilder::topN(ASTPtr order_exprs, ASTPtr limit_expr) { assert(root); @@ -92,36 +133,29 @@ DAGRequestBuilder & DAGRequestBuilder::topN(ASTPtr order_exprs, ASTPtr limit_exp return *this; } -DAGRequestBuilder & DAGRequestBuilder::topN(String col_name, bool desc, int limit) +DAGRequestBuilder & DAGRequestBuilder::topN(const String & col_name, bool desc, int limit) { assert(root); - root = compileTopN(root, - getExecutorIndex(), - AstExprBuilder().appendOrderByItem(col_name, desc).appendList().build(), - AstExprBuilder().appendLiteral(Field(static_cast(limit))).build()); + root = compileTopN(root, getExecutorIndex(), buildOrderByItemList({{col_name, desc}}), buildLiteral(Field(static_cast(limit)))); return *this; } -DAGRequestBuilder & DAGRequestBuilder::topN(std::initializer_list cols, bool desc, int limit) +DAGRequestBuilder & DAGRequestBuilder::topN(MockOrderByItems order_by_items, int limit) +{ + return topN(order_by_items, buildLiteral(Field(static_cast(limit)))); +} + +DAGRequestBuilder & DAGRequestBuilder::topN(MockOrderByItems order_by_items, ASTPtr limit_expr) { assert(root); - auto exp_list = std::make_shared(); - AstExprBuilder expr_builder; - for (const auto & name : cols) - expr_builder.appendOrderByItem(name, desc); - - root = compileTopN(root, - getExecutorIndex(), - expr_builder.appendList().build(), - AstExprBuilder().appendLiteral(Field(static_cast(limit))).build()); + root = compileTopN(root, getExecutorIndex(), buildOrderByItemList(order_by_items), limit_expr); return *this; } - DAGRequestBuilder & DAGRequestBuilder::project(String col_name) { assert(root); - root = compileProject(root, getExecutorIndex(), AstExprBuilder().appendColumnRef(col_name).appendList().build()); + root = compileProject(root, getExecutorIndex(), buildColumn(col_name)); return *this; } @@ -130,20 +164,17 @@ DAGRequestBuilder & DAGRequestBuilder::project(std::initializer_list col assert(root); auto exp_list = std::make_shared(); for (const auto & col : cols) - { exp_list->children.push_back(col); - } - root = compileProject(root, getExecutorIndex(), exp_list); return *this; } -DAGRequestBuilder & DAGRequestBuilder::project(std::initializer_list cols) +DAGRequestBuilder & DAGRequestBuilder::project(MockColumnNames col_names) { assert(root); auto exp_list = std::make_shared(); - for (const auto & name : cols) - exp_list->children.push_back(COL(name)); + for (const auto & name : col_names) + exp_list->children.push_back(col(name)); root = compileProject(root, getExecutorIndex(), exp_list); return *this; @@ -160,78 +191,4 @@ DAGRequestBuilder & DAGRequestBuilder::join(const DAGRequestBuilder & right, AST return *this; } -AstExprBuilder & AstExprBuilder::appendColumnRef(const String & column_name) -{ - vec.push_back(std::make_shared(column_name)); - return *this; -} - -AstExprBuilder & AstExprBuilder::appendLiteral(const Field & field) -{ - vec.push_back(std::make_shared(field)); - return *this; -} - -AstExprBuilder & AstExprBuilder::appendOrderByItem(const String & column_name, bool asc) -{ - int direction = asc ? 1 : -1; - ASTPtr locale_node; - auto order_by_item = std::make_shared(direction, direction, false, locale_node); - order_by_item->children.push_back(std::make_shared(column_name)); - vec.push_back(order_by_item); - return *this; -} - -AstExprBuilder & AstExprBuilder::appendList() -{ - auto exp_list = std::make_shared(); - for (const auto & ast_ptr : vec) - exp_list->children.push_back(ast_ptr); - vec.clear(); - vec.push_back(exp_list); - return *this; -} - -AstExprBuilder & AstExprBuilder::appendAlias(String alias) -{ - assert(vec.size() == 1); - auto i = std::make_shared(alias); - i->children.push_back(vec.back()); - vec.clear(); - return *this; -} - -AstExprBuilder & AstExprBuilder::appendFunction(const String & func_name) -{ - appendList(); - auto func = std::make_shared(); - func->name = func_name; - ASTPtr exp_list = vec.back(); - func->arguments = exp_list; - func->children.push_back(func->arguments); - vec.clear(); - vec.push_back(func); - return *this; -} - -ASTPtr AstExprBuilder::build() -{ - assert(vec.size() == 1); - ASTPtr ret = vec.back(); - vec.clear(); - return ret; -} - -ASTPtr buildFunction(std::initializer_list exprs, String name) -{ - auto func = std::make_shared(); - func->name = name; - auto expr_list = std::make_shared(); - for (const auto & expr : exprs) - expr_list->children.push_back(expr); - func->arguments = expr_list; - func->children.push_back(func->arguments); - return func; -} - } // namespace DB \ No newline at end of file diff --git a/dbms/src/TestUtils/mockExecutor.h b/dbms/src/TestUtils/mockExecutor.h index 277b4d7620f..7b76754a18a 100644 --- a/dbms/src/TestUtils/mockExecutor.h +++ b/dbms/src/TestUtils/mockExecutor.h @@ -11,8 +11,9 @@ namespace DB // using MockColumnInfo = std::pair; using MockTableName = std::pair; -class AstExprBuilder; - +using MockOrderByItem = std::pair; +using MockOrderByItems = std::initializer_list; +using MockColumnNames = std::initializer_list; class DAGRequestBuilder { @@ -28,53 +29,41 @@ class DAGRequestBuilder DAGRequestBuilder & mockTable(String db, String table, const std::vector & columns); DAGRequestBuilder & mockTable(MockTableName name, const std::vector & columns); - DAGRequestBuilder & filter(AstExprBuilder filter_expr); DAGRequestBuilder & filter(ASTPtr filter_expr); DAGRequestBuilder & limit(int limit); + DAGRequestBuilder & limit(ASTPtr limit_expr); DAGRequestBuilder & topN(ASTPtr order_exprs, ASTPtr limit_expr); - DAGRequestBuilder & topN(String col_name, bool desc, int limit); - DAGRequestBuilder & topN(std::initializer_list cols, bool desc, int limit); + DAGRequestBuilder & topN(const String & col_name, bool desc, int limit); + DAGRequestBuilder & topN(MockOrderByItems order_by_items, int limit); + DAGRequestBuilder & topN(MockOrderByItems order_by_items, ASTPtr limit_expr); DAGRequestBuilder & project(String col_name); DAGRequestBuilder & project(std::initializer_list cols); - DAGRequestBuilder & project(std::initializer_list cols); + DAGRequestBuilder & project(MockColumnNames col_names); // only support inner join // TODO support more joins DAGRequestBuilder & join(const DAGRequestBuilder & right, ASTPtr using_expr_list); private: + void initDAGRequest(tipb::DAGRequest & dag_request); + ExecutorPtr root; + DAGProperties properties; }; -class AstExprBuilder -{ -public: - AstExprBuilder & appendColumnRef(const String & column_name); - - AstExprBuilder & appendLiteral(const Field & field); - - AstExprBuilder & appendOrderByItem(const String & column_name, bool asc); - - AstExprBuilder & appendList(); - - AstExprBuilder & appendAlias(String alias); - - AstExprBuilder & appendFunction(const String & func_name); - - ASTPtr build(); - -private: - std::vector vec; -}; +ASTPtr buildColumn(const String & column_name); +ASTPtr buildLiteral(const Field & field); +ASTPtr buildFunction(std::initializer_list exprs, const String & name); +ASTPtr buildOrderByItemList(std::initializer_list> order_by_items); -ASTPtr buildFunction(std::initializer_list exprs, String name); -#define COL(name) AstExprBuilder().appendColumnRef((name)).build() -#define EQ(expr1, expr2) buildFunction({(expr1), (expr2)}, "equals") -#define NOT_EQ(expr1, expr2) buildFunction({(expr1), (expr2)}, "notEquals") +#define col(name) buildColumn((name)) +#define lit(field) buildLiteral((field)) +#define eq(expr1, expr2) buildFunction({(expr1), (expr2)}, "equals") +#define Not_eq(expr1, expr2) buildFunction({(expr1), (expr2)}, "notEquals") #define lt(expr1, expr2) buildFunction({(expr1), (expr2)}, "less") #define gt(expr1, expr2) buildFunction({(expr1), (expr2)}, "greater") #define And(expr1, expr2) buildFunction({(expr1), (expr2)}, "and") diff --git a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp index da46cbb91ee..051bc76f986 100644 --- a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp +++ b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp @@ -11,3 +11,163 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace tests +{ + + namespace +{ +String toTreeString(const tipb::Executor & root_executor, size_t level = 0); + +String toTreeString(const tipb::DAGRequest * dag_request) +{ + assert((dag_request->executors_size() > 0) != dag_request->has_root_executor()); + if (dag_request->has_root_executor()) + { + return toTreeString(dag_request->root_executor()); + } + else + { + FmtBuffer buffer; + String prefix; + traverseExecutors(dag_request, [&buffer, &prefix](const tipb::Executor & executor) { + assert(executor.has_executor_id()); + buffer.fmtAppend("{}{}\n", prefix, executor.executor_id()); + prefix.append(" "); + return true; + }); + return buffer.toString(); + } +} + +String toTreeString(const tipb::Executor & root_executor, size_t level) +{ + FmtBuffer buffer; + + auto append_str = [&buffer, &level](const tipb::Executor & executor) { + assert(executor.has_executor_id()); + for (size_t i = 0; i < level; ++i) + buffer.append(" "); + buffer.append(executor.executor_id()).append("\n"); + }; + + traverseExecutorTree(root_executor, [&](const tipb::Executor & executor) { + if (executor.has_join()) + { + for (const auto & child : executor.join().children()) + buffer.append(toTreeString(child, level)); + return false; + } + else + { + append_str(executor); + ++level; + return true; + } + }); + + return buffer.toString(); +} + +// String & trim(String & str) +// { +// if (str.empty()) +// { +// return str; +// } + +// str.erase(0, str.find_first_not_of(' ')); +// str.erase(str.find_last_not_of(' ') + 1); +// return str; +// } +} // namespace +class MockTiPBDAGRequestTest : public DB::tests::FunctionTest +{ +}; + +TEST_F(MockTiPBDAGRequestTest, Test1) +try +{ + String empty_str; + + MockTableName right_table{"r_db", "r_table"}; + std::vector r_columns; + r_columns.emplace_back("l_a", TiDB::TP::TypeString); + r_columns.emplace_back("l_b", TiDB::TP::TypeString); + + MockTableName left_table{"l_db", "l_table"}; + std::vector l_columns; + l_columns.emplace_back("l_a", TiDB::TP::TypeString); + l_columns.emplace_back("l_b", TiDB::TP::TypeLong); + + DAGRequestBuilder right_builder; + right_builder + .mockTable(right_table, r_columns) + .filter(eq(col("l_a"), col("l_b"))) + .project({col("l_a")}) + .topN("l_a", false, 20); + + + DAGRequestBuilder left_builder; + left_builder + .mockTable(left_table, l_columns) + .topN({{"l_a",false}}, 10) + .join(right_builder, col("l_a")) // ywq todo more types of join, should make sure have both field + .limit(10); + + auto dag_request = left_builder.build(context); + String to_tree_string = toTreeString(dag_request.get()); + std::cout << to_tree_string << std::endl; +} +CATCH + +TEST_F(MockTiPBDAGRequestTest, Test2) +try +{ + String empty_str; + + MockTableName right_table{"r_db", "r_table"}; + std::vector r_columns; + r_columns.emplace_back("l_a", TiDB::TP::TypeString); + r_columns.emplace_back("l_b", TiDB::TP::TypeString); + + MockTableName left_table{"l_db", "l_table"}; + std::vector l_columns; + l_columns.emplace_back("l_a", TiDB::TP::TypeString); + l_columns.emplace_back("l_b", TiDB::TP::TypeLong); + + DAGRequestBuilder right_builder; + right_builder + .mockTable(right_table, r_columns) + .filter(eq(col("l_a"), col("l_b"))) + .project({col("l_a")}) + .topN("l_a", false, 20); + + + DAGRequestBuilder left_builder; + left_builder + .mockTable(left_table, l_columns) + .topN("l_a", false, 10) + .join(right_builder, col("l_a")) + .limit(10); + + auto dag_request = left_builder.build(context); + String to_tree_string = toTreeString(dag_request.get()); + std::cout << to_tree_string << std::endl; + +} +CATCH + + +} // namespace tests +} // namespace DB \ No newline at end of file From 4830f5a2c2bc0c1669c586283eaacc61a27f67b2 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Tue, 12 Apr 2022 18:04:50 +0800 Subject: [PATCH 03/16] format. --- dbms/src/TestUtils/mockExecutor.cpp | 4 ++-- dbms/src/TestUtils/tests/gtest_mock_executors.cpp | 8 +++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/dbms/src/TestUtils/mockExecutor.cpp b/dbms/src/TestUtils/mockExecutor.cpp index 55b3787a14b..c5269317cbd 100644 --- a/dbms/src/TestUtils/mockExecutor.cpp +++ b/dbms/src/TestUtils/mockExecutor.cpp @@ -41,12 +41,12 @@ ASTPtr buildFunction(std::initializer_list exprs, const String & name) ASTPtr buildOrderByItemList(std::initializer_list> order_by_items) { std::vector vec; - for (auto item: order_by_items) + for (auto item : order_by_items) { int direction = item.second ? 1 : -1; ASTPtr locale_node; auto order_by_item = std::make_shared(direction, direction, false, locale_node); - order_by_item->children.push_back(std::make_shared(item.first)); + order_by_item->children.push_back(std::make_shared(item.first)); vec.push_back(order_by_item); } auto exp_list = std::make_shared(); diff --git a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp index 051bc76f986..12d2df79534 100644 --- a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp +++ b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp @@ -12,20 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. #include -#include #include #include #include #include #include #include +#include namespace DB { namespace tests { - - namespace +namespace { String toTreeString(const tipb::Executor & root_executor, size_t level = 0); @@ -121,7 +120,7 @@ try DAGRequestBuilder left_builder; left_builder .mockTable(left_table, l_columns) - .topN({{"l_a",false}}, 10) + .topN({{"l_a", false}}, 10) .join(right_builder, col("l_a")) // ywq todo more types of join, should make sure have both field .limit(10); @@ -164,7 +163,6 @@ try auto dag_request = left_builder.build(context); String to_tree_string = toTreeString(dag_request.get()); std::cout << to_tree_string << std::endl; - } CATCH From 5b326f8ff1a2979f65e7fafbd177d03a5212bebf Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Tue, 12 Apr 2022 18:16:25 +0800 Subject: [PATCH 04/16] update. --- dbms/src/TestUtils/mockExecutor.cpp | 17 ++++++++--------- dbms/src/TestUtils/mockExecutor.h | 14 ++++++++------ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/dbms/src/TestUtils/mockExecutor.cpp b/dbms/src/TestUtils/mockExecutor.cpp index c5269317cbd..f7289fa9906 100644 --- a/dbms/src/TestUtils/mockExecutor.cpp +++ b/dbms/src/TestUtils/mockExecutor.cpp @@ -11,7 +11,6 @@ #include #include -#include #include namespace DB @@ -26,7 +25,7 @@ ASTPtr buildLiteral(const Field & field) return std::make_shared(field); } -ASTPtr buildFunction(std::initializer_list exprs, const String & name) +ASTPtr buildFunction(MockAsts exprs, const String & name) { auto func = std::make_shared(); func->name = name; @@ -38,7 +37,7 @@ ASTPtr buildFunction(std::initializer_list exprs, const String & name) return func; } -ASTPtr buildOrderByItemList(std::initializer_list> order_by_items) +ASTPtr buildOrderByItemList(MockOrderByItems order_by_items) { std::vector vec; for (auto item : order_by_items) @@ -83,7 +82,7 @@ std::shared_ptr DAGRequestBuilder::build(Context & context) return dag_request_ptr; } -DAGRequestBuilder & DAGRequestBuilder::mockTable(String db, String table, const std::vector> & columns) +DAGRequestBuilder & DAGRequestBuilder::mockTable(const String & db, const String & table, const MockColumnInfos & columns) { assert(!columns.empty()); TableInfo table_info; @@ -100,7 +99,7 @@ DAGRequestBuilder & DAGRequestBuilder::mockTable(String db, String table, const return *this; } -DAGRequestBuilder & DAGRequestBuilder::mockTable(MockTableName name, const std::vector> & columns) +DAGRequestBuilder & DAGRequestBuilder::mockTable(const MockTableName & name, const std::vector> & columns) { return mockTable(name.first, name.second, columns); } @@ -152,19 +151,19 @@ DAGRequestBuilder & DAGRequestBuilder::topN(MockOrderByItems order_by_items, AST return *this; } -DAGRequestBuilder & DAGRequestBuilder::project(String col_name) +DAGRequestBuilder & DAGRequestBuilder::project(const String & col_name) { assert(root); root = compileProject(root, getExecutorIndex(), buildColumn(col_name)); return *this; } -DAGRequestBuilder & DAGRequestBuilder::project(std::initializer_list cols) +DAGRequestBuilder & DAGRequestBuilder::project(MockAsts exprs) { assert(root); auto exp_list = std::make_shared(); - for (const auto & col : cols) - exp_list->children.push_back(col); + for (const auto & expr : exprs) + exp_list->children.push_back(expr); root = compileProject(root, getExecutorIndex(), exp_list); return *this; } diff --git a/dbms/src/TestUtils/mockExecutor.h b/dbms/src/TestUtils/mockExecutor.h index 7b76754a18a..d5b65842791 100644 --- a/dbms/src/TestUtils/mockExecutor.h +++ b/dbms/src/TestUtils/mockExecutor.h @@ -10,10 +10,12 @@ namespace DB { // using MockColumnInfo = std::pair; +using MockColumnInfos = std::vector; using MockTableName = std::pair; using MockOrderByItem = std::pair; using MockOrderByItems = std::initializer_list; using MockColumnNames = std::initializer_list; +using MockAsts = std::initializer_list; class DAGRequestBuilder { @@ -26,8 +28,8 @@ class DAGRequestBuilder std::shared_ptr build(Context & context); - DAGRequestBuilder & mockTable(String db, String table, const std::vector & columns); - DAGRequestBuilder & mockTable(MockTableName name, const std::vector & columns); + DAGRequestBuilder & mockTable(const String & db, const String & table, const MockColumnInfos & columns); + DAGRequestBuilder & mockTable(const MockTableName & name, const MockColumnInfos & columns); DAGRequestBuilder & filter(ASTPtr filter_expr); @@ -39,8 +41,8 @@ class DAGRequestBuilder DAGRequestBuilder & topN(MockOrderByItems order_by_items, int limit); DAGRequestBuilder & topN(MockOrderByItems order_by_items, ASTPtr limit_expr); - DAGRequestBuilder & project(String col_name); - DAGRequestBuilder & project(std::initializer_list cols); + DAGRequestBuilder & project(const String & col_name); + DAGRequestBuilder & project(MockAsts expr); DAGRequestBuilder & project(MockColumnNames col_names); // only support inner join @@ -56,8 +58,8 @@ class DAGRequestBuilder ASTPtr buildColumn(const String & column_name); ASTPtr buildLiteral(const Field & field); -ASTPtr buildFunction(std::initializer_list exprs, const String & name); -ASTPtr buildOrderByItemList(std::initializer_list> order_by_items); +ASTPtr buildFunction(MockAsts exprs, const String & name); +ASTPtr buildOrderByItemList(MockOrderByItems order_by_items); #define col(name) buildColumn((name)) From 597923bc056f957d485dd09f35830760d9956a77 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Tue, 12 Apr 2022 20:05:33 +0800 Subject: [PATCH 05/16] update. --- dbms/src/TestUtils/InterpreterTestUtils.h | 45 ++++++++++++++++ dbms/src/TestUtils/mockExecutor.cpp | 18 ++++++- dbms/src/TestUtils/mockExecutor.h | 25 +++++++-- .../TestUtils/tests/gtest_mock_executors.cpp | 52 ++++++++++--------- 4 files changed, 112 insertions(+), 28 deletions(-) create mode 100644 dbms/src/TestUtils/InterpreterTestUtils.h diff --git a/dbms/src/TestUtils/InterpreterTestUtils.h b/dbms/src/TestUtils/InterpreterTestUtils.h new file mode 100644 index 00000000000..78275c3b60d --- /dev/null +++ b/dbms/src/TestUtils/InterpreterTestUtils.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace DB +{ +namespace tests +{ +class MockExecutorTest : public ::testing::Test +{ +protected: + void SetUp() override + { + initializeDAGContext(); + DAGRequestBuilder::executor_index = 0; + } + +public: + MockExecutorTest() + : context(TiFlashTestEnv::getContext()) + {} + + virtual void initializeDAGContext() + { + dag_context_ptr = std::make_unique(1024); + context.setDAGContext(dag_context_ptr.get()); + } + + DAGContext & getDAGContext() + { + assert(dag_context_ptr != nullptr); + return *dag_context_ptr; + } + +protected: + Context context; + std::unique_ptr dag_context_ptr; +}; + +} // namespace tests +} // namespace DB \ No newline at end of file diff --git a/dbms/src/TestUtils/mockExecutor.cpp b/dbms/src/TestUtils/mockExecutor.cpp index f7289fa9906..f2af77cb15f 100644 --- a/dbms/src/TestUtils/mockExecutor.cpp +++ b/dbms/src/TestUtils/mockExecutor.cpp @@ -1,3 +1,17 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include #include #include @@ -15,6 +29,8 @@ namespace DB { +namespace tests +{ ASTPtr buildColumn(const String & column_name) { return std::make_shared(column_name); @@ -189,5 +205,5 @@ DAGRequestBuilder & DAGRequestBuilder::join(const DAGRequestBuilder & right, AST root = compileJoin(getExecutorIndex(), root, right.root, join_ast); return *this; } - +} // namespace tests } // namespace DB \ No newline at end of file diff --git a/dbms/src/TestUtils/mockExecutor.h b/dbms/src/TestUtils/mockExecutor.h index d5b65842791..5c897318811 100644 --- a/dbms/src/TestUtils/mockExecutor.h +++ b/dbms/src/TestUtils/mockExecutor.h @@ -1,13 +1,30 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #include #include +#include #include #include namespace DB { +namespace tests +{ // using MockColumnInfo = std::pair; using MockColumnInfos = std::vector; @@ -20,9 +37,9 @@ using MockAsts = std::initializer_list; class DAGRequestBuilder { public: + static size_t executor_index; static size_t & getExecutorIndex() { - thread_local size_t executor_index = 0; return executor_index; } @@ -45,7 +62,7 @@ class DAGRequestBuilder DAGRequestBuilder & project(MockAsts expr); DAGRequestBuilder & project(MockColumnNames col_names); - // only support inner join + // Currentlt only support inner join. // TODO support more joins DAGRequestBuilder & join(const DAGRequestBuilder & right, ASTPtr using_expr_list); @@ -56,6 +73,8 @@ class DAGRequestBuilder DAGProperties properties; }; +size_t DAGRequestBuilder::executor_index = 0; + ASTPtr buildColumn(const String & column_name); ASTPtr buildLiteral(const Field & field); ASTPtr buildFunction(MockAsts exprs, const String & name); @@ -71,5 +90,5 @@ ASTPtr buildOrderByItemList(MockOrderByItems order_by_items); #define And(expr1, expr2) buildFunction({(expr1), (expr2)}, "and") #define Or(expr1, expr2) buildFunction({(expr1), (expr2)}, "or") #define NOT(expr) buildFunction({expr1}, "not") - +} // namespace tests } // namespace DB \ No newline at end of file diff --git a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp index 12d2df79534..c8c73d9723b 100644 --- a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp +++ b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp @@ -11,14 +11,13 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + #include #include #include #include #include -#include -#include -#include +#include namespace DB { @@ -78,23 +77,23 @@ String toTreeString(const tipb::Executor & root_executor, size_t level) return buffer.toString(); } -// String & trim(String & str) -// { -// if (str.empty()) -// { -// return str; -// } - -// str.erase(0, str.find_first_not_of(' ')); -// str.erase(str.find_last_not_of(' ') + 1); -// return str; -// } +String & trim(String & str) +{ + if (str.empty()) + { + return str; + } + + str.erase(0, str.find_first_not_of(' ')); + str.erase(str.find_last_not_of(' ') + 1); + return str; +} } // namespace -class MockTiPBDAGRequestTest : public DB::tests::FunctionTest +class MockDAGRequestTest : public DB::tests::MockExecutorTest { }; -TEST_F(MockTiPBDAGRequestTest, Test1) +TEST_F(MockDAGRequestTest, Test1) try { String empty_str; @@ -120,8 +119,8 @@ try DAGRequestBuilder left_builder; left_builder .mockTable(left_table, l_columns) - .topN({{"l_a", false}}, 10) - .join(right_builder, col("l_a")) // ywq todo more types of join, should make sure have both field + .topN("l_a", false, 10) + .join(right_builder, col("l_a")) .limit(10); auto dag_request = left_builder.build(context); @@ -130,11 +129,10 @@ try } CATCH -TEST_F(MockTiPBDAGRequestTest, Test2) + +TEST_F(MockDAGRequestTest, Test2) try { - String empty_str; - MockTableName right_table{"r_db", "r_table"}; std::vector r_columns; r_columns.emplace_back("l_a", TiDB::TP::TypeString); @@ -156,16 +154,22 @@ try DAGRequestBuilder left_builder; left_builder .mockTable(left_table, l_columns) - .topN("l_a", false, 10) + .topN({{"l_a", false}}, 10) .join(right_builder, col("l_a")) .limit(10); auto dag_request = left_builder.build(context); String to_tree_string = toTreeString(dag_request.get()); - std::cout << to_tree_string << std::endl; + String expected_string = "limit_7\n" + " topn_5\n" + " table_scan_4\n" + " topn_3\n" + " project_2\n" + " selection_1\n" + " table_scan_0\n"; + ASSERT_EQ(trim(to_tree_string), trim(expected_string)); } CATCH - } // namespace tests } // namespace DB \ No newline at end of file From 7aff4eff4f95b3d5a9689adb0327cb5cc0499feb Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Tue, 12 Apr 2022 20:23:48 +0800 Subject: [PATCH 06/16] fix executor_index error. --- dbms/src/TestUtils/InterpreterTestUtils.h | 78 +++++++++++++++++++ dbms/src/TestUtils/mockExecutor.cpp | 2 + dbms/src/TestUtils/mockExecutor.h | 2 - .../TestUtils/tests/gtest_mock_executors.cpp | 70 +---------------- 4 files changed, 83 insertions(+), 69 deletions(-) diff --git a/dbms/src/TestUtils/InterpreterTestUtils.h b/dbms/src/TestUtils/InterpreterTestUtils.h index 78275c3b60d..985d241404e 100644 --- a/dbms/src/TestUtils/InterpreterTestUtils.h +++ b/dbms/src/TestUtils/InterpreterTestUtils.h @@ -1,6 +1,21 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #include +#include #include #include #include @@ -10,6 +25,69 @@ namespace DB { namespace tests { +String toTreeString(const tipb::Executor & root_executor, size_t level = 0); + +String toTreeString(const tipb::DAGRequest * dag_request) +{ + assert((dag_request->executors_size() > 0) != dag_request->has_root_executor()); + if (dag_request->has_root_executor()) + { + return toTreeString(dag_request->root_executor()); + } + else + { + FmtBuffer buffer; + String prefix; + traverseExecutors(dag_request, [&buffer, &prefix](const tipb::Executor & executor) { + assert(executor.has_executor_id()); + buffer.fmtAppend("{}{}\n", prefix, executor.executor_id()); + prefix.append(" "); + return true; + }); + return buffer.toString(); + } +} + +String toTreeString(const tipb::Executor & root_executor, size_t level) +{ + FmtBuffer buffer; + + auto append_str = [&buffer, &level](const tipb::Executor & executor) { + assert(executor.has_executor_id()); + for (size_t i = 0; i < level; ++i) + buffer.append(" "); + buffer.append(executor.executor_id()).append("\n"); + }; + + traverseExecutorTree(root_executor, [&](const tipb::Executor & executor) { + if (executor.has_join()) + { + for (const auto & child : executor.join().children()) + buffer.append(toTreeString(child, level)); + return false; + } + else + { + append_str(executor); + ++level; + return true; + } + }); + + return buffer.toString(); +} + +String & trim(String & str) +{ + if (str.empty()) + { + return str; + } + + str.erase(0, str.find_first_not_of(' ')); + str.erase(str.find_last_not_of(' ') + 1); + return str; +} class MockExecutorTest : public ::testing::Test { protected: diff --git a/dbms/src/TestUtils/mockExecutor.cpp b/dbms/src/TestUtils/mockExecutor.cpp index f2af77cb15f..e30015b94b5 100644 --- a/dbms/src/TestUtils/mockExecutor.cpp +++ b/dbms/src/TestUtils/mockExecutor.cpp @@ -31,6 +31,8 @@ namespace DB { namespace tests { +size_t DAGRequestBuilder::executor_index = 0; + ASTPtr buildColumn(const String & column_name) { return std::make_shared(column_name); diff --git a/dbms/src/TestUtils/mockExecutor.h b/dbms/src/TestUtils/mockExecutor.h index 5c897318811..071f7f2cecc 100644 --- a/dbms/src/TestUtils/mockExecutor.h +++ b/dbms/src/TestUtils/mockExecutor.h @@ -73,8 +73,6 @@ class DAGRequestBuilder DAGProperties properties; }; -size_t DAGRequestBuilder::executor_index = 0; - ASTPtr buildColumn(const String & column_name); ASTPtr buildLiteral(const Field & field); ASTPtr buildFunction(MockAsts exprs, const String & name); diff --git a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp index c8c73d9723b..91924df3d72 100644 --- a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp +++ b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp @@ -14,81 +14,16 @@ #include #include -#include #include #include #include +#include + namespace DB { namespace tests { -namespace -{ -String toTreeString(const tipb::Executor & root_executor, size_t level = 0); - -String toTreeString(const tipb::DAGRequest * dag_request) -{ - assert((dag_request->executors_size() > 0) != dag_request->has_root_executor()); - if (dag_request->has_root_executor()) - { - return toTreeString(dag_request->root_executor()); - } - else - { - FmtBuffer buffer; - String prefix; - traverseExecutors(dag_request, [&buffer, &prefix](const tipb::Executor & executor) { - assert(executor.has_executor_id()); - buffer.fmtAppend("{}{}\n", prefix, executor.executor_id()); - prefix.append(" "); - return true; - }); - return buffer.toString(); - } -} - -String toTreeString(const tipb::Executor & root_executor, size_t level) -{ - FmtBuffer buffer; - - auto append_str = [&buffer, &level](const tipb::Executor & executor) { - assert(executor.has_executor_id()); - for (size_t i = 0; i < level; ++i) - buffer.append(" "); - buffer.append(executor.executor_id()).append("\n"); - }; - - traverseExecutorTree(root_executor, [&](const tipb::Executor & executor) { - if (executor.has_join()) - { - for (const auto & child : executor.join().children()) - buffer.append(toTreeString(child, level)); - return false; - } - else - { - append_str(executor); - ++level; - return true; - } - }); - - return buffer.toString(); -} - -String & trim(String & str) -{ - if (str.empty()) - { - return str; - } - - str.erase(0, str.find_first_not_of(' ')); - str.erase(str.find_last_not_of(' ') + 1); - return str; -} -} // namespace class MockDAGRequestTest : public DB::tests::MockExecutorTest { }; @@ -167,6 +102,7 @@ try " project_2\n" " selection_1\n" " table_scan_0\n"; + std::cout << to_tree_string << std::endl; ASSERT_EQ(trim(to_tree_string), trim(expected_string)); } CATCH From 36b14b1c876bb2317b11a77c02bd37ce18ebc310 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Wed, 13 Apr 2022 16:34:04 +0800 Subject: [PATCH 07/16] update. --- dbms/src/Debug/astToExecutor.cpp | 1 + dbms/src/TestUtils/mockExecutor.cpp | 59 ++++++++++++++-- dbms/src/TestUtils/mockExecutor.h | 15 +++- .../TestUtils/tests/gtest_mock_executors.cpp | 68 ++++++++++--------- 4 files changed, 101 insertions(+), 42 deletions(-) diff --git a/dbms/src/Debug/astToExecutor.cpp b/dbms/src/Debug/astToExecutor.cpp index 5f7567f0eff..cd01a1f0d1b 100644 --- a/dbms/src/Debug/astToExecutor.cpp +++ b/dbms/src/Debug/astToExecutor.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include diff --git a/dbms/src/TestUtils/mockExecutor.cpp b/dbms/src/TestUtils/mockExecutor.cpp index e30015b94b5..c32bf94e43f 100644 --- a/dbms/src/TestUtils/mockExecutor.cpp +++ b/dbms/src/TestUtils/mockExecutor.cpp @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include -#include #include #include #include @@ -22,11 +20,6 @@ #include #include #include -#include - -#include -#include - namespace DB { namespace tests @@ -122,6 +115,23 @@ DAGRequestBuilder & DAGRequestBuilder::mockTable(const MockTableName & name, con return mockTable(name.first, name.second, columns); } +DAGRequestBuilder & DAGRequestBuilder::mockTable(const MockTableName & name, const MockColumnInfoList & columns) +{ + assert(columns.size() != 0); + TableInfo table_info; + table_info.name = name.first + "." + name.second; + for (const auto & column: columns) + { + TiDB::ColumnInfo ret; + ret.tp = column.second; + ret.name = column.first; + table_info.columns.push_back(std::move(ret)); + } + String empty_alias; + root = compileTableScan(getExecutorIndex(), table_info, empty_alias, false); + return *this; +} + DAGRequestBuilder & DAGRequestBuilder::filter(ASTPtr filter_expr) { assert(root); @@ -198,14 +208,49 @@ DAGRequestBuilder & DAGRequestBuilder::project(MockColumnNames col_names) } DAGRequestBuilder & DAGRequestBuilder::join(const DAGRequestBuilder & right, ASTPtr using_expr_list) +{ + return join(right, using_expr_list, ASTTableJoin::Kind::Inner); +} + +DAGRequestBuilder & DAGRequestBuilder::join(const DAGRequestBuilder & right, ASTPtr using_expr_list, ASTTableJoin::Kind kind) { assert(root); assert(right.root); auto join_ast = std::make_shared(); join_ast->using_expression_list = using_expr_list; join_ast->strictness = ASTTableJoin::Strictness::All; + join_ast->kind = kind; root = compileJoin(getExecutorIndex(), root, right.root, join_ast); return *this; } + +DAGRequestBuilder & DAGRequestBuilder::aggregation(ASTPtr agg_func, ASTPtr group_by_expr) +{ + auto agg_funcs = std::make_shared(); + auto group_by_exprs = std::make_shared(); + agg_funcs->children.push_back(agg_func); + group_by_exprs->children.push_back(group_by_expr); + return buildAggregation(agg_funcs,group_by_exprs); +} + +DAGRequestBuilder & DAGRequestBuilder::aggregation(MockAsts agg_funcs, MockAsts group_by_exprs) +{ + auto agg_func_list = std::make_shared(); + auto group_by_expr_list = std::make_shared(); + for (const auto & func : agg_funcs) + agg_func_list->children.push_back(func); + for (const auto & group_by: group_by_exprs) + group_by_expr_list->children.push_back(group_by); + + return buildAggregation(agg_func_list, group_by_expr_list); +} + +DAGRequestBuilder & DAGRequestBuilder::buildAggregation(ASTPtr agg_funcs, ASTPtr group_by_exprs) +{ + assert(root); + root = compileAggregation(root, getExecutorIndex(), agg_funcs, group_by_exprs); + return *this; +} + } // namespace tests } // namespace DB \ No newline at end of file diff --git a/dbms/src/TestUtils/mockExecutor.h b/dbms/src/TestUtils/mockExecutor.h index 071f7f2cecc..c37f7a4b0df 100644 --- a/dbms/src/TestUtils/mockExecutor.h +++ b/dbms/src/TestUtils/mockExecutor.h @@ -25,9 +25,10 @@ namespace DB { namespace tests { -// + using MockColumnInfo = std::pair; using MockColumnInfos = std::vector; +using MockColumnInfoList = std::initializer_list; using MockTableName = std::pair; using MockOrderByItem = std::pair; using MockOrderByItems = std::initializer_list; @@ -45,8 +46,10 @@ class DAGRequestBuilder std::shared_ptr build(Context & context); + // ywq todo check arguments DAGRequestBuilder & mockTable(const String & db, const String & table, const MockColumnInfos & columns); DAGRequestBuilder & mockTable(const MockTableName & name, const MockColumnInfos & columns); + DAGRequestBuilder & mockTable(const MockTableName & name, const MockColumnInfoList & columns); DAGRequestBuilder & filter(ASTPtr filter_expr); @@ -62,12 +65,17 @@ class DAGRequestBuilder DAGRequestBuilder & project(MockAsts expr); DAGRequestBuilder & project(MockColumnNames col_names); - // Currentlt only support inner join. - // TODO support more joins + // Currentlt only support inner join, left join and right join. + // TODO support more types of join. DAGRequestBuilder & join(const DAGRequestBuilder & right, ASTPtr using_expr_list); + DAGRequestBuilder & join(const DAGRequestBuilder & right, ASTPtr using_expr_list, ASTTableJoin::Kind kind); + // aggregation + DAGRequestBuilder & aggregation(ASTPtr agg_func, ASTPtr group_by_expr); + DAGRequestBuilder & aggregation(MockAsts agg_funcs, MockAsts group_by_exprs); private: void initDAGRequest(tipb::DAGRequest & dag_request); + DAGRequestBuilder & buildAggregation(ASTPtr agg_funcs, ASTPtr group_by_exprs); ExecutorPtr root; DAGProperties properties; @@ -88,5 +96,6 @@ ASTPtr buildOrderByItemList(MockOrderByItems order_by_items); #define And(expr1, expr2) buildFunction({(expr1), (expr2)}, "and") #define Or(expr1, expr2) buildFunction({(expr1), (expr2)}, "or") #define NOT(expr) buildFunction({expr1}, "not") +#define Max(expr1, expr2) buildFunction({(expr1), (expr2)}, "max") } // namespace tests } // namespace DB \ No newline at end of file diff --git a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp index 91924df3d72..ce7d45a3a08 100644 --- a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp +++ b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp @@ -17,8 +17,7 @@ #include #include #include - -#include +#include namespace DB { @@ -31,12 +30,10 @@ class MockDAGRequestTest : public DB::tests::MockExecutorTest TEST_F(MockDAGRequestTest, Test1) try { - String empty_str; - MockTableName right_table{"r_db", "r_table"}; std::vector r_columns; - r_columns.emplace_back("l_a", TiDB::TP::TypeString); - r_columns.emplace_back("l_b", TiDB::TP::TypeString); + r_columns.emplace_back("r_a", TiDB::TP::TypeString); + r_columns.emplace_back("r_b", TiDB::TP::TypeString); MockTableName left_table{"l_db", "l_table"}; std::vector l_columns; @@ -46,32 +43,38 @@ try DAGRequestBuilder right_builder; right_builder .mockTable(right_table, r_columns) - .filter(eq(col("l_a"), col("l_b"))) - .project({col("l_a")}) - .topN("l_a", false, 20); + .filter(eq(col("r_a"), col("r_b"))) + .project({col("r_a")}) + .topN("r_a", false, 20); DAGRequestBuilder left_builder; left_builder .mockTable(left_table, l_columns) - .topN("l_a", false, 10) - .join(right_builder, col("l_a")) + .topN({{"l_a", false}}, 10) + .join(right_builder, col("l_a"), ASTTableJoin::Kind::Left) .limit(10); auto dag_request = left_builder.build(context); String to_tree_string = toTreeString(dag_request.get()); - std::cout << to_tree_string << std::endl; + String expected_string = "limit_7\n" + " topn_5\n" + " table_scan_4\n" + " topn_3\n" + " project_2\n" + " selection_1\n" + " table_scan_0\n"; + ASSERT_EQ(trim(to_tree_string), trim(expected_string)); } CATCH - TEST_F(MockDAGRequestTest, Test2) try { MockTableName right_table{"r_db", "r_table"}; std::vector r_columns; - r_columns.emplace_back("l_a", TiDB::TP::TypeString); - r_columns.emplace_back("l_b", TiDB::TP::TypeString); + r_columns.emplace_back("r_a", TiDB::TP::TypeString); + r_columns.emplace_back("r_b", TiDB::TP::TypeString); MockTableName left_table{"l_db", "l_table"}; std::vector l_columns; @@ -81,28 +84,29 @@ try DAGRequestBuilder right_builder; right_builder .mockTable(right_table, r_columns) - .filter(eq(col("l_a"), col("l_b"))) - .project({col("l_a")}) - .topN("l_a", false, 20); + .filter(eq(col("r_a"), col("r_b"))) + .project({col("r_a")}) + .topN("r_a", false, 20); - DAGRequestBuilder left_builder; - left_builder + DAGRequestBuilder join_builder; + join_builder .mockTable(left_table, l_columns) .topN({{"l_a", false}}, 10) - .join(right_builder, col("l_a")) + .aggregation({Max(col("l_a"), col("l_b"))}, {col("l_a")}) + .join(right_builder, col("l_a"), ASTTableJoin::Kind::Left) .limit(10); - - auto dag_request = left_builder.build(context); - String to_tree_string = toTreeString(dag_request.get()); - String expected_string = "limit_7\n" - " topn_5\n" - " table_scan_4\n" - " topn_3\n" - " project_2\n" - " selection_1\n" - " table_scan_0\n"; - std::cout << to_tree_string << std::endl; + auto dag_request = join_builder.build(context); + auto to_tree_string = toTreeString(dag_request.get()); + String expected_string = + "limit_8\n" + " aggregation_6\n" + " topn_5\n" + " table_scan_4\n" + " topn_3\n" + " project_2\n" + " selection_1\n" + " table_scan_0\n"; ASSERT_EQ(trim(to_tree_string), trim(expected_string)); } CATCH From 8e90d039dce0cb27c30283ddac99389b131c5972 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Wed, 13 Apr 2022 16:34:27 +0800 Subject: [PATCH 08/16] format. --- dbms/src/TestUtils/mockExecutor.cpp | 10 +++++----- dbms/src/TestUtils/mockExecutor.h | 2 +- .../TestUtils/tests/gtest_mock_executors.cpp | 19 +++++++++---------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/dbms/src/TestUtils/mockExecutor.cpp b/dbms/src/TestUtils/mockExecutor.cpp index c32bf94e43f..d5212cf87ba 100644 --- a/dbms/src/TestUtils/mockExecutor.cpp +++ b/dbms/src/TestUtils/mockExecutor.cpp @@ -115,12 +115,12 @@ DAGRequestBuilder & DAGRequestBuilder::mockTable(const MockTableName & name, con return mockTable(name.first, name.second, columns); } -DAGRequestBuilder & DAGRequestBuilder::mockTable(const MockTableName & name, const MockColumnInfoList & columns) +DAGRequestBuilder & DAGRequestBuilder::mockTable(const MockTableName & name, const MockColumnInfoList & columns) { assert(columns.size() != 0); TableInfo table_info; table_info.name = name.first + "." + name.second; - for (const auto & column: columns) + for (const auto & column : columns) { TiDB::ColumnInfo ret; ret.tp = column.second; @@ -230,7 +230,7 @@ DAGRequestBuilder & DAGRequestBuilder::aggregation(ASTPtr agg_func, ASTPtr group auto group_by_exprs = std::make_shared(); agg_funcs->children.push_back(agg_func); group_by_exprs->children.push_back(group_by_expr); - return buildAggregation(agg_funcs,group_by_exprs); + return buildAggregation(agg_funcs, group_by_exprs); } DAGRequestBuilder & DAGRequestBuilder::aggregation(MockAsts agg_funcs, MockAsts group_by_exprs) @@ -239,9 +239,9 @@ DAGRequestBuilder & DAGRequestBuilder::aggregation(MockAsts agg_funcs, MockAsts auto group_by_expr_list = std::make_shared(); for (const auto & func : agg_funcs) agg_func_list->children.push_back(func); - for (const auto & group_by: group_by_exprs) + for (const auto & group_by : group_by_exprs) group_by_expr_list->children.push_back(group_by); - + return buildAggregation(agg_func_list, group_by_expr_list); } diff --git a/dbms/src/TestUtils/mockExecutor.h b/dbms/src/TestUtils/mockExecutor.h index c37f7a4b0df..161a5bb9288 100644 --- a/dbms/src/TestUtils/mockExecutor.h +++ b/dbms/src/TestUtils/mockExecutor.h @@ -25,7 +25,6 @@ namespace DB { namespace tests { - using MockColumnInfo = std::pair; using MockColumnInfos = std::vector; using MockColumnInfoList = std::initializer_list; @@ -73,6 +72,7 @@ class DAGRequestBuilder // aggregation DAGRequestBuilder & aggregation(ASTPtr agg_func, ASTPtr group_by_expr); DAGRequestBuilder & aggregation(MockAsts agg_funcs, MockAsts group_by_exprs); + private: void initDAGRequest(tipb::DAGRequest & dag_request); DAGRequestBuilder & buildAggregation(ASTPtr agg_funcs, ASTPtr group_by_exprs); diff --git a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp index ce7d45a3a08..06c48ef6912 100644 --- a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp +++ b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp @@ -64,7 +64,7 @@ try " project_2\n" " selection_1\n" " table_scan_0\n"; - ASSERT_EQ(trim(to_tree_string), trim(expected_string)); + ASSERT_EQ(trim(to_tree_string), trim(expected_string)); } CATCH @@ -98,15 +98,14 @@ try .limit(10); auto dag_request = join_builder.build(context); auto to_tree_string = toTreeString(dag_request.get()); - String expected_string = - "limit_8\n" - " aggregation_6\n" - " topn_5\n" - " table_scan_4\n" - " topn_3\n" - " project_2\n" - " selection_1\n" - " table_scan_0\n"; + String expected_string = "limit_8\n" + " aggregation_6\n" + " topn_5\n" + " table_scan_4\n" + " topn_3\n" + " project_2\n" + " selection_1\n" + " table_scan_0\n"; ASSERT_EQ(trim(to_tree_string), trim(expected_string)); } CATCH From abb92b50ea32a7f8f849a8669636f268db006eda Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Wed, 13 Apr 2022 16:46:15 +0800 Subject: [PATCH 09/16] update --- dbms/src/TestUtils/tests/gtest_mock_executors.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp index 06c48ef6912..ed943b0a5d7 100644 --- a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp +++ b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp @@ -71,11 +71,6 @@ CATCH TEST_F(MockDAGRequestTest, Test2) try { - MockTableName right_table{"r_db", "r_table"}; - std::vector r_columns; - r_columns.emplace_back("r_a", TiDB::TP::TypeString); - r_columns.emplace_back("r_b", TiDB::TP::TypeString); - MockTableName left_table{"l_db", "l_table"}; std::vector l_columns; l_columns.emplace_back("l_a", TiDB::TP::TypeString); @@ -83,7 +78,7 @@ try DAGRequestBuilder right_builder; right_builder - .mockTable(right_table, r_columns) + .mockTable({"r_db", "r_table"}, {{"r_a", TiDB::TP::TypeString}, {"r_b", TiDB::TP::TypeString}}) .filter(eq(col("r_a"), col("r_b"))) .project({col("r_a")}) .topN("r_a", false, 20); From 5369192501e9eaf49437562afa03020ab553aaa5 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 14 Apr 2022 18:31:04 +0800 Subject: [PATCH 10/16] more tests. Address comments. --- dbms/src/TestUtils/InterpreterTestUtils.cpp | 77 ++++++++ dbms/src/TestUtils/InterpreterTestUtils.h | 80 ++------ dbms/src/TestUtils/mockExecutor.cpp | 32 ++-- dbms/src/TestUtils/mockExecutor.h | 23 ++- .../TestUtils/tests/gtest_mock_executors.cpp | 174 +++++++++++++----- 5 files changed, 249 insertions(+), 137 deletions(-) create mode 100644 dbms/src/TestUtils/InterpreterTestUtils.cpp diff --git a/dbms/src/TestUtils/InterpreterTestUtils.cpp b/dbms/src/TestUtils/InterpreterTestUtils.cpp new file mode 100644 index 00000000000..348859d15ff --- /dev/null +++ b/dbms/src/TestUtils/InterpreterTestUtils.cpp @@ -0,0 +1,77 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +namespace DB +{ +namespace tests +{ +String toTreeString(std::shared_ptr dag_request) +{ + assert((dag_request->executors_size() > 0) != dag_request->has_root_executor()); + if (dag_request->has_root_executor()) + { + return toTreeString(dag_request->root_executor()); + } + else + { + FmtBuffer buffer; + String prefix; + traverseExecutors(dag_request.get(), [&buffer, &prefix](const tipb::Executor & executor) { + assert(executor.has_executor_id()); + buffer.fmtAppend("{}{}\n", prefix, executor.executor_id()); + prefix.append(" "); + return true; + }); + return buffer.toString(); + } +} + +String toTreeString(const tipb::Executor & root_executor, size_t level) +{ + FmtBuffer buffer; + + auto append_str = [&buffer, &level](const tipb::Executor & executor) { + assert(executor.has_executor_id()); + buffer.append(String(level, ' ')); + buffer.append(executor.executor_id()).append("\n"); + }; + + traverseExecutorTree(root_executor, [&](const tipb::Executor & executor) { + if (executor.has_join()) + { + for (const auto & child : executor.join().children()) + buffer.append(toTreeString(child, level)); + return false; + } + else + { + append_str(executor); + ++level; + return true; + } + }); + + return buffer.toString(); +} + +void dagRequestEqual(String & expected_string, const std::shared_ptr & actual) +{ + String actual_string = toTreeString(actual); + ASSERT_EQ(Poco::trimInPlace(expected_string), Poco::trimInPlace(actual_string)); +} + +} // namespace tests +} // namespace DB \ No newline at end of file diff --git a/dbms/src/TestUtils/InterpreterTestUtils.h b/dbms/src/TestUtils/InterpreterTestUtils.h index 985d241404e..16d7641e55e 100644 --- a/dbms/src/TestUtils/InterpreterTestUtils.h +++ b/dbms/src/TestUtils/InterpreterTestUtils.h @@ -14,9 +14,11 @@ #pragma once +#include #include #include #include +#include #include #include #include @@ -25,76 +27,15 @@ namespace DB { namespace tests { +String toTreeString(std::shared_ptr dag_request); String toTreeString(const tipb::Executor & root_executor, size_t level = 0); - -String toTreeString(const tipb::DAGRequest * dag_request) -{ - assert((dag_request->executors_size() > 0) != dag_request->has_root_executor()); - if (dag_request->has_root_executor()) - { - return toTreeString(dag_request->root_executor()); - } - else - { - FmtBuffer buffer; - String prefix; - traverseExecutors(dag_request, [&buffer, &prefix](const tipb::Executor & executor) { - assert(executor.has_executor_id()); - buffer.fmtAppend("{}{}\n", prefix, executor.executor_id()); - prefix.append(" "); - return true; - }); - return buffer.toString(); - } -} - -String toTreeString(const tipb::Executor & root_executor, size_t level) -{ - FmtBuffer buffer; - - auto append_str = [&buffer, &level](const tipb::Executor & executor) { - assert(executor.has_executor_id()); - for (size_t i = 0; i < level; ++i) - buffer.append(" "); - buffer.append(executor.executor_id()).append("\n"); - }; - - traverseExecutorTree(root_executor, [&](const tipb::Executor & executor) { - if (executor.has_join()) - { - for (const auto & child : executor.join().children()) - buffer.append(toTreeString(child, level)); - return false; - } - else - { - append_str(executor); - ++level; - return true; - } - }); - - return buffer.toString(); -} - -String & trim(String & str) -{ - if (str.empty()) - { - return str; - } - - str.erase(0, str.find_first_not_of(' ')); - str.erase(str.find_last_not_of(' ') + 1); - return str; -} +void dagRequestEqual(String & expected_string, const std::shared_ptr & actual); class MockExecutorTest : public ::testing::Test { protected: void SetUp() override { initializeDAGContext(); - DAGRequestBuilder::executor_index = 0; } public: @@ -102,6 +43,18 @@ class MockExecutorTest : public ::testing::Test : context(TiFlashTestEnv::getContext()) {} + static void SetUpTestCase() + { + try + { + DB::registerFunctions(); + } + catch (DB::Exception &) + { + // Maybe another test has already registered, ignore exception here. + } + } + virtual void initializeDAGContext() { dag_context_ptr = std::make_unique(1024); @@ -119,5 +72,6 @@ class MockExecutorTest : public ::testing::Test std::unique_ptr dag_context_ptr; }; +#define ASSERT_DAGREQUEST_EQAUL(str, request) dagRequestEqual(str, request); } // namespace tests } // namespace DB \ No newline at end of file diff --git a/dbms/src/TestUtils/mockExecutor.cpp b/dbms/src/TestUtils/mockExecutor.cpp index d5212cf87ba..eb17b002635 100644 --- a/dbms/src/TestUtils/mockExecutor.cpp +++ b/dbms/src/TestUtils/mockExecutor.cpp @@ -19,13 +19,12 @@ #include #include #include +#include #include namespace DB { namespace tests { -size_t DAGRequestBuilder::executor_index = 0; - ASTPtr buildColumn(const String & column_name) { return std::make_shared(column_name); @@ -36,6 +35,7 @@ ASTPtr buildLiteral(const Field & field) return std::make_shared(field); } +// todo remove ASTPtr buildFunction(MockAsts exprs, const String & name) { auto func = std::make_shared(); @@ -60,8 +60,7 @@ ASTPtr buildOrderByItemList(MockOrderByItems order_by_items) vec.push_back(order_by_item); } auto exp_list = std::make_shared(); - for (const auto & ast_ptr : vec) - exp_list->children.push_back(ast_ptr); + exp_list->children.insert(exp_list->children.end(), vec.begin(), vec.end()); return exp_list; } @@ -89,7 +88,8 @@ std::shared_ptr DAGRequestBuilder::build(Context & context) tipb::DAGRequest & dag_request = *dag_request_ptr; initDAGRequest(dag_request); root->toTiPBExecutor(dag_request.mutable_root_executor(), properties.collator, mpp_info, context); - + root.reset(); + executor_index = 0; return dag_request_ptr; } @@ -117,19 +117,7 @@ DAGRequestBuilder & DAGRequestBuilder::mockTable(const MockTableName & name, con DAGRequestBuilder & DAGRequestBuilder::mockTable(const MockTableName & name, const MockColumnInfoList & columns) { - assert(columns.size() != 0); - TableInfo table_info; - table_info.name = name.first + "." + name.second; - for (const auto & column : columns) - { - TiDB::ColumnInfo ret; - ret.tp = column.second; - ret.name = column.first; - table_info.columns.push_back(std::move(ret)); - } - String empty_alias; - root = compileTableScan(getExecutorIndex(), table_info, empty_alias, false); - return *this; + return mockTable(name.first, name.second, columns); } DAGRequestBuilder & DAGRequestBuilder::filter(ASTPtr filter_expr) @@ -191,7 +179,13 @@ DAGRequestBuilder & DAGRequestBuilder::project(MockAsts exprs) assert(root); auto exp_list = std::make_shared(); for (const auto & expr : exprs) + { + if (typeid_cast(expr.get())) + { + throw TiFlashTestException(fmt::format("Can't include Function in project.")); + } exp_list->children.push_back(expr); + } root = compileProject(root, getExecutorIndex(), exp_list); return *this; } @@ -201,7 +195,9 @@ DAGRequestBuilder & DAGRequestBuilder::project(MockColumnNames col_names) assert(root); auto exp_list = std::make_shared(); for (const auto & name : col_names) + { exp_list->children.push_back(col(name)); + } root = compileProject(root, getExecutorIndex(), exp_list); return *this; diff --git a/dbms/src/TestUtils/mockExecutor.h b/dbms/src/TestUtils/mockExecutor.h index 161a5bb9288..b22f83de538 100644 --- a/dbms/src/TestUtils/mockExecutor.h +++ b/dbms/src/TestUtils/mockExecutor.h @@ -16,10 +16,9 @@ #include #include +#include -#include #include -#include namespace DB { @@ -37,12 +36,17 @@ using MockAsts = std::initializer_list; class DAGRequestBuilder { public: - static size_t executor_index; - static size_t & getExecutorIndex() + size_t & executor_index; + + size_t & getExecutorIndex() const { return executor_index; } + explicit DAGRequestBuilder(size_t & index) + : executor_index(index) + {} + std::shared_ptr build(Context & context); // ywq todo check arguments @@ -81,6 +85,15 @@ class DAGRequestBuilder DAGProperties properties; }; +class DAGRequestBuilderFactory +{ +public: + static DAGRequestBuilder createDAGRequestBuilder(size_t & executor_id) + { + return DAGRequestBuilder(executor_id); + } +}; + ASTPtr buildColumn(const String & column_name); ASTPtr buildLiteral(const Field & field); ASTPtr buildFunction(MockAsts exprs, const String & name); @@ -96,6 +109,6 @@ ASTPtr buildOrderByItemList(MockOrderByItems order_by_items); #define And(expr1, expr2) buildFunction({(expr1), (expr2)}, "and") #define Or(expr1, expr2) buildFunction({(expr1), (expr2)}, "or") #define NOT(expr) buildFunction({expr1}, "not") -#define Max(expr1, expr2) buildFunction({(expr1), (expr2)}, "max") +#define Max(expr) makeASTFunction("max", expr) } // namespace tests } // namespace DB \ No newline at end of file diff --git a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp index ed943b0a5d7..695aa5939cd 100644 --- a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp +++ b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp @@ -27,83 +27,155 @@ class MockDAGRequestTest : public DB::tests::MockExecutorTest { }; -TEST_F(MockDAGRequestTest, Test1) +TEST_F(MockDAGRequestTest, MockTable) try { - MockTableName right_table{"r_db", "r_table"}; - std::vector r_columns; - r_columns.emplace_back("r_a", TiDB::TP::TypeString); - r_columns.emplace_back("r_b", TiDB::TP::TypeString); + size_t index = 0; + auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(index); + MockTableName table_name("test_db", "test"); // db_name-->table_name + std::vector table_columns; + table_columns.emplace_back("t_l", TiDB::TP::TypeLong); + table_columns.emplace_back("t_s", TiDB::TP::TypeString); + builder.mockTable(table_name, table_columns); + auto dag_request_1 = builder.build(context); + String expected_string_1 = "table_scan_0\n"; + ASSERT_DAGREQUEST_EQAUL(expected_string_1, dag_request_1); - MockTableName left_table{"l_db", "l_table"}; - std::vector l_columns; - l_columns.emplace_back("l_a", TiDB::TP::TypeString); - l_columns.emplace_back("l_b", TiDB::TP::TypeLong); + builder.mockTable({"test_db", "test_table"}, {{"t_l", TiDB::TP::TypeLong}, {"t_s", TiDB::TP::TypeString}}); + auto dag_request_2 = builder.build(context); + String expected_string_2 = "table_scan_0\n"; + ASSERT_DAGREQUEST_EQAUL(expected_string_2, dag_request_2); +} +CATCH - DAGRequestBuilder right_builder; - right_builder - .mockTable(right_table, r_columns) - .filter(eq(col("r_a"), col("r_b"))) - .project({col("r_a")}) - .topN("r_a", false, 20); +TEST_F(MockDAGRequestTest, Filter) +try +{ + size_t index = 0; + auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(index); + auto request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) + .filter(eq(col("s1"), col("s2"))) + .build(context); + String expected_string = "selection_1\n" + " table_scan_0\n"; + ASSERT_DAGREQUEST_EQAUL(expected_string, request); + request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}, {"s3", TiDB::TP::TypeString}}) + .filter(And(eq(col("s1"), col("s2")), lt(col("s2"), col("s3")))) + .build(context); + ASSERT_DAGREQUEST_EQAUL(expected_string, request); +} +CATCH - DAGRequestBuilder left_builder; - left_builder - .mockTable(left_table, l_columns) - .topN({{"l_a", false}}, 10) - .join(right_builder, col("l_a"), ASTTableJoin::Kind::Left) - .limit(10); +TEST_F(MockDAGRequestTest, Projection) +try +{ + size_t index = 0; + auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(index); + auto request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) + .project("s1") + .build(context); + String expected_string = "project_1\n" + " table_scan_0\n"; + ASSERT_DAGREQUEST_EQAUL(expected_string, request); - auto dag_request = left_builder.build(context); - String to_tree_string = toTreeString(dag_request.get()); - String expected_string = "limit_7\n" - " topn_5\n" - " table_scan_4\n" - " topn_3\n" - " project_2\n" - " selection_1\n" - " table_scan_0\n"; - ASSERT_EQ(trim(to_tree_string), trim(expected_string)); + request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}, {"s3", TiDB::TP::TypeString}}) + .project({col("s3")}) + .build(context); + String expected_string_2 = "project_1\n" + " table_scan_0\n"; + ASSERT_DAGREQUEST_EQAUL(expected_string_2, request); + + request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}, {"s3", TiDB::TP::TypeString}}) + .project({"s1", "s2"}) + .build(context); + ASSERT_DAGREQUEST_EQAUL(expected_string, request); } CATCH -TEST_F(MockDAGRequestTest, Test2) +TEST_F(MockDAGRequestTest, Limit) try { - MockTableName left_table{"l_db", "l_table"}; - std::vector l_columns; - l_columns.emplace_back("l_a", TiDB::TP::TypeString); - l_columns.emplace_back("l_b", TiDB::TP::TypeLong); + size_t index = 0; + auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(index); + auto request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) + .limit(10) + .build(context); + String expected_string = "limit_1\n" + " table_scan_0\n"; + ASSERT_DAGREQUEST_EQAUL(expected_string, request); - DAGRequestBuilder right_builder; + request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) + .limit(lit(Field(static_cast(10)))) + .build(context); + ASSERT_DAGREQUEST_EQAUL(expected_string, request); +} +CATCH + +TEST_F(MockDAGRequestTest, TopN) +try +{ + size_t index = 0; + auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(index); + auto request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) + .topN({{"s1", false}}, 10) + .build(context); + String expected_string = "topn_1\n" + " table_scan_0\n"; + ASSERT_DAGREQUEST_EQAUL(expected_string, request); + + request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) + .topN("s1", false, 10) + .build(context); + ASSERT_DAGREQUEST_EQAUL(expected_string, request); +} +CATCH + +TEST_F(MockDAGRequestTest, Aggregation) +try +{ + size_t index = 0; + auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(index); + auto request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) + .aggregation(Max(col("s1")), col("s2")) + .build(context); + String expected_string = "aggregation_1\n" + " table_scan_0\n"; + ASSERT_DAGREQUEST_EQAUL(expected_string, request); +} +CATCH + +TEST_F(MockDAGRequestTest, Join) +try +{ + size_t index = 0; + DAGRequestBuilder right_builder = DAGRequestBuilderFactory().createDAGRequestBuilder(index); right_builder .mockTable({"r_db", "r_table"}, {{"r_a", TiDB::TP::TypeString}, {"r_b", TiDB::TP::TypeString}}) .filter(eq(col("r_a"), col("r_b"))) - .project({col("r_a")}) - .topN("r_a", false, 20); + .project({col("r_a"), col("r_b")}) + .aggregation(Max(col("r_a")), col("r_b")); - DAGRequestBuilder join_builder; - join_builder - .mockTable(left_table, l_columns) + DAGRequestBuilder left_builder = DAGRequestBuilderFactory().createDAGRequestBuilder(index); + left_builder + .mockTable({"l_db", "l_table"}, {{"l_a", TiDB::TP::TypeString}, {"l_b", TiDB::TP::TypeString}}) .topN({{"l_a", false}}, 10) - .aggregation({Max(col("l_a"), col("l_b"))}, {col("l_a")}) .join(right_builder, col("l_a"), ASTTableJoin::Kind::Left) .limit(10); - auto dag_request = join_builder.build(context); - auto to_tree_string = toTreeString(dag_request.get()); - String expected_string = "limit_8\n" - " aggregation_6\n" - " topn_5\n" - " table_scan_4\n" - " topn_3\n" + + auto request = left_builder.build(context); + String expected_string = "limit_7\n" + " topn_5\n" + " table_scan_4\n" + " aggregation_3\n" " project_2\n" " selection_1\n" " table_scan_0\n"; - ASSERT_EQ(trim(to_tree_string), trim(expected_string)); + ASSERT_DAGREQUEST_EQAUL(expected_string, request); } CATCH + } // namespace tests } // namespace DB \ No newline at end of file From e07135d0c4609cf43b8ac8530636cbc3d643794d Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 14 Apr 2022 19:10:19 +0800 Subject: [PATCH 11/16] update. --- dbms/src/Debug/astToExecutor.cpp | 4 ---- dbms/src/TestUtils/mockExecutor.cpp | 13 ------------- dbms/src/TestUtils/mockExecutor.h | 14 +++++++------- 3 files changed, 7 insertions(+), 24 deletions(-) diff --git a/dbms/src/Debug/astToExecutor.cpp b/dbms/src/Debug/astToExecutor.cpp index cd01a1f0d1b..11b90e60fb9 100644 --- a/dbms/src/Debug/astToExecutor.cpp +++ b/dbms/src/Debug/astToExecutor.cpp @@ -28,10 +28,6 @@ #include #include #include -#include - -#include -#include namespace DB { diff --git a/dbms/src/TestUtils/mockExecutor.cpp b/dbms/src/TestUtils/mockExecutor.cpp index eb17b002635..d3e87211857 100644 --- a/dbms/src/TestUtils/mockExecutor.cpp +++ b/dbms/src/TestUtils/mockExecutor.cpp @@ -35,19 +35,6 @@ ASTPtr buildLiteral(const Field & field) return std::make_shared(field); } -// todo remove -ASTPtr buildFunction(MockAsts exprs, const String & name) -{ - auto func = std::make_shared(); - func->name = name; - auto expr_list = std::make_shared(); - for (const auto & expr : exprs) - expr_list->children.push_back(expr); - func->arguments = expr_list; - func->children.push_back(func->arguments); - return func; -} - ASTPtr buildOrderByItemList(MockOrderByItems order_by_items) { std::vector vec; diff --git a/dbms/src/TestUtils/mockExecutor.h b/dbms/src/TestUtils/mockExecutor.h index b22f83de538..a137fa35929 100644 --- a/dbms/src/TestUtils/mockExecutor.h +++ b/dbms/src/TestUtils/mockExecutor.h @@ -102,13 +102,13 @@ ASTPtr buildOrderByItemList(MockOrderByItems order_by_items); #define col(name) buildColumn((name)) #define lit(field) buildLiteral((field)) -#define eq(expr1, expr2) buildFunction({(expr1), (expr2)}, "equals") -#define Not_eq(expr1, expr2) buildFunction({(expr1), (expr2)}, "notEquals") -#define lt(expr1, expr2) buildFunction({(expr1), (expr2)}, "less") -#define gt(expr1, expr2) buildFunction({(expr1), (expr2)}, "greater") -#define And(expr1, expr2) buildFunction({(expr1), (expr2)}, "and") -#define Or(expr1, expr2) buildFunction({(expr1), (expr2)}, "or") -#define NOT(expr) buildFunction({expr1}, "not") +#define eq(expr1, expr2) makeASTFunction("equals", (expr1), (expr2)) +#define Not_eq(expr1, expr2) makeASTFunction("notEquals", (expr1), (expr2)) +#define lt(expr1, expr2) makeASTFunction("less", (expr1), (expr2)) +#define gt(expr1, expr2) makeASTFunction("greater", (expr1), (expr2)) +#define And(expr1, expr2) makeASTFunction("and", (expr1), (expr2)) +#define Or(expr1, expr2) makeASTFunction("or", (expr1), (expr2)) +#define NOT(expr) makeASTFunction("not", (expr1), (expr2)) #define Max(expr) makeASTFunction("max", expr) } // namespace tests } // namespace DB \ No newline at end of file From c591503707bb36050e8be01194f44a79c4ecc3dd Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 14 Apr 2022 19:31:37 +0800 Subject: [PATCH 12/16] fix --- dbms/src/TestUtils/mockExecutor.h | 9 +++++-- .../TestUtils/tests/gtest_mock_executors.cpp | 24 +++++++------------ 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/dbms/src/TestUtils/mockExecutor.h b/dbms/src/TestUtils/mockExecutor.h index a137fa35929..3eb7ce45a6c 100644 --- a/dbms/src/TestUtils/mockExecutor.h +++ b/dbms/src/TestUtils/mockExecutor.h @@ -88,9 +88,14 @@ class DAGRequestBuilder class DAGRequestBuilderFactory { public: - static DAGRequestBuilder createDAGRequestBuilder(size_t & executor_id) + size_t index; + DAGRequestBuilderFactory() { - return DAGRequestBuilder(executor_id); + index = 0; + } + DAGRequestBuilder createDAGRequestBuilder() + { + return DAGRequestBuilder(index); } }; diff --git a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp index 695aa5939cd..bec3f303bfc 100644 --- a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp +++ b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp @@ -30,8 +30,7 @@ class MockDAGRequestTest : public DB::tests::MockExecutorTest TEST_F(MockDAGRequestTest, MockTable) try { - size_t index = 0; - auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(index); + auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(); MockTableName table_name("test_db", "test"); // db_name-->table_name std::vector table_columns; table_columns.emplace_back("t_l", TiDB::TP::TypeLong); @@ -51,8 +50,7 @@ CATCH TEST_F(MockDAGRequestTest, Filter) try { - size_t index = 0; - auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(index); + auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(); auto request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) .filter(eq(col("s1"), col("s2"))) .build(context); @@ -70,8 +68,7 @@ CATCH TEST_F(MockDAGRequestTest, Projection) try { - size_t index = 0; - auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(index); + auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(); auto request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) .project("s1") .build(context); @@ -96,8 +93,7 @@ CATCH TEST_F(MockDAGRequestTest, Limit) try { - size_t index = 0; - auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(index); + auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(); auto request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) .limit(10) .build(context); @@ -115,8 +111,7 @@ CATCH TEST_F(MockDAGRequestTest, TopN) try { - size_t index = 0; - auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(index); + auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(); auto request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) .topN({{"s1", false}}, 10) .build(context); @@ -134,8 +129,7 @@ CATCH TEST_F(MockDAGRequestTest, Aggregation) try { - size_t index = 0; - auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(index); + auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(); auto request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) .aggregation(Max(col("s1")), col("s2")) .build(context); @@ -148,8 +142,8 @@ CATCH TEST_F(MockDAGRequestTest, Join) try { - size_t index = 0; - DAGRequestBuilder right_builder = DAGRequestBuilderFactory().createDAGRequestBuilder(index); + auto builder_factory = DAGRequestBuilderFactory(); + DAGRequestBuilder right_builder = builder_factory.createDAGRequestBuilder(); right_builder .mockTable({"r_db", "r_table"}, {{"r_a", TiDB::TP::TypeString}, {"r_b", TiDB::TP::TypeString}}) .filter(eq(col("r_a"), col("r_b"))) @@ -157,7 +151,7 @@ try .aggregation(Max(col("r_a")), col("r_b")); - DAGRequestBuilder left_builder = DAGRequestBuilderFactory().createDAGRequestBuilder(index); + DAGRequestBuilder left_builder = builder_factory.createDAGRequestBuilder(); left_builder .mockTable({"l_db", "l_table"}, {{"l_a", TiDB::TP::TypeString}, {"l_b", TiDB::TP::TypeString}}) .topN({{"l_a", false}}, 10) From d71780949234f0e5e89e774bc41b5f5ed5c437dd Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Thu, 14 Apr 2022 20:11:30 +0800 Subject: [PATCH 13/16] annotation. --- dbms/src/TestUtils/mockExecutor.cpp | 2 ++ dbms/src/TestUtils/mockExecutor.h | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/dbms/src/TestUtils/mockExecutor.cpp b/dbms/src/TestUtils/mockExecutor.cpp index d3e87211857..365ea84a852 100644 --- a/dbms/src/TestUtils/mockExecutor.cpp +++ b/dbms/src/TestUtils/mockExecutor.cpp @@ -51,6 +51,7 @@ ASTPtr buildOrderByItemList(MockOrderByItems order_by_items) return exp_list; } +// a mock DAGRequest should prepare its time_zone, flags, encode_type and output_schema. void DAGRequestBuilder::initDAGRequest(tipb::DAGRequest & dag_request) { dag_request.set_time_zone_name(properties.tz_name); @@ -68,6 +69,7 @@ void DAGRequestBuilder::initDAGRequest(tipb::DAGRequest & dag_request) dag_request.add_output_offsets(i); } +// traval the AST tree to build tipb::Executor recursively. std::shared_ptr DAGRequestBuilder::build(Context & context) { MPPInfo mpp_info(properties.start_ts, -1, -1, {}, {}); diff --git a/dbms/src/TestUtils/mockExecutor.h b/dbms/src/TestUtils/mockExecutor.h index 3eb7ce45a6c..e927ce43d39 100644 --- a/dbms/src/TestUtils/mockExecutor.h +++ b/dbms/src/TestUtils/mockExecutor.h @@ -33,6 +33,13 @@ using MockOrderByItems = std::initializer_list; using MockColumnNames = std::initializer_list; using MockAsts = std::initializer_list; + +/** Responsible for Hand write tipb::DAGRequest + * Use this class to mock DAGRequest, then feed the DAGRequest into + * the Interpreter for test purpose. + * The mockTable() method must called first in order to generate the table schema. + * After construct all necessary operators in DAGRequest, call build() to generate DAGRequest。 + */ class DAGRequestBuilder { public: @@ -104,7 +111,6 @@ ASTPtr buildLiteral(const Field & field); ASTPtr buildFunction(MockAsts exprs, const String & name); ASTPtr buildOrderByItemList(MockOrderByItems order_by_items); - #define col(name) buildColumn((name)) #define lit(field) buildLiteral((field)) #define eq(expr1, expr2) makeASTFunction("equals", (expr1), (expr2)) From b5cfb6b17d785dff41f8c3aaf46f99241c279ebc Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Fri, 15 Apr 2022 14:26:10 +0800 Subject: [PATCH 14/16] address comments. --- dbms/src/TestUtils/InterpreterTestUtils.cpp | 9 +- dbms/src/TestUtils/InterpreterTestUtils.h | 16 ++-- dbms/src/TestUtils/mockExecutor.cpp | 38 ++++++--- dbms/src/TestUtils/mockExecutor.h | 27 ++++-- .../TestUtils/tests/gtest_mock_executors.cpp | 84 ++++++++----------- 5 files changed, 93 insertions(+), 81 deletions(-) diff --git a/dbms/src/TestUtils/InterpreterTestUtils.cpp b/dbms/src/TestUtils/InterpreterTestUtils.cpp index 348859d15ff..8374ee74017 100644 --- a/dbms/src/TestUtils/InterpreterTestUtils.cpp +++ b/dbms/src/TestUtils/InterpreterTestUtils.cpp @@ -14,10 +14,10 @@ #include -namespace DB -{ -namespace tests +namespace DB::tests { +String toTreeString(const tipb::Executor & root_executor, size_t level = 0); + String toTreeString(std::shared_ptr dag_request) { assert((dag_request->executors_size() > 0) != dag_request->has_root_executor()); @@ -73,5 +73,4 @@ void dagRequestEqual(String & expected_string, const std::shared_ptr #include #include - -namespace DB -{ -namespace tests +namespace DB::tests { -String toTreeString(std::shared_ptr dag_request); -String toTreeString(const tipb::Executor & root_executor, size_t level = 0); void dagRequestEqual(String & expected_string, const std::shared_ptr & actual); class MockExecutorTest : public ::testing::Test { protected: void SetUp() override { - initializeDAGContext(); + initializeContext(); } public: @@ -55,10 +50,11 @@ class MockExecutorTest : public ::testing::Test } } - virtual void initializeDAGContext() + virtual void initializeContext() { dag_context_ptr = std::make_unique(1024); context.setDAGContext(dag_context_ptr.get()); + mock_dag_request_context = MockDAGRequestContext(); } DAGContext & getDAGContext() @@ -69,9 +65,9 @@ class MockExecutorTest : public ::testing::Test protected: Context context; + MockDAGRequestContext mock_dag_request_context; std::unique_ptr dag_context_ptr; }; #define ASSERT_DAGREQUEST_EQAUL(str, request) dagRequestEqual(str, request); -} // namespace tests -} // namespace DB \ No newline at end of file +} // namespace DB::tests \ No newline at end of file diff --git a/dbms/src/TestUtils/mockExecutor.cpp b/dbms/src/TestUtils/mockExecutor.cpp index 365ea84a852..8295d161753 100644 --- a/dbms/src/TestUtils/mockExecutor.cpp +++ b/dbms/src/TestUtils/mockExecutor.cpp @@ -18,12 +18,9 @@ #include #include #include -#include #include #include -namespace DB -{ -namespace tests +namespace DB::tests { ASTPtr buildColumn(const String & column_name) { @@ -169,10 +166,6 @@ DAGRequestBuilder & DAGRequestBuilder::project(MockAsts exprs) auto exp_list = std::make_shared(); for (const auto & expr : exprs) { - if (typeid_cast(expr.get())) - { - throw TiFlashTestException(fmt::format("Can't include Function in project.")); - } exp_list->children.push_back(expr); } root = compileProject(root, getExecutorIndex(), exp_list); @@ -237,5 +230,30 @@ DAGRequestBuilder & DAGRequestBuilder::buildAggregation(ASTPtr agg_funcs, ASTPtr return *this; } -} // namespace tests -} // namespace DB \ No newline at end of file + +void MockDAGRequestContext::addMockTable(const MockTableName & name, const MockColumnInfoList & columns) +{ + std::vector v_column_info; + for (const auto & info : columns) + { + v_column_info.push_back(std::move(info)); + } + mock_tables[name.first + "." + name.second] = v_column_info; +} + +void MockDAGRequestContext::addMockTable(const String & db, const String & table, const MockColumnInfos & columns) +{ + mock_tables[db + "." + table] = columns; +} + +void MockDAGRequestContext::addMockTable(const MockTableName & name, const MockColumnInfos & columns) +{ + mock_tables[name.first + "." + name.second] = columns; +} + +DAGRequestBuilder MockDAGRequestContext::scan(String db_name, String table_name) +{ + return DAGRequestBuilder(index).mockTable({db_name, table_name}, mock_tables[db_name + "." + table_name]); +} + +} // namespace DB::tests \ No newline at end of file diff --git a/dbms/src/TestUtils/mockExecutor.h b/dbms/src/TestUtils/mockExecutor.h index e927ce43d39..0002d63768e 100644 --- a/dbms/src/TestUtils/mockExecutor.h +++ b/dbms/src/TestUtils/mockExecutor.h @@ -19,10 +19,9 @@ #include #include +#include -namespace DB -{ -namespace tests +namespace DB::tests { using MockColumnInfo = std::pair; using MockColumnInfos = std::vector; @@ -92,18 +91,30 @@ class DAGRequestBuilder DAGProperties properties; }; -class DAGRequestBuilderFactory +class MockDAGRequestContext { public: - size_t index; - DAGRequestBuilderFactory() + MockDAGRequestContext() { index = 0; } + DAGRequestBuilder createDAGRequestBuilder() { return DAGRequestBuilder(index); } + + void addMockTable(const MockTableName & name, const MockColumnInfoList & columns); + void addMockTable(const String & db, const String & table, const MockColumnInfos & columns); + void addMockTable(const MockTableName & name, const MockColumnInfos & columns); + + DAGRequestBuilder scan(String db_name, String table_name); + + size_t size() { return mock_tables.size(); } + +private: + size_t index; + std::unordered_map mock_tables; }; ASTPtr buildColumn(const String & column_name); @@ -121,5 +132,5 @@ ASTPtr buildOrderByItemList(MockOrderByItems order_by_items); #define Or(expr1, expr2) makeASTFunction("or", (expr1), (expr2)) #define NOT(expr) makeASTFunction("not", (expr1), (expr2)) #define Max(expr) makeASTFunction("max", expr) -} // namespace tests -} // namespace DB \ No newline at end of file + +} // namespace DB::tests \ No newline at end of file diff --git a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp index bec3f303bfc..bf2b7dbf720 100644 --- a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp +++ b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp @@ -12,10 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include -#include #include -#include #include #include @@ -25,40 +22,41 @@ namespace tests { class MockDAGRequestTest : public DB::tests::MockExecutorTest { +public: + void initializeContext() override + { + dag_context_ptr = std::make_unique(1024); + context.setDAGContext(dag_context_ptr.get()); + mock_dag_request_context = MockDAGRequestContext(); + mock_dag_request_context.addMockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}); + mock_dag_request_context.addMockTable({"test_db", "test_table_1"}, {{"s1", TiDB::TP::TypeLong}, {"s2", TiDB::TP::TypeString}, {"s3", TiDB::TP::TypeString}}); + mock_dag_request_context.addMockTable({"test_db", "r_table"}, {{"r_a", TiDB::TP::TypeLong}, {"r_b", TiDB::TP::TypeString}, {"r_c", TiDB::TP::TypeString}}); + mock_dag_request_context.addMockTable({"test_db", "l_table"}, {{"l_a", TiDB::TP::TypeLong}, {"l_b", TiDB::TP::TypeString}, {"l_c", TiDB::TP::TypeString}}); + } }; TEST_F(MockDAGRequestTest, MockTable) try { - auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(); - MockTableName table_name("test_db", "test"); // db_name-->table_name - std::vector table_columns; - table_columns.emplace_back("t_l", TiDB::TP::TypeLong); - table_columns.emplace_back("t_s", TiDB::TP::TypeString); - builder.mockTable(table_name, table_columns); - auto dag_request_1 = builder.build(context); + auto request = mock_dag_request_context.scan("test_db", "test_table").build(context); String expected_string_1 = "table_scan_0\n"; - ASSERT_DAGREQUEST_EQAUL(expected_string_1, dag_request_1); + ASSERT_DAGREQUEST_EQAUL(expected_string_1, request); - builder.mockTable({"test_db", "test_table"}, {{"t_l", TiDB::TP::TypeLong}, {"t_s", TiDB::TP::TypeString}}); - auto dag_request_2 = builder.build(context); + request = mock_dag_request_context.scan("test_db", "test_table_1").build(context); String expected_string_2 = "table_scan_0\n"; - ASSERT_DAGREQUEST_EQAUL(expected_string_2, dag_request_2); + ASSERT_DAGREQUEST_EQAUL(expected_string_2, request); } CATCH TEST_F(MockDAGRequestTest, Filter) try { - auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(); - auto request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) - .filter(eq(col("s1"), col("s2"))) - .build(context); + auto request = mock_dag_request_context.scan("test_db", "test_table").filter(eq(col("s1"), col("s2"))).build(context); String expected_string = "selection_1\n" " table_scan_0\n"; ASSERT_DAGREQUEST_EQAUL(expected_string, request); - request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}, {"s3", TiDB::TP::TypeString}}) + request = mock_dag_request_context.scan("test_db", "test_table_1") .filter(And(eq(col("s1"), col("s2")), lt(col("s2"), col("s3")))) .build(context); ASSERT_DAGREQUEST_EQAUL(expected_string, request); @@ -68,22 +66,21 @@ CATCH TEST_F(MockDAGRequestTest, Projection) try { - auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(); - auto request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) + auto request = mock_dag_request_context.scan("test_db", "test_table") .project("s1") .build(context); String expected_string = "project_1\n" " table_scan_0\n"; ASSERT_DAGREQUEST_EQAUL(expected_string, request); - request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}, {"s3", TiDB::TP::TypeString}}) - .project({col("s3")}) + request = mock_dag_request_context.scan("test_db", "test_table_1") + .project({col("s3"), eq(col("s1"), col("s2"))}) .build(context); String expected_string_2 = "project_1\n" " table_scan_0\n"; ASSERT_DAGREQUEST_EQAUL(expected_string_2, request); - request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}, {"s3", TiDB::TP::TypeString}}) + request = mock_dag_request_context.scan("test_db", "test_table_1") .project({"s1", "s2"}) .build(context); ASSERT_DAGREQUEST_EQAUL(expected_string, request); @@ -93,15 +90,14 @@ CATCH TEST_F(MockDAGRequestTest, Limit) try { - auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(); - auto request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) + auto request = mock_dag_request_context.scan("test_db", "test_table") .limit(10) .build(context); String expected_string = "limit_1\n" " table_scan_0\n"; ASSERT_DAGREQUEST_EQAUL(expected_string, request); - request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) + request = mock_dag_request_context.scan("test_db", "test_table_1") .limit(lit(Field(static_cast(10)))) .build(context); ASSERT_DAGREQUEST_EQAUL(expected_string, request); @@ -111,15 +107,14 @@ CATCH TEST_F(MockDAGRequestTest, TopN) try { - auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(); - auto request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) + auto request = mock_dag_request_context.scan("test_db", "test_table") .topN({{"s1", false}}, 10) .build(context); String expected_string = "topn_1\n" " table_scan_0\n"; ASSERT_DAGREQUEST_EQAUL(expected_string, request); - request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) + request = mock_dag_request_context.scan("test_db", "test_table") .topN("s1", false, 10) .build(context); ASSERT_DAGREQUEST_EQAUL(expected_string, request); @@ -129,8 +124,7 @@ CATCH TEST_F(MockDAGRequestTest, Aggregation) try { - auto builder = DAGRequestBuilderFactory().createDAGRequestBuilder(); - auto request = builder.mockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}) + auto request = mock_dag_request_context.scan("test_db", "test_table") .aggregation(Max(col("s1")), col("s2")) .build(context); String expected_string = "aggregation_1\n" @@ -142,21 +136,16 @@ CATCH TEST_F(MockDAGRequestTest, Join) try { - auto builder_factory = DAGRequestBuilderFactory(); - DAGRequestBuilder right_builder = builder_factory.createDAGRequestBuilder(); - right_builder - .mockTable({"r_db", "r_table"}, {{"r_a", TiDB::TP::TypeString}, {"r_b", TiDB::TP::TypeString}}) - .filter(eq(col("r_a"), col("r_b"))) - .project({col("r_a"), col("r_b")}) - .aggregation(Max(col("r_a")), col("r_b")); - - - DAGRequestBuilder left_builder = builder_factory.createDAGRequestBuilder(); - left_builder - .mockTable({"l_db", "l_table"}, {{"l_a", TiDB::TP::TypeString}, {"l_b", TiDB::TP::TypeString}}) - .topN({{"l_a", false}}, 10) - .join(right_builder, col("l_a"), ASTTableJoin::Kind::Left) - .limit(10); + DAGRequestBuilder right_builder = mock_dag_request_context.scan("test_db", "r_table") + .filter(eq(col("r_a"), col("r_b"))) + .project({col("r_a"), col("r_b")}) + .aggregation(Max(col("r_a")), col("r_b")); + + + DAGRequestBuilder left_builder = mock_dag_request_context.scan("test_db", "l_table") + .topN({{"l_a", false}}, 10) + .join(right_builder, col("l_a"), ASTTableJoin::Kind::Left) + .limit(10); auto request = left_builder.build(context); String expected_string = "limit_7\n" @@ -170,6 +159,5 @@ try } CATCH - } // namespace tests } // namespace DB \ No newline at end of file From 7111839c2e5ba1581eeea4808e5fb3d974d56f3c Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Fri, 15 Apr 2022 15:43:15 +0800 Subject: [PATCH 15/16] address comments. --- dbms/src/TestUtils/InterpreterTestUtils.cpp | 5 +++++ dbms/src/TestUtils/tests/gtest_mock_executors.cpp | 13 +++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/dbms/src/TestUtils/InterpreterTestUtils.cpp b/dbms/src/TestUtils/InterpreterTestUtils.cpp index 8374ee74017..27928822f79 100644 --- a/dbms/src/TestUtils/InterpreterTestUtils.cpp +++ b/dbms/src/TestUtils/InterpreterTestUtils.cpp @@ -16,6 +16,8 @@ namespace DB::tests { +namespace +{ String toTreeString(const tipb::Executor & root_executor, size_t level = 0); String toTreeString(std::shared_ptr dag_request) @@ -52,6 +54,8 @@ String toTreeString(const tipb::Executor & root_executor, size_t level) traverseExecutorTree(root_executor, [&](const tipb::Executor & executor) { if (executor.has_join()) { + append_str(executor); + ++level; for (const auto & child : executor.join().children()) buffer.append(toTreeString(child, level)); return false; @@ -66,6 +70,7 @@ String toTreeString(const tipb::Executor & root_executor, size_t level) return buffer.toString(); } +} // namespace void dagRequestEqual(String & expected_string, const std::shared_ptr & actual) { diff --git a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp index bf2b7dbf720..2be63311034 100644 --- a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp +++ b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp @@ -149,12 +149,13 @@ try auto request = left_builder.build(context); String expected_string = "limit_7\n" - " topn_5\n" - " table_scan_4\n" - " aggregation_3\n" - " project_2\n" - " selection_1\n" - " table_scan_0\n"; + " Join_6\n" + " topn_5\n" + " table_scan_4\n" + " aggregation_3\n" + " project_2\n" + " selection_1\n" + " table_scan_0\n"; ASSERT_DAGREQUEST_EQAUL(expected_string, request); } CATCH From 2a76598d7303b73037977f5c7e22871e919dcf51 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Fri, 15 Apr 2022 15:51:53 +0800 Subject: [PATCH 16/16] add annotations. --- dbms/src/TestUtils/InterpreterTestUtils.cpp | 1 + dbms/src/TestUtils/mockExecutor.h | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dbms/src/TestUtils/InterpreterTestUtils.cpp b/dbms/src/TestUtils/InterpreterTestUtils.cpp index 27928822f79..52ff5e1cb08 100644 --- a/dbms/src/TestUtils/InterpreterTestUtils.cpp +++ b/dbms/src/TestUtils/InterpreterTestUtils.cpp @@ -20,6 +20,7 @@ namespace { String toTreeString(const tipb::Executor & root_executor, size_t level = 0); +// serialize tipb::DAGRequest, print the executor name in a Tree format. String toTreeString(std::shared_ptr dag_request) { assert((dag_request->executors_size() > 0) != dag_request->has_root_executor()); diff --git a/dbms/src/TestUtils/mockExecutor.h b/dbms/src/TestUtils/mockExecutor.h index 0002d63768e..24d2df21f4a 100644 --- a/dbms/src/TestUtils/mockExecutor.h +++ b/dbms/src/TestUtils/mockExecutor.h @@ -55,7 +55,6 @@ class DAGRequestBuilder std::shared_ptr build(Context & context); - // ywq todo check arguments DAGRequestBuilder & mockTable(const String & db, const String & table, const MockColumnInfos & columns); DAGRequestBuilder & mockTable(const MockTableName & name, const MockColumnInfos & columns); DAGRequestBuilder & mockTable(const MockTableName & name, const MockColumnInfoList & columns); @@ -91,6 +90,10 @@ class DAGRequestBuilder DAGProperties properties; }; +/** Responsible for storing necessary arguments in order to Mock DAGRequest + * index: used in DAGRequestBuilder to identify executors + * mock_tables: DAGRequestBuilder uses it to mock TableScan executors + */ class MockDAGRequestContext { public: @@ -110,8 +113,6 @@ class MockDAGRequestContext DAGRequestBuilder scan(String db_name, String table_name); - size_t size() { return mock_tables.size(); } - private: size_t index; std::unordered_map mock_tables;