From 7fb76642a5ac1623d46033ee9b001348ae5602c6 Mon Sep 17 00:00:00 2001 From: JaySon Date: Thu, 1 Aug 2024 10:56:50 +0800 Subject: [PATCH] This is an automated cherry-pick of #9274 Signed-off-by: ti-chi-bot --- dbms/src/Debug/MockTiDB.cpp | 9 +- dbms/src/TiDB/Schema/SchemaBuilder.cpp | 331 +++++++++++++++++- dbms/src/TiDB/Schema/SchemaBuilder.h | 6 + .../ddl/rename_table_across_databases.test | 91 +++++ 4 files changed, 431 insertions(+), 6 deletions(-) diff --git a/dbms/src/Debug/MockTiDB.cpp b/dbms/src/Debug/MockTiDB.cpp index 3b7ec9f70d4..0a09c674197 100644 --- a/dbms/src/Debug/MockTiDB.cpp +++ b/dbms/src/Debug/MockTiDB.cpp @@ -667,17 +667,18 @@ TiDB::TableInfoPtr MockTiDB::getTableInfoByID(TableID table_id) TiDB::DBInfoPtr MockTiDB::getDBInfoByID(DatabaseID db_id) { - TiDB::DBInfoPtr db_ptr = std::make_shared(TiDB::DBInfo()); - db_ptr->id = db_id; for (const auto & database : databases) { if (database.second == db_id) { + TiDB::DBInfoPtr db_ptr = std::make_shared(TiDB::DBInfo()); + db_ptr->id = db_id; db_ptr->name = database.first; - break; + return db_ptr; } } - return db_ptr; + // If the database has been dropped in TiKV, TiFlash get a nullptr + return nullptr; } std::pair MockTiDB::getDBIDByName(const String & database_name) diff --git a/dbms/src/TiDB/Schema/SchemaBuilder.cpp b/dbms/src/TiDB/Schema/SchemaBuilder.cpp index 878f2b29c1b..21a447b10ce 100644 --- a/dbms/src/TiDB/Schema/SchemaBuilder.cpp +++ b/dbms/src/TiDB/Schema/SchemaBuilder.cpp @@ -23,16 +23,19 @@ #include #include #include +<<<<<<< HEAD #include +======= +#include +>>>>>>> 45eb5079cd (ddl: Fix rename come after database has been dropped (#9274)) #include -#include #include #include -#include #include #include #include #include +#include #include #include #include @@ -940,6 +943,268 @@ bool SchemaBuilder::applyCreateSchema(DatabaseID schema_id) auto db = getter.getDatabase(schema_id); if (db == nullptr) { +<<<<<<< HEAD +======= + LOG_INFO( + log, + "Altering non-partition table to be a partition table {} with database_id={}, table_id={}", + name_mapper.debugCanonicalName(local_table_info, database_id, keyspace_id), + database_id, + local_table_info.id); + } + + const auto & local_defs = local_table_info.partition.definitions; + const auto & new_defs = table_info->partition.definitions; + + std::unordered_set local_part_id_set, new_part_id_set; + std::for_each(local_defs.begin(), local_defs.end(), [&local_part_id_set](const auto & def) { + local_part_id_set.emplace(def.id); + }); + std::for_each(new_defs.begin(), new_defs.end(), [&new_part_id_set](const auto & def) { + new_part_id_set.emplace(def.id); + }); + + LOG_INFO( + log, + "Applying partition changes {} with database_id={}, table_id={}, old: {}, new: {}", + name_mapper.debugCanonicalName(*table_info, database_id, keyspace_id), + database_id, + table_info->id, + local_part_id_set, + new_part_id_set); + + if (local_part_id_set == new_part_id_set) + { + LOG_INFO( + log, + "No partition changes, partitions_size={} {} with database_id={}, table_id={}", + new_part_id_set.size(), + name_mapper.debugCanonicalName(*table_info, database_id, keyspace_id), + database_id, + table_info->id); + return; + } + + // Copy the local table info and update fields on the copy + auto updated_table_info = local_table_info; + updated_table_info.is_partition_table = true; + updated_table_info.belonging_table_id = table_info->belonging_table_id; + updated_table_info.partition = table_info->partition; + + /// Apply changes to physical tables. + auto reason = fmt::format("ApplyPartitionDiff-logical_table_id={}", local_table_info.id); + for (const auto & local_def : local_defs) + { + if (!new_part_id_set.contains(local_def.id)) + { + applyDropPhysicalTable(name_mapper.mapDatabaseName(database_id, keyspace_id), local_def.id, reason); + } + } + + for (const auto & new_def : new_defs) + { + if (!local_part_id_set.contains(new_def.id)) + { + table_id_map.emplacePartitionTableID(new_def.id, updated_table_info.id); + } + } + + auto alter_lock = storage->lockForAlter(getThreadNameAndID()); + storage->alterSchemaChange( + alter_lock, + updated_table_info, + name_mapper.mapDatabaseName(database_id, keyspace_id), + name_mapper.mapTableName(updated_table_info), + context); + + GET_METRIC(tiflash_schema_internal_ddl_count, type_apply_partition).Increment(); + LOG_INFO( + log, + "Applied partition changes {} with database_id={}, table_id={}", + name_mapper.debugCanonicalName(*table_info, database_id, keyspace_id), + database_id, + table_info->id); +} + +template +void SchemaBuilder::applyRenameTable(DatabaseID database_id, TableID table_id) +{ + // update the table_id_map no matter storage instance is created or not + table_id_map.emplaceTableID(table_id, database_id); + + auto & tmt_context = context.getTMTContext(); + auto storage = tmt_context.getStorages().get(keyspace_id, table_id); + if (storage == nullptr) + { + LOG_WARNING( + log, + "Storage instance is not exist in TiFlash, applyRenameTable is ignored, table_id={}", + table_id); + return; + } + + auto new_table_info = getter.getTableInfo(database_id, table_id); + if (unlikely(new_table_info == nullptr)) + { + LOG_ERROR(log, "table is not exist in TiKV, applyRenameTable is ignored, table_id={}", table_id); + return; + } + + String new_db_display_name = tryGetDatabaseDisplayNameFromLocal(database_id); + applyRenameLogicalTable(database_id, new_db_display_name, new_table_info, storage); +} + +template +void SchemaBuilder::applyRenameLogicalTable( + const DatabaseID new_database_id, + const String & new_database_display_name, + const TableInfoPtr & new_table_info, + const ManageableStoragePtr & storage) +{ + applyRenamePhysicalTable(new_database_id, new_database_display_name, *new_table_info, storage); + if (!new_table_info->isLogicalPartitionTable()) + return; + + // For partitioned table, try to execute rename on each partition (physical table) + auto & tmt_context = context.getTMTContext(); + for (const auto & part_def : new_table_info->partition.definitions) + { + auto part_storage = tmt_context.getStorages().get(keyspace_id, part_def.id); + if (part_storage == nullptr) + { + LOG_WARNING( + log, + "Storage instance is not exist in TiFlash, the partition is not created yet in this TiFlash instance, " + "applyRenamePhysicalTable is ignored, physical_table_id={} logical_table_id={}", + part_def.id, + new_table_info->id); + continue; // continue for next partition + } + + FAIL_POINT_TRIGGER_EXCEPTION(FailPoints::random_ddl_fail_when_rename_partitions); + auto part_table_info = new_table_info->producePartitionTableInfo(part_def.id, name_mapper); + applyRenamePhysicalTable(new_database_id, new_database_display_name, *part_table_info, part_storage); + } +} + +template +void SchemaBuilder::applyRenamePhysicalTable( + const DatabaseID new_database_id, + const String & new_database_display_name, + const TableInfo & new_table_info, + const ManageableStoragePtr & storage) +{ + const auto old_mapped_db_name = storage->getDatabaseName(); + const auto new_mapped_db_name = name_mapper.mapDatabaseName(new_database_id, keyspace_id); + const auto old_display_table_name = name_mapper.displayTableName(storage->getTableInfo()); + const auto new_display_table_name = name_mapper.displayTableName(new_table_info); + if (old_mapped_db_name == new_mapped_db_name && old_display_table_name == new_display_table_name) + { + LOG_DEBUG( + log, + "Table {} name identical, not renaming. database_id={} table_id={}", + name_mapper.debugCanonicalName(new_table_info, new_database_id, keyspace_id), + new_database_id, + new_table_info.id); + return; + } + + // There could be a chance that the target database has been dropped in TiKV before + // TiFlash accepts the "create database" schema diff. We need to ensure the local + // database exist before executing renaming. + const auto action = fmt::format("applyRenamePhysicalTable-table_id={}", new_table_info.id); + ensureLocalDatabaseExist(new_database_id, new_mapped_db_name, action); + const auto old_mapped_tbl_name = storage->getTableName(); + GET_METRIC(tiflash_schema_internal_ddl_count, type_rename_table).Increment(); + LOG_INFO( + log, + "Rename table {}.{} (display name: {}) to {} begin, database_id={} table_id={}", + old_mapped_db_name, + old_mapped_tbl_name, + old_display_table_name, + name_mapper.debugCanonicalName(new_table_info, new_database_id, keyspace_id), + new_database_id, + new_table_info.id); + + // Note that rename will update table info in table create statement by modifying original table info + // with "tidb_display.table" instead of using new_table_info directly, so that other changes + // (ALTER commands) won't be saved. Besides, no need to update schema_version as table name is not structural. + auto rename = std::make_shared(); + ASTRenameQuery::Element elem{ + .from = ASTRenameQuery::Table{old_mapped_db_name, old_mapped_tbl_name}, + .to = ASTRenameQuery::Table{new_mapped_db_name, name_mapper.mapTableName(new_table_info)}, + .tidb_display = ASTRenameQuery::Table{new_database_display_name, new_display_table_name}, + }; + rename->elements.emplace_back(std::move(elem)); + + InterpreterRenameQuery(rename, context, getThreadNameAndID()).execute(); + + LOG_INFO( + log, + "Rename table {}.{} (display name: {}) to {} end, database_id={} table_id={}", + old_mapped_db_name, + old_mapped_tbl_name, + old_display_table_name, + name_mapper.debugCanonicalName(new_table_info, new_database_id, keyspace_id), + new_database_id, + new_table_info.id); +} + +template +void SchemaBuilder::applyRecoverTable(DatabaseID database_id, TiDB::TableID table_id) +{ + auto table_info = getter.getTableInfo(database_id, table_id); + if (unlikely(table_info == nullptr)) + { + // this table is dropped. + LOG_INFO( + log, + "table is not exist in TiKV, may have been dropped, recover table is ignored, table_id={}", + table_id); + return; + } + + applyRecoverLogicalTable(database_id, table_info, "RecoverTable"); +} + +template +void SchemaBuilder::applyRecoverLogicalTable( + const DatabaseID database_id, + const TiDB::TableInfoPtr & table_info, + std::string_view action) +{ + assert(table_info != nullptr); + if (table_info->isLogicalPartitionTable()) + { + for (const auto & part_def : table_info->partition.definitions) + { + auto part_table_info = table_info->producePartitionTableInfo(part_def.id, name_mapper); + tryRecoverPhysicalTable(database_id, part_table_info, action); + } + } + + tryRecoverPhysicalTable(database_id, table_info, action); +} + +// Return true - the Storage instance exists and is recovered (or not tombstone) +// false - the Storage instance does not exist +template +bool SchemaBuilder::tryRecoverPhysicalTable( + const DatabaseID database_id, + const TiDB::TableInfoPtr & table_info, + std::string_view action) +{ + assert(table_info != nullptr); + auto & tmt_context = context.getTMTContext(); + auto storage = tmt_context.getStorages().get(keyspace_id, table_info->id); + if (storage == nullptr) + { + LOG_INFO( + log, + "Storage instance does not exist, tryRecoverPhysicalTable is ignored, table_id={} action={}", + table_info->id, + action); +>>>>>>> 45eb5079cd (ddl: Fix rename come after database has been dropped (#9274)) return false; } applyCreateSchema(db); @@ -1220,7 +1485,25 @@ void SchemaBuilder::applyCreatePhysicalTable(const DBInfoPtr String stmt = createTableStmt(*db_info, *table_info, name_mapper, log); +<<<<<<< HEAD LOG_INFO(log, "Creating table {} with statement: {}", name_mapper.debugCanonicalName(*db_info, *table_info), stmt); +======= + String stmt = createTableStmt(keyspace_id, database_id, *table_info, name_mapper, tombstone_ts, log); + + LOG_INFO( + log, + "Create table {} (database_id={} table_id={}) with statement: {}", + name_mapper.debugCanonicalName(*table_info, database_id, keyspace_id), + database_id, + table_info->id, + stmt); + + // If "CREATE DATABASE" is executed in TiFlash after user has executed "DROP DATABASE" + // in TiDB, then TiFlash may not create the IDatabase instance. Make sure we can access + // to the IDatabase when creating IStorage. + const auto database_mapped_name = name_mapper.mapDatabaseName(database_id, keyspace_id); + ensureLocalDatabaseExist(database_id, database_mapped_name, action); +>>>>>>> 45eb5079cd (ddl: Fix rename come after database has been dropped (#9274)) ParserCreateQuery parser; ASTPtr ast = parseQuery(parser, stmt.data(), stmt.data() + stmt.size(), "from syncSchema " + table_info->name, 0); @@ -1234,9 +1517,50 @@ void SchemaBuilder::applyCreatePhysicalTable(const DBInfoPtr interpreter.setInternal(true); interpreter.setForceRestoreData(false); interpreter.execute(); +<<<<<<< HEAD LOG_INFO(log, "Created table {}", name_mapper.debugCanonicalName(*db_info, *table_info)); } +======= + LOG_INFO( + log, + "Create table {} end, database_id={} table_id={} action={}", + name_mapper.debugCanonicalName(*table_info, database_id, keyspace_id), + database_id, + table_info->id, + action); +} + +template +void SchemaBuilder::ensureLocalDatabaseExist( + DatabaseID database_id, + const String & database_mapped_name, + std::string_view action) +{ + if (likely(context.isDatabaseExist(database_mapped_name))) + return; + + LOG_WARNING( + log, + "database instance is not exist, may has been dropped, create a database " + "with fake DatabaseInfo for it, database_id={} database_name={} action={}", + database_id, + database_mapped_name, + action); + // The database is dropped in TiKV and we can not fetch it. Generate a fake + // DatabaseInfo for it. It is OK because the DatabaseInfo will be updated + // when the database is `FLASHBACK`. + TiDB::DBInfoPtr database_info = std::make_shared(); + database_info->id = database_id; + database_info->keyspace_id = keyspace_id; + database_info->name = database_mapped_name; // use the mapped name because we don't known the actual name + database_info->charset = "utf8mb4"; // default value + database_info->collate = "utf8mb4_bin"; // default value + database_info->state = TiDB::StateNone; // special state + applyCreateDatabaseByInfo(database_info); +} + +>>>>>>> 45eb5079cd (ddl: Fix rename come after database has been dropped (#9274)) template void SchemaBuilder::applyCreateTable(const TiDB::DBInfoPtr & db_info, TableID table_id) { @@ -1496,6 +1820,9 @@ template struct SchemaBuilder; template struct SchemaBuilder; // unit test template struct SchemaBuilder; +<<<<<<< HEAD // end namespace +======= +>>>>>>> 45eb5079cd (ddl: Fix rename come after database has been dropped (#9274)) } // namespace DB diff --git a/dbms/src/TiDB/Schema/SchemaBuilder.h b/dbms/src/TiDB/Schema/SchemaBuilder.h index 8fac1141f64..ea5f6435c91 100644 --- a/dbms/src/TiDB/Schema/SchemaBuilder.h +++ b/dbms/src/TiDB/Schema/SchemaBuilder.h @@ -53,7 +53,13 @@ struct SchemaBuilder /// Parameter db_name should be mapped. void applyDropSchema(const String & db_name); +<<<<<<< HEAD void applyRecoverSchema(DatabaseID database_id); +======= + bool applyCreateDatabase(DatabaseID database_id); + void applyCreateDatabaseByInfo(const TiDB::DBInfoPtr & db_info); + void ensureLocalDatabaseExist(DatabaseID database_id, const String & database_mapped_name, std::string_view action); +>>>>>>> 45eb5079cd (ddl: Fix rename come after database has been dropped (#9274)) bool applyCreateSchema(DatabaseID schema_id); diff --git a/tests/fullstack-test2/ddl/rename_table_across_databases.test b/tests/fullstack-test2/ddl/rename_table_across_databases.test index 32ff3ab96e9..ca6895065af 100644 --- a/tests/fullstack-test2/ddl/rename_table_across_databases.test +++ b/tests/fullstack-test2/ddl/rename_table_across_databases.test @@ -61,3 +61,94 @@ mysql> set session tidb_isolation_read_engines='tiflash'; select * from test_new mysql> drop table if exists test.t; mysql> drop table if exists test_new.t2; mysql> drop database if exists test_new; +<<<<<<< HEAD +======= + +# (case 2) rename table across database +mysql> create database if not exists test +mysql> create database if not exists test_new +## (required) stop regular schema sync +=> DBGInvoke __enable_schema_sync_service('false') + +mysql> create table test.t(a int, b int); +mysql> insert into test.t values (1, 1); insert into test.t values (1, 2); +## (required) sync table id mapping to tiflash +=> DBGInvoke __refresh_schemas() +mysql> rename table test.t to test_new.t2; +mysql> alter table test_new.t2 set tiflash replica 1; +## new snapshot sync to tiflash, but the table id mapping is not updated +func> wait_table test_new t2 +mysql> set session tidb_isolation_read_engines='tiflash'; select * from test_new.t2; ++------+------+ +| a | b | ++------+------+ +| 1 | 1 | +| 1 | 2 | ++------+------+ + +mysql> drop table if exists test.t; +mysql> drop table if exists test_new.t2; +mysql> drop database if exists test_new; + +## (required) create a new table and sync to tiflash, check whether it can apply +mysql> drop table if exists test.t3; +mysql> create table test.t3(c int, d int); +mysql> insert into test.t3 values (3,3),(3,4); +mysql> alter table test.t3 set tiflash replica 1; +func> wait_table test t3 +mysql> set session tidb_isolation_read_engines='tiflash'; select * from test.t3; ++------+------+ +| c | d | ++------+------+ +| 3 | 3 | +| 3 | 4 | ++------+------+ + +mysql> drop table if exists test.t3; + +# (case 3) rename partitioned table across database +mysql> create database if not exists test_new; +mysql> drop table if exists test.part4; +mysql> CREATE TABLE test.part4 (id INT NOT NULL,store_id INT NOT NULL)PARTITION BY RANGE (store_id) (PARTITION p0 VALUES LESS THAN (6),PARTITION p1 VALUES LESS THAN (11),PARTITION p2 VALUES LESS THAN (16),PARTITION p3 VALUES LESS THAN (21)); +# (1,1),(2,2),(3,3) => p0; p1 is empty;(11,11) => p2;(16,16) => p3 +mysql> insert into test.part4(id, store_id) values(1,1),(2,2),(3,3),(11,11),(16,16); +mysql> alter table test.part4 set tiflash replica 1; +func> wait_table test part4 + +mysql> rename table test.part4 to test_new.part4; +mysql> alter table test_new.part4 add column c1 int; +mysql> set session tidb_isolation_read_engines='tiflash'; select * from test_new.part4 order by id; ++----+----------+------+ +| id | store_id | c1 | ++----+----------+------+ +| 1 | 1 | NULL | +| 2 | 2 | NULL | +| 3 | 3 | NULL | +| 11 | 11 | NULL | +| 16 | 16 | NULL | ++----+----------+------+ + +mysql> drop table if exists test_new.part4 + +# (case 4) rename partitioned table across database +# (required) stop regular schema sync +=> DBGInvoke __enable_schema_sync_service('false') +mysql> drop database if exists test_new; +mysql> drop table if exists test.part5; +mysql> CREATE TABLE test.part5 (id INT NOT NULL,store_id INT NOT NULL)PARTITION BY RANGE (store_id) (PARTITION p0 VALUES LESS THAN (6),PARTITION p1 VALUES LESS THAN (11),PARTITION p2 VALUES LESS THAN (16),PARTITION p3 VALUES LESS THAN (21)); +# (1,1),(2,2),(3,3) => p0; p1 is empty;(11,11) => p2;(16,16) => p3 +mysql> alter table test.part5 set tiflash replica 1; +func> wait_table test part5 + +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) +mysql> insert into test.part5(id, store_id) values(1,1),(2,2),(3,3),(11,11),(16,16); + +# create target db, rename table to target db, then drop target db +mysql> create database if not exists test_new; +mysql> rename table test.part5 to test_new.part5; +mysql> alter table test_new.part5 add column c1 int; +mysql> drop database if exists test_new; +# raft command comes after target db dropped, no crashes +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) +>> DBGInvoke __refresh_schemas() +>>>>>>> 45eb5079cd (ddl: Fix rename come after database has been dropped (#9274))