From 52c63c62ffc16826afd273fde7067b1210b31a17 Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Wed, 9 Nov 2022 14:42:35 +0800 Subject: [PATCH 1/7] fix cluster index --- br/tests/_utils/run_sql | 2 ++ br/tests/_utils/run_sql_file | 2 ++ 2 files changed, 4 insertions(+) diff --git a/br/tests/_utils/run_sql b/br/tests/_utils/run_sql index 39241157cade6..730b2a73ec0a6 100755 --- a/br/tests/_utils/run_sql +++ b/br/tests/_utils/run_sql @@ -19,7 +19,9 @@ set -euo pipefail SQL="$1" shift +prepare="set global tidb_enable_clustered_index = 'int_only';" echo "[$(date)] Executing SQL: $SQL" > "$TEST_DIR/sql_res.$TEST_NAME.txt" +mysql -uroot -h127.0.0.1 -P4000 --default-character-set utf8 -E -e "${prepare}" | tee -a "$OUT_DIR/sql_res.$TEST_NAME.txt" mysql -uroot -h127.0.0.1 -P4000 \ --ssl-ca="$TEST_DIR/certs/ca.pem" \ --ssl-cert="$TEST_DIR/certs/curl.pem" \ diff --git a/br/tests/_utils/run_sql_file b/br/tests/_utils/run_sql_file index 59942940c076a..2b9d7d662e55c 100755 --- a/br/tests/_utils/run_sql_file +++ b/br/tests/_utils/run_sql_file @@ -19,7 +19,9 @@ set -euo pipefail SQL_FILE="$1" shift +prepare="set global tidb_enable_clustered_index = 'int_only';" echo "[$(date)] Executing SQL_FILE: $SQL_FILE" > "$TEST_DIR/sql_res.$TEST_NAME.txt" +mysql -uroot -h127.0.0.1 -P4000 --default-character-set utf8 -E -e "${prepare}" | tee -a "$OUT_DIR/sql_res.$TEST_NAME.txt" cat $SQL_FILE | mysql -uroot -h127.0.0.1 -P4000 \ --ssl-ca="$TEST_DIR/certs/ca.pem" \ --ssl-cert="$TEST_DIR/certs/curl.pem" \ From e6102a094c147499e39eff548b87c4a230acb10d Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Wed, 9 Nov 2022 14:52:21 +0800 Subject: [PATCH 2/7] fix --- br/tests/_utils/run_sql | 2 +- br/tests/_utils/run_sql_file | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/br/tests/_utils/run_sql b/br/tests/_utils/run_sql index 730b2a73ec0a6..8fc9afbb53fb0 100755 --- a/br/tests/_utils/run_sql +++ b/br/tests/_utils/run_sql @@ -21,7 +21,7 @@ shift prepare="set global tidb_enable_clustered_index = 'int_only';" echo "[$(date)] Executing SQL: $SQL" > "$TEST_DIR/sql_res.$TEST_NAME.txt" -mysql -uroot -h127.0.0.1 -P4000 --default-character-set utf8 -E -e "${prepare}" | tee -a "$OUT_DIR/sql_res.$TEST_NAME.txt" +mysql -uroot -h127.0.0.1 -P4000 --default-character-set utf8 -E -e "${prepare}" | tee -a "$TEST_DIR/sql_res.$TEST_NAME.txt" mysql -uroot -h127.0.0.1 -P4000 \ --ssl-ca="$TEST_DIR/certs/ca.pem" \ --ssl-cert="$TEST_DIR/certs/curl.pem" \ diff --git a/br/tests/_utils/run_sql_file b/br/tests/_utils/run_sql_file index 2b9d7d662e55c..88718826bb2b5 100755 --- a/br/tests/_utils/run_sql_file +++ b/br/tests/_utils/run_sql_file @@ -21,7 +21,7 @@ shift prepare="set global tidb_enable_clustered_index = 'int_only';" echo "[$(date)] Executing SQL_FILE: $SQL_FILE" > "$TEST_DIR/sql_res.$TEST_NAME.txt" -mysql -uroot -h127.0.0.1 -P4000 --default-character-set utf8 -E -e "${prepare}" | tee -a "$OUT_DIR/sql_res.$TEST_NAME.txt" +mysql -uroot -h127.0.0.1 -P4000 --default-character-set utf8 -E -e "${prepare}" | tee -a "$TEST_DIR/sql_res.$TEST_NAME.txt" cat $SQL_FILE | mysql -uroot -h127.0.0.1 -P4000 \ --ssl-ca="$TEST_DIR/certs/ca.pem" \ --ssl-cert="$TEST_DIR/certs/curl.pem" \ From bfac66c98693e96efcbed7822e33f14c6af793ce Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Wed, 9 Nov 2022 17:12:13 +0800 Subject: [PATCH 3/7] fix test --- br/tests/_utils/run_sql | 3 +-- br/tests/_utils/run_sql_file | 3 +-- .../lightning_shard_rowid/data/shard_rowid.shr-schema.sql | 4 ++-- br/tests/lightning_tidb_rowid/data/rowid.non_pk-schema.sql | 2 +- .../data/rowid.non_pk_auto_inc-schema.sql | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/br/tests/_utils/run_sql b/br/tests/_utils/run_sql index 8fc9afbb53fb0..566eaa968620e 100755 --- a/br/tests/_utils/run_sql +++ b/br/tests/_utils/run_sql @@ -19,9 +19,8 @@ set -euo pipefail SQL="$1" shift -prepare="set global tidb_enable_clustered_index = 'int_only';" echo "[$(date)] Executing SQL: $SQL" > "$TEST_DIR/sql_res.$TEST_NAME.txt" -mysql -uroot -h127.0.0.1 -P4000 --default-character-set utf8 -E -e "${prepare}" | tee -a "$TEST_DIR/sql_res.$TEST_NAME.txt" + mysql -uroot -h127.0.0.1 -P4000 \ --ssl-ca="$TEST_DIR/certs/ca.pem" \ --ssl-cert="$TEST_DIR/certs/curl.pem" \ diff --git a/br/tests/_utils/run_sql_file b/br/tests/_utils/run_sql_file index 88718826bb2b5..bbd1efb4ca84c 100755 --- a/br/tests/_utils/run_sql_file +++ b/br/tests/_utils/run_sql_file @@ -19,9 +19,8 @@ set -euo pipefail SQL_FILE="$1" shift -prepare="set global tidb_enable_clustered_index = 'int_only';" echo "[$(date)] Executing SQL_FILE: $SQL_FILE" > "$TEST_DIR/sql_res.$TEST_NAME.txt" -mysql -uroot -h127.0.0.1 -P4000 --default-character-set utf8 -E -e "${prepare}" | tee -a "$TEST_DIR/sql_res.$TEST_NAME.txt" + cat $SQL_FILE | mysql -uroot -h127.0.0.1 -P4000 \ --ssl-ca="$TEST_DIR/certs/ca.pem" \ --ssl-cert="$TEST_DIR/certs/curl.pem" \ diff --git a/br/tests/lightning_shard_rowid/data/shard_rowid.shr-schema.sql b/br/tests/lightning_shard_rowid/data/shard_rowid.shr-schema.sql index 312b13c1c1118..d544b7fdb84c1 100644 --- a/br/tests/lightning_shard_rowid/data/shard_rowid.shr-schema.sql +++ b/br/tests/lightning_shard_rowid/data/shard_rowid.shr-schema.sql @@ -3,5 +3,5 @@ CREATE TABLE `test` ( `s1` char(10) NOT NULL, `s2` char(10) NOT NULL, `s3` char(10) DEFAULT NULL, - PRIMARY KEY (`s1`,`s2`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin/*!90000 SHARD_ROW_ID_BITS=3 PRE_SPLIT_REGIONS=3 */; \ No newline at end of file + PRIMARY KEY (`s1`,`s2`) /*T![clustered_index] NONCLUSTERED */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin/*!90000 SHARD_ROW_ID_BITS=3 PRE_SPLIT_REGIONS=3 */; diff --git a/br/tests/lightning_tidb_rowid/data/rowid.non_pk-schema.sql b/br/tests/lightning_tidb_rowid/data/rowid.non_pk-schema.sql index 5b5757644b6dd..52ee2729417a3 100644 --- a/br/tests/lightning_tidb_rowid/data/rowid.non_pk-schema.sql +++ b/br/tests/lightning_tidb_rowid/data/rowid.non_pk-schema.sql @@ -1 +1 @@ -create table non_pk (pk varchar(6) primary key); +create table non_pk (pk varchar(6) primary key /*T![clustered_index] NONCLUSTERED */); diff --git a/br/tests/lightning_tidb_rowid/data/rowid.non_pk_auto_inc-schema.sql b/br/tests/lightning_tidb_rowid/data/rowid.non_pk_auto_inc-schema.sql index a71be02c9e8f1..97aa81838b1bc 100644 --- a/br/tests/lightning_tidb_rowid/data/rowid.non_pk_auto_inc-schema.sql +++ b/br/tests/lightning_tidb_rowid/data/rowid.non_pk_auto_inc-schema.sql @@ -4,6 +4,6 @@ CREATE TABLE `non_pk_auto_inc` ( `pk` char(36) NOT NULL, `id` int(11) NOT NULL AUTO_INCREMENT, - PRIMARY KEY (`pk`), + PRIMARY KEY (`pk`) /*T![clustered_index] NONCLUSTERED */, UNIQUE KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; From 444ec2b7186679014f225b72079cc9a57d170e5b Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Wed, 9 Nov 2022 17:12:58 +0800 Subject: [PATCH 4/7] fix --- br/tests/_utils/run_sql | 1 - br/tests/_utils/run_sql_file | 1 - 2 files changed, 2 deletions(-) diff --git a/br/tests/_utils/run_sql b/br/tests/_utils/run_sql index 566eaa968620e..39241157cade6 100755 --- a/br/tests/_utils/run_sql +++ b/br/tests/_utils/run_sql @@ -20,7 +20,6 @@ SQL="$1" shift echo "[$(date)] Executing SQL: $SQL" > "$TEST_DIR/sql_res.$TEST_NAME.txt" - mysql -uroot -h127.0.0.1 -P4000 \ --ssl-ca="$TEST_DIR/certs/ca.pem" \ --ssl-cert="$TEST_DIR/certs/curl.pem" \ diff --git a/br/tests/_utils/run_sql_file b/br/tests/_utils/run_sql_file index bbd1efb4ca84c..59942940c076a 100755 --- a/br/tests/_utils/run_sql_file +++ b/br/tests/_utils/run_sql_file @@ -20,7 +20,6 @@ SQL_FILE="$1" shift echo "[$(date)] Executing SQL_FILE: $SQL_FILE" > "$TEST_DIR/sql_res.$TEST_NAME.txt" - cat $SQL_FILE | mysql -uroot -h127.0.0.1 -P4000 \ --ssl-ca="$TEST_DIR/certs/ca.pem" \ --ssl-cert="$TEST_DIR/certs/curl.pem" \ From 4587e88964b080ecc730cf1816e05aed144e5cac Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Wed, 9 Nov 2022 20:18:54 +0800 Subject: [PATCH 5/7] fix test --- br/tests/lightning_exotic_filenames/data/xfn.etn-schema.sql | 2 +- br/tests/lightning_exotic_filenames/data/zwk.zwb-schema.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/br/tests/lightning_exotic_filenames/data/xfn.etn-schema.sql b/br/tests/lightning_exotic_filenames/data/xfn.etn-schema.sql index e2d94bbdf8f32..d004fa92e0b64 100644 --- a/br/tests/lightning_exotic_filenames/data/xfn.etn-schema.sql +++ b/br/tests/lightning_exotic_filenames/data/xfn.etn-schema.sql @@ -1 +1 @@ -create table `exotic``table````name` (a varchar(6) primary key, b int unique auto_increment) auto_increment=80000; \ No newline at end of file +create table `exotic``table````name` (a varchar(6) primary key /*T![clustered_index] NONCLUSTERED */, b int unique auto_increment) auto_increment=80000; diff --git a/br/tests/lightning_exotic_filenames/data/zwk.zwb-schema.sql b/br/tests/lightning_exotic_filenames/data/zwk.zwb-schema.sql index 449584777c299..d9fae1aad0373 100644 --- a/br/tests/lightning_exotic_filenames/data/zwk.zwb-schema.sql +++ b/br/tests/lightning_exotic_filenames/data/zwk.zwb-schema.sql @@ -1 +1 @@ -create table 中文表(a int primary key); +create table 中文表(a int primary key /*T![clustered_index] NONCLUSTERED */); From 4863a2cab2ac889c89e8bae7240db36d21194dca Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Thu, 10 Nov 2022 15:23:46 +0800 Subject: [PATCH 6/7] cherry-pick https://github.com/pingcap/tidb/pull/39051 --- br/pkg/lightning/restore/meta_manager.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/br/pkg/lightning/restore/meta_manager.go b/br/pkg/lightning/restore/meta_manager.go index 659a33c579ef0..2d9875ad56960 100644 --- a/br/pkg/lightning/restore/meta_manager.go +++ b/br/pkg/lightning/restore/meta_manager.go @@ -1186,9 +1186,12 @@ func getGlobalAutoIDAlloc(store kv.Storage, dbID int64, tblInfo *model.TableInfo return nil, errors.New("internal error: dbID should not be 0") } - // We don't need the cache here because we allocate all IDs at once. - // The argument for CustomAutoIncCacheOption is the cache step. step 1 means no cache. - noCache := autoid.CustomAutoIncCacheOption(1) + // We don't need autoid cache here because we allocate all IDs at once. + // The argument for CustomAutoIncCacheOption is the cache step. Step 1 means no cache, + // but step 1 will enable an experimental feature, so we use step 2 here. + // + // See https://github.com/pingcap/tidb/issues/38442 for more details. + noCache := autoid.CustomAutoIncCacheOption(2) tblVer := autoid.AllocOptionTableInfoVersion(tblInfo.Version) hasRowID := common.TableHasAutoRowID(tblInfo) From 9d45a8cfeb0144082b3ba5ea9dc7eec736f75a7f Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Thu, 10 Nov 2022 17:01:46 +0800 Subject: [PATCH 7/7] trigger