From 2964eed415c9ff8a52d28d03263a25224e7c9fb9 Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Mon, 6 May 2024 15:40:00 +0800 Subject: [PATCH 01/14] add tests --- pkg/executor/test/admintest/admin_test.go | 44 +++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/pkg/executor/test/admintest/admin_test.go b/pkg/executor/test/admintest/admin_test.go index 470d50443f0f8..962cc104ab025 100644 --- a/pkg/executor/test/admintest/admin_test.go +++ b/pkg/executor/test/admintest/admin_test.go @@ -34,6 +34,7 @@ import ( "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/table/tables" + "github.com/pingcap/tidb/pkg/tablecodec" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/testkit/testutil" "github.com/pingcap/tidb/pkg/types" @@ -1773,3 +1774,46 @@ func TestAdminCheckTableErrorLocateForClusterIndex(t *testing.T) { tk.MustExec("admin check table admin_test") } } + +func TestAdminCheckGlobalIndex(t *testing.T) { + store, domain := testkit.CreateMockStoreAndDomain(t) + + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists admin_test") + + tk.MustExec("set tidb_enable_global_index = true") + tk.MustExec("set tidb_enable_fast_table_check = false") + + tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a)) partition by hash(c) partitions 5") + tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)") + + // Make some corrupted index. Build the index information. + sctx := mock.NewContext() + sctx.Store = store + // ctx := sctx.GetTableCtx() + is := domain.InfoSchema() + dbName := model.NewCIStr("test") + tblName := model.NewCIStr("admin_test") + tbl, err := is.TableByName(dbName, tblName) + require.NoError(t, err) + tblInfo := tbl.Meta() + // idxInfo := tblInfo.Indices[0] + df := tblInfo.GetPartitionInfo().Definitions[0] + // indexOpr := tables.NewIndex(df.ID, tblInfo, idxInfo) + tk.Session().GetSessionVars().IndexLookupSize = 3 + tk.Session().GetSessionVars().MaxChunkSize = 3 + + // Reduce one row of index. + // Table count > index count. + // Index c2 is missing 2. + txn, err := store.Begin() + require.NoError(t, err) + txn.Delete(tablecodec.EncodeRowKey(df.ID, kv.IntHandle(4).Encoded())) + err = txn.Commit(context.Background()) + require.NoError(t, err) + err = tk.ExecToErr("admin check table admin_test") + require.Error(t, err) + require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 4, index-values:\"\" != record-values:\"handle: -1, values: [KindInt64 -10]\"") + require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err)) +} From be1637b4faa0544431d4c4e58c8e012ae11096aa Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Tue, 7 May 2024 22:30:55 +0800 Subject: [PATCH 02/14] update --- pkg/executor/builder.go | 6 ++++++ pkg/executor/distsql.go | 7 +++---- pkg/executor/test/admintest/admin_test.go | 3 ++- pkg/planner/core/planbuilder.go | 13 ++++++++----- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/pkg/executor/builder.go b/pkg/executor/builder.go index beeb5e1fc90c9..96fa677743982 100644 --- a/pkg/executor/builder.go +++ b/pkg/executor/builder.go @@ -449,6 +449,9 @@ func buildIndexLookUpChecker(b *executorBuilder, p *plannercore.PhysicalIndexLoo if !e.isCommonHandle() { fullColLen++ } + if e.index.Global { + fullColLen++ + } e.dagPB.OutputOffsets = make([]uint32, fullColLen) for i := 0; i < fullColLen; i++ { e.dagPB.OutputOffsets[i] = uint32(i) @@ -468,6 +471,9 @@ func buildIndexLookUpChecker(b *executorBuilder, p *plannercore.PhysicalIndexLoo if !e.isCommonHandle() { tps = append(tps, types.NewFieldType(mysql.TypeLonglong)) } + if e.index.Global { + tps = append(tps, types.NewFieldType(mysql.TypeLonglong)) + } e.checkIndexValue = &checkIndexValue{idxColTps: tps} diff --git a/pkg/executor/distsql.go b/pkg/executor/distsql.go index 41b9857b6a3e2..5d6a8bc09eefc 100644 --- a/pkg/executor/distsql.go +++ b/pkg/executor/distsql.go @@ -594,17 +594,16 @@ func (e *IndexLookUpExecutor) needPartitionHandle(tp getHandleType) (bool, error outputOffsets := e.tableRequest.OutputOffsets col = cols[outputOffsets[len(outputOffsets)-1]] - // For TableScan, need partitionHandle in `indexOrder` when e.keepOrder == true - needPartitionHandle = (e.index.Global || e.partitionTableMode) && e.keepOrder + // For TableScan, need partitionHandle in `indexOrder` when e.keepOrder == true or execute `admin check [table|index]` + needPartitionHandle = ((e.index.Global || e.partitionTableMode) && e.keepOrder) || e.checkIndexValue != nil // no ExtraPidColID here, because TableScan shouldn't contain them. hasExtraCol = col.ID == model.ExtraPhysTblID } - // TODO: fix global index related bugs later // There will be two needPartitionHandle != hasExtraCol situations. // Only `needPartitionHandle` == true and `hasExtraCol` == false are not allowed. // `ExtraPhysTblID` will be used in `SelectLock` when `needPartitionHandle` == false and `hasExtraCol` == true. - if needPartitionHandle && !hasExtraCol && !e.index.Global { + if needPartitionHandle && !hasExtraCol { return needPartitionHandle, errors.Errorf("Internal error, needPartitionHandle != ret, tp(%d)", tp) } return needPartitionHandle, nil diff --git a/pkg/executor/test/admintest/admin_test.go b/pkg/executor/test/admintest/admin_test.go index 962cc104ab025..6718e92536a4b 100644 --- a/pkg/executor/test/admintest/admin_test.go +++ b/pkg/executor/test/admintest/admin_test.go @@ -1787,6 +1787,7 @@ func TestAdminCheckGlobalIndex(t *testing.T) { tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a)) partition by hash(c) partitions 5") tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)") + tk.MustExec("analyze table admin_test") // Make some corrupted index. Build the index information. sctx := mock.NewContext() @@ -1814,6 +1815,6 @@ func TestAdminCheckGlobalIndex(t *testing.T) { require.NoError(t, err) err = tk.ExecToErr("admin check table admin_test") require.Error(t, err) - require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 4, index-values:\"\" != record-values:\"handle: -1, values: [KindInt64 -10]\"") + require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 4, index-values:\"handle: 4, values: [KindInt64 2 KindInt64 4 KindInt64 105]\" != record-values:\"\"") require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err)) } diff --git a/pkg/planner/core/planbuilder.go b/pkg/planner/core/planbuilder.go index 476e43fce3319..98c307a2fd60f 100644 --- a/pkg/planner/core/planbuilder.go +++ b/pkg/planner/core/planbuilder.go @@ -1481,7 +1481,7 @@ func (b *PlanBuilder) buildAdmin(ctx context.Context, as *ast.AdminStmt) (base.P func (b *PlanBuilder) buildPhysicalIndexLookUpReader(_ context.Context, dbName model.CIStr, tbl table.Table, idx *model.IndexInfo) (base.Plan, error) { tblInfo := tbl.Meta() - physicalID, isPartition := getPhysicalID(tbl) + physicalID, isPartition := getPhysicalID(tbl, idx.Global) fullExprCols, _, err := expression.TableInfo2SchemaAndNames(b.ctx.GetExprCtx(), dbName, tblInfo) if err != nil { return nil, err @@ -1555,6 +1555,9 @@ func (b *PlanBuilder) buildPhysicalIndexLookUpReader(_ context.Context, dbName m } } } + if is.Index.Global { + ts.Columns, ts.schema, _ = AddExtraPhysTblIDColumn(b.ctx, ts.Columns, ts.schema) + } cop := &CopTask{ indexPlan: is, @@ -1591,9 +1594,9 @@ func getIndexColsSchema(tblInfo *model.TableInfo, idx *model.IndexInfo, allColSc return schema } -func getPhysicalID(t table.Table) (physicalID int64, isPartition bool) { +func getPhysicalID(t table.Table, isGlobalIndex bool) (physicalID int64, isPartition bool) { tblInfo := t.Meta() - if tblInfo.GetPartitionInfo() != nil { + if !isGlobalIndex && tblInfo.GetPartitionInfo() != nil { pid := t.(table.PhysicalTable).GetPhysicalID() return pid, true } @@ -1681,8 +1684,8 @@ func (b *PlanBuilder) buildPhysicalIndexLookUpReaders(ctx context.Context, dbNam } } indexInfos = append(indexInfos, idxInfo) - // For partition tables. - if pi := tbl.Meta().GetPartitionInfo(); pi != nil { + // For partition tables except global index. + if pi := tbl.Meta().GetPartitionInfo(); pi != nil && !idxInfo.Global { for _, def := range pi.Definitions { t := tbl.(table.PartitionedTable).GetPartition(def.ID) reader, err := b.buildPhysicalIndexLookUpReader(ctx, dbName, t, idxInfo) From 04d577b7dfa12f44da0de5b62138671bc7639d35 Mon Sep 17 00:00:00 2001 From: Jason Mo Date: Wed, 8 May 2024 15:52:52 +0800 Subject: [PATCH 03/14] update tests --- pkg/executor/test/admintest/admin_test.go | 111 +++++++++++++++------- 1 file changed, 77 insertions(+), 34 deletions(-) diff --git a/pkg/executor/test/admintest/admin_test.go b/pkg/executor/test/admintest/admin_test.go index 6718e92536a4b..1fb119febc84d 100644 --- a/pkg/executor/test/admintest/admin_test.go +++ b/pkg/executor/test/admintest/admin_test.go @@ -43,6 +43,7 @@ import ( "github.com/pingcap/tidb/pkg/util/logutil/consistency" "github.com/pingcap/tidb/pkg/util/mock" "github.com/pingcap/tidb/pkg/util/redact" + "github.com/pingcap/tidb/pkg/util/rowcodec" "github.com/stretchr/testify/require" "go.uber.org/zap" ) @@ -1779,42 +1780,84 @@ func TestAdminCheckGlobalIndex(t *testing.T) { store, domain := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) - tk.MustExec("use test") - tk.MustExec("drop table if exists admin_test") + var enable_fast_check = []bool{false, true} + for _, enabled := range enable_fast_check { + tk.MustExec("use test") + tk.MustExec("drop table if exists admin_test") - tk.MustExec("set tidb_enable_global_index = true") - tk.MustExec("set tidb_enable_fast_table_check = false") + tk.MustExec("set tidb_enable_global_index = true") + tk.MustExec(fmt.Sprintf("set tidb_enable_fast_table_check = %v", enabled)) - tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a)) partition by hash(c) partitions 5") - tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)") - tk.MustExec("analyze table admin_test") + tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a)) partition by hash(c) partitions 5") + tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)") + tk.MustExec("analyze table admin_test") - // Make some corrupted index. Build the index information. - sctx := mock.NewContext() - sctx.Store = store - // ctx := sctx.GetTableCtx() - is := domain.InfoSchema() - dbName := model.NewCIStr("test") - tblName := model.NewCIStr("admin_test") - tbl, err := is.TableByName(dbName, tblName) - require.NoError(t, err) - tblInfo := tbl.Meta() - // idxInfo := tblInfo.Indices[0] - df := tblInfo.GetPartitionInfo().Definitions[0] - // indexOpr := tables.NewIndex(df.ID, tblInfo, idxInfo) - tk.Session().GetSessionVars().IndexLookupSize = 3 - tk.Session().GetSessionVars().MaxChunkSize = 3 + // Make some corrupted index. Build the index information. + sctx := mock.NewContext() + sctx.Store = store + is := domain.InfoSchema() + dbName := model.NewCIStr("test") + tblName := model.NewCIStr("admin_test") + tbl, err := is.TableByName(dbName, tblName) + require.NoError(t, err) + tblInfo := tbl.Meta() + idxInfo := tblInfo.Indices[0] + require.True(t, idxInfo.Global) + df := tblInfo.GetPartitionInfo().Definitions[0] - // Reduce one row of index. - // Table count > index count. - // Index c2 is missing 2. - txn, err := store.Begin() - require.NoError(t, err) - txn.Delete(tablecodec.EncodeRowKey(df.ID, kv.IntHandle(4).Encoded())) - err = txn.Commit(context.Background()) - require.NoError(t, err) - err = tk.ExecToErr("admin check table admin_test") - require.Error(t, err) - require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 4, index-values:\"handle: 4, values: [KindInt64 2 KindInt64 4 KindInt64 105]\" != record-values:\"\"") - require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err)) + // Reduce one row of table. + // Index count > table count, (2, 12, 0) is deleted. + txn, err := store.Begin() + require.NoError(t, err) + txn.Delete(tablecodec.EncodeRowKey(df.ID, kv.IntHandle(4).Encoded())) + err = txn.Commit(context.Background()) + require.NoError(t, err) + err = tk.ExecToErr("admin check table admin_test") + require.Error(t, err) + require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err)) + require.ErrorContains(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 4, index-values:\"handle: 4, values: [KindInt64 2") + + // Remove corresponding index key/value. + // Admin check table will success. + idxVal, err := codec.EncodeKey(tk.Session().GetSessionVars().StmtCtx.TimeZone(), nil, types.NewIntDatum(2)) + require.NoError(t, err) + txn, err = store.Begin() + require.NoError(t, err) + txn.Delete(tablecodec.EncodeIndexSeekKey(tblInfo.ID, idxInfo.ID, idxVal)) + err = txn.Commit(context.Background()) + require.NoError(t, err) + tk.MustExec("admin check table admin_test") + + // Reduce one row of index. + // Index count < table count, (-1, -10, 2) is deleted. + idxVal, err = codec.EncodeKey(tk.Session().GetSessionVars().StmtCtx.TimeZone(), nil, types.NewIntDatum(-1)) + require.NoError(t, err) + txn, err = store.Begin() + require.NoError(t, err) + txn.Delete(tablecodec.EncodeIndexSeekKey(tblInfo.ID, idxInfo.ID, idxVal)) + err = txn.Commit(context.Background()) + require.NoError(t, err) + err = tk.ExecToErr("admin check table admin_test") + require.Error(t, err) + require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err)) + require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 2, index-values:\"\" != record-values:\"handle: 2, values: [KindInt64 -1]\"") + + // Add one row with inconsistent value. + // Index count = table count, but data is different. + rd := rowcodec.Encoder{Enable: true} + value, err := tablecodec.EncodeRow(tk.Session().GetSessionVars().StmtCtx.TimeZone(), + []types.Datum{types.NewIntDatum(100), types.NewIntDatum(200), types.NewIntDatum(300)}, + []int64{tblInfo.Cols()[0].ID, tblInfo.Cols()[1].ID, tblInfo.Cols()[2].ID}, nil, nil, &rd) + require.NoError(t, err) + txn, err = store.Begin() + require.NoError(t, err) + err = txn.Set(tablecodec.EncodeRowKeyWithHandle(df.ID, kv.IntHandle(2)), value) + require.NoError(t, err) + err = txn.Commit(context.Background()) + require.NoError(t, err) + err = tk.ExecToErr("admin check table admin_test") + require.Error(t, err) + require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err)) + require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 2, index-values:\"\" != record-values:\"handle: 2, values: [KindInt64 100]\"") + } } From 4b9a9664b362d11ed83fc91e2dd0dd032459d964 Mon Sep 17 00:00:00 2001 From: Jason Mo Date: Wed, 8 May 2024 16:02:18 +0800 Subject: [PATCH 04/14] add tests --- pkg/executor/test/admintest/admin_test.go | 86 +++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/pkg/executor/test/admintest/admin_test.go b/pkg/executor/test/admintest/admin_test.go index 1fb119febc84d..2d62424291115 100644 --- a/pkg/executor/test/admintest/admin_test.go +++ b/pkg/executor/test/admintest/admin_test.go @@ -1861,3 +1861,89 @@ func TestAdminCheckGlobalIndex(t *testing.T) { require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 2, index-values:\"\" != record-values:\"handle: 2, values: [KindInt64 100]\"") } } + +func TestAdminCheckGlobalIndexWithClusterIndex(t *testing.T) { + store, domain := testkit.CreateMockStoreAndDomain(t) + + tk := testkit.NewTestKit(t, store) + var enable_fast_check = []bool{false, true} + for _, enabled := range enable_fast_check { + tk.MustExec("use test") + tk.MustExec("drop table if exists admin_test") + + tk.MustExec("set tidb_enable_global_index = true") + tk.MustExec(fmt.Sprintf("set tidb_enable_fast_table_check = %v", enabled)) + + tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a), primary key(c)) partition by hash(c) partitions 5") + tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)") + tk.MustExec("analyze table admin_test") + + // Make some corrupted index. Build the index information. + sctx := mock.NewContext() + sctx.Store = store + is := domain.InfoSchema() + dbName := model.NewCIStr("test") + tblName := model.NewCIStr("admin_test") + tbl, err := is.TableByName(dbName, tblName) + require.NoError(t, err) + tblInfo := tbl.Meta() + idxInfo := tblInfo.Indices[0] + require.True(t, idxInfo.Global) + df := tblInfo.GetPartitionInfo().Definitions[0] + + // Reduce one row of table. + // Index count > table count, (2, 12, 0) is deleted. + txn, err := store.Begin() + require.NoError(t, err) + txn.Delete(tablecodec.EncodeRowKey(df.ID, kv.IntHandle(0).Encoded())) + err = txn.Commit(context.Background()) + require.NoError(t, err) + err = tk.ExecToErr("admin check table admin_test") + require.Error(t, err) + require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err)) + require.ErrorContains(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 0, index-values:\"handle: 0, values: [KindInt64 2") + + // Remove corresponding index key/value. + // Admin check table will success. + idxVal, err := codec.EncodeKey(tk.Session().GetSessionVars().StmtCtx.TimeZone(), nil, types.NewIntDatum(2)) + require.NoError(t, err) + txn, err = store.Begin() + require.NoError(t, err) + txn.Delete(tablecodec.EncodeIndexSeekKey(tblInfo.ID, idxInfo.ID, idxVal)) + err = txn.Commit(context.Background()) + require.NoError(t, err) + tk.MustExec("admin check table admin_test") + + // Reduce one row of index. + // Index count < table count, (-1, -10, 2) is deleted. + idxVal, err = codec.EncodeKey(tk.Session().GetSessionVars().StmtCtx.TimeZone(), nil, types.NewIntDatum(-1)) + require.NoError(t, err) + txn, err = store.Begin() + require.NoError(t, err) + txn.Delete(tablecodec.EncodeIndexSeekKey(tblInfo.ID, idxInfo.ID, idxVal)) + err = txn.Commit(context.Background()) + require.NoError(t, err) + err = tk.ExecToErr("admin check table admin_test") + require.Error(t, err) + require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err)) + require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 2, index-values:\"\" != record-values:\"handle: 2, values: [KindInt64 -1]\"") + + // Add one row with inconsistent value. + // Index count = table count, but data is different. + rd := rowcodec.Encoder{Enable: true} + value, err := tablecodec.EncodeRow(tk.Session().GetSessionVars().StmtCtx.TimeZone(), + []types.Datum{types.NewIntDatum(100), types.NewIntDatum(200), types.NewIntDatum(300)}, + []int64{tblInfo.Cols()[0].ID, tblInfo.Cols()[1].ID, tblInfo.Cols()[2].ID}, nil, nil, &rd) + require.NoError(t, err) + txn, err = store.Begin() + require.NoError(t, err) + err = txn.Set(tablecodec.EncodeRowKeyWithHandle(df.ID, kv.IntHandle(2)), value) + require.NoError(t, err) + err = txn.Commit(context.Background()) + require.NoError(t, err) + err = tk.ExecToErr("admin check table admin_test") + require.Error(t, err) + require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err)) + require.ErrorContains(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, ") + } +} From 08ff11ea03fcd9c6e04aa7e144b606ee8d2f746b Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Fri, 10 May 2024 11:27:24 +0800 Subject: [PATCH 05/14] add tests --- pkg/ddl/index.go | 6 +++ pkg/executor/test/admintest/admin_test.go | 51 +++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/pkg/ddl/index.go b/pkg/ddl/index.go index f0468554be25b..9ef0091af88ac 100644 --- a/pkg/ddl/index.go +++ b/pkg/ddl/index.go @@ -2471,6 +2471,12 @@ func (w *cleanUpIndexWorker) BackfillData(handleRange reorgBackfillTask) (taskCt return nil }) logSlowOperations(time.Since(oprStartTime), "cleanUpIndexBackfillDataInTxn", 3000) + failpoint.Inject("mockDMLExecution", func(val failpoint.Value) { + //nolint:forcetypeassert + if val.(bool) && MockDMLExecution != nil { + MockDMLExecution() + } + }) return } diff --git a/pkg/executor/test/admintest/admin_test.go b/pkg/executor/test/admintest/admin_test.go index 2d62424291115..88da4fb0eebe0 100644 --- a/pkg/executor/test/admintest/admin_test.go +++ b/pkg/executor/test/admintest/admin_test.go @@ -24,6 +24,9 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/pkg/ddl" + "github.com/pingcap/tidb/pkg/ddl/util/callback" "github.com/pingcap/tidb/pkg/domain" mysql "github.com/pingcap/tidb/pkg/errno" "github.com/pingcap/tidb/pkg/executor" @@ -44,6 +47,7 @@ import ( "github.com/pingcap/tidb/pkg/util/mock" "github.com/pingcap/tidb/pkg/util/redact" "github.com/pingcap/tidb/pkg/util/rowcodec" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" ) @@ -1947,3 +1951,50 @@ func TestAdminCheckGlobalIndexWithClusterIndex(t *testing.T) { require.ErrorContains(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, ") } } + +func TestAdminCheckGlobalIndexDuringDDL(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + originalHook := dom.DDL().GetHook() + tk := testkit.NewTestKit(t, store) + + var schemaMap = make(map[model.SchemaState]struct{}) + + hook := &callback.TestDDLCallback{Do: dom} + onJobUpdatedExportedFunc := func(job *model.Job) { + schemaMap[job.SchemaState] = struct{}{} + _, err := tk.Exec("admin check table admin_test") + assert.NoError(t, err) + } + hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc) + + // check table after delete some index key/value pairs. + ddl.MockDMLExecution = func() { + _, err := tk.Exec("admin check table admin_test") + assert.NoError(t, err) + } + + var enable_fast_check = []bool{false, true} + for _, enabled := range enable_fast_check { + tk.MustExec("use test") + tk.MustExec("drop table if exists admin_test") + + tk.MustExec("set tidb_enable_global_index = true") + tk.MustExec(fmt.Sprintf("set tidb_enable_fast_table_check = %v", enabled)) + + tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a), primary key(c)) partition by hash(c) partitions 5") + tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)") + tk.MustExec("analyze table admin_test") + + dom.DDL().SetHook(hook) + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/mockDMLExecution", "1*return(true)->return(false)")) + tk.MustExec("alter table admin_test truncate partition p0") + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/mockDMLExecution")) + dom.DDL().SetHook(originalHook) + + // Should have 3 different schema states, `none`, `deleteOnly`, `deleteReorg` + require.Len(t, schemaMap, 3) + for ss := range schemaMap { + delete(schemaMap, ss) + } + } +} From cab8cab132fe77a6a2ff6a828a7b74c4d2ec966f Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Fri, 10 May 2024 11:38:34 +0800 Subject: [PATCH 06/14] update --- pkg/executor/test/admintest/BUILD.bazel | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/executor/test/admintest/BUILD.bazel b/pkg/executor/test/admintest/BUILD.bazel index 40cdf88fe1826..12279e3c4da9a 100644 --- a/pkg/executor/test/admintest/BUILD.bazel +++ b/pkg/executor/test/admintest/BUILD.bazel @@ -8,9 +8,11 @@ go_test( "main_test.go", ], flaky = True, - shard_count = 18, + shard_count = 21, deps = [ "//pkg/config", + "//pkg/ddl", + "//pkg/ddl/util/callback", "//pkg/domain", "//pkg/errno", "//pkg/executor", @@ -22,6 +24,7 @@ go_test( "//pkg/sessionctx/variable", "//pkg/table", "//pkg/table/tables", + "//pkg/tablecodec", "//pkg/testkit", "//pkg/testkit/testsetup", "//pkg/testkit/testutil", @@ -31,7 +34,10 @@ go_test( "//pkg/util/logutil/consistency", "//pkg/util/mock", "//pkg/util/redact", + "//pkg/util/rowcodec", "@com_github_pingcap_errors//:errors", + "@com_github_pingcap_failpoint//:failpoint", + "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", "@com_github_tikv_client_go_v2//tikv", "@org_uber_go_goleak//:goleak", From 6520fe64c4a7ce63f0a8758ac2ef0f3a4219510f Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Fri, 10 May 2024 11:45:13 +0800 Subject: [PATCH 07/14] update --- pkg/executor/test/admintest/admin_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/executor/test/admintest/admin_test.go b/pkg/executor/test/admintest/admin_test.go index 88da4fb0eebe0..54125e10b5486 100644 --- a/pkg/executor/test/admintest/admin_test.go +++ b/pkg/executor/test/admintest/admin_test.go @@ -1784,8 +1784,8 @@ func TestAdminCheckGlobalIndex(t *testing.T) { store, domain := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) - var enable_fast_check = []bool{false, true} - for _, enabled := range enable_fast_check { + var enableFastCheck = []bool{false, true} + for _, enabled := range enableFastCheck { tk.MustExec("use test") tk.MustExec("drop table if exists admin_test") @@ -1870,8 +1870,8 @@ func TestAdminCheckGlobalIndexWithClusterIndex(t *testing.T) { store, domain := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) - var enable_fast_check = []bool{false, true} - for _, enabled := range enable_fast_check { + var enableFastCheck = []bool{false, true} + for _, enabled := range enableFastCheck { tk.MustExec("use test") tk.MustExec("drop table if exists admin_test") @@ -1973,8 +1973,8 @@ func TestAdminCheckGlobalIndexDuringDDL(t *testing.T) { assert.NoError(t, err) } - var enable_fast_check = []bool{false, true} - for _, enabled := range enable_fast_check { + var enableFastCheck = []bool{false, true} + for _, enabled := range enableFastCheck { tk.MustExec("use test") tk.MustExec("drop table if exists admin_test") From 540236e29c53061e07b8c9414f4e86565bb490cd Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Fri, 10 May 2024 12:16:09 +0800 Subject: [PATCH 08/14] update --- pkg/executor/distsql.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/executor/distsql.go b/pkg/executor/distsql.go index 5d6a8bc09eefc..f9f2b02f40240 100644 --- a/pkg/executor/distsql.go +++ b/pkg/executor/distsql.go @@ -600,10 +600,11 @@ func (e *IndexLookUpExecutor) needPartitionHandle(tp getHandleType) (bool, error hasExtraCol = col.ID == model.ExtraPhysTblID } + // TODO: fix global index related bugs later // There will be two needPartitionHandle != hasExtraCol situations. // Only `needPartitionHandle` == true and `hasExtraCol` == false are not allowed. // `ExtraPhysTblID` will be used in `SelectLock` when `needPartitionHandle` == false and `hasExtraCol` == true. - if needPartitionHandle && !hasExtraCol { + if needPartitionHandle && !hasExtraCol && !e.index.Global { return needPartitionHandle, errors.Errorf("Internal error, needPartitionHandle != ret, tp(%d)", tp) } return needPartitionHandle, nil From d2321f6e394d96fb3110fe08afaa5f77bdfcf663 Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Fri, 10 May 2024 12:30:52 +0800 Subject: [PATCH 09/14] update --- pkg/executor/distsql.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/executor/distsql.go b/pkg/executor/distsql.go index f9f2b02f40240..57750abe30390 100644 --- a/pkg/executor/distsql.go +++ b/pkg/executor/distsql.go @@ -594,17 +594,16 @@ func (e *IndexLookUpExecutor) needPartitionHandle(tp getHandleType) (bool, error outputOffsets := e.tableRequest.OutputOffsets col = cols[outputOffsets[len(outputOffsets)-1]] - // For TableScan, need partitionHandle in `indexOrder` when e.keepOrder == true or execute `admin check [table|index]` - needPartitionHandle = ((e.index.Global || e.partitionTableMode) && e.keepOrder) || e.checkIndexValue != nil + // For TableScan, need partitionHandle in `indexOrder` when e.keepOrder == true or execute `admin check [table|index]` with global index + needPartitionHandle = ((e.index.Global || e.partitionTableMode) && e.keepOrder) || (e.index.Global && e.checkIndexValue != nil) // no ExtraPidColID here, because TableScan shouldn't contain them. hasExtraCol = col.ID == model.ExtraPhysTblID } - // TODO: fix global index related bugs later // There will be two needPartitionHandle != hasExtraCol situations. // Only `needPartitionHandle` == true and `hasExtraCol` == false are not allowed. // `ExtraPhysTblID` will be used in `SelectLock` when `needPartitionHandle` == false and `hasExtraCol` == true. - if needPartitionHandle && !hasExtraCol && !e.index.Global { + if needPartitionHandle && !hasExtraCol { return needPartitionHandle, errors.Errorf("Internal error, needPartitionHandle != ret, tp(%d)", tp) } return needPartitionHandle, nil From 360373876e84cef12ea7baecd1697161437d0c7b Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Fri, 10 May 2024 13:22:37 +0800 Subject: [PATCH 10/14] update --- tests/integrationtest/r/executor/sample.result | 3 +++ tests/integrationtest/t/executor/sample.test | 2 ++ 2 files changed, 5 insertions(+) diff --git a/tests/integrationtest/r/executor/sample.result b/tests/integrationtest/r/executor/sample.result index 0707a4ac14d0b..0970ef6f6e5ce 100644 --- a/tests/integrationtest/r/executor/sample.result +++ b/tests/integrationtest/r/executor/sample.result @@ -210,6 +210,9 @@ set @@global.tidb_scatter_region=default; drop table if exists t; create table t (a int, b varchar(255), primary key (a)) partition by hash(a) partitions 2; insert into t values (1, '1'), (2, '2'), (3, '3'); +select sleep(0.5); +sleep(0.5) +0 set @@tidb_partition_prune_mode='static'; select * from t tablesample regions() order by a; a b diff --git a/tests/integrationtest/t/executor/sample.test b/tests/integrationtest/t/executor/sample.test index 8a237c8539f17..bd61b2121b804 100644 --- a/tests/integrationtest/t/executor/sample.test +++ b/tests/integrationtest/t/executor/sample.test @@ -136,6 +136,8 @@ set @@global.tidb_scatter_region=default; drop table if exists t; create table t (a int, b varchar(255), primary key (a)) partition by hash(a) partitions 2; insert into t values (1, '1'), (2, '2'), (3, '3'); +# https://github.com/pingcap/tidb/issues/52897 +select sleep(0.5); set @@tidb_partition_prune_mode='static'; select * from t tablesample regions() order by a; set @@tidb_partition_prune_mode='dynamic'; From 70dba8db6cbc610b696549750be4413d17b97764 Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Fri, 10 May 2024 16:36:57 +0800 Subject: [PATCH 11/14] update --- pkg/executor/test/admintest/admin_test.go | 3 --- pkg/session/session.go | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/executor/test/admintest/admin_test.go b/pkg/executor/test/admintest/admin_test.go index 54125e10b5486..5161c5977261d 100644 --- a/pkg/executor/test/admintest/admin_test.go +++ b/pkg/executor/test/admintest/admin_test.go @@ -1794,7 +1794,6 @@ func TestAdminCheckGlobalIndex(t *testing.T) { tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a)) partition by hash(c) partitions 5") tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)") - tk.MustExec("analyze table admin_test") // Make some corrupted index. Build the index information. sctx := mock.NewContext() @@ -1880,7 +1879,6 @@ func TestAdminCheckGlobalIndexWithClusterIndex(t *testing.T) { tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a), primary key(c)) partition by hash(c) partitions 5") tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)") - tk.MustExec("analyze table admin_test") // Make some corrupted index. Build the index information. sctx := mock.NewContext() @@ -1983,7 +1981,6 @@ func TestAdminCheckGlobalIndexDuringDDL(t *testing.T) { tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a), primary key(c)) partition by hash(c) partitions 5") tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)") - tk.MustExec("analyze table admin_test") dom.DDL().SetHook(hook) require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/mockDMLExecution", "1*return(true)->return(false)")) diff --git a/pkg/session/session.go b/pkg/session/session.go index 6f9c66bd94828..bef8c477b3bae 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -1872,6 +1872,9 @@ func (s *session) getInternalSession(execOption sqlexec.ExecOption) (*session, f } se.sessionVars.OptimizerUseInvisibleIndexes = s.sessionVars.OptimizerUseInvisibleIndexes + preSkipStats := s.sessionVars.SkipMissingPartitionStats + se.sessionVars.SkipMissingPartitionStats = s.sessionVars.SkipMissingPartitionStats + if execOption.SnapshotTS != 0 { if err := se.sessionVars.SetSystemVar(variable.TiDBSnapshot, strconv.FormatUint(execOption.SnapshotTS, 10)); err != nil { return nil, nil, err @@ -1913,6 +1916,7 @@ func (s *session) getInternalSession(execOption sqlexec.ExecOption) (*session, f } se.sessionVars.PartitionPruneMode.Store(prePruneMode) se.sessionVars.OptimizerUseInvisibleIndexes = false + se.sessionVars.SkipMissingPartitionStats = preSkipStats se.sessionVars.InspectionTableCache = nil se.sessionVars.MemTracker.Detach() s.sysSessionPool().Put(tmp) From b91d3ee1bd9eb442f9bf7a837f36a02acd8ac189 Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Sat, 11 May 2024 16:48:35 +0800 Subject: [PATCH 12/14] update tests --- pkg/executor/test/admintest/admin_test.go | 77 +++++++++++++---------- 1 file changed, 44 insertions(+), 33 deletions(-) diff --git a/pkg/executor/test/admintest/admin_test.go b/pkg/executor/test/admintest/admin_test.go index 5161c5977261d..dbc6eb73adc40 100644 --- a/pkg/executor/test/admintest/admin_test.go +++ b/pkg/executor/test/admintest/admin_test.go @@ -46,7 +46,6 @@ import ( "github.com/pingcap/tidb/pkg/util/logutil/consistency" "github.com/pingcap/tidb/pkg/util/mock" "github.com/pingcap/tidb/pkg/util/redact" - "github.com/pingcap/tidb/pkg/util/rowcodec" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -1806,13 +1805,15 @@ func TestAdminCheckGlobalIndex(t *testing.T) { tblInfo := tbl.Meta() idxInfo := tblInfo.Indices[0] require.True(t, idxInfo.Global) - df := tblInfo.GetPartitionInfo().Definitions[0] + idx := tbl.Indices()[0] + require.NotNil(t, idx) // Reduce one row of table. // Index count > table count, (2, 12, 0) is deleted. txn, err := store.Begin() require.NoError(t, err) - txn.Delete(tablecodec.EncodeRowKey(df.ID, kv.IntHandle(4).Encoded())) + err = txn.Delete(tablecodec.EncodeRowKey(tblInfo.GetPartitionInfo().Definitions[0].ID, kv.IntHandle(4).Encoded())) + require.NoError(t, err) err = txn.Commit(context.Background()) require.NoError(t, err) err = tk.ExecToErr("admin check table admin_test") @@ -1820,24 +1821,25 @@ func TestAdminCheckGlobalIndex(t *testing.T) { require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err)) require.ErrorContains(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 4, index-values:\"handle: 4, values: [KindInt64 2") + indexOpr := tables.NewIndex(tblInfo.GetPartitionInfo().Definitions[0].ID, tblInfo, idxInfo) // Remove corresponding index key/value. // Admin check table will success. - idxVal, err := codec.EncodeKey(tk.Session().GetSessionVars().StmtCtx.TimeZone(), nil, types.NewIntDatum(2)) - require.NoError(t, err) txn, err = store.Begin() require.NoError(t, err) - txn.Delete(tablecodec.EncodeIndexSeekKey(tblInfo.ID, idxInfo.ID, idxVal)) + err = indexOpr.Delete(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(2)}, kv.IntHandle(4)) + require.NoError(t, err) err = txn.Commit(context.Background()) require.NoError(t, err) tk.MustExec("admin check table admin_test") + indexOpr = tables.NewIndex(tblInfo.GetPartitionInfo().Definitions[2].ID, tblInfo, idxInfo) + // Reduce one row of index. // Index count < table count, (-1, -10, 2) is deleted. - idxVal, err = codec.EncodeKey(tk.Session().GetSessionVars().StmtCtx.TimeZone(), nil, types.NewIntDatum(-1)) - require.NoError(t, err) txn, err = store.Begin() require.NoError(t, err) - txn.Delete(tablecodec.EncodeIndexSeekKey(tblInfo.ID, idxInfo.ID, idxVal)) + err = indexOpr.Delete(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(-1)}, kv.IntHandle(2)) + require.NoError(t, err) err = txn.Commit(context.Background()) require.NoError(t, err) err = tk.ExecToErr("admin check table admin_test") @@ -1845,23 +1847,23 @@ func TestAdminCheckGlobalIndex(t *testing.T) { require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err)) require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 2, index-values:\"\" != record-values:\"handle: 2, values: [KindInt64 -1]\"") - // Add one row with inconsistent value. + // Add one row of index with inconsistent value. // Index count = table count, but data is different. - rd := rowcodec.Encoder{Enable: true} - value, err := tablecodec.EncodeRow(tk.Session().GetSessionVars().StmtCtx.TimeZone(), - []types.Datum{types.NewIntDatum(100), types.NewIntDatum(200), types.NewIntDatum(300)}, - []int64{tblInfo.Cols()[0].ID, tblInfo.Cols()[1].ID, tblInfo.Cols()[2].ID}, nil, nil, &rd) - require.NoError(t, err) txn, err = store.Begin() require.NoError(t, err) - err = txn.Set(tablecodec.EncodeRowKeyWithHandle(df.ID, kv.IntHandle(2)), value) + _, err = indexOpr.Create(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(100)}, kv.IntHandle(2), nil) require.NoError(t, err) err = txn.Commit(context.Background()) require.NoError(t, err) err = tk.ExecToErr("admin check table admin_test") require.Error(t, err) - require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err)) - require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 2, index-values:\"\" != record-values:\"handle: 2, values: [KindInt64 100]\"") + if !enabled { + require.True(t, consistency.ErrAdminCheckInconsistentWithColInfo.Equal(err)) + require.EqualError(t, err, "[executor:8134]data inconsistency in table: admin_test, index: uidx_a, col: a, handle: \"2\", index-values:\"KindInt64 100\" != record-values:\"KindInt64 -1\", compare err:") + } else { + require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err)) + require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 2, index-values:\"handle: 2, values: [KindInt64 100]\" != record-values:\"handle: 2, values: [KindInt64 -1]\"") + } } } @@ -1869,6 +1871,15 @@ func TestAdminCheckGlobalIndexWithClusterIndex(t *testing.T) { store, domain := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) + + getCommonHandle := func(row int) *kv.CommonHandle { + h, err := codec.EncodeKey(tk.Session().GetSessionVars().StmtCtx.TimeZone(), nil, types.MakeDatums(row)...) + require.NoError(t, err) + ch, err := kv.NewCommonHandle(h) + require.NoError(t, err) + return ch + } + var enableFastCheck = []bool{false, true} for _, enabled := range enableFastCheck { tk.MustExec("use test") @@ -1905,24 +1916,24 @@ func TestAdminCheckGlobalIndexWithClusterIndex(t *testing.T) { require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err)) require.ErrorContains(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 0, index-values:\"handle: 0, values: [KindInt64 2") + indexOpr := tables.NewIndex(tblInfo.GetPartitionInfo().Definitions[0].ID, tblInfo, idxInfo) // Remove corresponding index key/value. // Admin check table will success. - idxVal, err := codec.EncodeKey(tk.Session().GetSessionVars().StmtCtx.TimeZone(), nil, types.NewIntDatum(2)) - require.NoError(t, err) txn, err = store.Begin() require.NoError(t, err) - txn.Delete(tablecodec.EncodeIndexSeekKey(tblInfo.ID, idxInfo.ID, idxVal)) + err = indexOpr.Delete(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(2)}, getCommonHandle(0)) + require.NoError(t, err) err = txn.Commit(context.Background()) require.NoError(t, err) tk.MustExec("admin check table admin_test") + indexOpr = tables.NewIndex(tblInfo.GetPartitionInfo().Definitions[2].ID, tblInfo, idxInfo) // Reduce one row of index. // Index count < table count, (-1, -10, 2) is deleted. - idxVal, err = codec.EncodeKey(tk.Session().GetSessionVars().StmtCtx.TimeZone(), nil, types.NewIntDatum(-1)) - require.NoError(t, err) txn, err = store.Begin() require.NoError(t, err) - txn.Delete(tablecodec.EncodeIndexSeekKey(tblInfo.ID, idxInfo.ID, idxVal)) + err = indexOpr.Delete(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(-1)}, getCommonHandle(2)) + require.NoError(t, err) err = txn.Commit(context.Background()) require.NoError(t, err) err = tk.ExecToErr("admin check table admin_test") @@ -1932,21 +1943,21 @@ func TestAdminCheckGlobalIndexWithClusterIndex(t *testing.T) { // Add one row with inconsistent value. // Index count = table count, but data is different. - rd := rowcodec.Encoder{Enable: true} - value, err := tablecodec.EncodeRow(tk.Session().GetSessionVars().StmtCtx.TimeZone(), - []types.Datum{types.NewIntDatum(100), types.NewIntDatum(200), types.NewIntDatum(300)}, - []int64{tblInfo.Cols()[0].ID, tblInfo.Cols()[1].ID, tblInfo.Cols()[2].ID}, nil, nil, &rd) - require.NoError(t, err) txn, err = store.Begin() require.NoError(t, err) - err = txn.Set(tablecodec.EncodeRowKeyWithHandle(df.ID, kv.IntHandle(2)), value) + _, err = indexOpr.Create(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(100)}, getCommonHandle(2), nil) require.NoError(t, err) err = txn.Commit(context.Background()) require.NoError(t, err) err = tk.ExecToErr("admin check table admin_test") require.Error(t, err) - require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err)) - require.ErrorContains(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, ") + if !enabled { + require.True(t, consistency.ErrAdminCheckInconsistentWithColInfo.Equal(err)) + require.EqualError(t, err, "[executor:8134]data inconsistency in table: admin_test, index: uidx_a, col: a, handle: \"2\", index-values:\"KindInt64 100\" != record-values:\"KindInt64 -1\", compare err:") + } else { + require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err)) + require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 2, index-values:\"handle: 2, values: [KindInt64 100]\" != record-values:\"handle: 2, values: [KindInt64 -1]\"") + } } } @@ -1984,7 +1995,7 @@ func TestAdminCheckGlobalIndexDuringDDL(t *testing.T) { dom.DDL().SetHook(hook) require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/mockDMLExecution", "1*return(true)->return(false)")) - tk.MustExec("alter table admin_test truncate partition p0") + tk.MustExec("alter table admin_test truncate partition p1") require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/mockDMLExecution")) dom.DDL().SetHook(originalHook) From e4da1dd6b155f9418a1ca51e557b546d5b78ccfe Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Sat, 11 May 2024 16:55:12 +0800 Subject: [PATCH 13/14] update --- pkg/executor/test/admintest/BUILD.bazel | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/executor/test/admintest/BUILD.bazel b/pkg/executor/test/admintest/BUILD.bazel index 12279e3c4da9a..310ac6fb0c6ff 100644 --- a/pkg/executor/test/admintest/BUILD.bazel +++ b/pkg/executor/test/admintest/BUILD.bazel @@ -34,7 +34,6 @@ go_test( "//pkg/util/logutil/consistency", "//pkg/util/mock", "//pkg/util/redact", - "//pkg/util/rowcodec", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", "@com_github_stretchr_testify//assert", From 1353407ccb01223b9b3bd0950f5b17ead9a71644 Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Sat, 11 May 2024 17:27:05 +0800 Subject: [PATCH 14/14] update --- pkg/executor/test/admintest/admin_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/executor/test/admintest/admin_test.go b/pkg/executor/test/admintest/admin_test.go index dbc6eb73adc40..23b396ff45f18 100644 --- a/pkg/executor/test/admintest/admin_test.go +++ b/pkg/executor/test/admintest/admin_test.go @@ -1982,6 +1982,9 @@ func TestAdminCheckGlobalIndexDuringDDL(t *testing.T) { assert.NoError(t, err) } + batchSize := 32 + tk.MustExec(fmt.Sprintf("set global tidb_ddl_reorg_batch_size = %d", batchSize)) + var enableFastCheck = []bool{false, true} for _, enabled := range enableFastCheck { tk.MustExec("use test") @@ -1992,6 +1995,9 @@ func TestAdminCheckGlobalIndexDuringDDL(t *testing.T) { tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a), primary key(c)) partition by hash(c) partitions 5") tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)") + for i := 1; i <= batchSize*2; i++ { + tk.MustExec(fmt.Sprintf("insert admin_test values (%d, %d, %d)", i*5+1, i, i*5+1)) + } dom.DDL().SetHook(hook) require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/mockDMLExecution", "1*return(true)->return(false)"))