From 69a68935699036a483d4186ba4c79b502f1d0dfa Mon Sep 17 00:00:00 2001 From: tangenta Date: Mon, 3 Jun 2024 18:26:54 +0800 Subject: [PATCH 1/4] This is an automated cherry-pick of #53720 Signed-off-by: ti-chi-bot --- ddl/ddl.go | 40 ++++- ddl/ddl_api_test.go | 44 +++++ ddl/job_table.go | 33 ++++ ddl/placement_policy.go | 9 +- ddl/schema.go | 2 +- ddl/table.go | 100 +++++++++++- ddl/table_test.go | 3 +- pkg/ddl/BUILD.bazel | 348 ++++++++++++++++++++++++++++++++++++++++ 8 files changed, 566 insertions(+), 13 deletions(-) create mode 100644 pkg/ddl/BUILD.bazel diff --git a/ddl/ddl.go b/ddl/ddl.go index 1da09796b27d1..3bb1cf1768e93 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -33,6 +33,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/kvrpcpb" +<<<<<<< HEAD:ddl/ddl.go "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl/ingest" sess "github.com/pingcap/tidb/ddl/internal/session" @@ -65,6 +66,40 @@ import ( "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/mathutil" "github.com/pingcap/tidb/util/syncutil" +======= + "github.com/pingcap/tidb/pkg/config" + "github.com/pingcap/tidb/pkg/ddl/ingest" + sess "github.com/pingcap/tidb/pkg/ddl/internal/session" + "github.com/pingcap/tidb/pkg/ddl/logutil" + "github.com/pingcap/tidb/pkg/ddl/syncer" + "github.com/pingcap/tidb/pkg/ddl/util" + "github.com/pingcap/tidb/pkg/disttask/framework/proto" + "github.com/pingcap/tidb/pkg/disttask/framework/scheduler" + "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor" + "github.com/pingcap/tidb/pkg/domain/infosync" + "github.com/pingcap/tidb/pkg/infoschema" + "github.com/pingcap/tidb/pkg/kv" + "github.com/pingcap/tidb/pkg/meta" + "github.com/pingcap/tidb/pkg/meta/autoid" + "github.com/pingcap/tidb/pkg/metrics" + "github.com/pingcap/tidb/pkg/owner" + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/model" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/parser/terror" + "github.com/pingcap/tidb/pkg/sessionctx" + "github.com/pingcap/tidb/pkg/sessionctx/binloginfo" + "github.com/pingcap/tidb/pkg/sessionctx/variable" + "github.com/pingcap/tidb/pkg/statistics/handle" + statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util" + "github.com/pingcap/tidb/pkg/table" + pumpcli "github.com/pingcap/tidb/pkg/tidb-binlog/pump_client" + tidbutil "github.com/pingcap/tidb/pkg/util" + "github.com/pingcap/tidb/pkg/util/dbterror" + "github.com/pingcap/tidb/pkg/util/dbterror/exeerrors" + "github.com/pingcap/tidb/pkg/util/gcutil" + "github.com/pingcap/tidb/pkg/util/generic" +>>>>>>> 44c9096efbc (ddl: get latest old table ID before replace view (#53720)):pkg/ddl/ddl.go "github.com/tikv/client-go/v2/tikvrpc" clientv3 "go.etcd.io/etcd/client/v3" atomicutil "go.uber.org/atomic" @@ -367,11 +402,6 @@ type ddlCtx struct { // reorgCtx is used for reorganization. reorgCtx reorgContexts - // backfillCtx is used for backfill workers. - backfillCtx struct { - syncutil.RWMutex - jobCtxMap map[int64]*JobContext - } jobCtx struct { sync.RWMutex diff --git a/ddl/ddl_api_test.go b/ddl/ddl_api_test.go index 7492d5bd4ca1b..3681861b65e5f 100644 --- a/ddl/ddl_api_test.go +++ b/ddl/ddl_api_test.go @@ -30,7 +30,11 @@ import ( "github.com/pingcap/tidb/testkit" "github.com/pingcap/tidb/util/chunk" "github.com/stretchr/testify/require" +<<<<<<< HEAD:ddl/ddl_api_test.go "golang.org/x/exp/slices" +======= + "golang.org/x/sync/errgroup" +>>>>>>> 44c9096efbc (ddl: get latest old table ID before replace view (#53720)):pkg/ddl/ddl_api_test.go ) func TestGetDDLJobs(t *testing.T) { @@ -152,6 +156,46 @@ func enQueueDDLJobs(t *testing.T, sess session.Session, txn kv.Transaction, jobT } } +func TestCreateViewConcurrently(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + + tk.MustExec("create table t (a int);") + tk.MustExec("create view v as select * from t;") + var ( + counterErr error + counter int + ) + failpoint.EnableCall("github.com/pingcap/tidb/pkg/ddl/onDDLCreateView", func(job *model.Job) { + counter++ + if counter > 1 { + counterErr = fmt.Errorf("create view job should not run concurrently") + return + } + }) + failpoint.EnableCall("github.com/pingcap/tidb/pkg/ddl/afterDelivery2Worker", func(job *model.Job) { + if job.Type == model.ActionCreateView { + counter-- + } + }) + var eg errgroup.Group + for i := 0; i < 5; i++ { + eg.Go(func() error { + newTk := testkit.NewTestKit(t, store) + _, err := newTk.Exec("use test") + if err != nil { + return err + } + _, err = newTk.Exec("create or replace view v as select * from t;") + return err + }) + } + err := eg.Wait() + require.NoError(t, err) + require.NoError(t, counterErr) +} + func TestCreateDropCreateTable(t *testing.T) { store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) diff --git a/ddl/job_table.go b/ddl/job_table.go index 6fa5d641b5733..9386628f6bd68 100644 --- a/ddl/job_table.go +++ b/ddl/job_table.go @@ -362,6 +362,39 @@ func (d *ddl) delivery2worker(wk *worker, pool *workerPool, job *model.Job) { asyncNotify(d.ddlJobCh) metrics.DDLRunningJobCount.WithLabelValues(pool.tp().String()).Dec() }() +<<<<<<< HEAD:ddl/job_table.go +======= + + err := wk.HandleLocalDDLJob(d.ddlCtx, job) + pool.put(wk) + if err != nil { + logutil.DDLLogger().Info("handle ddl job failed", zap.Error(err), zap.Stringer("job", job)) + } + task.NotifyError(err) + }) +} + +// delivery2Worker owns the worker, need to put it back to the pool in this function. +func (s *jobScheduler) delivery2Worker(wk *worker, pool *workerPool, job *model.Job) { + injectFailPointForGetJob(job) + s.runningJobs.add(job) + s.wg.Run(func() { + metrics.DDLRunningJobCount.WithLabelValues(pool.tp().String()).Inc() + defer func() { + failpoint.InjectCall("afterDelivery2Worker", job) + s.runningJobs.remove(job) + asyncNotify(s.ddlJobNotifyCh) + metrics.DDLRunningJobCount.WithLabelValues(pool.tp().String()).Dec() + if wk.ctx.Err() != nil && ingest.LitBackCtxMgr != nil { + // if ctx cancelled, i.e. owner changed, we need to Unregister the backend + // as litBackendCtx is holding this very 'ctx', and it cannot reuse now. + // TODO make LitBackCtxMgr a local value of the job scheduler, it makes + // it much harder to test multiple owners in 1 unit test. + ingest.LitBackCtxMgr.Unregister(job.ID) + } + }() + ownerID := s.ownerManager.ID() +>>>>>>> 44c9096efbc (ddl: get latest old table ID before replace view (#53720)):pkg/ddl/job_table.go // check if this ddl job is synced to all servers. if !job.NotStarted() && (!d.isSynced(job) || !d.maybeAlreadyRunOnce(job.ID)) { if variable.EnableMDL.Load() { diff --git a/ddl/placement_policy.go b/ddl/placement_policy.go index eedb4bbf09dcf..ef6a57f2e8faa 100644 --- a/ddl/placement_policy.go +++ b/ddl/placement_policy.go @@ -118,7 +118,7 @@ func getPlacementPolicyByName(d *ddlCtx, t *meta.Meta, policyName model.CIStr) ( } is := d.infoCache.GetLatest() - if is.SchemaMetaVersion() == currVer { + if is != nil && is.SchemaMetaVersion() == currVer { // Use cached policy. policy, ok := is.PolicyByName(policyName) if ok { @@ -319,8 +319,15 @@ func checkPlacementPolicyNotInUse(d *ddlCtx, t *meta.Meta, policy *model.PolicyI return err } is := d.infoCache.GetLatest() +<<<<<<< HEAD:ddl/placement_policy.go if is.SchemaMetaVersion() == currVer { return CheckPlacementPolicyNotInUseFromInfoSchema(is, policy) +======= + if is != nil && is.SchemaMetaVersion() == currVer { + err = CheckPlacementPolicyNotInUseFromInfoSchema(is, policy) + } else { + err = CheckPlacementPolicyNotInUseFromMeta(t, policy) +>>>>>>> 44c9096efbc (ddl: get latest old table ID before replace view (#53720)):pkg/ddl/placement_policy.go } return CheckPlacementPolicyNotInUseFromMeta(t, policy) diff --git a/ddl/schema.go b/ddl/schema.go index e9cb1e6579635..2a50e33ca2a9e 100644 --- a/ddl/schema.go +++ b/ddl/schema.go @@ -76,7 +76,7 @@ func checkSchemaNotExists(d *ddlCtx, t *meta.Meta, schemaID int64, dbInfo *model return err } is := d.infoCache.GetLatest() - if is.SchemaMetaVersion() == currVer { + if is != nil && is.SchemaMetaVersion() == currVer { return checkSchemaNotExistsFromInfoSchema(is, schemaID, dbInfo) } return checkSchemaNotExistsFromStore(t, schemaID, dbInfo) diff --git a/ddl/table.go b/ddl/table.go index ca700d1369b42..0076587380364 100644 --- a/ddl/table.go +++ b/ddl/table.go @@ -274,14 +274,19 @@ func onCreateView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) schemaID := job.SchemaID tbInfo := &model.TableInfo{} var orReplace bool - var oldTbInfoID int64 - if err := job.DecodeArgs(tbInfo, &orReplace, &oldTbInfoID); err != nil { + var _placeholder int64 // oldTblInfoID + if err := job.DecodeArgs(tbInfo, &orReplace, &_placeholder); err != nil { // Invalid arguments, cancel this job. job.State = model.JobStateCancelled return ver, errors.Trace(err) } tbInfo.State = model.StateNone - err := checkTableNotExists(d, t, schemaID, tbInfo.Name.L) + + oldTableID, err := findTableIDByName(d, t, schemaID, tbInfo.Name.L) + if infoschema.ErrTableNotExists.Equal(err) { + err = nil + } + failpoint.InjectCall("onDDLCreateView", job) if err != nil { if infoschema.ErrDatabaseNotExists.Equal(err) { job.State = model.JobStateCancelled @@ -304,13 +309,18 @@ func onCreateView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) // none -> public tbInfo.State = model.StatePublic tbInfo.UpdateTS = t.StartTS +<<<<<<< HEAD:ddl/table.go if oldTbInfoID > 0 && orReplace { err = t.DropTableOrView(schemaID, oldTbInfoID) +======= + if oldTableID > 0 && orReplace { + err = t.DropTableOrView(schemaID, job.SchemaName, oldTableID, tbInfo.Name.L) +>>>>>>> 44c9096efbc (ddl: get latest old table ID before replace view (#53720)):pkg/ddl/table.go if err != nil { job.State = model.JobStateCancelled return ver, errors.Trace(err) } - err = t.GetAutoIDAccessors(schemaID, oldTbInfoID).Del() + err = t.GetAutoIDAccessors(schemaID, oldTableID).Del() if err != nil { job.State = model.JobStateCancelled return ver, errors.Trace(err) @@ -1463,13 +1473,51 @@ func checkTableNotExists(d *ddlCtx, t *meta.Meta, schemaID int64, tableName stri return err } is := d.infoCache.GetLatest() - if is.SchemaMetaVersion() == currVer { + if is != nil && is.SchemaMetaVersion() == currVer { return checkTableNotExistsFromInfoSchema(is, schemaID, tableName) } return checkTableNotExistsFromStore(t, schemaID, tableName) } +<<<<<<< HEAD:ddl/table.go +======= +func checkTableNotExistsByName(d *ddlCtx, t *meta.Meta, schemaID int64, schemaName, tableName string) error { + // Try to use memory schema info to check first. + currVer, err := t.GetSchemaVersion() + if err != nil { + return err + } + is := d.infoCache.GetLatest() + if is != nil && is.SchemaMetaVersion() == currVer { + return checkTableNotExistsFromInfoSchema(is, schemaID, tableName) + } + return t.CheckTableNameNotExists(t.TableNameKey(schemaName, tableName)) +} + +func checkConstraintNamesNotExists(t *meta.Meta, schemaID int64, constraints []*model.ConstraintInfo) error { + if len(constraints) == 0 { + return nil + } + tbInfos, err := t.ListTables(schemaID) + if err != nil { + return err + } + + for _, tb := range tbInfos { + for _, constraint := range constraints { + if constraint.State != model.StateWriteOnly { + if constraintInfo := tb.FindConstraintInfoByName(constraint.Name.L); constraintInfo != nil { + return infoschema.ErrCheckConstraintDupName.GenWithStackByArgs(constraint.Name.L) + } + } + } + } + + return nil +} + +>>>>>>> 44c9096efbc (ddl: get latest old table ID before replace view (#53720)):pkg/ddl/table.go func checkTableIDNotExists(t *meta.Meta, schemaID, tableID int64) error { tbl, err := t.GetTable(schemaID, tableID) if err != nil { @@ -1516,6 +1564,48 @@ func checkTableNotExistsFromStore(t *meta.Meta, schemaID int64, tableName string return nil } +func findTableIDByName(d *ddlCtx, t *meta.Meta, schemaID int64, tableName string) (int64, error) { + // Try to use memory schema info to check first. + currVer, err := t.GetSchemaVersion() + if err != nil { + return 0, err + } + is := d.infoCache.GetLatest() + if is != nil && is.SchemaMetaVersion() == currVer { + return findTableIDFromInfoSchema(is, schemaID, tableName) + } + + return findTableIDFromStore(t, schemaID, tableName) +} + +func findTableIDFromInfoSchema(is infoschema.InfoSchema, schemaID int64, tableName string) (int64, error) { + schema, ok := is.SchemaByID(schemaID) + if !ok { + return 0, infoschema.ErrDatabaseNotExists.GenWithStackByArgs("") + } + tbl, err := is.TableByName(schema.Name, model.NewCIStr(tableName)) + if err != nil { + return 0, err + } + return tbl.Meta().ID, nil +} + +func findTableIDFromStore(t *meta.Meta, schemaID int64, tableName string) (int64, error) { + tbls, err := t.ListSimpleTables(schemaID) + if err != nil { + if meta.ErrDBNotExists.Equal(err) { + return 0, infoschema.ErrDatabaseNotExists.GenWithStackByArgs("") + } + return 0, errors.Trace(err) + } + for _, tbl := range tbls { + if tbl.Name.L == tableName { + return tbl.ID, nil + } + } + return 0, infoschema.ErrTableNotExists.FastGenByArgs(tableName) +} + // updateVersionAndTableInfoWithCheck checks table info validate and updates the schema version and the table information func updateVersionAndTableInfoWithCheck(d *ddlCtx, t *meta.Meta, job *model.Job, tblInfo *model.TableInfo, shouldUpdateVer bool, multiInfos ...schemaIDAndTableInfo) ( ver int64, err error) { diff --git a/ddl/table_test.go b/ddl/table_test.go index 7d150cbc86d9f..48eb9f7639f2f 100644 --- a/ddl/table_test.go +++ b/ddl/table_test.go @@ -309,7 +309,8 @@ func TestCreateView(t *testing.T) { } ctx.SetValue(sessionctx.QueryString, "skip") err = d.DoDDLJob(ctx, job) - require.Error(t, err) + // The non-existing table id in job args will not be considered anymore. + require.NoError(t, err) } func checkTableCacheTest(t *testing.T, store kv.Storage, dbInfo *model.DBInfo, tblInfo *model.TableInfo) { diff --git a/pkg/ddl/BUILD.bazel b/pkg/ddl/BUILD.bazel new file mode 100644 index 0000000000000..f5cddf95e7fad --- /dev/null +++ b/pkg/ddl/BUILD.bazel @@ -0,0 +1,348 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +package_group( + name = "ddl_friend", + packages = [ + "-//pkg/planner/...", + "//...", + ], +) + +go_library( + name = "ddl", + srcs = [ + "backfilling.go", + "backfilling_clean_s3.go", + "backfilling_dist_executor.go", + "backfilling_dist_scheduler.go", + "backfilling_import_cloud.go", + "backfilling_merge_sort.go", + "backfilling_operators.go", + "backfilling_read_index.go", + "backfilling_scheduler.go", + "bdr.go", + "callback.go", + "cluster.go", + "column.go", + "constant.go", + "constraint.go", + "ddl.go", + "ddl_algorithm.go", + "ddl_api.go", + "ddl_history.go", + "ddl_running_jobs.go", + "ddl_tiflash_api.go", + "ddl_worker.go", + "ddl_workerpool.go", + "delete_range.go", + "delete_range_util.go", + "dist_owner.go", + "foreign_key.go", + "generated_column.go", + "index.go", + "index_cop.go", + "index_merge_tmp.go", + "job_table.go", + "mock.go", + "multi_schema_change.go", + "options.go", + "partition.go", + "placement_policy.go", + "reorg.go", + "resource_group.go", + "rollingback.go", + "sanity_check.go", + "schema.go", + "schema_version.go", + "sequence.go", + "split_region.go", + "stat.go", + "table.go", + "table_lock.go", + "ttl.go", + ], + importpath = "github.com/pingcap/tidb/pkg/ddl", + visibility = [ + ":ddl_friend", + ], + deps = [ + "//br/pkg/storage", + "//pkg/config", + "//pkg/ddl/copr", + "//pkg/ddl/ingest", + "//pkg/ddl/internal/session", + "//pkg/ddl/label", + "//pkg/ddl/logutil", + "//pkg/ddl/placement", + "//pkg/ddl/resourcegroup", + "//pkg/ddl/syncer", + "//pkg/ddl/util", + "//pkg/distsql", + "//pkg/distsql/context", + "//pkg/disttask/framework/handle", + "//pkg/disttask/framework/proto", + "//pkg/disttask/framework/scheduler", + "//pkg/disttask/framework/storage", + "//pkg/disttask/framework/taskexecutor", + "//pkg/disttask/framework/taskexecutor/execute", + "//pkg/disttask/operator", + "//pkg/domain/infosync", + "//pkg/domain/resourcegroup", + "//pkg/errctx", + "//pkg/expression", + "//pkg/expression/context", + "//pkg/expression/contextstatic", + "//pkg/infoschema", + "//pkg/kv", + "//pkg/lightning/backend", + "//pkg/lightning/backend/external", + "//pkg/lightning/backend/local", + "//pkg/lightning/common", + "//pkg/lightning/config", + "//pkg/meta", + "//pkg/meta/autoid", + "//pkg/metrics", + "//pkg/owner", + "//pkg/parser", + "//pkg/parser/ast", + "//pkg/parser/charset", + "//pkg/parser/format", + "//pkg/parser/model", + "//pkg/parser/mysql", + "//pkg/parser/opcode", + "//pkg/parser/terror", + "//pkg/parser/types", + "//pkg/privilege", + "//pkg/resourcemanager/pool/workerpool", + "//pkg/resourcemanager/util", + "//pkg/sessionctx", + "//pkg/sessionctx/binloginfo", + "//pkg/sessionctx/stmtctx", + "//pkg/sessionctx/variable", + "//pkg/sessiontxn", + "//pkg/statistics", + "//pkg/statistics/handle", + "//pkg/statistics/handle/util", + "//pkg/store/copr", + "//pkg/store/driver/backoff", + "//pkg/store/driver/txn", + "//pkg/store/helper", + "//pkg/table", + "//pkg/table/tables", + "//pkg/tablecodec", + "//pkg/tidb-binlog/pump_client", + "//pkg/ttl/cache", + "//pkg/types", + "//pkg/types/parser_driver", + "//pkg/util", + "//pkg/util/backoff", + "//pkg/util/chunk", + "//pkg/util/codec", + "//pkg/util/collate", + "//pkg/util/context", + "//pkg/util/dbterror", + "//pkg/util/dbterror/exeerrors", + "//pkg/util/domainutil", + "//pkg/util/engine", + "//pkg/util/execdetails", + "//pkg/util/filter", + "//pkg/util/gcutil", + "//pkg/util/generic", + "//pkg/util/hack", + "//pkg/util/intest", + "//pkg/util/logutil", + "//pkg/util/mathutil", + "//pkg/util/memory", + "//pkg/util/mock", + "//pkg/util/ranger", + "//pkg/util/resourcegrouptag", + "//pkg/util/rowDecoder", + "//pkg/util/rowcodec", + "//pkg/util/set", + "//pkg/util/size", + "//pkg/util/slice", + "//pkg/util/sqlexec", + "//pkg/util/sqlkiller", + "//pkg/util/stringutil", + "//pkg/util/tiflash", + "//pkg/util/timeutil", + "//pkg/util/topsql", + "//pkg/util/topsql/state", + "@com_github_docker_go_units//:go-units", + "@com_github_google_uuid//:uuid", + "@com_github_ngaut_pools//:pools", + "@com_github_pingcap_errors//:errors", + "@com_github_pingcap_failpoint//:failpoint", + "@com_github_pingcap_kvproto//pkg/errorpb", + "@com_github_pingcap_kvproto//pkg/kvrpcpb", + "@com_github_pingcap_kvproto//pkg/metapb", + "@com_github_pingcap_log//:log", + "@com_github_pingcap_tipb//go-tipb", + "@com_github_prometheus_client_golang//prometheus", + "@com_github_tikv_client_go_v2//error", + "@com_github_tikv_client_go_v2//kv", + "@com_github_tikv_client_go_v2//oracle", + "@com_github_tikv_client_go_v2//tikv", + "@com_github_tikv_client_go_v2//tikvrpc", + "@com_github_tikv_client_go_v2//txnkv/rangetask", + "@com_github_tikv_client_go_v2//util", + "@com_github_tikv_pd_client//:client", + "@com_github_tikv_pd_client//http", + "@io_etcd_go_etcd_client_v3//:client", + "@io_etcd_go_etcd_client_v3//concurrency", + "@org_golang_x_sync//errgroup", + "@org_uber_go_atomic//:atomic", + "@org_uber_go_zap//:zap", + ], +) + +go_test( + name = "ddl_test", + timeout = "moderate", + srcs = [ + "attributes_sql_test.go", + "backfilling_dist_scheduler_test.go", + "backfilling_test.go", + "bdr_test.go", + "bench_test.go", + "cancel_test.go", + "cluster_test.go", + "column_change_test.go", + "column_modify_test.go", + "column_test.go", + "column_type_change_test.go", + "constraint_test.go", + "db_cache_test.go", + "db_change_failpoints_test.go", + "db_change_test.go", + "db_integration_test.go", + "db_rename_test.go", + "db_table_test.go", + "db_test.go", + "ddl_algorithm_test.go", + "ddl_api_test.go", + "ddl_error_test.go", + "ddl_history_test.go", + "ddl_running_jobs_test.go", + "ddl_test.go", + "ddl_worker_test.go", + "ddl_workerpool_test.go", + "export_test.go", + "fail_test.go", + "foreign_key_test.go", + "index_change_test.go", + "index_cop_test.go", + "index_modify_test.go", + "integration_test.go", + "job_table_test.go", + "main_test.go", + "modify_column_test.go", + "multi_schema_change_test.go", + "mv_index_test.go", + "options_test.go", + "partition_test.go", + "placement_policy_ddl_test.go", + "placement_policy_test.go", + "placement_sql_test.go", + "primary_key_handle_test.go", + "reorg_partition_test.go", + "repair_table_test.go", + "restart_test.go", + "rollingback_test.go", + "schema_test.go", + "sequence_test.go", + "stat_test.go", + "table_modify_test.go", + "table_split_test.go", + "table_test.go", + "tiflash_replica_test.go", + "ttl_test.go", + ], + embed = [":ddl"], + flaky = True, + shard_count = 50, + deps = [ + "//pkg/autoid_service", + "//pkg/config", + "//pkg/ddl/copr", + "//pkg/ddl/ingest", + "//pkg/ddl/internal/session", + "//pkg/ddl/logutil", + "//pkg/ddl/placement", + "//pkg/ddl/schematracker", + "//pkg/ddl/syncer", + "//pkg/ddl/testutil", + "//pkg/ddl/util", + "//pkg/ddl/util/callback", + "//pkg/disttask/framework/proto", + "//pkg/disttask/framework/scheduler", + "//pkg/disttask/framework/storage", + "//pkg/domain", + "//pkg/domain/infosync", + "//pkg/errctx", + "//pkg/errno", + "//pkg/executor", + "//pkg/expression", + "//pkg/infoschema", + "//pkg/keyspace", + "//pkg/kv", + "//pkg/lightning/backend/external", + "//pkg/meta", + "//pkg/meta/autoid", + "//pkg/parser", + "//pkg/parser/ast", + "//pkg/parser/auth", + "//pkg/parser/charset", + "//pkg/parser/model", + "//pkg/parser/mysql", + "//pkg/parser/terror", + "//pkg/parser/types", + "//pkg/server", + "//pkg/session", + "//pkg/session/types", + "//pkg/sessionctx", + "//pkg/sessionctx/variable", + "//pkg/sessiontxn", + "//pkg/store/gcworker", + "//pkg/store/helper", + "//pkg/store/mockstore", + "//pkg/table", + "//pkg/table/tables", + "//pkg/tablecodec", + "//pkg/testkit", + "//pkg/testkit/external", + "//pkg/testkit/testfailpoint", + "//pkg/testkit/testsetup", + "//pkg/testkit/testutil", + "//pkg/types", + "//pkg/util", + "//pkg/util/chunk", + "//pkg/util/codec", + "//pkg/util/collate", + "//pkg/util/context", + "//pkg/util/dbterror", + "//pkg/util/dbterror/plannererrors", + "//pkg/util/domainutil", + "//pkg/util/gcutil", + "//pkg/util/mathutil", + "//pkg/util/mock", + "//pkg/util/sem", + "//pkg/util/sqlexec", + "//pkg/util/timeutil", + "@com_github_ngaut_pools//:pools", + "@com_github_pingcap_errors//:errors", + "@com_github_pingcap_failpoint//:failpoint", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + "@com_github_tikv_client_go_v2//oracle", + "@com_github_tikv_client_go_v2//testutils", + "@com_github_tikv_client_go_v2//tikv", + "@com_github_tikv_client_go_v2//util", + "@io_etcd_go_etcd_client_v3//:client", + "@org_golang_google_grpc//:grpc", + "@org_golang_x_sync//errgroup", + "@org_uber_go_atomic//:atomic", + "@org_uber_go_goleak//:goleak", + "@org_uber_go_zap//:zap", + ], +) From d50d066084c2f6e97dc0af4b4cfd11ef0b96edd9 Mon Sep 17 00:00:00 2001 From: Ruihao Chen Date: Mon, 11 Nov 2024 14:05:19 +0800 Subject: [PATCH 2/4] Fix conflict --- ddl/BUILD.bazel | 1 + ddl/ddl.go | 40 +---- ddl/ddl_api_test.go | 30 ++-- ddl/job_table.go | 39 +---- ddl/placement_policy.go | 9 +- ddl/table.go | 54 +------ pkg/ddl/BUILD.bazel | 348 ---------------------------------------- 7 files changed, 41 insertions(+), 480 deletions(-) delete mode 100644 pkg/ddl/BUILD.bazel diff --git a/ddl/BUILD.bazel b/ddl/BUILD.bazel index 08d1d30b70a1f..0280847139a6e 100644 --- a/ddl/BUILD.bazel +++ b/ddl/BUILD.bazel @@ -292,6 +292,7 @@ go_test( "@io_etcd_go_etcd_client_v3//:client", "@org_golang_google_grpc//:grpc", "@org_golang_x_exp//slices", + "@org_golang_x_sync//errgroup", "@org_uber_go_atomic//:atomic", "@org_uber_go_goleak//:goleak", "@org_uber_go_zap//:zap", diff --git a/ddl/ddl.go b/ddl/ddl.go index 3bb1cf1768e93..1da09796b27d1 100644 --- a/ddl/ddl.go +++ b/ddl/ddl.go @@ -33,7 +33,6 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/kvrpcpb" -<<<<<<< HEAD:ddl/ddl.go "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/ddl/ingest" sess "github.com/pingcap/tidb/ddl/internal/session" @@ -66,40 +65,6 @@ import ( "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/mathutil" "github.com/pingcap/tidb/util/syncutil" -======= - "github.com/pingcap/tidb/pkg/config" - "github.com/pingcap/tidb/pkg/ddl/ingest" - sess "github.com/pingcap/tidb/pkg/ddl/internal/session" - "github.com/pingcap/tidb/pkg/ddl/logutil" - "github.com/pingcap/tidb/pkg/ddl/syncer" - "github.com/pingcap/tidb/pkg/ddl/util" - "github.com/pingcap/tidb/pkg/disttask/framework/proto" - "github.com/pingcap/tidb/pkg/disttask/framework/scheduler" - "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor" - "github.com/pingcap/tidb/pkg/domain/infosync" - "github.com/pingcap/tidb/pkg/infoschema" - "github.com/pingcap/tidb/pkg/kv" - "github.com/pingcap/tidb/pkg/meta" - "github.com/pingcap/tidb/pkg/meta/autoid" - "github.com/pingcap/tidb/pkg/metrics" - "github.com/pingcap/tidb/pkg/owner" - "github.com/pingcap/tidb/pkg/parser/ast" - "github.com/pingcap/tidb/pkg/parser/model" - "github.com/pingcap/tidb/pkg/parser/mysql" - "github.com/pingcap/tidb/pkg/parser/terror" - "github.com/pingcap/tidb/pkg/sessionctx" - "github.com/pingcap/tidb/pkg/sessionctx/binloginfo" - "github.com/pingcap/tidb/pkg/sessionctx/variable" - "github.com/pingcap/tidb/pkg/statistics/handle" - statsutil "github.com/pingcap/tidb/pkg/statistics/handle/util" - "github.com/pingcap/tidb/pkg/table" - pumpcli "github.com/pingcap/tidb/pkg/tidb-binlog/pump_client" - tidbutil "github.com/pingcap/tidb/pkg/util" - "github.com/pingcap/tidb/pkg/util/dbterror" - "github.com/pingcap/tidb/pkg/util/dbterror/exeerrors" - "github.com/pingcap/tidb/pkg/util/gcutil" - "github.com/pingcap/tidb/pkg/util/generic" ->>>>>>> 44c9096efbc (ddl: get latest old table ID before replace view (#53720)):pkg/ddl/ddl.go "github.com/tikv/client-go/v2/tikvrpc" clientv3 "go.etcd.io/etcd/client/v3" atomicutil "go.uber.org/atomic" @@ -402,6 +367,11 @@ type ddlCtx struct { // reorgCtx is used for reorganization. reorgCtx reorgContexts + // backfillCtx is used for backfill workers. + backfillCtx struct { + syncutil.RWMutex + jobCtxMap map[int64]*JobContext + } jobCtx struct { sync.RWMutex diff --git a/ddl/ddl_api_test.go b/ddl/ddl_api_test.go index 3681861b65e5f..74e079091cb4d 100644 --- a/ddl/ddl_api_test.go +++ b/ddl/ddl_api_test.go @@ -15,8 +15,10 @@ package ddl_test import ( + "cmp" "context" "fmt" + "slices" "sync" "testing" @@ -30,11 +32,7 @@ import ( "github.com/pingcap/tidb/testkit" "github.com/pingcap/tidb/util/chunk" "github.com/stretchr/testify/require" -<<<<<<< HEAD:ddl/ddl_api_test.go - "golang.org/x/exp/slices" -======= "golang.org/x/sync/errgroup" ->>>>>>> 44c9096efbc (ddl: get latest old table ID before replace view (#53720)):pkg/ddl/ddl_api_test.go ) func TestGetDDLJobs(t *testing.T) { @@ -115,8 +113,8 @@ func TestGetDDLJobsIsSort(t *testing.T) { require.NoError(t, err) require.Len(t, currJobs, 15) - isSort := slices.IsSortedFunc(currJobs, func(i, j *model.Job) bool { - return i.ID <= j.ID + isSort := slices.IsSortedFunc(currJobs, func(i, j *model.Job) int { + return cmp.Compare(i.ID, j.ID) }) require.True(t, isSort) @@ -163,22 +161,34 @@ func TestCreateViewConcurrently(t *testing.T) { tk.MustExec("create table t (a int);") tk.MustExec("create view v as select * from t;") + tk.MustExec("set global tidb_enable_metadata_lock = 1;") + tk.MustExec("set global tidb_enable_concurrent_ddl = 1;") var ( counterErr error counter int ) - failpoint.EnableCall("github.com/pingcap/tidb/pkg/ddl/onDDLCreateView", func(job *model.Job) { + ddl.OnCreateViewForTest = func(job *model.Job) { counter++ if counter > 1 { counterErr = fmt.Errorf("create view job should not run concurrently") return } - }) - failpoint.EnableCall("github.com/pingcap/tidb/pkg/ddl/afterDelivery2Worker", func(job *model.Job) { + } + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/onDDLCreateView", "return")) + defer func() { + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/onDDLCreateView")) + }() + + ddl.AfterDelivery2WorkerForTest = func(job *model.Job) { if job.Type == model.ActionCreateView { counter-- } - }) + } + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/afterDelivery2Worker", "return")) + defer func() { + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/afterDelivery2Worker")) + }() + var eg errgroup.Group for i := 0; i < 5; i++ { eg.Go(func() error { diff --git a/ddl/job_table.go b/ddl/job_table.go index 9386628f6bd68..5295dccabc2ac 100644 --- a/ddl/job_table.go +++ b/ddl/job_table.go @@ -351,6 +351,9 @@ func (d *ddl) loadDDLJobAndRun(se *sess.Session, pool *workerPool, getJob func(* d.delivery2worker(wk, pool, job) } +// AfterDelivery2WorkerForTest is only used for test. +var AfterDelivery2WorkerForTest func(*model.Job) + // delivery2worker owns the worker, need to put it back to the pool in this function. func (d *ddl) delivery2worker(wk *worker, pool *workerPool, job *model.Job) { injectFailPointForGetJob(job) @@ -358,43 +361,13 @@ func (d *ddl) delivery2worker(wk *worker, pool *workerPool, job *model.Job) { d.wg.Run(func() { metrics.DDLRunningJobCount.WithLabelValues(pool.tp().String()).Inc() defer func() { + failpoint.Inject("afterDelivery2Worker", func() { + AfterDelivery2WorkerForTest(job) + }) d.runningJobs.remove(job) asyncNotify(d.ddlJobCh) metrics.DDLRunningJobCount.WithLabelValues(pool.tp().String()).Dec() }() -<<<<<<< HEAD:ddl/job_table.go -======= - - err := wk.HandleLocalDDLJob(d.ddlCtx, job) - pool.put(wk) - if err != nil { - logutil.DDLLogger().Info("handle ddl job failed", zap.Error(err), zap.Stringer("job", job)) - } - task.NotifyError(err) - }) -} - -// delivery2Worker owns the worker, need to put it back to the pool in this function. -func (s *jobScheduler) delivery2Worker(wk *worker, pool *workerPool, job *model.Job) { - injectFailPointForGetJob(job) - s.runningJobs.add(job) - s.wg.Run(func() { - metrics.DDLRunningJobCount.WithLabelValues(pool.tp().String()).Inc() - defer func() { - failpoint.InjectCall("afterDelivery2Worker", job) - s.runningJobs.remove(job) - asyncNotify(s.ddlJobNotifyCh) - metrics.DDLRunningJobCount.WithLabelValues(pool.tp().String()).Dec() - if wk.ctx.Err() != nil && ingest.LitBackCtxMgr != nil { - // if ctx cancelled, i.e. owner changed, we need to Unregister the backend - // as litBackendCtx is holding this very 'ctx', and it cannot reuse now. - // TODO make LitBackCtxMgr a local value of the job scheduler, it makes - // it much harder to test multiple owners in 1 unit test. - ingest.LitBackCtxMgr.Unregister(job.ID) - } - }() - ownerID := s.ownerManager.ID() ->>>>>>> 44c9096efbc (ddl: get latest old table ID before replace view (#53720)):pkg/ddl/job_table.go // check if this ddl job is synced to all servers. if !job.NotStarted() && (!d.isSynced(job) || !d.maybeAlreadyRunOnce(job.ID)) { if variable.EnableMDL.Load() { diff --git a/ddl/placement_policy.go b/ddl/placement_policy.go index ef6a57f2e8faa..47bfe63b11784 100644 --- a/ddl/placement_policy.go +++ b/ddl/placement_policy.go @@ -319,15 +319,8 @@ func checkPlacementPolicyNotInUse(d *ddlCtx, t *meta.Meta, policy *model.PolicyI return err } is := d.infoCache.GetLatest() -<<<<<<< HEAD:ddl/placement_policy.go - if is.SchemaMetaVersion() == currVer { - return CheckPlacementPolicyNotInUseFromInfoSchema(is, policy) -======= if is != nil && is.SchemaMetaVersion() == currVer { - err = CheckPlacementPolicyNotInUseFromInfoSchema(is, policy) - } else { - err = CheckPlacementPolicyNotInUseFromMeta(t, policy) ->>>>>>> 44c9096efbc (ddl: get latest old table ID before replace view (#53720)):pkg/ddl/placement_policy.go + return CheckPlacementPolicyNotInUseFromInfoSchema(is, policy) } return CheckPlacementPolicyNotInUseFromMeta(t, policy) diff --git a/ddl/table.go b/ddl/table.go index 0076587380364..50666e072071b 100644 --- a/ddl/table.go +++ b/ddl/table.go @@ -270,6 +270,9 @@ func repairTableOrViewWithCheck(t *meta.Meta, job *model.Job, schemaID int64, tb return t.UpdateTable(schemaID, tbInfo) } +// OnCreateViewForTest is only used for test. +var OnCreateViewForTest func(*model.Job) + func onCreateView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) { schemaID := job.SchemaID tbInfo := &model.TableInfo{} @@ -286,7 +289,9 @@ func onCreateView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) if infoschema.ErrTableNotExists.Equal(err) { err = nil } - failpoint.InjectCall("onDDLCreateView", job) + failpoint.Inject("onDDLCreateView", func() { + OnCreateViewForTest(job) + }) if err != nil { if infoschema.ErrDatabaseNotExists.Equal(err) { job.State = model.JobStateCancelled @@ -309,13 +314,8 @@ func onCreateView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) // none -> public tbInfo.State = model.StatePublic tbInfo.UpdateTS = t.StartTS -<<<<<<< HEAD:ddl/table.go - if oldTbInfoID > 0 && orReplace { - err = t.DropTableOrView(schemaID, oldTbInfoID) -======= if oldTableID > 0 && orReplace { - err = t.DropTableOrView(schemaID, job.SchemaName, oldTableID, tbInfo.Name.L) ->>>>>>> 44c9096efbc (ddl: get latest old table ID before replace view (#53720)):pkg/ddl/table.go + err = t.DropTableOrView(schemaID, oldTableID) if err != nil { job.State = model.JobStateCancelled return ver, errors.Trace(err) @@ -1480,44 +1480,6 @@ func checkTableNotExists(d *ddlCtx, t *meta.Meta, schemaID int64, tableName stri return checkTableNotExistsFromStore(t, schemaID, tableName) } -<<<<<<< HEAD:ddl/table.go -======= -func checkTableNotExistsByName(d *ddlCtx, t *meta.Meta, schemaID int64, schemaName, tableName string) error { - // Try to use memory schema info to check first. - currVer, err := t.GetSchemaVersion() - if err != nil { - return err - } - is := d.infoCache.GetLatest() - if is != nil && is.SchemaMetaVersion() == currVer { - return checkTableNotExistsFromInfoSchema(is, schemaID, tableName) - } - return t.CheckTableNameNotExists(t.TableNameKey(schemaName, tableName)) -} - -func checkConstraintNamesNotExists(t *meta.Meta, schemaID int64, constraints []*model.ConstraintInfo) error { - if len(constraints) == 0 { - return nil - } - tbInfos, err := t.ListTables(schemaID) - if err != nil { - return err - } - - for _, tb := range tbInfos { - for _, constraint := range constraints { - if constraint.State != model.StateWriteOnly { - if constraintInfo := tb.FindConstraintInfoByName(constraint.Name.L); constraintInfo != nil { - return infoschema.ErrCheckConstraintDupName.GenWithStackByArgs(constraint.Name.L) - } - } - } - } - - return nil -} - ->>>>>>> 44c9096efbc (ddl: get latest old table ID before replace view (#53720)):pkg/ddl/table.go func checkTableIDNotExists(t *meta.Meta, schemaID, tableID int64) error { tbl, err := t.GetTable(schemaID, tableID) if err != nil { @@ -1591,7 +1553,7 @@ func findTableIDFromInfoSchema(is infoschema.InfoSchema, schemaID int64, tableNa } func findTableIDFromStore(t *meta.Meta, schemaID int64, tableName string) (int64, error) { - tbls, err := t.ListSimpleTables(schemaID) + tbls, err := t.ListTables(schemaID) if err != nil { if meta.ErrDBNotExists.Equal(err) { return 0, infoschema.ErrDatabaseNotExists.GenWithStackByArgs("") diff --git a/pkg/ddl/BUILD.bazel b/pkg/ddl/BUILD.bazel deleted file mode 100644 index f5cddf95e7fad..0000000000000 --- a/pkg/ddl/BUILD.bazel +++ /dev/null @@ -1,348 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -package_group( - name = "ddl_friend", - packages = [ - "-//pkg/planner/...", - "//...", - ], -) - -go_library( - name = "ddl", - srcs = [ - "backfilling.go", - "backfilling_clean_s3.go", - "backfilling_dist_executor.go", - "backfilling_dist_scheduler.go", - "backfilling_import_cloud.go", - "backfilling_merge_sort.go", - "backfilling_operators.go", - "backfilling_read_index.go", - "backfilling_scheduler.go", - "bdr.go", - "callback.go", - "cluster.go", - "column.go", - "constant.go", - "constraint.go", - "ddl.go", - "ddl_algorithm.go", - "ddl_api.go", - "ddl_history.go", - "ddl_running_jobs.go", - "ddl_tiflash_api.go", - "ddl_worker.go", - "ddl_workerpool.go", - "delete_range.go", - "delete_range_util.go", - "dist_owner.go", - "foreign_key.go", - "generated_column.go", - "index.go", - "index_cop.go", - "index_merge_tmp.go", - "job_table.go", - "mock.go", - "multi_schema_change.go", - "options.go", - "partition.go", - "placement_policy.go", - "reorg.go", - "resource_group.go", - "rollingback.go", - "sanity_check.go", - "schema.go", - "schema_version.go", - "sequence.go", - "split_region.go", - "stat.go", - "table.go", - "table_lock.go", - "ttl.go", - ], - importpath = "github.com/pingcap/tidb/pkg/ddl", - visibility = [ - ":ddl_friend", - ], - deps = [ - "//br/pkg/storage", - "//pkg/config", - "//pkg/ddl/copr", - "//pkg/ddl/ingest", - "//pkg/ddl/internal/session", - "//pkg/ddl/label", - "//pkg/ddl/logutil", - "//pkg/ddl/placement", - "//pkg/ddl/resourcegroup", - "//pkg/ddl/syncer", - "//pkg/ddl/util", - "//pkg/distsql", - "//pkg/distsql/context", - "//pkg/disttask/framework/handle", - "//pkg/disttask/framework/proto", - "//pkg/disttask/framework/scheduler", - "//pkg/disttask/framework/storage", - "//pkg/disttask/framework/taskexecutor", - "//pkg/disttask/framework/taskexecutor/execute", - "//pkg/disttask/operator", - "//pkg/domain/infosync", - "//pkg/domain/resourcegroup", - "//pkg/errctx", - "//pkg/expression", - "//pkg/expression/context", - "//pkg/expression/contextstatic", - "//pkg/infoschema", - "//pkg/kv", - "//pkg/lightning/backend", - "//pkg/lightning/backend/external", - "//pkg/lightning/backend/local", - "//pkg/lightning/common", - "//pkg/lightning/config", - "//pkg/meta", - "//pkg/meta/autoid", - "//pkg/metrics", - "//pkg/owner", - "//pkg/parser", - "//pkg/parser/ast", - "//pkg/parser/charset", - "//pkg/parser/format", - "//pkg/parser/model", - "//pkg/parser/mysql", - "//pkg/parser/opcode", - "//pkg/parser/terror", - "//pkg/parser/types", - "//pkg/privilege", - "//pkg/resourcemanager/pool/workerpool", - "//pkg/resourcemanager/util", - "//pkg/sessionctx", - "//pkg/sessionctx/binloginfo", - "//pkg/sessionctx/stmtctx", - "//pkg/sessionctx/variable", - "//pkg/sessiontxn", - "//pkg/statistics", - "//pkg/statistics/handle", - "//pkg/statistics/handle/util", - "//pkg/store/copr", - "//pkg/store/driver/backoff", - "//pkg/store/driver/txn", - "//pkg/store/helper", - "//pkg/table", - "//pkg/table/tables", - "//pkg/tablecodec", - "//pkg/tidb-binlog/pump_client", - "//pkg/ttl/cache", - "//pkg/types", - "//pkg/types/parser_driver", - "//pkg/util", - "//pkg/util/backoff", - "//pkg/util/chunk", - "//pkg/util/codec", - "//pkg/util/collate", - "//pkg/util/context", - "//pkg/util/dbterror", - "//pkg/util/dbterror/exeerrors", - "//pkg/util/domainutil", - "//pkg/util/engine", - "//pkg/util/execdetails", - "//pkg/util/filter", - "//pkg/util/gcutil", - "//pkg/util/generic", - "//pkg/util/hack", - "//pkg/util/intest", - "//pkg/util/logutil", - "//pkg/util/mathutil", - "//pkg/util/memory", - "//pkg/util/mock", - "//pkg/util/ranger", - "//pkg/util/resourcegrouptag", - "//pkg/util/rowDecoder", - "//pkg/util/rowcodec", - "//pkg/util/set", - "//pkg/util/size", - "//pkg/util/slice", - "//pkg/util/sqlexec", - "//pkg/util/sqlkiller", - "//pkg/util/stringutil", - "//pkg/util/tiflash", - "//pkg/util/timeutil", - "//pkg/util/topsql", - "//pkg/util/topsql/state", - "@com_github_docker_go_units//:go-units", - "@com_github_google_uuid//:uuid", - "@com_github_ngaut_pools//:pools", - "@com_github_pingcap_errors//:errors", - "@com_github_pingcap_failpoint//:failpoint", - "@com_github_pingcap_kvproto//pkg/errorpb", - "@com_github_pingcap_kvproto//pkg/kvrpcpb", - "@com_github_pingcap_kvproto//pkg/metapb", - "@com_github_pingcap_log//:log", - "@com_github_pingcap_tipb//go-tipb", - "@com_github_prometheus_client_golang//prometheus", - "@com_github_tikv_client_go_v2//error", - "@com_github_tikv_client_go_v2//kv", - "@com_github_tikv_client_go_v2//oracle", - "@com_github_tikv_client_go_v2//tikv", - "@com_github_tikv_client_go_v2//tikvrpc", - "@com_github_tikv_client_go_v2//txnkv/rangetask", - "@com_github_tikv_client_go_v2//util", - "@com_github_tikv_pd_client//:client", - "@com_github_tikv_pd_client//http", - "@io_etcd_go_etcd_client_v3//:client", - "@io_etcd_go_etcd_client_v3//concurrency", - "@org_golang_x_sync//errgroup", - "@org_uber_go_atomic//:atomic", - "@org_uber_go_zap//:zap", - ], -) - -go_test( - name = "ddl_test", - timeout = "moderate", - srcs = [ - "attributes_sql_test.go", - "backfilling_dist_scheduler_test.go", - "backfilling_test.go", - "bdr_test.go", - "bench_test.go", - "cancel_test.go", - "cluster_test.go", - "column_change_test.go", - "column_modify_test.go", - "column_test.go", - "column_type_change_test.go", - "constraint_test.go", - "db_cache_test.go", - "db_change_failpoints_test.go", - "db_change_test.go", - "db_integration_test.go", - "db_rename_test.go", - "db_table_test.go", - "db_test.go", - "ddl_algorithm_test.go", - "ddl_api_test.go", - "ddl_error_test.go", - "ddl_history_test.go", - "ddl_running_jobs_test.go", - "ddl_test.go", - "ddl_worker_test.go", - "ddl_workerpool_test.go", - "export_test.go", - "fail_test.go", - "foreign_key_test.go", - "index_change_test.go", - "index_cop_test.go", - "index_modify_test.go", - "integration_test.go", - "job_table_test.go", - "main_test.go", - "modify_column_test.go", - "multi_schema_change_test.go", - "mv_index_test.go", - "options_test.go", - "partition_test.go", - "placement_policy_ddl_test.go", - "placement_policy_test.go", - "placement_sql_test.go", - "primary_key_handle_test.go", - "reorg_partition_test.go", - "repair_table_test.go", - "restart_test.go", - "rollingback_test.go", - "schema_test.go", - "sequence_test.go", - "stat_test.go", - "table_modify_test.go", - "table_split_test.go", - "table_test.go", - "tiflash_replica_test.go", - "ttl_test.go", - ], - embed = [":ddl"], - flaky = True, - shard_count = 50, - deps = [ - "//pkg/autoid_service", - "//pkg/config", - "//pkg/ddl/copr", - "//pkg/ddl/ingest", - "//pkg/ddl/internal/session", - "//pkg/ddl/logutil", - "//pkg/ddl/placement", - "//pkg/ddl/schematracker", - "//pkg/ddl/syncer", - "//pkg/ddl/testutil", - "//pkg/ddl/util", - "//pkg/ddl/util/callback", - "//pkg/disttask/framework/proto", - "//pkg/disttask/framework/scheduler", - "//pkg/disttask/framework/storage", - "//pkg/domain", - "//pkg/domain/infosync", - "//pkg/errctx", - "//pkg/errno", - "//pkg/executor", - "//pkg/expression", - "//pkg/infoschema", - "//pkg/keyspace", - "//pkg/kv", - "//pkg/lightning/backend/external", - "//pkg/meta", - "//pkg/meta/autoid", - "//pkg/parser", - "//pkg/parser/ast", - "//pkg/parser/auth", - "//pkg/parser/charset", - "//pkg/parser/model", - "//pkg/parser/mysql", - "//pkg/parser/terror", - "//pkg/parser/types", - "//pkg/server", - "//pkg/session", - "//pkg/session/types", - "//pkg/sessionctx", - "//pkg/sessionctx/variable", - "//pkg/sessiontxn", - "//pkg/store/gcworker", - "//pkg/store/helper", - "//pkg/store/mockstore", - "//pkg/table", - "//pkg/table/tables", - "//pkg/tablecodec", - "//pkg/testkit", - "//pkg/testkit/external", - "//pkg/testkit/testfailpoint", - "//pkg/testkit/testsetup", - "//pkg/testkit/testutil", - "//pkg/types", - "//pkg/util", - "//pkg/util/chunk", - "//pkg/util/codec", - "//pkg/util/collate", - "//pkg/util/context", - "//pkg/util/dbterror", - "//pkg/util/dbterror/plannererrors", - "//pkg/util/domainutil", - "//pkg/util/gcutil", - "//pkg/util/mathutil", - "//pkg/util/mock", - "//pkg/util/sem", - "//pkg/util/sqlexec", - "//pkg/util/timeutil", - "@com_github_ngaut_pools//:pools", - "@com_github_pingcap_errors//:errors", - "@com_github_pingcap_failpoint//:failpoint", - "@com_github_stretchr_testify//assert", - "@com_github_stretchr_testify//require", - "@com_github_tikv_client_go_v2//oracle", - "@com_github_tikv_client_go_v2//testutils", - "@com_github_tikv_client_go_v2//tikv", - "@com_github_tikv_client_go_v2//util", - "@io_etcd_go_etcd_client_v3//:client", - "@org_golang_google_grpc//:grpc", - "@org_golang_x_sync//errgroup", - "@org_uber_go_atomic//:atomic", - "@org_uber_go_goleak//:goleak", - "@org_uber_go_zap//:zap", - ], -) From 2043253fa76f59592a14db7f4a8d38a12a3abee9 Mon Sep 17 00:00:00 2001 From: Ruihao Chen Date: Mon, 11 Nov 2024 14:58:40 +0800 Subject: [PATCH 3/4] Fix --- ddl/ddl_api_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ddl/ddl_api_test.go b/ddl/ddl_api_test.go index 7072eb9377335..8c996d5dd0b46 100644 --- a/ddl/ddl_api_test.go +++ b/ddl/ddl_api_test.go @@ -15,10 +15,8 @@ package ddl_test import ( - "cmp" "context" "fmt" - "slices" "sync" "testing" @@ -33,6 +31,7 @@ import ( "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/cmp" "github.com/stretchr/testify/require" + "golang.org/x/exp/slices" "golang.org/x/sync/errgroup" ) From 2ee15c579ed8e4d2a4e5378415f4a0ee5e3216df Mon Sep 17 00:00:00 2001 From: Ruihao Chen Date: Tue, 12 Nov 2024 07:32:25 +0800 Subject: [PATCH 4/4] Update test --- ddl/ddl_api_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/ddl/ddl_api_test.go b/ddl/ddl_api_test.go index 8c996d5dd0b46..3ae596031a9a6 100644 --- a/ddl/ddl_api_test.go +++ b/ddl/ddl_api_test.go @@ -162,7 +162,6 @@ func TestCreateViewConcurrently(t *testing.T) { tk.MustExec("create table t (a int);") tk.MustExec("create view v as select * from t;") tk.MustExec("set global tidb_enable_metadata_lock = 1;") - tk.MustExec("set global tidb_enable_concurrent_ddl = 1;") var ( counterErr error counter int