From 8973dddc9ed52e732fc3ef437a7e30aacd5a989c Mon Sep 17 00:00:00 2001 From: Jianjun Liao <36503113+Leavrth@users.noreply.github.com> Date: Thu, 25 Apr 2024 11:00:11 +0800 Subject: [PATCH 01/20] br: no domain to run log command (#52127) close pingcap/tidb#52088 --- br/pkg/config/kv.go | 16 ++++ br/pkg/conn/BUILD.bazel | 2 +- br/pkg/conn/conn.go | 19 ++++ br/pkg/conn/conn_test.go | 163 +++++++++++++++++++++++++++++++++ br/pkg/stream/stream_status.go | 9 ++ br/pkg/task/stream.go | 104 +++++++++++++-------- br/pkg/utils/BUILD.bazel | 2 +- br/pkg/utils/db.go | 48 ---------- br/pkg/utils/db_test.go | 70 -------------- 9 files changed, 273 insertions(+), 160 deletions(-) diff --git a/br/pkg/config/kv.go b/br/pkg/config/kv.go index 0283c8d3cc721..e211887099854 100644 --- a/br/pkg/config/kv.go +++ b/br/pkg/config/kv.go @@ -56,3 +56,19 @@ func ParseMergeRegionSizeFromConfig(resp []byte) (uint64, uint64, error) { urs := uint64(rs) return urs, c.Cop.RegionSplitKeys, nil } + +func ParseLogBackupEnableFromConfig(resp []byte) (bool, error) { + type logbackup struct { + Enable bool `json:"enable"` + } + + type config struct { + LogBackup logbackup `json:"log-backup"` + } + var c config + e := json.Unmarshal(resp, &c) + if e != nil { + return false, e + } + return c.LogBackup.Enable, nil +} diff --git a/br/pkg/conn/BUILD.bazel b/br/pkg/conn/BUILD.bazel index 0c7dc388ada74..8f25055538375 100644 --- a/br/pkg/conn/BUILD.bazel +++ b/br/pkg/conn/BUILD.bazel @@ -46,7 +46,7 @@ go_test( ], embed = [":conn"], flaky = True, - shard_count = 7, + shard_count = 8, deps = [ "//br/pkg/config", "//br/pkg/conn/util", diff --git a/br/pkg/conn/conn.go b/br/pkg/conn/conn.go index 16c86e18c8985..87accda625a22 100644 --- a/br/pkg/conn/conn.go +++ b/br/pkg/conn/conn.go @@ -352,6 +352,25 @@ func (mgr *Mgr) ProcessTiKVConfigs(ctx context.Context, cfg *kvconfig.KVConfig, } } +// IsLogBackupEnabled is used for br to check whether tikv has enabled log backup. +func (mgr *Mgr) IsLogBackupEnabled(ctx context.Context, client *http.Client) (bool, error) { + logbackupEnable := true + err := mgr.GetConfigFromTiKV(ctx, client, func(resp *http.Response) error { + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + enable, err := kvconfig.ParseLogBackupEnableFromConfig(respBytes) + if err != nil { + log.Warn("Failed to parse log-backup enable from config", logutil.ShortError(err)) + return err + } + logbackupEnable = logbackupEnable && enable + return nil + }) + return logbackupEnable, errors.Trace(err) +} + // GetConfigFromTiKV get configs from all alive tikv stores. func (mgr *Mgr) GetConfigFromTiKV(ctx context.Context, cli *http.Client, fn func(*http.Response) error) error { allStores, err := GetAllTiKVStoresWithRetry(ctx, mgr.GetPDClient(), util.SkipTiFlash) diff --git a/br/pkg/conn/conn_test.go b/br/pkg/conn/conn_test.go index 17b678f6c268f..00fe21d60f1e0 100644 --- a/br/pkg/conn/conn_test.go +++ b/br/pkg/conn/conn_test.go @@ -455,6 +455,169 @@ func TestGetMergeRegionSizeAndCount(t *testing.T) { } } +func TestIsLogBackupEnabled(t *testing.T) { + cases := []struct { + stores []*metapb.Store + content []string + enable bool + err bool + }{ + { + stores: []*metapb.Store{ + { + Id: 1, + State: metapb.StoreState_Up, + Labels: []*metapb.StoreLabel{ + { + Key: "engine", + Value: "tiflash", + }, + }, + }, + }, + content: []string{""}, + enable: true, + err: false, + }, + { + stores: []*metapb.Store{ + { + Id: 1, + State: metapb.StoreState_Up, + Labels: []*metapb.StoreLabel{ + { + Key: "engine", + Value: "tiflash", + }, + }, + }, + { + Id: 2, + State: metapb.StoreState_Up, + Labels: []*metapb.StoreLabel{ + { + Key: "engine", + Value: "tikv", + }, + }, + }, + }, + content: []string{ + "", + // Assuming the TiKV has failed due to some reason. + "", + }, + enable: false, + err: true, + }, + { + stores: []*metapb.Store{ + { + Id: 1, + State: metapb.StoreState_Up, + Labels: []*metapb.StoreLabel{ + { + Key: "engine", + Value: "tikv", + }, + }, + }, + }, + content: []string{ + "{\"log-level\": \"debug\", \"log-backup\": {\"enable\": true}}", + }, + enable: true, + err: false, + }, + { + stores: []*metapb.Store{ + { + Id: 1, + State: metapb.StoreState_Up, + Labels: []*metapb.StoreLabel{ + { + Key: "engine", + Value: "tikv", + }, + }, + }, + }, + content: []string{ + "{\"log-level\": \"debug\", \"log-backup\": {\"enable\": false}}", + }, + enable: false, + err: false, + }, + { + stores: []*metapb.Store{ + { + Id: 1, + State: metapb.StoreState_Up, + Labels: []*metapb.StoreLabel{ + { + Key: "engine", + Value: "tikv", + }, + }, + }, + { + Id: 2, + State: metapb.StoreState_Up, + Labels: []*metapb.StoreLabel{ + { + Key: "engine", + Value: "tikv", + }, + }, + }, + }, + content: []string{ + "{\"log-level\": \"debug\", \"log-backup\": {\"enable\": true}}", + "{\"log-level\": \"debug\", \"log-backup\": {\"enable\": false}}", + }, + enable: false, + err: false, + }, + } + + pctx := context.Background() + for _, ca := range cases { + ctx, cancel := context.WithCancel(pctx) + pdCli := utils.FakePDClient{Stores: ca.stores} + require.Equal(t, len(ca.content), len(ca.stores)) + count := 0 + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch strings.TrimSpace(r.URL.Path) { + case "/config": + if len(ca.content[count]) == 0 { + cancel() + } + _, _ = fmt.Fprint(w, ca.content[count]) + default: + http.NotFoundHandler().ServeHTTP(w, r) + } + count++ + })) + + for _, s := range ca.stores { + s.Address = mockServer.URL + s.StatusAddress = mockServer.URL + } + + httpCli := mockServer.Client() + mgr := &Mgr{PdController: &pdutil.PdController{}} + mgr.PdController.SetPDClient(pdCli) + enable, err := mgr.IsLogBackupEnabled(ctx, httpCli) + if ca.err { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, ca.enable, enable) + } + mockServer.Close() + } +} + func TestHandleTiKVAddress(t *testing.T) { cases := []struct { store *metapb.Store diff --git a/br/pkg/stream/stream_status.go b/br/pkg/stream/stream_status.go index 7090e0622fabc..186567cc86f6c 100644 --- a/br/pkg/stream/stream_status.go +++ b/br/pkg/stream/stream_status.go @@ -344,6 +344,15 @@ func NewStatusController(meta *MetaDataClient, mgr PDInfoProvider, view TaskPrin } } +func (ctl *StatusController) Close() error { + if ctl.meta != nil { + if err := ctl.meta.Close(); err != nil { + return errors.Trace(err) + } + } + return nil +} + // fillTask queries and fills the extra information for a raw task. func (ctl *StatusController) fillTask(ctx context.Context, task Task) (TaskStatus, error) { var err error diff --git a/br/pkg/task/stream.go b/br/pkg/task/stream.go index 4f2144fae2c1e..5362fce37f19a 100644 --- a/br/pkg/task/stream.go +++ b/br/pkg/task/stream.go @@ -293,7 +293,7 @@ type streamMgr struct { func NewStreamMgr(ctx context.Context, cfg *StreamConfig, g glue.Glue, isStreamStart bool) (*streamMgr, error) { mgr, err := NewMgr(ctx, g, cfg.PD, cfg.TLS, GetKeepalive(&cfg.Config), - cfg.CheckRequirements, true, conn.StreamVersionChecker) + cfg.CheckRequirements, false, conn.StreamVersionChecker) if err != nil { return nil, errors.Trace(err) } @@ -371,8 +371,8 @@ func (s *streamMgr) adjustAndCheckStartTS(ctx context.Context) error { } // checkImportTaskRunning checks whether there is any import task running. -func (s *streamMgr) checkImportTaskRunning(ctx context.Context) error { - list, err := utils.GetImportTasksFrom(ctx, s.mgr.GetDomain().GetEtcdClient()) +func (s *streamMgr) checkImportTaskRunning(ctx context.Context, etcdCLI *clientv3.Client) error { + list, err := utils.GetImportTasksFrom(ctx, etcdCLI) if err != nil { return errors.Trace(err) } @@ -402,7 +402,7 @@ func (s *streamMgr) setGCSafePoint(ctx context.Context, sp utils.BRServiceSafePo return nil } -func (s *streamMgr) buildObserveRanges(ctx context.Context) ([]kv.KeyRange, error) { +func (s *streamMgr) buildObserveRanges() ([]kv.KeyRange, error) { dRanges, err := stream.BuildObserveDataRanges( s.mgr.GetStorage(), s.cfg.FilterStr, @@ -422,13 +422,13 @@ func (s *streamMgr) buildObserveRanges(ctx context.Context) ([]kv.KeyRange, erro return rs, nil } -func (s *streamMgr) backupFullSchemas(ctx context.Context, g glue.Glue) error { +func (s *streamMgr) backupFullSchemas(ctx context.Context) error { clusterVersion, err := s.mgr.GetClusterVersion(ctx) if err != nil { return errors.Trace(err) } - metaWriter := metautil.NewMetaWriter(s.bc.GetStorage(), metautil.MetaFileSize, false, metautil.MetaFile, nil) + metaWriter := metautil.NewMetaWriter(s.bc.GetStorage(), metautil.MetaFileSize, true, metautil.MetaFile, nil) metaWriter.Update(func(m *backuppb.BackupMeta) { // save log startTS to backupmeta file m.StartVersion = s.cfg.StartTS @@ -454,13 +454,8 @@ func (s *streamMgr) backupFullSchemas(ctx context.Context, g glue.Glue) error { return nil } -func (s *streamMgr) checkStreamStartEnable(g glue.Glue) error { - se, err := g.CreateSession(s.mgr.GetStorage()) - if err != nil { - return errors.Trace(err) - } - execCtx := se.GetSessionCtx().GetRestrictedSQLExecutor() - supportStream, err := utils.IsLogBackupEnabled(execCtx) +func (s *streamMgr) checkStreamStartEnable(ctx context.Context) error { + supportStream, err := s.mgr.IsLogBackupEnabled(ctx, s.httpCli) if err != nil { return errors.Trace(err) } @@ -555,17 +550,26 @@ func RunStreamStart( } defer streamMgr.close() - if err = streamMgr.checkStreamStartEnable(g); err != nil { + if err = streamMgr.checkStreamStartEnable(ctx); err != nil { return errors.Trace(err) } if err = streamMgr.adjustAndCheckStartTS(ctx); err != nil { return errors.Trace(err) } - if err = streamMgr.checkImportTaskRunning(ctx); err != nil { + + etcdCLI, err := dialEtcdWithCfg(ctx, cfg.Config) + if err != nil { + return errors.Trace(err) + } + cli := streamhelper.NewMetaDataClient(etcdCLI) + defer func() { + if closeErr := cli.Close(); closeErr != nil { + log.Warn("failed to close etcd client", zap.Error(closeErr)) + } + }() + if err = streamMgr.checkImportTaskRunning(ctx, cli.Client); err != nil { return errors.Trace(err) } - - cli := streamhelper.NewMetaDataClient(streamMgr.mgr.GetDomain().GetEtcdClient()) // It supports single stream log task currently. if count, err := cli.GetTaskCount(ctx); err != nil { return errors.Trace(err) @@ -614,12 +618,12 @@ func RunStreamStart( if err = streamMgr.setLock(ctx); err != nil { return errors.Trace(err) } - if err = streamMgr.backupFullSchemas(ctx, g); err != nil { + if err = streamMgr.backupFullSchemas(ctx); err != nil { return errors.Trace(err) } } - ranges, err := streamMgr.buildObserveRanges(ctx) + ranges, err := streamMgr.buildObserveRanges() if err != nil { return errors.Trace(err) } else if len(ranges) == 0 { @@ -707,7 +711,16 @@ func RunStreamStop( } defer streamMgr.close() - cli := streamhelper.NewMetaDataClient(streamMgr.mgr.GetDomain().GetEtcdClient()) + etcdCLI, err := dialEtcdWithCfg(ctx, cfg.Config) + if err != nil { + return errors.Trace(err) + } + cli := streamhelper.NewMetaDataClient(etcdCLI) + defer func() { + if closeErr := cli.Close(); closeErr != nil { + log.Warn("failed to close etcd client", zap.Error(closeErr)) + } + }() // to add backoff ti, err := cli.GetTask(ctx, cfg.TaskName) if err != nil { @@ -757,7 +770,16 @@ func RunStreamPause( } defer streamMgr.close() - cli := streamhelper.NewMetaDataClient(streamMgr.mgr.GetDomain().GetEtcdClient()) + etcdCLI, err := dialEtcdWithCfg(ctx, cfg.Config) + if err != nil { + return errors.Trace(err) + } + cli := streamhelper.NewMetaDataClient(etcdCLI) + defer func() { + if closeErr := cli.Close(); closeErr != nil { + log.Warn("failed to close etcd client", zap.Error(closeErr)) + } + }() // to add backoff ti, isPaused, err := cli.GetTaskWithPauseStatus(ctx, cfg.TaskName) if err != nil { @@ -815,7 +837,16 @@ func RunStreamResume( } defer streamMgr.close() - cli := streamhelper.NewMetaDataClient(streamMgr.mgr.GetDomain().GetEtcdClient()) + etcdCLI, err := dialEtcdWithCfg(ctx, cfg.Config) + if err != nil { + return errors.Trace(err) + } + cli := streamhelper.NewMetaDataClient(etcdCLI) + defer func() { + if closeErr := cli.Close(); closeErr != nil { + log.Warn("failed to close etcd client", zap.Error(closeErr)) + } + }() // to add backoff ti, isPaused, err := cli.GetTaskWithPauseStatus(ctx, cfg.TaskName) if err != nil { @@ -943,6 +974,11 @@ func RunStreamStatus( if err != nil { return err } + defer func() { + if closeErr := ctl.Close(); closeErr != nil { + log.Warn("failed to close etcd client", zap.Error(closeErr)) + } + }() return ctl.PrintStatusOfTask(ctx, cfg.TaskName) } @@ -1109,7 +1145,7 @@ func RunStreamRestore( if err != nil { return errors.Trace(err) } - logInfo, err := getLogRangeWithStorage(ctx, &cfg.Config, s) + logInfo, err := getLogRangeWithStorage(ctx, s) if err != nil { return errors.Trace(err) } @@ -1179,7 +1215,7 @@ func RunStreamRestore( } // restore log. cfg.adjustRestoreConfigForStreamRestore() - if err := restoreStream(ctx, g, cfg, curTaskInfo, logInfo.logMinTS, logInfo.logMaxTS); err != nil { + if err := restoreStream(ctx, g, cfg, curTaskInfo); err != nil { return errors.Trace(err) } return nil @@ -1191,7 +1227,6 @@ func restoreStream( g glue.Glue, cfg *RestoreConfig, taskInfo *checkpoint.CheckpointTaskInfoForLogRestore, - logMinTS, logMaxTS uint64, ) (err error) { var ( totalKVCount uint64 @@ -1312,7 +1347,7 @@ func restoreStream( } // get full backup meta storage to generate rewrite rules. - fullBackupStorage, err := parseFullBackupTablesStorage(ctx, cfg) + fullBackupStorage, err := parseFullBackupTablesStorage(cfg) if err != nil { return errors.Trace(err) } @@ -1531,15 +1566,6 @@ func withProgress(p glue.Progress, cc func(p glue.Progress) error) error { return cc(p) } -// nolint: unused, deadcode -func countIndices(ts map[int64]*metautil.Table) int64 { - result := int64(0) - for _, t := range ts { - result += int64(len(t.Info.Indices)) - } - return result -} - type backupLogInfo struct { logMaxTS uint64 logMinTS uint64 @@ -1555,12 +1581,11 @@ func getLogRange( if err != nil { return backupLogInfo{}, errors.Trace(err) } - return getLogRangeWithStorage(ctx, cfg, s) + return getLogRangeWithStorage(ctx, s) } func getLogRangeWithStorage( ctx context.Context, - cfg *Config, s storage.ExternalStorage, ) (backupLogInfo, error) { // logStartTS: Get log start ts from backupmeta file. @@ -1644,7 +1669,6 @@ func getFullBackupTS( } func parseFullBackupTablesStorage( - ctx context.Context, cfg *RestoreConfig, ) (*restore.FullBackupStorageConfig, error) { var storageName string @@ -1733,7 +1757,7 @@ func buildPauseSafePointName(taskName string) string { return fmt.Sprintf("%s_pause_safepoint", taskName) } -func checkPiTRRequirements(ctx context.Context, g glue.Glue, cfg *RestoreConfig, mgr *conn.Mgr) error { +func checkPiTRRequirements(mgr *conn.Mgr) error { userDBs := restore.GetExistedUserDBs(mgr.GetDomain()) if len(userDBs) > 0 { userDBNames := make([]string, 0, len(userDBs)) @@ -1810,7 +1834,7 @@ func checkPiTRTaskInfo( // Only when use checkpoint and not the first execution, // skip checking requirements. log.Info("check pitr requirements for the first execution") - if err := checkPiTRRequirements(ctx, g, cfg, mgr); err != nil { + if err := checkPiTRRequirements(mgr); err != nil { if len(errTaskMsg) > 0 { err = errors.Annotatef(err, "The current restore task is regarded as %s. "+ "If you ensure that no changes have been made to the cluster since the last execution, "+ diff --git a/br/pkg/utils/BUILD.bazel b/br/pkg/utils/BUILD.bazel index 0c24b46141179..edbbc4dcf2bb5 100644 --- a/br/pkg/utils/BUILD.bazel +++ b/br/pkg/utils/BUILD.bazel @@ -77,7 +77,7 @@ go_test( ], embed = [":utils"], flaky = True, - shard_count = 33, + shard_count = 32, deps = [ "//br/pkg/errors", "//pkg/kv", diff --git a/br/pkg/utils/db.go b/br/pkg/utils/db.go index d9ee2cefeba1a..f6d0b66dc7a54 100644 --- a/br/pkg/utils/db.go +++ b/br/pkg/utils/db.go @@ -5,7 +5,6 @@ package utils import ( "context" "strconv" - "strings" "github.com/docker/go-units" "github.com/pingcap/errors" @@ -26,53 +25,6 @@ var ( logBackupTaskCount = atomic.NewInt32(0) ) -// CheckLogBackupEnabled checks if LogBackup is enabled in cluster. -// this mainly used in three places. -// 1. GC worker resolve locks to scan more locks after safepoint. (every minute) -// 2. Add index skipping use lightning.(every add index ddl) -// 3. Telemetry of log backup feature usage (every 6 hours). -// NOTE: this result shouldn't be cached by caller. because it may change every time in one cluster. -func CheckLogBackupEnabled(ctx sessionctx.Context) bool { - executor := ctx.GetRestrictedSQLExecutor() - enabled, err := IsLogBackupEnabled(executor) - if err != nil { - // if it failed by any reason. we can simply return true this time. - // for GC worker it will scan more locks in one tick. - // for Add index it will skip using lightning this time. - // for Telemetry it will get a false positive usage count. - log.Warn("check log backup config failed, ignore it", zap.String("category", "backup"), zap.Error(err)) - return true - } - return enabled -} - -// IsLogBackupEnabled is used for br to check whether tikv has enabled log backup. -// we use `sqlexec.RestrictedSQLExecutor` as parameter because it's easy to mock. -// it should return error. -func IsLogBackupEnabled(ctx sqlexec.RestrictedSQLExecutor) (bool, error) { - valStr := "show config where name = 'log-backup.enable' and type = 'tikv'" - internalCtx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnBR) - rows, fields, errSQL := ctx.ExecRestrictedSQL(internalCtx, nil, valStr) - if errSQL != nil { - return false, errSQL - } - if len(rows) == 0 { - // no rows mean not support log backup. - return false, nil - } - for _, row := range rows { - d := row.GetDatum(3, &fields[3].Column.FieldType) - value, errField := d.ToString() - if errField != nil { - return false, errField - } - if strings.ToLower(value) == "false" { - return false, nil - } - } - return true, nil -} - func GetRegionSplitInfo(ctx sqlexec.RestrictedSQLExecutor) (uint64, int64) { return GetSplitSize(ctx), GetSplitKeys(ctx) } diff --git a/br/pkg/utils/db_test.go b/br/pkg/utils/db_test.go index a43a8840c9cc4..a3d67a6a7667e 100644 --- a/br/pkg/utils/db_test.go +++ b/br/pkg/utils/db_test.go @@ -51,76 +51,6 @@ func (m *mockRestrictedSQLExecutor) ExecRestrictedSQL(ctx context.Context, opts return nil, nil, nil } -func TestIsLogBackupEnabled(t *testing.T) { - // config format: - // MySQL [(none)]> show config where name="log-backup.enable"; - // +------+-----------------+-------------------+-------+ - // | Type | Instance | Name | Value | - // +------+-----------------+-------------------+-------+ - // | tikv | 127.0.0.1:20161 | log-backup.enable | false | - // | tikv | 127.0.0.1:20162 | log-backup.enable | false | - // | tikv | 127.0.0.1:20160 | log-backup.enable | false | - // +------+-----------------+-------------------+-------+ - fields := make([]*ast.ResultField, 4) - tps := []*types.FieldType{ - types.NewFieldType(mysql.TypeString), - types.NewFieldType(mysql.TypeString), - types.NewFieldType(mysql.TypeString), - types.NewFieldType(mysql.TypeString), - } - for i := 0; i < len(tps); i++ { - rf := new(ast.ResultField) - rf.Column = new(model.ColumnInfo) - rf.Column.FieldType = *tps[i] - fields[i] = rf - } - rows := make([]chunk.Row, 0, 1) - - // case 1: non of tikvs enabled log-backup expected false - // tikv | 127.0.0.1:20161 | log-backup.enable | false | - row := chunk.MutRowFromValues("tikv", " 127.0.0.1:20161", "log-backup.enable", "false").ToRow() - rows = append(rows, row) - s := &mockRestrictedSQLExecutor{rows: rows, fields: fields} - enabled, err := utils.IsLogBackupEnabled(s) - require.NoError(t, err) - require.False(t, enabled) - - // case 2: one of tikvs enabled log-backup expected false - // tikv | 127.0.0.1:20161 | log-backup.enable | false | - // tikv | 127.0.0.1:20162 | log-backup.enable | true | - rows = nil - row = chunk.MutRowFromValues("tikv", " 127.0.0.1:20161", "log-backup.enable", "false").ToRow() - rows = append(rows, row) - row = chunk.MutRowFromValues("tikv", " 127.0.0.1:20162", "log-backup.enable", "true").ToRow() - rows = append(rows, row) - s = &mockRestrictedSQLExecutor{rows: rows, fields: fields} - enabled, err = utils.IsLogBackupEnabled(s) - require.NoError(t, err) - require.False(t, enabled) - - // case 3: all of tikvs enabled log-backup expected true - // tikv | 127.0.0.1:20161 | log-backup.enable | true | - // tikv | 127.0.0.1:20162 | log-backup.enable | true | - // tikv | 127.0.0.1:20163 | log-backup.enable | true | - rows = nil - row = chunk.MutRowFromValues("tikv", " 127.0.0.1:20161", "log-backup.enable", "true").ToRow() - rows = append(rows, row) - row = chunk.MutRowFromValues("tikv", " 127.0.0.1:20162", "log-backup.enable", "true").ToRow() - rows = append(rows, row) - row = chunk.MutRowFromValues("tikv", " 127.0.0.1:20163", "log-backup.enable", "true").ToRow() - rows = append(rows, row) - s = &mockRestrictedSQLExecutor{rows: rows, fields: fields} - enabled, err = utils.IsLogBackupEnabled(s) - require.NoError(t, err) - require.True(t, enabled) - - // case 4: met error and expected false. - s = &mockRestrictedSQLExecutor{errHappen: true} - enabled, err = utils.IsLogBackupEnabled(s) - require.Error(t, err) - require.False(t, enabled) -} - func TestCheckLogBackupTaskExist(t *testing.T) { require.False(t, utils.CheckLogBackupTaskExist()) utils.LogBackupTaskCountInc() From 24990b5ddd60053985b2d8c5c56bf8c2e522b5ad Mon Sep 17 00:00:00 2001 From: YangKeao Date: Thu, 25 Apr 2024 11:40:41 +0800 Subject: [PATCH 02/20] server: handle the case the type of param is set for the param sent by `SEND_LONG_DATA` (#52720) close pingcap/tidb#52711 --- pkg/server/conn_stmt_params.go | 19 +++++- .../internal/testserverclient/BUILD.bazel | 1 + .../testserverclient/server_client.go | 61 +++++++++++++++++++ pkg/server/tests/commontest/BUILD.bazel | 2 +- pkg/server/tests/commontest/tidb_test.go | 5 ++ 5 files changed, 86 insertions(+), 2 deletions(-) diff --git a/pkg/server/conn_stmt_params.go b/pkg/server/conn_stmt_params.go index 5b625c683faac..29fbfd9b058c4 100644 --- a/pkg/server/conn_stmt_params.go +++ b/pkg/server/conn_stmt_params.go @@ -40,7 +40,24 @@ func parseBinaryParams(params []param.BinaryParam, boundParams [][]byte, nullBit if boundParams[i] != nil { params[i] = param.BinaryParam{ Tp: mysql.TypeBlob, - Val: enc.DecodeInput(boundParams[i]), + Val: boundParams[i], + } + + // The legacy logic is kept: if the `paramTypes` somehow didn't contain the type information, it will be treated as + // BLOB type. We didn't return `mysql.ErrMalformPacket` to keep compatibility with older versions, though it's + // meaningless if every clients work properly. + if (i<<1)+1 < len(paramTypes) { + // Only TEXT or BLOB type will be sent through `SEND_LONG_DATA`. + tp := paramTypes[i<<1] + + switch tp { + case mysql.TypeVarchar, mysql.TypeVarString, mysql.TypeString, mysql.TypeBit: + params[i].Tp = tp + params[i].Val = enc.DecodeInput(boundParams[i]) + case mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob: + params[i].Tp = tp + params[i].Val = boundParams[i] + } } continue } diff --git a/pkg/server/internal/testserverclient/BUILD.bazel b/pkg/server/internal/testserverclient/BUILD.bazel index 4625be01ab8f3..f90bad7322d82 100644 --- a/pkg/server/internal/testserverclient/BUILD.bazel +++ b/pkg/server/internal/testserverclient/BUILD.bazel @@ -20,6 +20,7 @@ go_library( "@com_github_pingcap_log//:log", "@com_github_prometheus_client_model//go", "@com_github_stretchr_testify//require", + "@org_golang_x_text//encoding/simplifiedchinese", "@org_uber_go_zap//:zap", ], ) diff --git a/pkg/server/internal/testserverclient/server_client.go b/pkg/server/internal/testserverclient/server_client.go index f3e205eb37912..fa6d42f42935e 100644 --- a/pkg/server/internal/testserverclient/server_client.go +++ b/pkg/server/internal/testserverclient/server_client.go @@ -48,6 +48,7 @@ import ( dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/require" "go.uber.org/zap" + "golang.org/x/text/encoding/simplifiedchinese" ) //revive:disable:exported @@ -2727,4 +2728,64 @@ func (cli *TestServerClient) RunTestConnectionCount(t *testing.T) { }) } +func (cli *TestServerClient) RunTestTypeAndCharsetOfSendLongData(t *testing.T) { + cli.RunTests(t, func(config *mysql.Config) { + config.MaxAllowedPacket = 1024 + }, func(dbt *testkit.DBTestKit) { + ctx := context.Background() + + conn, err := dbt.GetDB().Conn(ctx) + require.NoError(t, err) + _, err = conn.ExecContext(ctx, "CREATE TABLE t (j JSON);") + require.NoError(t, err) + + str := `"` + strings.Repeat("a", 1024) + `"` + stmt, err := conn.PrepareContext(ctx, "INSERT INTO t VALUES (cast(? as JSON));") + require.NoError(t, err) + _, err = stmt.ExecContext(ctx, str) + require.NoError(t, err) + result, err := conn.QueryContext(ctx, "SELECT j FROM t;") + require.NoError(t, err) + + for result.Next() { + var j string + require.NoError(t, result.Scan(&j)) + require.Equal(t, str, j) + } + }) + + str := strings.Repeat("你好", 1024) + enc := simplifiedchinese.GBK.NewEncoder() + gbkStr, err := enc.String(str) + require.NoError(t, err) + + cli.RunTests(t, func(config *mysql.Config) { + config.MaxAllowedPacket = 1024 + config.Params["charset"] = "gbk" + }, func(dbt *testkit.DBTestKit) { + ctx := context.Background() + + conn, err := dbt.GetDB().Conn(ctx) + require.NoError(t, err) + _, err = conn.ExecContext(ctx, "drop table t") + require.NoError(t, err) + _, err = conn.ExecContext(ctx, "CREATE TABLE t (t TEXT);") + require.NoError(t, err) + + stmt, err := conn.PrepareContext(ctx, "INSERT INTO t VALUES (?);") + require.NoError(t, err) + _, err = stmt.ExecContext(ctx, gbkStr) + require.NoError(t, err) + + result, err := conn.QueryContext(ctx, "SELECT * FROM t;") + require.NoError(t, err) + + for result.Next() { + var txt string + require.NoError(t, result.Scan(&txt)) + require.Equal(t, gbkStr, txt) + } + }) +} + //revive:enable:exported diff --git a/pkg/server/tests/commontest/BUILD.bazel b/pkg/server/tests/commontest/BUILD.bazel index d2b4d8ed82490..8e1baedf9abbf 100644 --- a/pkg/server/tests/commontest/BUILD.bazel +++ b/pkg/server/tests/commontest/BUILD.bazel @@ -8,7 +8,7 @@ go_test( "tidb_test.go", ], flaky = True, - shard_count = 49, + shard_count = 50, deps = [ "//pkg/config", "//pkg/ddl/util", diff --git a/pkg/server/tests/commontest/tidb_test.go b/pkg/server/tests/commontest/tidb_test.go index 5e8b0f431b3c3..d1c1846bf6408 100644 --- a/pkg/server/tests/commontest/tidb_test.go +++ b/pkg/server/tests/commontest/tidb_test.go @@ -3082,3 +3082,8 @@ func TestConnectionCount(t *testing.T) { ts := servertestkit.CreateTidbTestSuite(t) ts.RunTestConnectionCount(t) } + +func TestTypeAndCharsetOfSendLongData(t *testing.T) { + ts := servertestkit.CreateTidbTestSuite(t) + ts.RunTestTypeAndCharsetOfSendLongData(t) +} From b0d756d18a9f6f151b777ac035b628f1b422dbfc Mon Sep 17 00:00:00 2001 From: lance6716 Date: Thu, 25 Apr 2024 13:09:10 +0800 Subject: [PATCH 03/20] ddl: refactor the goroutines of writePhysicalTableRecord (#52851) close pingcap/tidb#52868 --- pkg/ddl/backfilling.go | 327 ++++++++---------- pkg/ddl/backfilling_operators.go | 5 +- pkg/ddl/backfilling_scheduler.go | 101 +++--- pkg/ddl/column.go | 6 +- pkg/ddl/ddl.go | 2 + pkg/ddl/index_cop.go | 12 +- pkg/ddl/reorg.go | 7 +- .../pool/workerpool/workerpool.go | 5 +- 8 files changed, 229 insertions(+), 236 deletions(-) diff --git a/pkg/ddl/backfilling.go b/pkg/ddl/backfilling.go index 5a7266fdb8ed8..8a176c96b7f6f 100644 --- a/pkg/ddl/backfilling.go +++ b/pkg/ddl/backfilling.go @@ -339,31 +339,44 @@ func (w *backfillWorker) handleBackfillTask(d *ddlCtx, task *reorgBackfillTask, return result } +func (w *backfillWorker) sendResult(result *backfillResult) { + select { + case <-w.ctx.Done(): + case w.resultCh <- result: + } +} + func (w *backfillWorker) run(d *ddlCtx, bf backfiller, job *model.Job) { - logutil.BgLogger().Info("backfill worker start", zap.String("category", "ddl"), zap.Stringer("worker", w)) - var curTaskID int + logger := ddlLogger.With(zap.Stringer("worker", w), zap.Int64("jobID", job.ID)) + var ( + curTaskID int + task *reorgBackfillTask + ok bool + ) + defer w.wg.Done() defer util.Recover(metrics.LabelDDL, "backfillWorker.run", func() { - w.resultCh <- &backfillResult{taskID: curTaskID, err: dbterror.ErrReorgPanic} + w.sendResult(&backfillResult{taskID: curTaskID, err: dbterror.ErrReorgPanic}) }, false) for { - if util.HasCancelled(w.ctx) { - logutil.BgLogger().Info("backfill worker exit on context done", zap.String("category", "ddl"), zap.Stringer("worker", w)) + select { + case <-w.ctx.Done(): + logger.Info("backfill worker exit on context done") return + case task, ok = <-w.taskCh: } - task, more := <-w.taskCh - if !more { - logutil.BgLogger().Info("backfill worker exit", zap.String("category", "ddl"), zap.Stringer("worker", w)) + if !ok { + logger.Info("backfill worker exit") return } curTaskID = task.id d.setDDLLabelForTopSQL(job.ID, job.Query) - logutil.BgLogger().Debug("backfill worker got task", zap.String("category", "ddl"), zap.Int("workerID", w.GetCtx().id), zap.String("task", task.String())) + logger.Debug("backfill worker got task", zap.Int("workerID", w.GetCtx().id), zap.Stringer("task", task)) failpoint.Inject("mockBackfillRunErr", func() { if w.GetCtx().id == 0 { result := &backfillResult{taskID: task.id, addedCount: 0, nextKey: nil, err: errors.Errorf("mock backfill error")} - w.resultCh <- result + w.sendResult(result) failpoint.Continue() } }) @@ -380,10 +393,11 @@ func (w *backfillWorker) run(d *ddlCtx, bf backfiller, job *model.Job) { // Change the batch size dynamically. w.GetCtx().batchCnt = int(variable.GetDDLReorgBatchSize()) result := w.handleBackfillTask(d, task, bf) - w.resultCh <- result + w.sendResult(result) + if result.err != nil { - logutil.BgLogger().Info("backfill worker exit on error", zap.String("category", "ddl"), - zap.Stringer("worker", w), zap.Error(result.err)) + logger.Info("backfill worker exit on error", + zap.Error(result.err)) return } } @@ -392,7 +406,13 @@ func (w *backfillWorker) run(d *ddlCtx, bf backfiller, job *model.Job) { // splitTableRanges uses PD region's key ranges to split the backfilling table key range space, // to speed up backfilling data in table with disperse handle. // The `t` should be a non-partitioned table or a partition. -func splitTableRanges(t table.PhysicalTable, store kv.Storage, startKey, endKey kv.Key, limit int) ([]kv.KeyRange, error) { +func splitTableRanges( + ctx context.Context, + t table.PhysicalTable, + store kv.Storage, + startKey, endKey kv.Key, + limit int, +) ([]kv.KeyRange, error) { logutil.BgLogger().Info("split table range from PD", zap.String("category", "ddl"), zap.Int64("physicalTableID", t.GetPhysicalID()), zap.String("start key", hex.EncodeToString(startKey)), @@ -410,7 +430,7 @@ func splitTableRanges(t table.PhysicalTable, store kv.Storage, startKey, endKey } maxSleep := 10000 // ms - bo := backoff.NewBackofferWithVars(context.Background(), maxSleep, nil) + bo := backoff.NewBackofferWithVars(ctx, maxSleep, nil) rc := copr.NewRegionCache(s.GetRegionCache()) ranges, err := rc.SplitRegionRanges(bo, []kv.KeyRange{kvRange}, limit) if err != nil { @@ -423,127 +443,6 @@ func splitTableRanges(t table.PhysicalTable, store kv.Storage, startKey, endKey return ranges, nil } -type resultConsumer struct { - dc *ddlCtx - wg *sync.WaitGroup - err error - hasError *atomic.Bool - reorgInfo *reorgInfo // reorgInfo is used to update the reorg handle. - sessPool *sess.Pool // sessPool is used to get the session to update the reorg handle. - distribute bool -} - -func newResultConsumer(dc *ddlCtx, reorgInfo *reorgInfo, sessPool *sess.Pool, distribute bool) *resultConsumer { - return &resultConsumer{ - dc: dc, - wg: &sync.WaitGroup{}, - hasError: &atomic.Bool{}, - reorgInfo: reorgInfo, - sessPool: sessPool, - distribute: distribute, - } -} - -func (s *resultConsumer) run(scheduler backfillScheduler, start kv.Key, totalAddedCount *int64) { - s.wg.Add(1) - go func() { - reorgInfo := s.reorgInfo - err := consumeResults(scheduler, s, start, totalAddedCount) - if err != nil { - logutil.BgLogger().Warn("backfill worker handle tasks failed", zap.String("category", "ddl"), - zap.Int64("total added count", *totalAddedCount), - zap.String("start key", hex.EncodeToString(start)), - zap.String("task failed error", err.Error())) - s.err = err - } else { - logutil.BgLogger().Info("backfill workers successfully processed", zap.String("category", "ddl"), - zap.Stringer("element", reorgInfo.currElement), - zap.Int64("total added count", *totalAddedCount), - zap.String("start key", hex.EncodeToString(start))) - } - s.wg.Done() - }() -} - -func (s *resultConsumer) getResult() error { - s.wg.Wait() - return s.err -} - -func (s *resultConsumer) shouldAbort() bool { - return s.hasError.Load() -} - -func consumeResults(scheduler backfillScheduler, consumer *resultConsumer, start kv.Key, totalAddedCount *int64) error { - keeper := newDoneTaskKeeper(start) - handledTaskCnt := 0 - var firstErr error - for { - result, ok := scheduler.receiveResult() - if !ok { - return firstErr - } - err := handleOneResult(result, scheduler, consumer, keeper, totalAddedCount, handledTaskCnt) - handledTaskCnt++ - if err != nil && firstErr == nil { - consumer.hasError.Store(true) - firstErr = err - } - } -} - -func handleOneResult(result *backfillResult, scheduler backfillScheduler, consumer *resultConsumer, - keeper *doneTaskKeeper, totalAddedCount *int64, taskSeq int) error { - reorgInfo := consumer.reorgInfo - if result.err != nil { - logutil.BgLogger().Warn("backfill worker failed", zap.String("category", "ddl"), - zap.Int64("job ID", reorgInfo.ID), - zap.String("result next key", hex.EncodeToString(result.nextKey)), - zap.Error(result.err)) - scheduler.drainTasks() // Make it quit early. - return result.err - } - if result.totalCount > 0 { - *totalAddedCount = int64(result.totalCount) - } else { - *totalAddedCount += int64(result.addedCount) - } - if !consumer.distribute { - reorgCtx := consumer.dc.getReorgCtx(reorgInfo.Job.ID) - reorgCtx.setRowCount(*totalAddedCount) - } - keeper.updateNextKey(result.taskID, result.nextKey) - if taskSeq%(scheduler.currentWorkerSize()*4) == 0 { - if !consumer.distribute { - err := consumer.dc.isReorgRunnable(reorgInfo.ID, consumer.distribute) - if err != nil { - logutil.BgLogger().Warn("backfill worker is not runnable", zap.String("category", "ddl"), zap.Error(err)) - scheduler.drainTasks() // Make it quit early. - return err - } - failpoint.Inject("MockGetIndexRecordErr", func() { - // Make sure this job didn't failed because by the "Write conflict" error. - if dbterror.ErrNotOwner.Equal(err) { - time.Sleep(50 * time.Millisecond) - } - }) - err = reorgInfo.UpdateReorgMeta(keeper.nextKey, consumer.sessPool) - if err != nil { - logutil.BgLogger().Warn("update reorg meta failed", zap.String("category", "ddl"), - zap.Int64("job ID", reorgInfo.ID), zap.Error(err)) - } - } - // We try to adjust the worker size regularly to reduce - // the overhead of loading the DDL related global variables. - err := scheduler.adjustWorkerSize() - if err != nil { - logutil.BgLogger().Warn("cannot adjust backfill worker size", zap.String("category", "ddl"), - zap.Int64("job ID", reorgInfo.ID), zap.Error(err)) - } - } - return nil -} - func getBatchTasks(t table.Table, reorgInfo *reorgInfo, kvRanges []kv.KeyRange, taskIDAlloc *taskIDAllocator) []*reorgBackfillTask { batchTasks := make([]*reorgBackfillTask, 0, len(kvRanges)) @@ -592,15 +491,20 @@ func getBatchTasks(t table.Table, reorgInfo *reorgInfo, kvRanges []kv.KeyRange, } // sendTasks sends tasks to workers, and returns remaining kvRanges that is not handled. -func sendTasks(scheduler backfillScheduler, consumer *resultConsumer, - t table.PhysicalTable, kvRanges []kv.KeyRange, reorgInfo *reorgInfo, taskIDAlloc *taskIDAllocator) { +func sendTasks( + scheduler backfillScheduler, + t table.PhysicalTable, + kvRanges []kv.KeyRange, + reorgInfo *reorgInfo, + taskIDAlloc *taskIDAllocator, +) error { batchTasks := getBatchTasks(t, reorgInfo, kvRanges, taskIDAlloc) for _, task := range batchTasks { - if consumer.shouldAbort() { - return + if err := scheduler.sendTask(task); err != nil { + return errors.Trace(err) } - scheduler.sendTask(task) } + return nil } var ( @@ -678,10 +582,12 @@ func SetBackfillTaskChanSizeForTest(n int) { // // The above operations are completed in a transaction. // Finally, update the concurrent processing of the total number of rows, and store the completed handle value. -func (dc *ddlCtx) writePhysicalTableRecord(sessPool *sess.Pool, t table.PhysicalTable, bfWorkerType backfillerType, reorgInfo *reorgInfo) error { - job := reorgInfo.Job - totalAddedCount := job.GetRowCount() - +func (dc *ddlCtx) writePhysicalTableRecord( + sessPool *sess.Pool, + t table.PhysicalTable, + bfWorkerType backfillerType, + reorgInfo *reorgInfo, +) error { startKey, endKey := reorgInfo.StartKey, reorgInfo.EndKey if err := dc.isReorgRunnable(reorgInfo.Job.ID, false); err != nil { @@ -691,53 +597,120 @@ func (dc *ddlCtx) writePhysicalTableRecord(sessPool *sess.Pool, t table.Physical failpoint.Inject("MockCaseWhenParseFailure", func(val failpoint.Value) { //nolint:forcetypeassert if val.(bool) { - failpoint.Return(errors.New("job.ErrCount:" + strconv.Itoa(int(job.ErrorCount)) + ", mock unknown type: ast.whenClause.")) + failpoint.Return(errors.New("job.ErrCount:" + strconv.Itoa(int(reorgInfo.Job.ErrorCount)) + ", mock unknown type: ast.whenClause.")) } }) jc := reorgInfo.NewJobContext() - sessCtx := newContext(reorgInfo.d.store) - scheduler, err := newBackfillScheduler(dc.ctx, reorgInfo, sessPool, bfWorkerType, t, sessCtx, jc) + sessCtx := newReorgSessCtx(reorgInfo.d.store) + + eg, egCtx := util.NewErrorGroupWithRecoverWithCtx(dc.ctx) + + scheduler, err := newBackfillScheduler(egCtx, reorgInfo, sessPool, bfWorkerType, t, sessCtx, jc) if err != nil { return errors.Trace(err) } defer scheduler.close(true) - - consumer := newResultConsumer(dc, reorgInfo, sessPool, false) - consumer.run(scheduler, startKey, &totalAddedCount) - err = scheduler.setupWorkers() if err != nil { return errors.Trace(err) } - taskIDAlloc := newTaskIDAllocator() - for { - kvRanges, err := splitTableRanges(t, reorgInfo.d.store, startKey, endKey, backfillTaskChanSize) - if err != nil { - return errors.Trace(err) - } - if len(kvRanges) == 0 { - break - } - logutil.BgLogger().Info("start backfill workers to reorg record", zap.String("category", "ddl"), - zap.Stringer("type", bfWorkerType), - zap.Int("workerCnt", scheduler.currentWorkerSize()), - zap.Int("regionCnt", len(kvRanges)), - zap.String("startKey", hex.EncodeToString(startKey)), - zap.String("endKey", hex.EncodeToString(endKey))) - - sendTasks(scheduler, consumer, t, kvRanges, reorgInfo, taskIDAlloc) - if consumer.shouldAbort() { - break + // process result goroutine + eg.Go(func() error { + totalAddedCount := reorgInfo.Job.GetRowCount() + keeper := newDoneTaskKeeper(startKey) + cnt := 0 + + for { + select { + case <-egCtx.Done(): + return egCtx.Err() + case result, ok := <-scheduler.resultChan(): + if !ok { + ddlLogger.Info("backfill workers successfully processed", + zap.Stringer("element", reorgInfo.currElement), + zap.Int64("total added count", totalAddedCount), + zap.String("start key", hex.EncodeToString(startKey))) + return nil + } + cnt++ + + if result.err != nil { + ddlLogger.Warn("backfill worker failed", + zap.Int64("job ID", reorgInfo.ID), + zap.Int64("total added count", totalAddedCount), + zap.String("start key", hex.EncodeToString(startKey)), + zap.String("result next key", hex.EncodeToString(result.nextKey)), + zap.Error(result.err)) + return result.err + } + + if result.totalCount > 0 { + totalAddedCount = int64(result.totalCount) + } else { + totalAddedCount += int64(result.addedCount) + } + dc.getReorgCtx(reorgInfo.Job.ID).setRowCount(totalAddedCount) + + keeper.updateNextKey(result.taskID, result.nextKey) + + if cnt%(scheduler.currentWorkerSize()*4) == 0 { + err2 := reorgInfo.UpdateReorgMeta(keeper.nextKey, sessPool) + if err2 != nil { + ddlLogger.Warn("update reorg meta failed", + zap.Int64("job ID", reorgInfo.ID), + zap.Error(err2)) + } + // We try to adjust the worker size regularly to reduce + // the overhead of loading the DDL related global variables. + err2 = scheduler.adjustWorkerSize() + if err2 != nil { + ddlLogger.Warn("cannot adjust backfill worker size", + zap.Int64("job ID", reorgInfo.ID), + zap.Error(err2)) + } + } + } } - startKey = kvRanges[len(kvRanges)-1].EndKey - if startKey.Cmp(endKey) >= 0 { - break + }) + + // generate task goroutine + eg.Go(func() error { + // we will modify the startKey in this goroutine, so copy them to avoid race. + start, end := startKey, endKey + taskIDAlloc := newTaskIDAllocator() + for { + kvRanges, err2 := splitTableRanges(egCtx, t, reorgInfo.d.store, start, end, backfillTaskChanSize) + if err2 != nil { + return errors.Trace(err2) + } + if len(kvRanges) == 0 { + break + } + ddlLogger.Info("start backfill workers to reorg record", + zap.Stringer("type", bfWorkerType), + zap.Int("workerCnt", scheduler.currentWorkerSize()), + zap.Int("regionCnt", len(kvRanges)), + zap.String("startKey", hex.EncodeToString(start)), + zap.String("endKey", hex.EncodeToString(end))) + + err2 = sendTasks(scheduler, t, kvRanges, reorgInfo, taskIDAlloc) + if err2 != nil { + return errors.Trace(err2) + } + + start = kvRanges[len(kvRanges)-1].EndKey + if start.Cmp(end) >= 0 { + break + } } - } - scheduler.close(false) - return consumer.getResult() + + scheduler.close(false) + return nil + }) + + return eg.Wait() } func injectCheckBackfillWorkerNum(curWorkerSize int, isMergeWorker bool) error { diff --git a/pkg/ddl/backfilling_operators.go b/pkg/ddl/backfilling_operators.go index 4c1cfe7d60bce..a95dc8b426eef 100644 --- a/pkg/ddl/backfilling_operators.go +++ b/pkg/ddl/backfilling_operators.go @@ -249,7 +249,7 @@ type TableScanTask struct { } // String implement fmt.Stringer interface. -func (t *TableScanTask) String() string { +func (t TableScanTask) String() string { return fmt.Sprintf("TableScanTask: id=%d, startKey=%s, endKey=%s", t.ID, hex.EncodeToString(t.Start), hex.EncodeToString(t.End)) } @@ -311,6 +311,7 @@ func (src *TableScanTaskSource) generateTasks() error { endKey := src.endKey for { kvRanges, err := splitTableRanges( + src.ctx, src.tbl, src.store, startKey, @@ -448,7 +449,7 @@ var OperatorCallBackForTest func() func (w *tableScanWorker) scanRecords(task TableScanTask, sender func(IndexRecordChunk)) { logutil.Logger(w.ctx).Info("start a table scan task", - zap.Int("id", task.ID), zap.String("task", task.String())) + zap.Int("id", task.ID), zap.Stringer("task", task)) var idxResult IndexRecordChunk err := wrapInBeginRollback(w.se, func(startTS uint64) error { diff --git a/pkg/ddl/backfilling_scheduler.go b/pkg/ddl/backfilling_scheduler.go index ec4b655468e5b..57fbb997e8b01 100644 --- a/pkg/ddl/backfilling_scheduler.go +++ b/pkg/ddl/backfilling_scheduler.go @@ -50,9 +50,8 @@ type backfillScheduler interface { setupWorkers() error close(force bool) - sendTask(task *reorgBackfillTask) - drainTasks() - receiveResult() (*backfillResult, bool) + sendTask(*reorgBackfillTask) error + resultChan() <-chan *backfillResult currentWorkerSize() int adjustWorkerSize() error @@ -82,9 +81,15 @@ type txnBackfillScheduler struct { closed bool } -func newBackfillScheduler(ctx context.Context, info *reorgInfo, sessPool *sess.Pool, - tp backfillerType, tbl table.PhysicalTable, sessCtx sessionctx.Context, - jobCtx *JobContext) (backfillScheduler, error) { +func newBackfillScheduler( + ctx context.Context, + info *reorgInfo, + sessPool *sess.Pool, + tp backfillerType, + tbl table.PhysicalTable, + sessCtx sessionctx.Context, + jobCtx *JobContext, +) (backfillScheduler, error) { if tp == typeAddIndexWorker && info.ReorgMeta.ReorgTp == model.ReorgTypeLitMerge { ctx = logutil.WithCategory(ctx, "ddl-ingest") return newIngestBackfillScheduler(ctx, info, sessPool, tbl) @@ -117,19 +122,17 @@ func (b *txnBackfillScheduler) setupWorkers() error { return b.adjustWorkerSize() } -func (b *txnBackfillScheduler) sendTask(task *reorgBackfillTask) { - b.taskCh <- task -} - -func (b *txnBackfillScheduler) drainTasks() { - for len(b.taskCh) > 0 { - <-b.taskCh +func (b *txnBackfillScheduler) sendTask(task *reorgBackfillTask) error { + select { + case <-b.ctx.Done(): + return b.ctx.Err() + case b.taskCh <- task: + return nil } } -func (b *txnBackfillScheduler) receiveResult() (*backfillResult, bool) { - ret, ok := <-b.resultCh - return ret, ok +func (b *txnBackfillScheduler) resultChan() <-chan *backfillResult { + return b.resultCh } func newSessCtx( @@ -138,7 +141,7 @@ func newSessCtx( tzLocation *model.TimeZoneLocation, resourceGroupName string, ) (sessionctx.Context, error) { - sessCtx := newContext(store) + sessCtx := newReorgSessCtx(store) if err := initSessCtx(sessCtx, sqlMode, tzLocation, resourceGroupName); err != nil { return nil, errors.Trace(err) } @@ -248,12 +251,12 @@ func (b *txnBackfillScheduler) adjustWorkerSize() error { if err != nil { return err } - runner = newBackfillWorker(jc.ddlJobCtx, idxWorker) + runner = newBackfillWorker(b.ctx, idxWorker) worker = idxWorker case typeAddIndexMergeTmpWorker: backfillCtx := newBackfillCtx(reorgInfo.d, i, sessCtx, job.SchemaName, b.tbl, jc, "merge_tmp_idx_rate", false) tmpIdxWorker := newMergeTempIndexWorker(backfillCtx, b.tbl, reorgInfo.elements) - runner = newBackfillWorker(jc.ddlJobCtx, tmpIdxWorker) + runner = newBackfillWorker(b.ctx, tmpIdxWorker) worker = tmpIdxWorker case typeUpdateColumnWorker: // Setting InCreateOrAlterStmt tells the difference between SELECT casting and ALTER COLUMN casting. @@ -262,18 +265,18 @@ func (b *txnBackfillScheduler) adjustWorkerSize() error { sessCtx.GetSessionVars().StmtCtx.TypeFlags(). WithIgnoreZeroDateErr(!reorgInfo.ReorgMeta.SQLMode.HasStrictMode())) updateWorker := newUpdateColumnWorker(sessCtx, i, b.tbl, b.decodeColMap, reorgInfo, jc) - runner = newBackfillWorker(jc.ddlJobCtx, updateWorker) + runner = newBackfillWorker(b.ctx, updateWorker) worker = updateWorker case typeCleanUpIndexWorker: idxWorker := newCleanUpIndexWorker(sessCtx, i, b.tbl, b.decodeColMap, reorgInfo, jc) - runner = newBackfillWorker(jc.ddlJobCtx, idxWorker) + runner = newBackfillWorker(b.ctx, idxWorker) worker = idxWorker case typeReorgPartitionWorker: partWorker, err := newReorgPartitionWorker(sessCtx, i, b.tbl, b.decodeColMap, reorgInfo, jc) if err != nil { return err } - runner = newBackfillWorker(jc.ddlJobCtx, partWorker) + runner = newBackfillWorker(b.ctx, partWorker) worker = partWorker default: return errors.New("unknown backfill type") @@ -323,7 +326,6 @@ type ingestBackfillScheduler struct { writerPool *workerpool.WorkerPool[IndexRecordChunk, workerpool.None] writerMaxID int - poolErr chan error backendCtx ingest.BackendCtx checkpointMgr *ingest.CheckpointManager } @@ -348,7 +350,6 @@ func newIngestBackfillScheduler( avgRowSize: avgRowSize, taskCh: make(chan *reorgBackfillTask, backfillTaskChanSize), resultCh: make(chan *backfillResult, backfillTaskChanSize), - poolErr: make(chan error), }, nil } @@ -398,10 +399,10 @@ func (b *ingestBackfillScheduler) close(force bool) { b.checkpointMgr.Sync() // Get the latest status after all workers are closed so that the result is more accurate. cnt, nextKey := b.checkpointMgr.Status() - b.resultCh <- &backfillResult{ + b.sendResult(&backfillResult{ totalCount: cnt, nextKey: nextKey, - } + }) } close(b.resultCh) if intest.InTest && len(b.copReqSenderPool.srcChkPool) != copReadChunkPoolSize() { @@ -413,25 +414,26 @@ func (b *ingestBackfillScheduler) close(force bool) { } } -func (b *ingestBackfillScheduler) sendTask(task *reorgBackfillTask) { - b.taskCh <- task -} - -func (b *ingestBackfillScheduler) drainTasks() { - for len(b.taskCh) > 0 { - <-b.taskCh +func (b *ingestBackfillScheduler) sendTask(task *reorgBackfillTask) error { + select { + case <-b.ctx.Done(): + return b.ctx.Err() + case b.taskCh <- task: + return nil } } -func (b *ingestBackfillScheduler) receiveResult() (*backfillResult, bool) { +func (b *ingestBackfillScheduler) sendResult(res *backfillResult) { select { - case err := <-b.poolErr: - return &backfillResult{err: err}, true - case rs, ok := <-b.resultCh: - return rs, ok + case <-b.ctx.Done(): + case b.resultCh <- res: } } +func (b *ingestBackfillScheduler) resultChan() <-chan *backfillResult { + return b.resultCh +} + func (b *ingestBackfillScheduler) currentWorkerSize() int { return int(b.writerPool.Cap()) } @@ -448,7 +450,7 @@ func (b *ingestBackfillScheduler) createWorker() workerpool.Worker[IndexRecordCh job := reorgInfo.Job sessCtx, err := newSessCtx(reorgInfo.d.store, reorgInfo.ReorgMeta.SQLMode, reorgInfo.ReorgMeta.Location, reorgInfo.ReorgMeta.ResourceGroupName) if err != nil { - b.poolErr <- err + b.sendResult(&backfillResult{err: err}) return nil } bcCtx := b.backendCtx @@ -459,7 +461,7 @@ func (b *ingestBackfillScheduler) createWorker() workerpool.Worker[IndexRecordCh if err != nil { // Return an error only if it is the first worker. if b.writerMaxID == 0 { - b.poolErr <- err + b.sendResult(&backfillResult{err: err}) return nil } logutil.Logger(b.ctx).Warn("cannot create new writer", zap.Error(err), @@ -477,7 +479,7 @@ func (b *ingestBackfillScheduler) createWorker() workerpool.Worker[IndexRecordCh if err != nil { // Return an error only if it is the first worker. if b.writerMaxID == 0 { - b.poolErr <- err + b.sendResult(&backfillResult{err: err}) return nil } logutil.Logger(b.ctx).Warn("cannot create new writer", zap.Error(err), @@ -540,9 +542,16 @@ func expectedIngestWorkerCnt(concurrency, avgRowSize int) (readerCnt, writerCnt return readerCnt, writerCnt } +func (w *addIndexIngestWorker) sendResult(res *backfillResult) { + select { + case <-w.ctx.Done(): + case w.resultCh <- res: + } +} + func (w *addIndexIngestWorker) HandleTask(rs IndexRecordChunk, _ func(workerpool.None)) { defer util.Recover(metrics.LabelDDL, "ingestWorker.HandleTask", func() { - w.resultCh <- &backfillResult{taskID: rs.ID, err: dbterror.ErrReorgPanic} + w.sendResult(&backfillResult{taskID: rs.ID, err: dbterror.ErrReorgPanic}) }, false) defer w.copReqSenderPool.recycleChunk(rs.Chunk) result := &backfillResult{ @@ -552,19 +561,19 @@ func (w *addIndexIngestWorker) HandleTask(rs IndexRecordChunk, _ func(workerpool if result.err != nil { logutil.Logger(w.ctx).Error("encounter error when handle index chunk", zap.Int("id", rs.ID), zap.Error(rs.Err)) - w.resultCh <- result + w.sendResult(result) return } err := w.d.isReorgRunnable(w.jobID, false) if err != nil { result.err = err - w.resultCh <- result + w.sendResult(result) return } count, nextKey, err := w.WriteLocal(&rs) if err != nil { result.err = err - w.resultCh <- result + w.sendResult(result) return } if count == 0 { @@ -584,7 +593,7 @@ func (w *addIndexIngestWorker) HandleTask(rs IndexRecordChunk, _ func(workerpool if ResultCounterForTest != nil && result.err == nil { ResultCounterForTest.Add(1) } - w.resultCh <- result + w.sendResult(result) } func (*addIndexIngestWorker) Close() {} diff --git a/pkg/ddl/column.go b/pkg/ddl/column.go index ab3b313d9532c..ef79ca5a34675 100644 --- a/pkg/ddl/column.go +++ b/pkg/ddl/column.go @@ -580,7 +580,7 @@ func (w *worker) onModifyColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in changingCol.Name = newColName changingCol.ChangeStateInfo = &model.ChangeStateInfo{DependencyColumnOffset: oldCol.Offset} - originDefVal, err := GetOriginDefaultValueForModifyColumn(newContext(d.store), changingCol, oldCol) + originDefVal, err := GetOriginDefaultValueForModifyColumn(newReorgSessCtx(d.store), changingCol, oldCol) if err != nil { return ver, errors.Trace(err) } @@ -1800,7 +1800,7 @@ func updateColumnDefaultValue(d *ddlCtx, t *meta.Meta, job *model.Job, newCol *m return ver, infoschema.ErrColumnNotExists.GenWithStackByArgs(newCol.Name, tblInfo.Name) } - if hasDefaultValue, _, err := checkColumnDefaultValue(newContext(d.store), table.ToColumn(oldCol.Clone()), newCol.DefaultValue); err != nil { + if hasDefaultValue, _, err := checkColumnDefaultValue(newReorgSessCtx(d.store), table.ToColumn(oldCol.Clone()), newCol.DefaultValue); err != nil { job.State = model.JobStateCancelled return ver, errors.Trace(err) } else if !hasDefaultValue { @@ -1816,7 +1816,7 @@ func updateColumnDefaultValue(d *ddlCtx, t *meta.Meta, job *model.Job, newCol *m oldCol.AddFlag(mysql.NoDefaultValueFlag) } else { oldCol.DelFlag(mysql.NoDefaultValueFlag) - sctx := newContext(d.store) + sctx := newReorgSessCtx(d.store) err = checkDefaultValue(sctx, table.ToColumn(oldCol), true) if err != nil { job.State = model.JobStateCancelled diff --git a/pkg/ddl/ddl.go b/pkg/ddl/ddl.go index 48c99cfae5f90..faad5a126f207 100644 --- a/pkg/ddl/ddl.go +++ b/pkg/ddl/ddl.go @@ -101,6 +101,8 @@ const ( recoverCheckFlagDisableGC ) +var ddlLogger = logutil.BgLogger().With(zap.String("category", "ddl")) + // OnExist specifies what to do when a new object has a name collision. type OnExist uint8 diff --git a/pkg/ddl/index_cop.go b/pkg/ddl/index_cop.go index ab8802686b69a..58d6980ad9788 100644 --- a/pkg/ddl/index_cop.go +++ b/pkg/ddl/index_cop.go @@ -107,11 +107,17 @@ func (c *copReqSender) run() { } se := sess.NewSession(sessCtx) defer p.sessPool.Put(sessCtx) + var ( + task *reorgBackfillTask + ok bool + ) + for { - if util.HasCancelled(c.ctx) { + select { + case <-c.ctx.Done(): return + case task, ok = <-p.tasksCh: } - task, ok := <-p.tasksCh if !ok { return } @@ -131,7 +137,7 @@ func (c *copReqSender) run() { func scanRecords(p *copReqSenderPool, task *reorgBackfillTask, se *sess.Session) error { logutil.Logger(p.ctx).Info("start a cop-request task", - zap.Int("id", task.id), zap.String("task", task.String())) + zap.Int("id", task.id), zap.Stringer("task", task)) return wrapInBeginRollback(se, func(startTS uint64) error { rs, err := buildTableScan(p.ctx, p.copCtx.GetBase(), startTS, task.startKey, task.endKey) diff --git a/pkg/ddl/reorg.go b/pkg/ddl/reorg.go index 8ab9d3cd0beb6..4f99100c8c1c6 100644 --- a/pkg/ddl/reorg.go +++ b/pkg/ddl/reorg.go @@ -73,8 +73,7 @@ type reorgCtx struct { references atomicutil.Int32 } -// newContext gets a context. It is only used for adding column in reorganization state. -func newContext(store kv.Storage) sessionctx.Context { +func newReorgSessCtx(store kv.Storage) sessionctx.Context { c := mock.NewContext() c.Store = store c.GetSessionVars().SetStatusFlag(mysql.ServerStatusAutocommit, false) @@ -515,7 +514,7 @@ func getColumnsTypes(columns []*model.ColumnInfo) []*types.FieldType { // buildDescTableScan builds a desc table scan upon tblInfo. func (dc *ddlCtx) buildDescTableScan(ctx *JobContext, startTS uint64, tbl table.PhysicalTable, handleCols []*model.ColumnInfo, limit uint64) (distsql.SelectResult, error) { - sctx := newContext(dc.store) + sctx := newReorgSessCtx(dc.store) dagPB, err := buildDescTableScanDAG(sctx, tbl, handleCols, limit) if err != nil { return nil, errors.Trace(err) @@ -594,7 +593,7 @@ func (dc *ddlCtx) GetTableMaxHandle(ctx *JobContext, startTS uint64, tbl table.P // empty table return nil, true, nil } - sessCtx := newContext(dc.store) + sessCtx := newReorgSessCtx(dc.store) row := chk.GetRow(0) if tblInfo.IsCommonHandle { maxHandle, err = buildCommonHandleFromChunkRow(sessCtx.GetSessionVars().StmtCtx, tblInfo, pkIdx, handleCols, row) diff --git a/pkg/resourcemanager/pool/workerpool/workerpool.go b/pkg/resourcemanager/pool/workerpool/workerpool.go index 6acb814a3fbed..1a2c877845d15 100644 --- a/pkg/resourcemanager/pool/workerpool/workerpool.go +++ b/pkg/resourcemanager/pool/workerpool/workerpool.go @@ -158,7 +158,10 @@ func (p *WorkerPool[T, R]) runAWorker() { // AddTask adds a task to the pool. func (p *WorkerPool[T, R]) AddTask(task T) { - p.taskChan <- task + select { + case <-p.ctx.Done(): + case p.taskChan <- task: + } } // GetResultChan gets the result channel from the pool. From 6eb1712feff6b2232f8b5c33acea60e812057e8f Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Thu, 25 Apr 2024 13:57:11 +0800 Subject: [PATCH 04/20] pkg/infoschema: revert cache policy ref (#52873) ref pingcap/tidb#50959 --- pkg/infoschema/bench_test.go | 8 +--- pkg/infoschema/infoschema.go | 3 -- pkg/infoschema/infoschema_test.go | 33 +++++++------- pkg/infoschema/infoschema_v2.go | 74 ++++++++----------------------- 4 files changed, 36 insertions(+), 82 deletions(-) diff --git a/pkg/infoschema/bench_test.go b/pkg/infoschema/bench_test.go index ae71f6e1d3c71..85f395a008ac9 100644 --- a/pkg/infoschema/bench_test.go +++ b/pkg/infoschema/bench_test.go @@ -42,7 +42,7 @@ var ( // // GOOS=linux GOARCH=amd64 go test -tags intest -c -o bench.test ./pkg/infoschema // -// ./bench.test -test.v -run ^$ -test.bench=BenchmarkInfoschemaOverhead --with-tikv "upstream-pd:2379?disableGC=true" +// bench.test -test.v -run ^$ -test.bench=BenchmarkInfoschemaOverhead --with-tikv "upstream-pd:2379?disableGC=true" func BenchmarkInfoschemaOverhead(b *testing.B) { wg := testkit.MockTiDBStatusPort(context.Background(), b, *port) @@ -69,11 +69,7 @@ func BenchmarkInfoschemaOverhead(b *testing.B) { } startTime := time.Now() for j := 0; j < *tableCnt; j++ { - delta := true - if j == 0 { - delta = false - } - tc.runCreateTable("test"+strconv.Itoa(j), delta) + tc.runCreateTable("test" + strconv.Itoa(j)) } logutil.BgLogger().Info("all table created", zap.Duration("cost time", time.Since(startTime))) // TODO: add more scenes. diff --git a/pkg/infoschema/infoschema.go b/pkg/infoschema/infoschema.go index ae9cd9c5a08b0..5b5cad6969e76 100644 --- a/pkg/infoschema/infoschema.go +++ b/pkg/infoschema/infoschema.go @@ -68,9 +68,6 @@ type infoSchemaMisc struct { // ruleBundleMap stores all placement rules ruleBundleMap map[int64]*placement.Bundle - // policyRefMap stores all policyRefInfo for tables. - policyRefMap map[int64]*model.PolicyRefInfo - // policyMap stores all placement policies. policyMutex sync.RWMutex policyMap map[string]*model.PolicyInfo diff --git a/pkg/infoschema/infoschema_test.go b/pkg/infoschema/infoschema_test.go index 9d4cc57f1c98f..82db0f61798a7 100644 --- a/pkg/infoschema/infoschema_test.go +++ b/pkg/infoschema/infoschema_test.go @@ -1011,7 +1011,7 @@ func (tc *infoschemaTestContext) createSchema() { func (tc *infoschemaTestContext) runCreateSchema() { // create schema tc.createSchema() - tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionCreateSchema, SchemaID: tc.dbInfo.ID}, false, func(tc *infoschemaTestContext) { + tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionCreateSchema, SchemaID: tc.dbInfo.ID}, func(tc *infoschemaTestContext) { dbInfo, ok := tc.is.SchemaByID(tc.dbInfo.ID) require.True(tc.t, ok) require.Equal(tc.t, dbInfo.Name, tc.dbInfo.Name) @@ -1023,7 +1023,7 @@ func (tc *infoschemaTestContext) runDropSchema() { tc.runCreateSchema() // drop schema internal.DropDB(tc.t, tc.re.Store(), tc.dbInfo) - tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionDropSchema, SchemaID: tc.dbInfo.ID}, false, func(tc *infoschemaTestContext) { + tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionDropSchema, SchemaID: tc.dbInfo.ID}, func(tc *infoschemaTestContext) { _, ok := tc.is.SchemaByID(tc.dbInfo.ID) require.False(tc.t, ok) }) @@ -1033,14 +1033,14 @@ func (tc *infoschemaTestContext) runRecoverSchema() { tc.runDropSchema() // recover schema internal.AddDB(tc.t, tc.re.Store(), tc.dbInfo) - tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionRecoverSchema, SchemaID: tc.dbInfo.ID}, false, func(tc *infoschemaTestContext) { + tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionRecoverSchema, SchemaID: tc.dbInfo.ID}, func(tc *infoschemaTestContext) { dbInfo, ok := tc.is.SchemaByID(tc.dbInfo.ID) require.True(tc.t, ok) require.Equal(tc.t, dbInfo.Name, tc.dbInfo.Name) }) } -func (tc *infoschemaTestContext) runCreateTable(tblName string, delta bool) int64 { +func (tc *infoschemaTestContext) runCreateTable(tblName string) int64 { if tc.dbInfo == nil { tc.runCreateSchema() } @@ -1048,7 +1048,7 @@ func (tc *infoschemaTestContext) runCreateTable(tblName string, delta bool) int6 tblInfo := internal.MockTableInfo(tc.t, tc.re.Store(), tblName) internal.AddTable(tc.t, tc.re.Store(), tc.dbInfo, tblInfo) - tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionCreateTable, SchemaID: tc.dbInfo.ID, TableID: tblInfo.ID}, delta, func(tc *infoschemaTestContext) { + tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionCreateTable, SchemaID: tc.dbInfo.ID, TableID: tblInfo.ID}, func(tc *infoschemaTestContext) { tbl, ok := tc.is.TableByID(tblInfo.ID) require.True(tc.t, ok) require.Equal(tc.t, tbl.Meta().Name.O, tblName) @@ -1071,7 +1071,7 @@ func (tc *infoschemaTestContext) runCreateTables(tblNames []string) { } } - tc.applyDiffAndCheck(&diff, false, func(tc *infoschemaTestContext) { + tc.applyDiffAndCheck(&diff, func(tc *infoschemaTestContext) { for i, opt := range diff.AffectedOpts { tbl, ok := tc.is.TableByID(opt.TableID) require.True(tc.t, ok) @@ -1082,11 +1082,11 @@ func (tc *infoschemaTestContext) runCreateTables(tblNames []string) { func (tc *infoschemaTestContext) runDropTable(tblName string) { // createTable - tblID := tc.runCreateTable(tblName, false) + tblID := tc.runCreateTable(tblName) // dropTable internal.DropTable(tc.t, tc.re.Store(), tc.dbInfo, tblID, tblName) - tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionDropTable, SchemaID: tc.dbInfo.ID, TableID: tblID}, false, func(tc *infoschemaTestContext) { + tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionDropTable, SchemaID: tc.dbInfo.ID, TableID: tblID}, func(tc *infoschemaTestContext) { tbl, ok := tc.is.TableByID(tblID) require.False(tc.t, ok) require.Nil(tc.t, tbl) @@ -1109,7 +1109,7 @@ func (tc *infoschemaTestContext) runAddColumn(tblName string) { require.NoError(tc.t, err) tc.addColumn(tbl.Meta()) - tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionAddColumn, SchemaID: tc.dbInfo.ID, TableID: tbl.Meta().ID}, false, func(tc *infoschemaTestContext) { + tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionAddColumn, SchemaID: tc.dbInfo.ID, TableID: tbl.Meta().ID}, func(tc *infoschemaTestContext) { tbl, ok := tc.is.TableByID(tbl.Meta().ID) require.True(tc.t, ok) require.Equal(tc.t, 2, len(tbl.Cols())) @@ -1142,7 +1142,7 @@ func (tc *infoschemaTestContext) runModifyColumn(tblName string) { require.NoError(tc.t, err) tc.modifyColumn(tbl.Meta()) - tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionModifyColumn, SchemaID: tc.dbInfo.ID, TableID: tbl.Meta().ID}, false, func(tc *infoschemaTestContext) { + tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionModifyColumn, SchemaID: tc.dbInfo.ID, TableID: tbl.Meta().ID}, func(tc *infoschemaTestContext) { tbl, ok := tc.is.TableByID(tbl.Meta().ID) require.True(tc.t, ok) require.Equal(tc.t, "test", tbl.Cols()[0].Comment) @@ -1165,7 +1165,7 @@ func (tc *infoschemaTestContext) runModifySchemaCharsetAndCollate(charset, colla tc.dbInfo.Charset = charset tc.dbInfo.Collate = collate internal.UpdateDB(tc.t, tc.re.Store(), tc.dbInfo) - tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionModifySchemaCharsetAndCollate, SchemaID: tc.dbInfo.ID}, false, func(tc *infoschemaTestContext) { + tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionModifySchemaCharsetAndCollate, SchemaID: tc.dbInfo.ID}, func(tc *infoschemaTestContext) { schema, ok := tc.is.SchemaByID(tc.dbInfo.ID) require.True(tc.t, ok) require.Equal(tc.t, charset, schema.Charset) @@ -1176,22 +1176,19 @@ func (tc *infoschemaTestContext) runModifySchemaCharsetAndCollate(charset, colla func (tc *infoschemaTestContext) runModifySchemaDefaultPlacement(policy *model.PolicyRefInfo) { tc.dbInfo.PlacementPolicyRef = policy internal.UpdateDB(tc.t, tc.re.Store(), tc.dbInfo) - tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionModifySchemaDefaultPlacement, SchemaID: tc.dbInfo.ID}, false, func(tc *infoschemaTestContext) { + tc.applyDiffAndCheck(&model.SchemaDiff{Type: model.ActionModifySchemaDefaultPlacement, SchemaID: tc.dbInfo.ID}, func(tc *infoschemaTestContext) { schema, ok := tc.is.SchemaByID(tc.dbInfo.ID) require.True(tc.t, ok) require.Equal(tc.t, policy, schema.PlacementPolicyRef) }) } -func (tc *infoschemaTestContext) applyDiffAndCheck(diff *model.SchemaDiff, delta bool, checkFn func(tc *infoschemaTestContext)) { +func (tc *infoschemaTestContext) applyDiffAndCheck(diff *model.SchemaDiff, checkFn func(tc *infoschemaTestContext)) { txn, err := tc.re.Store().Begin() require.NoError(tc.t, err) builder, err := infoschema.NewBuilder(tc.re, nil, tc.data).InitWithOldInfoSchema(tc.is) require.NoError(tc.t, err) - if delta { - builder.SetDeltaUpdateBundles() - } // applyDiff _, err = builder.ApplyDiff(meta.NewMeta(txn), diff) require.NoError(tc.t, err) @@ -1229,12 +1226,12 @@ func TestApplyDiff(t *testing.T) { tc.clear() tc.runDropSchema() tc.clear() - tc.runCreateTable("test", false) + tc.runCreateTable("test") tc.clear() tc.runDropTable("test") tc.clear() - tc.runCreateTable("test", false) + tc.runCreateTable("test") tc.runModifyTable("test", model.ActionAddColumn) tc.runModifyTable("test", model.ActionModifyColumn) diff --git a/pkg/infoschema/infoschema_v2.go b/pkg/infoschema/infoschema_v2.go index d1729709cdef9..c433dce3772f1 100644 --- a/pkg/infoschema/infoschema_v2.go +++ b/pkg/infoschema/infoschema_v2.go @@ -28,10 +28,8 @@ import ( "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/table/tables" "github.com/pingcap/tidb/pkg/util" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/size" "github.com/tidwall/btree" - "go.uber.org/zap" "golang.org/x/sync/singleflight" ) @@ -889,59 +887,17 @@ func (b *bundleInfoBuilder) updateInfoSchemaBundlesV2(is *infoschemaV2) { if b.deltaUpdate { b.completeUpdateTablesV2(is) for tblID := range b.updateTables { - b.updateTableBundlesV2(is, tblID) + b.updateTableBundles(is, tblID) } return } - // do full update bundles and policyRefs. + // do full update bundles // TODO: This is quite inefficient! we need some better way or avoid this API. is.ruleBundleMap = make(map[int64]*placement.Bundle) for _, dbInfo := range is.AllSchemas() { for _, tbl := range is.SchemaTables(dbInfo.Name) { - b.updateTableBundlesV2(is, tbl.Meta().ID) - } - } -} - -func (b *bundleInfoBuilder) updateTableBundlesV2(infoSchemaInterface InfoSchema, tableID int64) { - is := infoSchemaInterface.base() - tbl, ok := infoSchemaInterface.TableByID(tableID) - if !ok { - b.deleteBundle(is, tableID) - delete(is.policyRefMap, tableID) - return - } - - getter := &policyGetter{is: is} - bundle, err := placement.NewTableBundle(getter, tbl.Meta()) - if err != nil { - logutil.BgLogger().Error("create table bundle failed", zap.Error(err)) - } else if bundle != nil { - is.policyRefMap[tableID] = tbl.Meta().PlacementPolicyRef - is.ruleBundleMap[tableID] = bundle - } else { - b.deleteBundle(is, tableID) - delete(is.policyRefMap, tableID) - } - - if tbl.Meta().Partition == nil { - return - } - - for _, par := range tbl.Meta().Partition.Definitions { - bundle, err = placement.NewPartitionBundle(getter, par) - if err != nil { - logutil.BgLogger().Error("create partition bundle failed", - zap.Error(err), - zap.Int64("partition id", par.ID), - ) - } else if bundle != nil { - is.policyRefMap[par.ID] = par.PlacementPolicyRef - is.ruleBundleMap[par.ID] = bundle - } else { - b.deleteBundle(is, par.ID) - delete(is.policyRefMap, par.ID) + b.updateTableBundles(is, tbl.Meta().ID) } } } @@ -951,15 +907,23 @@ func (b *bundleInfoBuilder) completeUpdateTablesV2(is *infoschemaV2) { return } - for id := range b.updatePolicies { - if _, ok := is.policyRefMap[id]; ok { - b.markTableBundleShouldUpdate(id) - } - } + // TODO: This is quite inefficient! we need some better way or avoid this API. + for _, dbInfo := range is.AllSchemas() { + for _, tbl := range is.SchemaTables(dbInfo.Name) { + tblInfo := tbl.Meta() + if tblInfo.PlacementPolicyRef != nil { + if _, ok := b.updatePolicies[tblInfo.PlacementPolicyRef.ID]; ok { + b.markTableBundleShouldUpdate(tblInfo.ID) + } + } - for id := range b.updatePartitions { - if _, ok := is.policyRefMap[id]; ok { - b.markTableBundleShouldUpdate(id) + if tblInfo.Partition != nil { + for _, par := range tblInfo.Partition.Definitions { + if _, ok := b.updatePartitions[par.ID]; ok { + b.markTableBundleShouldUpdate(tblInfo.ID) + } + } + } } } } From 8426c23cb1e1f63fc94ddb64c407e85ca46ca6cf Mon Sep 17 00:00:00 2001 From: ekexium Date: Thu, 25 Apr 2024 14:39:12 +0800 Subject: [PATCH 05/20] topsql: set RPC interceptor immediately after a txn is activated (#51795) close pingcap/tidb#52845 --- pkg/sessiontxn/isolation/base.go | 4 ++++ pkg/store/mockstore/unistore/testutil.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/pkg/sessiontxn/isolation/base.go b/pkg/sessiontxn/isolation/base.go index da1b6de9f5390..6f5a516d2754e 100644 --- a/pkg/sessiontxn/isolation/base.go +++ b/pkg/sessiontxn/isolation/base.go @@ -489,6 +489,10 @@ func (p *baseTxnContextProvider) SetOptionsOnTxnActive(txn kv.Transaction) { txn.SetDiskFullOpt(sessVars.DiskFullOpt) } txn.SetOption(kv.InfoSchema, sessVars.TxnCtx.InfoSchema) + if sessVars.StmtCtx.KvExecCounter != nil { + // Bind an interceptor for client-go to count the number of SQL executions of each TiKV. + txn.SetOption(kv.RPCInterceptor, sessVars.StmtCtx.KvExecCounter.RPCInterceptor()) + } txn.SetOption(kv.ResourceGroupTagger, sessVars.StmtCtx.GetResourceGroupTagger()) txn.SetOption(kv.ExplicitRequestSourceType, sessVars.ExplicitRequestSourceType) diff --git a/pkg/store/mockstore/unistore/testutil.go b/pkg/store/mockstore/unistore/testutil.go index fd957434f0afa..db345bd5273c7 100644 --- a/pkg/store/mockstore/unistore/testutil.go +++ b/pkg/store/mockstore/unistore/testutil.go @@ -102,6 +102,10 @@ func getReqStartKey(req *tikvrpc.Request) ([]byte, error) { case tikvrpc.CmdResolveLock, tikvrpc.CmdCheckTxnStatus, tikvrpc.CmdPessimisticRollback: // TODO: add resource tag for those request. https://github.com/pingcap/tidb/issues/33621 return nil, nil + case tikvrpc.CmdFlush: + return req.Flush().GetMutations()[0].GetKey(), nil + case tikvrpc.CmdBufferBatchGet: + return req.BufferBatchGet().GetKeys()[0], nil default: return nil, errors.New("unknown request, check the new type RPC request here") } From 896afa74a1e56f5e2d8deb8c87d0ecdce14dfd3a Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Thu, 25 Apr 2024 14:39:19 +0800 Subject: [PATCH 06/20] autoid_service: fix non-continuous autoid for allocating N IDs (#52517) close pingcap/tidb#52465 --- pkg/autoid_service/autoid.go | 30 ++++--- .../integrationtest/r/executor/autoid.result | 80 +++++++++++++++++++ tests/integrationtest/t/executor/autoid.test | 24 ++++++ 3 files changed, 125 insertions(+), 9 deletions(-) diff --git a/pkg/autoid_service/autoid.go b/pkg/autoid_service/autoid.go index 87839c9173c78..5fa3ea1cffba2 100644 --- a/pkg/autoid_service/autoid.go +++ b/pkg/autoid_service/autoid.go @@ -73,11 +73,11 @@ func (alloc *autoIDValue) alloc4Unsigned(ctx context.Context, store kv.Storage, // calcNeededBatchSize calculates the total batch size needed. n1 := calcNeededBatchSize(alloc.base, int64(n), increment, offset, isUnsigned) - // The local rest is not enough for alloc, skip it. + // The local rest is not enough for alloc. if uint64(alloc.base)+uint64(n1) > uint64(alloc.end) || alloc.base == 0 { var newBase, newEnd int64 nextStep := int64(batch) - // Although it may skip a segment here, we still treat it as consumed. + fromBase := alloc.base ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnMeta) err := kv.RunInNewTxn(ctx, store, true, func(_ context.Context, txn kv.Transaction) error { @@ -88,7 +88,12 @@ func (alloc *autoIDValue) alloc4Unsigned(ctx context.Context, store kv.Storage, return err1 } // calcNeededBatchSize calculates the total batch size needed on new base. - n1 = calcNeededBatchSize(newBase, int64(n), increment, offset, isUnsigned) + if alloc.base == 0 || newBase != alloc.end { + alloc.base = newBase + alloc.end = newBase + n1 = calcNeededBatchSize(newBase, int64(n), increment, offset, isUnsigned) + } + // Although the step is customized by user, we still need to make sure nextStep is big enough for insert batch. if nextStep < n1 { nextStep = n1 @@ -111,11 +116,11 @@ func (alloc *autoIDValue) alloc4Unsigned(ctx context.Context, store kv.Storage, zap.String("category", "autoid service"), zap.Int64("dbID", dbID), zap.Int64("tblID", tblID), - zap.Int64("from base", alloc.base), + zap.Int64("from base", fromBase), zap.Int64("from end", alloc.end), zap.Int64("to base", newBase), zap.Int64("to end", newEnd)) - alloc.base, alloc.end = newBase, newEnd + alloc.end = newEnd } min = alloc.base // Use uint64 n directly. @@ -142,11 +147,12 @@ func (alloc *autoIDValue) alloc4Signed(ctx context.Context, return 0, 0, errAutoincReadFailed } - // The local rest is not enough for allocN, skip it. + // The local rest is not enough for allocN. // If alloc.base is 0, the alloc may not be initialized, force fetch from remote. if alloc.base+n1 > alloc.end || alloc.base == 0 { var newBase, newEnd int64 nextStep := int64(batch) + fromBase := alloc.base ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnMeta) err := kv.RunInNewTxn(ctx, store, true, func(_ context.Context, txn kv.Transaction) error { @@ -157,7 +163,13 @@ func (alloc *autoIDValue) alloc4Signed(ctx context.Context, return err1 } // calcNeededBatchSize calculates the total batch size needed on global base. - n1 = calcNeededBatchSize(newBase, int64(n), increment, offset, isUnsigned) + // alloc.base == 0 means uninitialized + // newBase != alloc.end means something abnormal, maybe transaction conflict and retry? + if alloc.base == 0 || newBase != alloc.end { + alloc.base = newBase + alloc.end = newBase + n1 = calcNeededBatchSize(newBase, int64(n), increment, offset, isUnsigned) + } // Although the step is customized by user, we still need to make sure nextStep is big enough for insert batch. if nextStep < n1 { nextStep = n1 @@ -180,11 +192,11 @@ func (alloc *autoIDValue) alloc4Signed(ctx context.Context, zap.String("category", "autoid service"), zap.Int64("dbID", dbID), zap.Int64("tblID", tblID), - zap.Int64("from base", alloc.base), + zap.Int64("from base", fromBase), zap.Int64("from end", alloc.end), zap.Int64("to base", newBase), zap.Int64("to end", newEnd)) - alloc.base, alloc.end = newBase, newEnd + alloc.end = newEnd } min = alloc.base alloc.base += n1 diff --git a/tests/integrationtest/r/executor/autoid.result b/tests/integrationtest/r/executor/autoid.result index c4b7ad388217f..cc57ca2854fe4 100644 --- a/tests/integrationtest/r/executor/autoid.result +++ b/tests/integrationtest/r/executor/autoid.result @@ -737,3 +737,83 @@ auto_increment_increment 65535 auto_increment_offset 65535 set auto_increment_offset = default; set auto_increment_increment = default; +drop table if exists issue52465; +create table issue52465 (id int primary key auto_increment, k int) AUTO_ID_CACHE=1; +insert into issue52465 (k) values (1); +insert into issue52465 values (3997, 2); +select * from issue52465 t; +id k +1 1 +3997 2 +insert into issue52465 (k) values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10); +select * from issue52465; +id k +1 1 +3997 2 +3998 1 +3999 2 +4000 3 +4001 4 +4002 5 +4003 6 +4004 7 +4005 8 +4006 9 +4007 10 +insert into issue52465 (k) values (11); +select * from issue52465; +id k +1 1 +3997 2 +3998 1 +3999 2 +4000 3 +4001 4 +4002 5 +4003 6 +4004 7 +4005 8 +4006 9 +4007 10 +4008 11 +drop table issue52465; +drop table if exists issue52465; +create table issue52465 (id int unsigned primary key auto_increment, k int) AUTO_ID_CACHE=1; +insert into issue52465 (k) values (1); +insert into issue52465 values (3997, 2); +select * from issue52465 t; +id k +1 1 +3997 2 +insert into issue52465 (k) values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10); +select * from issue52465; +id k +1 1 +3997 2 +3998 1 +3999 2 +4000 3 +4001 4 +4002 5 +4003 6 +4004 7 +4005 8 +4006 9 +4007 10 +insert into issue52465 (k) values (11); +select * from issue52465; +id k +1 1 +3997 2 +3998 1 +3999 2 +4000 3 +4001 4 +4002 5 +4003 6 +4004 7 +4005 8 +4006 9 +4007 10 +4008 11 +drop table issue52465; diff --git a/tests/integrationtest/t/executor/autoid.test b/tests/integrationtest/t/executor/autoid.test index 7adb3104caf80..833de0f25aaa0 100644 --- a/tests/integrationtest/t/executor/autoid.test +++ b/tests/integrationtest/t/executor/autoid.test @@ -483,3 +483,27 @@ show variables like 'auto_increment%'; set auto_increment_offset = default; set auto_increment_increment = default; + +## Test for issue 52465 +drop table if exists issue52465; +create table issue52465 (id int primary key auto_increment, k int) AUTO_ID_CACHE=1; +insert into issue52465 (k) values (1); +insert into issue52465 values (3997, 2); +select * from issue52465 t; +insert into issue52465 (k) values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10); +select * from issue52465; +insert into issue52465 (k) values (11); +select * from issue52465; +drop table issue52465; + + +drop table if exists issue52465; +create table issue52465 (id int unsigned primary key auto_increment, k int) AUTO_ID_CACHE=1; +insert into issue52465 (k) values (1); +insert into issue52465 values (3997, 2); +select * from issue52465 t; +insert into issue52465 (k) values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10); +select * from issue52465; +insert into issue52465 (k) values (11); +select * from issue52465; +drop table issue52465; From e8c397ff93ff55da30e553b77bd1a25f6b841510 Mon Sep 17 00:00:00 2001 From: Arenatlx <314806019@qq.com> Date: Thu, 25 Apr 2024 16:10:43 +0800 Subject: [PATCH 07/20] planner: collect and move correlated utility to util pkg (#52866) ref pingcap/tidb#51664, ref pingcap/tidb#52714 --- pkg/executor/BUILD.bazel | 1 + pkg/executor/builder.go | 5 +- pkg/planner/cascades/BUILD.bazel | 1 + pkg/planner/cascades/transformation_rules.go | 3 +- pkg/planner/core/BUILD.bazel | 5 +- pkg/planner/core/base/BUILD.bazel | 3 +- pkg/planner/core/base/plan_base.go | 23 +- pkg/planner/core/common_plans.go | 15 +- pkg/planner/core/expression_rewriter.go | 13 +- pkg/planner/core/find_best_task.go | 56 +-- pkg/planner/core/find_best_task_test.go | 18 +- pkg/planner/core/logical_plan_builder.go | 5 +- pkg/planner/core/logical_plans.go | 59 +-- pkg/planner/core/optimizer.go | 10 +- pkg/planner/core/physical_plans.go | 29 +- pkg/planner/core/plan.go | 9 +- pkg/planner/core/plan_cost_detail.go | 14 +- pkg/planner/core/plan_cost_ver1.go | 121 +++--- pkg/planner/core/plan_cost_ver2.go | 393 +++++++++--------- pkg/planner/core/plan_cost_ver2_test.go | 7 +- pkg/planner/core/point_get_plan.go | 11 +- .../core/rule_aggregation_elimination.go | 12 +- .../core/rule_aggregation_push_down.go | 16 +- .../core/rule_aggregation_skew_rewrite.go | 8 +- pkg/planner/core/rule_build_key_info.go | 4 +- pkg/planner/core/rule_collect_plan_stats.go | 6 +- pkg/planner/core/rule_column_pruning.go | 55 +-- pkg/planner/core/rule_constant_propagation.go | 12 +- pkg/planner/core/rule_decorrelate.go | 49 +-- .../core/rule_derive_topn_from_window.go | 10 +- pkg/planner/core/rule_eliminate_projection.go | 10 +- .../core/rule_generate_column_substitute.go | 14 +- pkg/planner/core/rule_join_elimination.go | 12 +- pkg/planner/core/rule_join_reorder.go | 8 +- pkg/planner/core/rule_max_min_eliminate.go | 16 +- pkg/planner/core/rule_partition_processor.go | 20 +- pkg/planner/core/rule_predicate_push_down.go | 46 +- .../core/rule_predicate_simplification.go | 8 +- pkg/planner/core/rule_push_down_sequence.go | 4 +- .../core/rule_resolve_grouping_expand.go | 6 +- pkg/planner/core/rule_result_reorder.go | 4 +- pkg/planner/core/rule_semi_join_rewrite.go | 4 +- pkg/planner/core/rule_topn_push_down.go | 36 +- pkg/planner/core/util.go | 4 +- pkg/planner/util/coreusage/BUILD.bazel | 6 +- pkg/planner/util/coreusage/correlated_misc.go | 95 +++++ pkg/planner/util/costusage/BUILD.bazel | 9 + .../{coreusage => costusage}/cost_misc.go | 27 +- pkg/planner/util/optimizetrace/BUILD.bazel | 9 + .../opt_tracer.go | 27 +- 50 files changed, 720 insertions(+), 618 deletions(-) create mode 100644 pkg/planner/util/coreusage/correlated_misc.go create mode 100644 pkg/planner/util/costusage/BUILD.bazel rename pkg/planner/util/{coreusage => costusage}/cost_misc.go (80%) create mode 100644 pkg/planner/util/optimizetrace/BUILD.bazel rename pkg/planner/util/{coreusage => optimizetrace}/opt_tracer.go (81%) diff --git a/pkg/executor/BUILD.bazel b/pkg/executor/BUILD.bazel index cd969a0233082..a89134b00a255 100644 --- a/pkg/executor/BUILD.bazel +++ b/pkg/executor/BUILD.bazel @@ -156,6 +156,7 @@ go_library( "//pkg/planner/core", "//pkg/planner/core/base", "//pkg/planner/util", + "//pkg/planner/util/coreusage", "//pkg/planner/util/fixcontrol", "//pkg/plugin", "//pkg/privilege", diff --git a/pkg/executor/builder.go b/pkg/executor/builder.go index 210c801585913..18d228e8af218 100644 --- a/pkg/executor/builder.go +++ b/pkg/executor/builder.go @@ -61,6 +61,7 @@ import ( plannercore "github.com/pingcap/tidb/pkg/planner/core" "github.com/pingcap/tidb/pkg/planner/core/base" plannerutil "github.com/pingcap/tidb/pkg/planner/util" + "github.com/pingcap/tidb/pkg/planner/util/coreusage" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" @@ -2179,7 +2180,7 @@ func (b *executorBuilder) buildApply(v *plannercore.PhysicalApply) exec.Executor innerPlan = v.Children()[1] outerPlan = v.Children()[0] } - v.OuterSchema = plannercore.ExtractCorColumnsBySchema4PhysicalPlan(innerPlan, outerPlan.Schema()) + v.OuterSchema = coreusage.ExtractCorColumnsBySchema4PhysicalPlan(innerPlan, outerPlan.Schema()) leftChild := b.build(v.Children()[0]) if b.err != nil { return nil @@ -2230,7 +2231,7 @@ func (b *executorBuilder) buildApply(v *plannercore.PhysicalApply) exec.Executor b.err = nil return serialExec } - corCol := plannercore.ExtractCorColumnsBySchema4PhysicalPlan(clonedInnerPlan, outerPlan.Schema()) + corCol := coreusage.ExtractCorColumnsBySchema4PhysicalPlan(clonedInnerPlan, outerPlan.Schema()) clonedInnerExec := b.build(clonedInnerPlan) if b.err != nil { b.err = nil diff --git a/pkg/planner/cascades/BUILD.bazel b/pkg/planner/cascades/BUILD.bazel index da067459c045a..81454a2258f89 100644 --- a/pkg/planner/cascades/BUILD.bazel +++ b/pkg/planner/cascades/BUILD.bazel @@ -25,6 +25,7 @@ go_library( "//pkg/planner/pattern", "//pkg/planner/property", "//pkg/planner/util", + "//pkg/planner/util/coreusage", "//pkg/types", "//pkg/util/dbterror/plannererrors", "//pkg/util/ranger", diff --git a/pkg/planner/cascades/transformation_rules.go b/pkg/planner/cascades/transformation_rules.go index c78f717b02d6e..81d5d48beca5c 100644 --- a/pkg/planner/cascades/transformation_rules.go +++ b/pkg/planner/cascades/transformation_rules.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/tidb/pkg/planner/memo" "github.com/pingcap/tidb/pkg/planner/pattern" "github.com/pingcap/tidb/pkg/planner/util" + "github.com/pingcap/tidb/pkg/planner/util/coreusage" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/ranger" "github.com/pingcap/tidb/pkg/util/set" @@ -2455,7 +2456,7 @@ func (r *TransformApplyToJoin) OnTransform(old *memo.ExprIter) (newExprs []*memo func (r *TransformApplyToJoin) extractCorColumnsBySchema(innerGroup *memo.Group, outerSchema *expression.Schema) []*expression.CorrelatedColumn { corCols := r.extractCorColumnsFromGroup(innerGroup) - return plannercore.ExtractCorColumnsBySchema(corCols, outerSchema, false) + return coreusage.ExtractCorColumnsBySchema(corCols, outerSchema, false) } func (r *TransformApplyToJoin) extractCorColumnsFromGroup(g *memo.Group) []*expression.CorrelatedColumn { diff --git a/pkg/planner/core/BUILD.bazel b/pkg/planner/core/BUILD.bazel index 23d4c096d98c4..e9b9b99b70b0b 100644 --- a/pkg/planner/core/BUILD.bazel +++ b/pkg/planner/core/BUILD.bazel @@ -119,8 +119,10 @@ go_library( "//pkg/planner/property", "//pkg/planner/util", "//pkg/planner/util/coreusage", + "//pkg/planner/util/costusage", "//pkg/planner/util/debugtrace", "//pkg/planner/util/fixcontrol", + "//pkg/planner/util/optimizetrace", "//pkg/privilege", "//pkg/sessionctx", "//pkg/sessionctx/stmtctx", @@ -254,7 +256,8 @@ go_test( "//pkg/planner/property", "//pkg/planner/util", "//pkg/planner/util/coretestsdk", - "//pkg/planner/util/coreusage", + "//pkg/planner/util/costusage", + "//pkg/planner/util/optimizetrace", "//pkg/session", "//pkg/session/types", "//pkg/sessionctx", diff --git a/pkg/planner/core/base/BUILD.bazel b/pkg/planner/core/base/BUILD.bazel index bab81b6a5a87f..162cb34897459 100644 --- a/pkg/planner/core/base/BUILD.bazel +++ b/pkg/planner/core/base/BUILD.bazel @@ -15,7 +15,8 @@ go_library( "//pkg/planner/context", "//pkg/planner/funcdep", "//pkg/planner/property", - "//pkg/planner/util/coreusage", + "//pkg/planner/util/costusage", + "//pkg/planner/util/optimizetrace", "//pkg/types", "//pkg/util/execdetails", "//pkg/util/tracing", diff --git a/pkg/planner/core/base/plan_base.go b/pkg/planner/core/base/plan_base.go index 4ec3a8065b5d1..4cd26483a238c 100644 --- a/pkg/planner/core/base/plan_base.go +++ b/pkg/planner/core/base/plan_base.go @@ -22,7 +22,8 @@ import ( "github.com/pingcap/tidb/pkg/planner/context" fd "github.com/pingcap/tidb/pkg/planner/funcdep" "github.com/pingcap/tidb/pkg/planner/property" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/costusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/execdetails" "github.com/pingcap/tidb/pkg/util/tracing" @@ -82,10 +83,10 @@ type PhysicalPlan interface { Plan // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost on model ver1. - GetPlanCostVer1(taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) + GetPlanCostVer1(taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) // GetPlanCostVer2 calculates the cost of the plan if it has not been calculated yet and returns the cost on model ver2. - GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) + GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) // Attach2Task makes the current physical plan as the father of task's physicalPlan and updates the cost of // current task. If the child's task is cop task, some operator may close this task and return a new rootTask. @@ -129,7 +130,7 @@ type PhysicalPlan interface { // AppendChildCandidate append child physicalPlan into tracer in order to track each child physicalPlan which can't // be tracked during findBestTask or enumeratePhysicalPlans4Task - AppendChildCandidate(op *coreusage.PhysicalOptimizeOp) + AppendChildCandidate(op *optimizetrace.PhysicalOptimizeOp) // MemoryUsage return the memory usage of PhysicalPlan MemoryUsage() int64 @@ -191,10 +192,10 @@ type LogicalPlan interface { // PredicatePushDown pushes down the predicates in the where/on/having clauses as deeply as possible. // It will accept a predicate that is an expression slice, and return the expressions that can't be pushed. // Because it might change the root if the having clause exists, we need to return a plan that represents a new root. - PredicatePushDown([]expression.Expression, *coreusage.LogicalOptimizeOp) ([]expression.Expression, LogicalPlan) + PredicatePushDown([]expression.Expression, *optimizetrace.LogicalOptimizeOp) ([]expression.Expression, LogicalPlan) // PruneColumns prunes the unused columns, and return the new logical plan if changed, otherwise it's same. - PruneColumns([]*expression.Column, *coreusage.LogicalOptimizeOp) (LogicalPlan, error) + PruneColumns([]*expression.Column, *optimizetrace.LogicalOptimizeOp) (LogicalPlan, error) // FindBestTask converts the logical plan to the physical plan. It's a new interface. // It is called recursively from the parent to the children to create the result physical plan. @@ -204,7 +205,7 @@ type LogicalPlan interface { // If planCounter > 0, the clock_th plan generated in this function will be returned. // If planCounter = 0, the plan generated in this function will not be considered. // If planCounter = -1, then we will not force plan. - FindBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, op *coreusage.PhysicalOptimizeOp) (Task, int64, error) + FindBestTask(prop *property.PhysicalProperty, planCounter *PlanCounterTp, op *optimizetrace.PhysicalOptimizeOp) (Task, int64, error) // BuildKeyInfo will collect the information of unique keys into schema. // Because this method is also used in cascades planner, we cannot use @@ -214,16 +215,16 @@ type LogicalPlan interface { // PushDownTopN will push down the topN or limit operator during logical optimization. // interface definition should depend on concrete implementation type. - PushDownTopN(topN LogicalPlan, opt *coreusage.LogicalOptimizeOp) LogicalPlan + PushDownTopN(topN LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) LogicalPlan // DeriveTopN derives an implicit TopN from a filter on row_number window function... - DeriveTopN(opt *coreusage.LogicalOptimizeOp) LogicalPlan + DeriveTopN(opt *optimizetrace.LogicalOptimizeOp) LogicalPlan // PredicateSimplification consolidates different predcicates on a column and its equivalence classes. - PredicateSimplification(opt *coreusage.LogicalOptimizeOp) LogicalPlan + PredicateSimplification(opt *optimizetrace.LogicalOptimizeOp) LogicalPlan // ConstantPropagation generate new constant predicate according to column equivalence relation - ConstantPropagation(parentPlan LogicalPlan, currentChildIdx int, opt *coreusage.LogicalOptimizeOp) (newRoot LogicalPlan) + ConstantPropagation(parentPlan LogicalPlan, currentChildIdx int, opt *optimizetrace.LogicalOptimizeOp) (newRoot LogicalPlan) // PullUpConstantPredicates recursive find constant predicate, used for the constant propagation rule PullUpConstantPredicates() []expression.Expression diff --git a/pkg/planner/core/common_plans.go b/pkg/planner/core/common_plans.go index 392755ecfb9f1..ec6ab30e1542a 100644 --- a/pkg/planner/core/common_plans.go +++ b/pkg/planner/core/common_plans.go @@ -29,7 +29,8 @@ import ( "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/property" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/costusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/table" @@ -860,12 +861,12 @@ func (e *Explain) RenderResult() error { pp, ok := e.TargetPlan.(base.PhysicalPlan) if ok { if _, err := getPlanCost(pp, property.RootTaskType, - coreusage.NewDefaultPlanCostOption().WithCostFlag(coreusage.CostFlagRecalculate|coreusage.CostFlagUseTrueCardinality|coreusage.CostFlagTrace)); err != nil { + optimizetrace.NewDefaultPlanCostOption().WithCostFlag(costusage.CostFlagRecalculate|costusage.CostFlagUseTrueCardinality|costusage.CostFlagTrace)); err != nil { return err } if pp.SCtx().GetSessionVars().CostModelVersion == modelVer2 { // output cost formula and factor costs through warning under model ver2 and true_card_cost mode for cost calibration. - cost, _ := pp.GetPlanCostVer2(property.RootTaskType, coreusage.NewDefaultPlanCostOption()) + cost, _ := pp.GetPlanCostVer2(property.RootTaskType, optimizetrace.NewDefaultPlanCostOption()) if cost.GetTrace() != nil { trace := cost.GetTrace() pp.SCtx().GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackErrorf("cost formula: %v", trace.GetFormula())) @@ -899,7 +900,7 @@ func (e *Explain) RenderResult() error { if pp, ok := e.TargetPlan.(base.PhysicalPlan); ok { // trigger getPlanCost again with CostFlagTrace to record all cost formulas if _, err := getPlanCost(pp, property.RootTaskType, - coreusage.NewDefaultPlanCostOption().WithCostFlag(coreusage.CostFlagRecalculate|coreusage.CostFlagTrace)); err != nil { + optimizetrace.NewDefaultPlanCostOption().WithCostFlag(costusage.CostFlagRecalculate|costusage.CostFlagTrace)); err != nil { return err } } @@ -1146,13 +1147,13 @@ func (e *Explain) getOperatorInfo(p base.Plan, id string) (estRows, estCost, cos if isPhysicalPlan { estRows = strconv.FormatFloat(pp.GetEstRowCountForDisplay(), 'f', 2, 64) if e.SCtx() != nil && e.SCtx().GetSessionVars().CostModelVersion == modelVer2 { - costVer2, _ := pp.GetPlanCostVer2(property.RootTaskType, coreusage.NewDefaultPlanCostOption()) + costVer2, _ := pp.GetPlanCostVer2(property.RootTaskType, optimizetrace.NewDefaultPlanCostOption()) estCost = strconv.FormatFloat(costVer2.GetCost(), 'f', 2, 64) if costVer2.GetTrace() != nil { costFormula = costVer2.GetTrace().GetFormula() } } else { - planCost, _ := getPlanCost(pp, property.RootTaskType, coreusage.NewDefaultPlanCostOption()) + planCost, _ := getPlanCost(pp, property.RootTaskType, optimizetrace.NewDefaultPlanCostOption()) estCost = strconv.FormatFloat(planCost, 'f', 2, 64) } } else if si := p.StatsInfo(); si != nil { @@ -1256,7 +1257,7 @@ func binaryOpFromFlatOp(explainCtx base.PlanContext, fop *FlatOperator, out *tip if fop.IsPhysicalPlan { p := fop.Origin.(base.PhysicalPlan) - out.Cost, _ = getPlanCost(p, property.RootTaskType, coreusage.NewDefaultPlanCostOption()) + out.Cost, _ = getPlanCost(p, property.RootTaskType, optimizetrace.NewDefaultPlanCostOption()) out.EstRows = p.GetEstRowCountForDisplay() } else if statsInfo := fop.Origin.StatsInfo(); statsInfo != nil { out.EstRows = statsInfo.RowCount diff --git a/pkg/planner/core/expression_rewriter.go b/pkg/planner/core/expression_rewriter.go index 873295ca99ea7..e6a40ffe149a3 100644 --- a/pkg/planner/core/expression_rewriter.go +++ b/pkg/planner/core/expression_rewriter.go @@ -34,6 +34,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/parser/opcode" "github.com/pingcap/tidb/pkg/planner/core/base" + "github.com/pingcap/tidb/pkg/planner/util/coreusage" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/table/tables" @@ -700,7 +701,7 @@ func (er *expressionRewriter) handleCompareSubquery(ctx context.Context, planCtx } noDecorrelate := hintFlags&hint.HintFlagNoDecorrelate > 0 - if noDecorrelate && len(extractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema())) == 0 { + if noDecorrelate && len(coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema())) == 0 { b.ctx.GetSessionVars().StmtCtx.SetHintWarning( "NO_DECORRELATE() is inapplicable because there are no correlated columns.") noDecorrelate = false @@ -1017,7 +1018,7 @@ func (er *expressionRewriter) handleExistSubquery(ctx context.Context, planCtx * np = er.popExistsSubPlan(planCtx, np) noDecorrelate := hintFlags&hint.HintFlagNoDecorrelate > 0 - if noDecorrelate && len(extractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema())) == 0 { + if noDecorrelate && len(coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema())) == 0 { b.ctx.GetSessionVars().StmtCtx.SetHintWarning( "NO_DECORRELATE() is inapplicable because there are no correlated columns.") noDecorrelate = false @@ -1030,7 +1031,7 @@ func (er *expressionRewriter) handleExistSubquery(ctx context.Context, planCtx * semiJoinRewrite = false } - if b.disableSubQueryPreprocessing || len(ExtractCorrelatedCols4LogicalPlan(np)) > 0 || hasCTEConsumerInSubPlan(np) { + if b.disableSubQueryPreprocessing || len(coreusage.ExtractCorrelatedCols4LogicalPlan(np)) > 0 || hasCTEConsumerInSubPlan(np) { planCtx.plan, er.err = b.buildSemiApply(planCtx.plan, np, nil, er.asScalar, v.Not, semiJoinRewrite, noDecorrelate) if er.err != nil || !er.asScalar { return v, true @@ -1192,7 +1193,7 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, planCtx *exp collFlag := collate.CompatibleCollate(lt.GetCollate(), rt.GetCollate()) noDecorrelate := hintFlags&hint.HintFlagNoDecorrelate > 0 - corCols := extractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) + corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) if len(corCols) == 0 && noDecorrelate { planCtx.builder.ctx.GetSessionVars().StmtCtx.SetHintWarning( "NO_DECORRELATE() is inapplicable because there are no correlated columns.") @@ -1254,13 +1255,13 @@ func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, planCtx np = planCtx.builder.buildMaxOneRow(np) noDecorrelate := hintFlags&hint.HintFlagNoDecorrelate > 0 - if noDecorrelate && len(extractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema())) == 0 { + if noDecorrelate && len(coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema())) == 0 { planCtx.builder.ctx.GetSessionVars().StmtCtx.SetHintWarning( "NO_DECORRELATE() is inapplicable because there are no correlated columns.") noDecorrelate = false } - if planCtx.builder.disableSubQueryPreprocessing || len(ExtractCorrelatedCols4LogicalPlan(np)) > 0 || hasCTEConsumerInSubPlan(np) { + if planCtx.builder.disableSubQueryPreprocessing || len(coreusage.ExtractCorrelatedCols4LogicalPlan(np)) > 0 || hasCTEConsumerInSubPlan(np) { planCtx.plan = planCtx.builder.buildApplyWithJoinType(planCtx.plan, np, LeftOuterJoin, noDecorrelate) if np.Schema().Len() > 1 { newCols := make([]expression.Expression, 0, np.Schema().Len()) diff --git a/pkg/planner/core/find_best_task.go b/pkg/planner/core/find_best_task.go index f597aa936c390..8a263d642772a 100644 --- a/pkg/planner/core/find_best_task.go +++ b/pkg/planner/core/find_best_task.go @@ -33,8 +33,8 @@ import ( "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/property" "github.com/pingcap/tidb/pkg/planner/util" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" "github.com/pingcap/tidb/pkg/planner/util/fixcontrol" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/types" tidbutil "github.com/pingcap/tidb/pkg/util" @@ -120,7 +120,7 @@ func GetPropByOrderByItemsContainScalarFunc(items []*util.ByItems) (*property.Ph } // FindBestTask implements the LogicalPlan. -func (p *LogicalTableDual) FindBestTask(prop *property.PhysicalProperty, planCounter *base.PlanCounterTp, opt *coreusage.PhysicalOptimizeOp) (base.Task, int64, error) { +func (p *LogicalTableDual) FindBestTask(prop *property.PhysicalProperty, planCounter *base.PlanCounterTp, opt *optimizetrace.PhysicalOptimizeOp) (base.Task, int64, error) { // If the required property is not empty and the row count > 1, // we cannot ensure this required property. // But if the row count is 0 or 1, we don't need to care about the property. @@ -140,7 +140,7 @@ func (p *LogicalTableDual) FindBestTask(prop *property.PhysicalProperty, planCou } // FindBestTask implements the LogicalPlan interface. -func (p *LogicalShow) FindBestTask(prop *property.PhysicalProperty, planCounter *base.PlanCounterTp, _ *coreusage.PhysicalOptimizeOp) (base.Task, int64, error) { +func (p *LogicalShow) FindBestTask(prop *property.PhysicalProperty, planCounter *base.PlanCounterTp, _ *optimizetrace.PhysicalOptimizeOp) (base.Task, int64, error) { if !prop.IsSortItemEmpty() || planCounter.Empty() { return invalidTask, 0, nil } @@ -153,7 +153,7 @@ func (p *LogicalShow) FindBestTask(prop *property.PhysicalProperty, planCounter } // FindBestTask implements the LogicalPlan interface. -func (p *LogicalShowDDLJobs) FindBestTask(prop *property.PhysicalProperty, planCounter *base.PlanCounterTp, _ *coreusage.PhysicalOptimizeOp) (base.Task, int64, error) { +func (p *LogicalShowDDLJobs) FindBestTask(prop *property.PhysicalProperty, planCounter *base.PlanCounterTp, _ *optimizetrace.PhysicalOptimizeOp) (base.Task, int64, error) { if !prop.IsSortItemEmpty() || planCounter.Empty() { return invalidTask, 0, nil } @@ -166,7 +166,7 @@ func (p *LogicalShowDDLJobs) FindBestTask(prop *property.PhysicalProperty, planC } // rebuildChildTasks rebuilds the childTasks to make the clock_th combination. -func (p *baseLogicalPlan) rebuildChildTasks(childTasks *[]base.Task, pp base.PhysicalPlan, childCnts []int64, planCounter int64, ts uint64, opt *coreusage.PhysicalOptimizeOp) error { +func (p *baseLogicalPlan) rebuildChildTasks(childTasks *[]base.Task, pp base.PhysicalPlan, childCnts []int64, planCounter int64, ts uint64, opt *optimizetrace.PhysicalOptimizeOp) error { // The taskMap of children nodes should be rolled back first. for _, child := range p.children { child.RollBackTaskMap(ts) @@ -202,7 +202,7 @@ func (p *baseLogicalPlan) enumeratePhysicalPlans4Task( prop *property.PhysicalProperty, addEnforcer bool, planCounter *base.PlanCounterTp, - opt *coreusage.PhysicalOptimizeOp, + opt *optimizetrace.PhysicalOptimizeOp, ) (base.Task, int64, error) { var bestTask base.Task = invalidTask var curCntPlan, cntPlan int64 @@ -285,7 +285,7 @@ func (p *baseLogicalPlan) iteratePhysicalPlan( childTasks []base.Task, childCnts []int64, _ *property.PhysicalProperty, - opt *coreusage.PhysicalOptimizeOp, + opt *optimizetrace.PhysicalOptimizeOp, ) ([]base.Task, int64, []int64, error) { // Find best child tasks firstly. childTasks = childTasks[:0] @@ -318,7 +318,7 @@ func (p *LogicalSequence) iterateChildPlan( childTasks []base.Task, childCnts []int64, prop *property.PhysicalProperty, - opt *coreusage.PhysicalOptimizeOp, + opt *optimizetrace.PhysicalOptimizeOp, ) ([]base.Task, int64, []int64, error) { // Find best child tasks firstly. childTasks = childTasks[:0] @@ -371,7 +371,7 @@ func (p *LogicalSequence) iterateChildPlan( } // compareTaskCost compares cost of curTask and bestTask and returns whether curTask's cost is smaller than bestTask's. -func compareTaskCost(curTask, bestTask base.Task, op *coreusage.PhysicalOptimizeOp) (curIsBetter bool, err error) { +func compareTaskCost(curTask, bestTask base.Task, op *optimizetrace.PhysicalOptimizeOp) (curIsBetter bool, err error) { curCost, curInvalid, err := getTaskPlanCost(curTask, op) if err != nil { return false, err @@ -392,7 +392,7 @@ func compareTaskCost(curTask, bestTask base.Task, op *coreusage.PhysicalOptimize // getTaskPlanCost returns the cost of this task. // The new cost interface will be used if EnableNewCostInterface is true. // The second returned value indicates whether this task is valid. -func getTaskPlanCost(t base.Task, pop *coreusage.PhysicalOptimizeOp) (float64, bool, error) { +func getTaskPlanCost(t base.Task, pop *optimizetrace.PhysicalOptimizeOp) (float64, bool, error) { if t.Invalid() { return math.MaxFloat64, true, nil } @@ -411,15 +411,15 @@ func getTaskPlanCost(t base.Task, pop *coreusage.PhysicalOptimizeOp) (float64, b taskType = property.CopMultiReadTaskType // keep compatible with the old cost interface, for CopMultiReadTask, the cost is idxCost + tblCost. if !cop.indexPlanFinished { // only consider index cost in this case - idxCost, err := getPlanCost(cop.indexPlan, taskType, coreusage.NewDefaultPlanCostOption().WithOptimizeTracer(pop)) + idxCost, err := getPlanCost(cop.indexPlan, taskType, optimizetrace.NewDefaultPlanCostOption().WithOptimizeTracer(pop)) return idxCost, false, err } // consider both sides - idxCost, err := getPlanCost(cop.indexPlan, taskType, coreusage.NewDefaultPlanCostOption().WithOptimizeTracer(pop)) + idxCost, err := getPlanCost(cop.indexPlan, taskType, optimizetrace.NewDefaultPlanCostOption().WithOptimizeTracer(pop)) if err != nil { return 0, false, err } - tblCost, err := getPlanCost(cop.tablePlan, taskType, coreusage.NewDefaultPlanCostOption().WithOptimizeTracer(pop)) + tblCost, err := getPlanCost(cop.tablePlan, taskType, optimizetrace.NewDefaultPlanCostOption().WithOptimizeTracer(pop)) if err != nil { return 0, false, err } @@ -445,7 +445,7 @@ func getTaskPlanCost(t base.Task, pop *coreusage.PhysicalOptimizeOp) (float64, b // cost about table plan. if cop.indexPlanFinished && len(cop.idxMergePartPlans) != 0 { for _, partialScan := range cop.idxMergePartPlans { - partialCost, err := getPlanCost(partialScan, taskType, coreusage.NewDefaultPlanCostOption().WithOptimizeTracer(pop)) + partialCost, err := getPlanCost(partialScan, taskType, optimizetrace.NewDefaultPlanCostOption().WithOptimizeTracer(pop)) if err != nil { return 0, false, err } @@ -463,7 +463,7 @@ func getTaskPlanCost(t base.Task, pop *coreusage.PhysicalOptimizeOp) (float64, b cost := 0.0 copTsk := t.(*CopTask) for _, partialScan := range copTsk.idxMergePartPlans { - partialCost, err := getPlanCost(partialScan, taskType, coreusage.NewDefaultPlanCostOption().WithOptimizeTracer(pop)) + partialCost, err := getPlanCost(partialScan, taskType, optimizetrace.NewDefaultPlanCostOption().WithOptimizeTracer(pop)) if err != nil { return 0, false, err } @@ -471,11 +471,11 @@ func getTaskPlanCost(t base.Task, pop *coreusage.PhysicalOptimizeOp) (float64, b } return cost, false, nil } - cost, err := getPlanCost(t.Plan(), taskType, coreusage.NewDefaultPlanCostOption().WithOptimizeTracer(pop)) + cost, err := getPlanCost(t.Plan(), taskType, optimizetrace.NewDefaultPlanCostOption().WithOptimizeTracer(pop)) return cost + indexPartialCost, false, err } -func appendCandidate4PhysicalOptimizeOp(pop *coreusage.PhysicalOptimizeOp, lp base.LogicalPlan, pp base.PhysicalPlan, prop *property.PhysicalProperty) { +func appendCandidate4PhysicalOptimizeOp(pop *optimizetrace.PhysicalOptimizeOp, lp base.LogicalPlan, pp base.PhysicalPlan, prop *property.PhysicalProperty) { if pop == nil || pop.GetTracer() == nil || pp == nil { return } @@ -513,7 +513,7 @@ func appendCandidate4PhysicalOptimizeOp(pop *coreusage.PhysicalOptimizeOp, lp ba pp.AppendChildCandidate(pop) } -func appendPlanCostDetail4PhysicalOptimizeOp(pop *coreusage.PhysicalOptimizeOp, detail *tracing.PhysicalPlanCostDetail) { +func appendPlanCostDetail4PhysicalOptimizeOp(pop *optimizetrace.PhysicalOptimizeOp, detail *tracing.PhysicalPlanCostDetail) { if pop == nil || pop.GetTracer() == nil { return } @@ -521,7 +521,7 @@ func appendPlanCostDetail4PhysicalOptimizeOp(pop *coreusage.PhysicalOptimizeOp, } // FindBestTask implements LogicalPlan interface. -func (p *baseLogicalPlan) FindBestTask(prop *property.PhysicalProperty, planCounter *base.PlanCounterTp, opt *coreusage.PhysicalOptimizeOp) (bestTask base.Task, cntPlan int64, err error) { +func (p *baseLogicalPlan) FindBestTask(prop *property.PhysicalProperty, planCounter *base.PlanCounterTp, opt *optimizetrace.PhysicalOptimizeOp) (bestTask base.Task, cntPlan int64, err error) { // If p is an inner plan in an IndexJoin, the IndexJoin will generate an inner plan by itself, // and set inner child prop nil, so here we do nothing. if prop == nil { @@ -621,7 +621,7 @@ END: } // FindBestTask implements the LogicalPlan interface. -func (p *LogicalMemTable) FindBestTask(prop *property.PhysicalProperty, planCounter *base.PlanCounterTp, opt *coreusage.PhysicalOptimizeOp) (t base.Task, cntPlan int64, err error) { +func (p *LogicalMemTable) FindBestTask(prop *property.PhysicalProperty, planCounter *base.PlanCounterTp, opt *optimizetrace.PhysicalOptimizeOp) (t base.Task, cntPlan int64, err error) { if prop.MPPPartitionTp != property.AnyType { return invalidTask, 0, nil } @@ -1254,7 +1254,7 @@ func (ds *DataSource) exploreEnforcedPlan() bool { // FindBestTask implements the PhysicalPlan interface. // It will enumerate all the available indices and choose a plan with least cost. -func (ds *DataSource) FindBestTask(prop *property.PhysicalProperty, planCounter *base.PlanCounterTp, opt *coreusage.PhysicalOptimizeOp) (t base.Task, cntPlan int64, err error) { +func (ds *DataSource) FindBestTask(prop *property.PhysicalProperty, planCounter *base.PlanCounterTp, opt *optimizetrace.PhysicalOptimizeOp) (t base.Task, cntPlan int64, err error) { // If ds is an inner plan in an IndexJoin, the IndexJoin will generate an inner plan by itself, // and set inner child prop nil, so here we do nothing. if prop == nil { @@ -1568,7 +1568,7 @@ func (ds *DataSource) FindBestTask(prop *property.PhysicalProperty, planCounter } // convertToIndexMergeScan builds the index merge scan for intersection or union cases. -func (ds *DataSource) convertToIndexMergeScan(prop *property.PhysicalProperty, candidate *candidatePath, _ *coreusage.PhysicalOptimizeOp) (task base.Task, err error) { +func (ds *DataSource) convertToIndexMergeScan(prop *property.PhysicalProperty, candidate *candidatePath, _ *optimizetrace.PhysicalOptimizeOp) (task base.Task, err error) { if prop.IsFlashProp() || prop.TaskTp == property.CopSingleReadTaskType { return invalidTask, nil } @@ -1971,7 +1971,7 @@ func (ts *PhysicalTableScan) appendExtraHandleCol(ds *DataSource) (*expression.C // convertToIndexScan converts the DataSource to index scan with idx. func (ds *DataSource) convertToIndexScan(prop *property.PhysicalProperty, - candidate *candidatePath, _ *coreusage.PhysicalOptimizeOp) (task base.Task, err error) { + candidate *candidatePath, _ *optimizetrace.PhysicalOptimizeOp) (task base.Task, err error) { if candidate.path.Index.MVIndex { // MVIndex is special since different index rows may return the same _row_id and this can break some assumptions of IndexReader. // Currently only support using IndexMerge to access MVIndex instead of IndexReader. @@ -2361,7 +2361,7 @@ func (ds *DataSource) isPointGetPath(path *util.AccessPath) bool { } // convertToTableScan converts the DataSource to table scan. -func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candidate *candidatePath, _ *coreusage.PhysicalOptimizeOp) (base.Task, error) { +func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candidate *candidatePath, _ *optimizetrace.PhysicalOptimizeOp) (base.Task, error) { // It will be handled in convertToIndexScan. if prop.TaskTp == property.CopMultiReadTaskType { return invalidTask, nil @@ -2502,7 +2502,7 @@ func (ds *DataSource) convertToTableScan(prop *property.PhysicalProperty, candid } func (ds *DataSource) convertToSampleTable(prop *property.PhysicalProperty, - candidate *candidatePath, _ *coreusage.PhysicalOptimizeOp) (base.Task, error) { + candidate *candidatePath, _ *optimizetrace.PhysicalOptimizeOp) (base.Task, error) { if prop.TaskTp == property.CopMultiReadTaskType { return invalidTask, nil } @@ -2820,7 +2820,7 @@ func (ds *DataSource) getOriginalPhysicalIndexScan(prop *property.PhysicalProper } // FindBestTask implements the LogicalPlan interface. -func (p *LogicalCTE) FindBestTask(prop *property.PhysicalProperty, counter *base.PlanCounterTp, pop *coreusage.PhysicalOptimizeOp) (t base.Task, cntPlan int64, err error) { +func (p *LogicalCTE) FindBestTask(prop *property.PhysicalProperty, counter *base.PlanCounterTp, pop *optimizetrace.PhysicalOptimizeOp) (t base.Task, cntPlan int64, err error) { if len(p.children) > 0 { return p.baseLogicalPlan.FindBestTask(prop, counter, pop) } @@ -2854,7 +2854,7 @@ func (p *LogicalCTE) FindBestTask(prop *property.PhysicalProperty, counter *base } // FindBestTask implements the LogicalPlan interface. -func (p *LogicalCTETable) FindBestTask(prop *property.PhysicalProperty, _ *base.PlanCounterTp, _ *coreusage.PhysicalOptimizeOp) (t base.Task, cntPlan int64, err error) { +func (p *LogicalCTETable) FindBestTask(prop *property.PhysicalProperty, _ *base.PlanCounterTp, _ *optimizetrace.PhysicalOptimizeOp) (t base.Task, cntPlan int64, err error) { if !prop.IsSortItemEmpty() { return nil, 1, nil } @@ -2867,7 +2867,7 @@ func (p *LogicalCTETable) FindBestTask(prop *property.PhysicalProperty, _ *base. return t, 1, nil } -func appendCandidate(lp base.LogicalPlan, task base.Task, prop *property.PhysicalProperty, opt *coreusage.PhysicalOptimizeOp) { +func appendCandidate(lp base.LogicalPlan, task base.Task, prop *property.PhysicalProperty, opt *optimizetrace.PhysicalOptimizeOp) { if task == nil || task.Invalid() { return } diff --git a/pkg/planner/core/find_best_task_test.go b/pkg/planner/core/find_best_task_test.go index ac346b29afd68..1ac8bcb6ccffc 100644 --- a/pkg/planner/core/find_best_task_test.go +++ b/pkg/planner/core/find_best_task_test.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/property" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/stretchr/testify/require" ) @@ -35,7 +35,7 @@ func (ds mockDataSource) Init(ctx base.PlanContext) *mockDataSource { return &ds } -func (ds *mockDataSource) FindBestTask(prop *property.PhysicalProperty, planCounter *base.PlanCounterTp, opt *coreusage.PhysicalOptimizeOp) (base.Task, int64, error) { +func (ds *mockDataSource) FindBestTask(prop *property.PhysicalProperty, planCounter *base.PlanCounterTp, opt *optimizetrace.PhysicalOptimizeOp) (base.Task, int64, error) { // It can satisfy any of the property! // Just use a TableDual for convenience. p := PhysicalTableDual{}.Init(ds.SCtx(), &property.StatsInfo{RowCount: 1}, 0) @@ -148,7 +148,7 @@ func TestCostOverflow(t *testing.T) { mockPlan.SetChildren(mockDS) // An empty property is enough for this test. prop := property.NewPhysicalProperty(property.RootTaskType, nil, false, 0, false) - task, _, err := mockPlan.FindBestTask(prop, &PlanCounterDisabled, coreusage.DefaultPhysicalOptimizeOption()) + task, _, err := mockPlan.FindBestTask(prop, &PlanCounterDisabled, optimizetrace.DefaultPhysicalOptimizeOption()) require.NoError(t, err) // The cost should be overflowed, but the task shouldn't be invalid. require.False(t, task.Invalid()) @@ -177,7 +177,7 @@ func TestEnforcedProperty(t *testing.T) { CanAddEnforcer: false, } // should return invalid task because no physical plan can match this property. - task, _, err := mockPlan.FindBestTask(prop0, &PlanCounterDisabled, coreusage.DefaultPhysicalOptimizeOption()) + task, _, err := mockPlan.FindBestTask(prop0, &PlanCounterDisabled, optimizetrace.DefaultPhysicalOptimizeOption()) require.NoError(t, err) require.True(t, task.Invalid()) @@ -186,7 +186,7 @@ func TestEnforcedProperty(t *testing.T) { CanAddEnforcer: true, } // should return the valid task when the property is enforced. - task, _, err = mockPlan.FindBestTask(prop1, &PlanCounterDisabled, coreusage.DefaultPhysicalOptimizeOption()) + task, _, err = mockPlan.FindBestTask(prop1, &PlanCounterDisabled, optimizetrace.DefaultPhysicalOptimizeOption()) require.NoError(t, err) require.False(t, task.Invalid()) } @@ -212,7 +212,7 @@ func TestHintCannotFitProperty(t *testing.T) { SortItems: items, CanAddEnforcer: true, } - task, _, err := mockPlan0.FindBestTask(prop0, &PlanCounterDisabled, coreusage.DefaultPhysicalOptimizeOption()) + task, _, err := mockPlan0.FindBestTask(prop0, &PlanCounterDisabled, optimizetrace.DefaultPhysicalOptimizeOption()) require.NoError(t, err) require.False(t, task.Invalid()) _, enforcedSort := task.Plan().(*PhysicalSort) @@ -228,7 +228,7 @@ func TestHintCannotFitProperty(t *testing.T) { SortItems: items, CanAddEnforcer: false, } - task, _, err = mockPlan0.FindBestTask(prop1, &PlanCounterDisabled, coreusage.DefaultPhysicalOptimizeOption()) + task, _, err = mockPlan0.FindBestTask(prop1, &PlanCounterDisabled, optimizetrace.DefaultPhysicalOptimizeOption()) require.NoError(t, err) require.False(t, task.Invalid()) _, enforcedSort = task.Plan().(*PhysicalSort) @@ -249,7 +249,7 @@ func TestHintCannotFitProperty(t *testing.T) { canGeneratePlan2: false, }.Init(ctx) mockPlan1.SetChildren(mockDS) - task, _, err = mockPlan1.FindBestTask(prop2, &PlanCounterDisabled, coreusage.DefaultPhysicalOptimizeOption()) + task, _, err = mockPlan1.FindBestTask(prop2, &PlanCounterDisabled, optimizetrace.DefaultPhysicalOptimizeOption()) require.NoError(t, err) require.False(t, task.Invalid()) require.Equal(t, uint16(1), ctx.GetSessionVars().StmtCtx.WarningCount()) @@ -265,7 +265,7 @@ func TestHintCannotFitProperty(t *testing.T) { SortItems: items, CanAddEnforcer: true, } - task, _, err = mockPlan1.FindBestTask(prop3, &PlanCounterDisabled, coreusage.DefaultPhysicalOptimizeOption()) + task, _, err = mockPlan1.FindBestTask(prop3, &PlanCounterDisabled, optimizetrace.DefaultPhysicalOptimizeOption()) require.NoError(t, err) require.False(t, task.Invalid()) require.Equal(t, uint16(1), ctx.GetSessionVars().StmtCtx.WarningCount()) diff --git a/pkg/planner/core/logical_plan_builder.go b/pkg/planner/core/logical_plan_builder.go index c8a6ca525a356..ba450c36a36c3 100644 --- a/pkg/planner/core/logical_plan_builder.go +++ b/pkg/planner/core/logical_plan_builder.go @@ -45,6 +45,7 @@ import ( fd "github.com/pingcap/tidb/pkg/planner/funcdep" "github.com/pingcap/tidb/pkg/planner/property" "github.com/pingcap/tidb/pkg/planner/util" + "github.com/pingcap/tidb/pkg/planner/util/coreusage" "github.com/pingcap/tidb/pkg/planner/util/debugtrace" "github.com/pingcap/tidb/pkg/planner/util/fixcontrol" "github.com/pingcap/tidb/pkg/privilege" @@ -2812,7 +2813,7 @@ func (b *PlanBuilder) resolveHavingAndOrderBy(ctx context.Context, sel *ast.Sele if err != nil { return nil, nil, errors.Trace(err) } - correlatedCols := ExtractCorrelatedCols4LogicalPlan(np) + correlatedCols := coreusage.ExtractCorrelatedCols4LogicalPlan(np) for _, cone := range correlatedCols { var colName *ast.ColumnName for idx, pone := range p.Schema().Columns { @@ -5574,7 +5575,7 @@ func (b *PlanBuilder) buildSemiApply(outerPlan, innerPlan base.LogicalPlan, cond func setIsInApplyForCTE(p base.LogicalPlan, apSchema *expression.Schema) { switch x := p.(type) { case *LogicalCTE: - if len(extractCorColumnsBySchema4LogicalPlan(p, apSchema)) > 0 { + if len(coreusage.ExtractCorColumnsBySchema4LogicalPlan(p, apSchema)) > 0 { x.cte.IsInApply = true } setIsInApplyForCTE(x.cte.seedPartLogicalPlan, apSchema) diff --git a/pkg/planner/core/logical_plans.go b/pkg/planner/core/logical_plans.go index 828ad7809dd39..b7512116812e4 100644 --- a/pkg/planner/core/logical_plans.go +++ b/pkg/planner/core/logical_plans.go @@ -32,6 +32,7 @@ import ( fd "github.com/pingcap/tidb/pkg/planner/funcdep" "github.com/pingcap/tidb/pkg/planner/property" "github.com/pingcap/tidb/pkg/planner/util" + "github.com/pingcap/tidb/pkg/planner/util/coreusage" "github.com/pingcap/tidb/pkg/planner/util/debugtrace" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/statistics" @@ -1334,7 +1335,7 @@ func (la *LogicalApply) ExtractCorrelatedCols() []*expression.CorrelatedColumn { func (la *LogicalApply) ExtractFD() *fd.FDSet { innerPlan := la.children[1] // build the join correlated equal condition for apply join, this equal condition is used for deriving the transitive FD between outer and inner side. - correlatedCols := ExtractCorrelatedCols4LogicalPlan(innerPlan) + correlatedCols := coreusage.ExtractCorrelatedCols4LogicalPlan(innerPlan) deduplicateCorrelatedCols := make(map[int64]*expression.CorrelatedColumn) for _, cc := range correlatedCols { if _, ok := deduplicateCorrelatedCols[cc.UniqueID]; !ok { @@ -2185,58 +2186,6 @@ func (p *LogicalWindow) GetWindowResultColumns() []*expression.Column { return p.schema.Columns[p.schema.Len()-len(p.WindowFuncDescs):] } -// ExtractCorColumnsBySchema only extracts the correlated columns that match the specified schema. -// e.g. If the correlated columns from plan are [t1.a, t2.a, t3.a] and specified schema is [t2.a, t2.b, t2.c], -// only [t2.a] is returned. -func ExtractCorColumnsBySchema(corCols []*expression.CorrelatedColumn, schema *expression.Schema, resolveIndex bool) []*expression.CorrelatedColumn { - resultCorCols := make([]*expression.CorrelatedColumn, schema.Len()) - for _, corCol := range corCols { - idx := schema.ColumnIndex(&corCol.Column) - if idx != -1 { - if resultCorCols[idx] == nil { - resultCorCols[idx] = &expression.CorrelatedColumn{ - Column: *schema.Columns[idx], - Data: new(types.Datum), - } - } - corCol.Data = resultCorCols[idx].Data - } - } - // Shrink slice. e.g. [col1, nil, col2, nil] will be changed to [col1, col2]. - length := 0 - for _, col := range resultCorCols { - if col != nil { - resultCorCols[length] = col - length++ - } - } - resultCorCols = resultCorCols[:length] - - if resolveIndex { - for _, corCol := range resultCorCols { - corCol.Index = schema.ColumnIndex(&corCol.Column) - } - } - - return resultCorCols -} - -// extractCorColumnsBySchema4LogicalPlan only extracts the correlated columns that match the specified schema. -// e.g. If the correlated columns from plan are [t1.a, t2.a, t3.a] and specified schema is [t2.a, t2.b, t2.c], -// only [t2.a] is returned. -func extractCorColumnsBySchema4LogicalPlan(p base.LogicalPlan, schema *expression.Schema) []*expression.CorrelatedColumn { - corCols := ExtractCorrelatedCols4LogicalPlan(p) - return ExtractCorColumnsBySchema(corCols, schema, false) -} - -// ExtractCorColumnsBySchema4PhysicalPlan only extracts the correlated columns that match the specified schema. -// e.g. If the correlated columns from plan are [t1.a, t2.a, t3.a] and specified schema is [t2.a, t2.b, t2.c], -// only [t2.a] is returned. -func ExtractCorColumnsBySchema4PhysicalPlan(p base.PhysicalPlan, schema *expression.Schema) []*expression.CorrelatedColumn { - corCols := ExtractCorrelatedCols4PhysicalPlan(p) - return ExtractCorColumnsBySchema(corCols, schema, true) -} - // ShowContents stores the contents for the `SHOW` statement. type ShowContents struct { Tp ast.ShowStmtType // Databases/Tables/Columns/.... @@ -2365,9 +2314,9 @@ type LogicalCTETable struct { // ExtractCorrelatedCols implements LogicalPlan interface. func (p *LogicalCTE) ExtractCorrelatedCols() []*expression.CorrelatedColumn { - corCols := ExtractCorrelatedCols4LogicalPlan(p.cte.seedPartLogicalPlan) + corCols := coreusage.ExtractCorrelatedCols4LogicalPlan(p.cte.seedPartLogicalPlan) if p.cte.recursivePartLogicalPlan != nil { - corCols = append(corCols, ExtractCorrelatedCols4LogicalPlan(p.cte.recursivePartLogicalPlan)...) + corCols = append(corCols, coreusage.ExtractCorrelatedCols4LogicalPlan(p.cte.recursivePartLogicalPlan)...) } return corCols } diff --git a/pkg/planner/core/optimizer.go b/pkg/planner/core/optimizer.go index aaea5d42403a4..5ad5b78bc2f0d 100644 --- a/pkg/planner/core/optimizer.go +++ b/pkg/planner/core/optimizer.go @@ -41,8 +41,8 @@ import ( "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/property" plannerutil "github.com/pingcap/tidb/pkg/planner/util" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" "github.com/pingcap/tidb/pkg/planner/util/debugtrace" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/privilege" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" @@ -139,7 +139,7 @@ type logicalOptRule interface { The default value is false. It means that no interaction rule will be triggered. 3. error: If there is error during the rule optimizer, it will be thrown */ - optimize(context.Context, base.LogicalPlan, *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) + optimize(context.Context, base.LogicalPlan, *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) name() string } @@ -978,7 +978,7 @@ func logicalOptimize(ctx context.Context, flag uint64, logic base.LogicalPlan) ( debugtrace.EnterContextCommon(logic.SCtx()) defer debugtrace.LeaveContextCommon(logic.SCtx()) } - opt := coreusage.DefaultLogicalOptimizeOption() + opt := optimizetrace.DefaultLogicalOptimizeOption() vars := logic.SCtx().GetSessionVars() if vars.StmtCtx.EnableOptimizeTrace { vars.StmtCtx.OptimizeTracer = &tracing.OptimizeTracer{} @@ -1046,7 +1046,7 @@ func physicalOptimize(logic base.LogicalPlan, planCounter *base.PlanCounterTp) ( ExpectedCnt: math.MaxFloat64, } - opt := coreusage.DefaultPhysicalOptimizeOption() + opt := optimizetrace.DefaultPhysicalOptimizeOption() stmtCtx := logic.SCtx().GetSessionVars().StmtCtx if stmtCtx.EnableOptimizeTrace { tracer := &tracing.PhysicalOptimizeTracer{ @@ -1085,7 +1085,7 @@ func physicalOptimize(logic base.LogicalPlan, planCounter *base.PlanCounterTp) ( if err = t.Plan().ResolveIndices(); err != nil { return nil, 0, err } - cost, err = getPlanCost(t.Plan(), property.RootTaskType, coreusage.NewDefaultPlanCostOption()) + cost, err = getPlanCost(t.Plan(), property.RootTaskType, optimizetrace.NewDefaultPlanCostOption()) return t.Plan(), cost, err } diff --git a/pkg/planner/core/physical_plans.go b/pkg/planner/core/physical_plans.go index 3f74d7b54b5ac..b83c33ccba9c7 100644 --- a/pkg/planner/core/physical_plans.go +++ b/pkg/planner/core/physical_plans.go @@ -32,6 +32,7 @@ import ( "github.com/pingcap/tidb/pkg/planner/property" "github.com/pingcap/tidb/pkg/planner/util" "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" "github.com/pingcap/tidb/pkg/statistics" @@ -286,7 +287,7 @@ func (p *PhysicalTableReader) SetChildren(children ...base.PhysicalPlan) { // ExtractCorrelatedCols implements op.PhysicalPlan interface. func (p *PhysicalTableReader) ExtractCorrelatedCols() (corCols []*expression.CorrelatedColumn) { for _, child := range p.TablePlans { - corCols = append(corCols, ExtractCorrelatedCols4PhysicalPlan(child)...) + corCols = append(corCols, coreusage.ExtractCorrelatedCols4PhysicalPlan(child)...) } return corCols } @@ -301,7 +302,7 @@ func (p *PhysicalTableReader) BuildPlanTrace() *tracing.PlanTrace { } // AppendChildCandidate implements PhysicalPlan interface. -func (p *PhysicalTableReader) AppendChildCandidate(op *coreusage.PhysicalOptimizeOp) { +func (p *PhysicalTableReader) AppendChildCandidate(op *optimizetrace.PhysicalOptimizeOp) { p.basePhysicalPlan.AppendChildCandidate(op) appendChildCandidate(p, p.tablePlan, op) } @@ -363,7 +364,7 @@ func (p *PhysicalIndexReader) SetChildren(children ...base.PhysicalPlan) { // ExtractCorrelatedCols implements op.PhysicalPlan interface. func (p *PhysicalIndexReader) ExtractCorrelatedCols() (corCols []*expression.CorrelatedColumn) { for _, child := range p.IndexPlans { - corCols = append(corCols, ExtractCorrelatedCols4PhysicalPlan(child)...) + corCols = append(corCols, coreusage.ExtractCorrelatedCols4PhysicalPlan(child)...) } return corCols } @@ -378,7 +379,7 @@ func (p *PhysicalIndexReader) BuildPlanTrace() *tracing.PlanTrace { } // AppendChildCandidate implements PhysicalPlan interface. -func (p *PhysicalIndexReader) AppendChildCandidate(op *coreusage.PhysicalOptimizeOp) { +func (p *PhysicalIndexReader) AppendChildCandidate(op *optimizetrace.PhysicalOptimizeOp) { p.basePhysicalPlan.AppendChildCandidate(op) if p.indexPlan != nil { appendChildCandidate(p, p.indexPlan, op) @@ -499,10 +500,10 @@ func (p *PhysicalIndexLookUpReader) Clone() (base.PhysicalPlan, error) { // ExtractCorrelatedCols implements op.PhysicalPlan interface. func (p *PhysicalIndexLookUpReader) ExtractCorrelatedCols() (corCols []*expression.CorrelatedColumn) { for _, child := range p.TablePlans { - corCols = append(corCols, ExtractCorrelatedCols4PhysicalPlan(child)...) + corCols = append(corCols, coreusage.ExtractCorrelatedCols4PhysicalPlan(child)...) } for _, child := range p.IndexPlans { - corCols = append(corCols, ExtractCorrelatedCols4PhysicalPlan(child)...) + corCols = append(corCols, coreusage.ExtractCorrelatedCols4PhysicalPlan(child)...) } return corCols } @@ -530,7 +531,7 @@ func (p *PhysicalIndexLookUpReader) BuildPlanTrace() *tracing.PlanTrace { } // AppendChildCandidate implements PhysicalPlan interface. -func (p *PhysicalIndexLookUpReader) AppendChildCandidate(op *coreusage.PhysicalOptimizeOp) { +func (p *PhysicalIndexLookUpReader) AppendChildCandidate(op *optimizetrace.PhysicalOptimizeOp) { p.basePhysicalPlan.AppendChildCandidate(op) if p.indexPlan != nil { appendChildCandidate(p, p.indexPlan, op) @@ -614,14 +615,14 @@ func (p *PhysicalIndexMergeReader) GetAvgTableRowSize() float64 { // ExtractCorrelatedCols implements op.PhysicalPlan interface. func (p *PhysicalIndexMergeReader) ExtractCorrelatedCols() (corCols []*expression.CorrelatedColumn) { for _, child := range p.TablePlans { - corCols = append(corCols, ExtractCorrelatedCols4PhysicalPlan(child)...) + corCols = append(corCols, coreusage.ExtractCorrelatedCols4PhysicalPlan(child)...) } for _, child := range p.partialPlans { - corCols = append(corCols, ExtractCorrelatedCols4PhysicalPlan(child)...) + corCols = append(corCols, coreusage.ExtractCorrelatedCols4PhysicalPlan(child)...) } for _, PartialPlan := range p.PartialPlans { for _, child := range PartialPlan { - corCols = append(corCols, ExtractCorrelatedCols4PhysicalPlan(child)...) + corCols = append(corCols, coreusage.ExtractCorrelatedCols4PhysicalPlan(child)...) } } return corCols @@ -640,7 +641,7 @@ func (p *PhysicalIndexMergeReader) BuildPlanTrace() *tracing.PlanTrace { } // AppendChildCandidate implements PhysicalPlan interface. -func (p *PhysicalIndexMergeReader) AppendChildCandidate(op *coreusage.PhysicalOptimizeOp) { +func (p *PhysicalIndexMergeReader) AppendChildCandidate(op *optimizetrace.PhysicalOptimizeOp) { p.basePhysicalPlan.AppendChildCandidate(op) if p.tablePlan != nil { appendChildCandidate(p, p.tablePlan, op) @@ -2588,9 +2589,9 @@ type PhysicalCTETable struct { // ExtractCorrelatedCols implements op.PhysicalPlan interface. func (p *PhysicalCTE) ExtractCorrelatedCols() []*expression.CorrelatedColumn { - corCols := ExtractCorrelatedCols4PhysicalPlan(p.SeedPlan) + corCols := coreusage.ExtractCorrelatedCols4PhysicalPlan(p.SeedPlan) if p.RecurPlan != nil { - corCols = append(corCols, ExtractCorrelatedCols4PhysicalPlan(p.RecurPlan)...) + corCols = append(corCols, coreusage.ExtractCorrelatedCols4PhysicalPlan(p.RecurPlan)...) } return corCols } @@ -2767,7 +2768,7 @@ func (p *PhysicalCTEStorage) Clone() (base.PhysicalPlan, error) { return (*PhysicalCTEStorage)(cloned.(*PhysicalCTE)), nil } -func appendChildCandidate(origin base.PhysicalPlan, pp base.PhysicalPlan, op *coreusage.PhysicalOptimizeOp) { +func appendChildCandidate(origin base.PhysicalPlan, pp base.PhysicalPlan, op *optimizetrace.PhysicalOptimizeOp) { candidate := &tracing.CandidatePlanTrace{ PlanTrace: &tracing.PlanTrace{ ID: pp.ID(), diff --git a/pkg/planner/core/plan.go b/pkg/planner/core/plan.go index 2b0bc374a35ad..ab2b60d152eac 100644 --- a/pkg/planner/core/plan.go +++ b/pkg/planner/core/plan.go @@ -25,7 +25,8 @@ import ( fd "github.com/pingcap/tidb/pkg/planner/funcdep" "github.com/pingcap/tidb/pkg/planner/property" "github.com/pingcap/tidb/pkg/planner/util" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/costusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/execdetails" @@ -296,7 +297,7 @@ type basePhysicalPlan struct { // used by the new cost interface planCostInit bool planCost float64 - planCostVer2 coreusage.CostVer2 + planCostVer2 costusage.CostVer2 // probeParents records the IndexJoins and Applys with this operator in their inner children. // Please see comments in op.PhysicalPlan for details. @@ -521,7 +522,7 @@ func (*baseLogicalPlan) ExtractCorrelatedCols() []*expression.CorrelatedColumn { } // PruneColumns implements LogicalPlan interface. -func (p *baseLogicalPlan) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (p *baseLogicalPlan) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { if len(p.children) == 0 { return p.self, nil } @@ -607,7 +608,7 @@ func (p *baseLogicalPlan) BuildPlanTrace() *tracing.PlanTrace { } // AppendChildCandidate implements PhysicalPlan interface. -func (p *basePhysicalPlan) AppendChildCandidate(op *coreusage.PhysicalOptimizeOp) { +func (p *basePhysicalPlan) AppendChildCandidate(op *optimizetrace.PhysicalOptimizeOp) { if len(p.Children()) < 1 { return } diff --git a/pkg/planner/core/plan_cost_detail.go b/pkg/planner/core/plan_cost_detail.go index a5c6b35597267..84e008ff780c2 100644 --- a/pkg/planner/core/plan_cost_detail.go +++ b/pkg/planner/core/plan_cost_detail.go @@ -19,7 +19,7 @@ import ( "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/planner/core/base" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/util/tracing" ) @@ -89,7 +89,7 @@ const ( MemQuotaLbl = "memQuota" ) -func setPointGetPlanCostDetail(p *PointGetPlan, opt *coreusage.PhysicalOptimizeOp, +func setPointGetPlanCostDetail(p *PointGetPlan, opt *optimizetrace.PhysicalOptimizeOp, rowSize, networkFactor, seekFactor float64) { if opt == nil { return @@ -102,7 +102,7 @@ func setPointGetPlanCostDetail(p *PointGetPlan, opt *coreusage.PhysicalOptimizeO appendPlanCostDetail4PhysicalOptimizeOp(opt, detail) } -func setBatchPointGetPlanCostDetail(p *BatchPointGetPlan, opt *coreusage.PhysicalOptimizeOp, +func setBatchPointGetPlanCostDetail(p *BatchPointGetPlan, opt *optimizetrace.PhysicalOptimizeOp, rowCount, rowSize, networkFactor, seekFactor float64, scanConcurrency int) { if opt == nil { return @@ -118,7 +118,7 @@ func setBatchPointGetPlanCostDetail(p *BatchPointGetPlan, opt *coreusage.Physica appendPlanCostDetail4PhysicalOptimizeOp(opt, detail) } -func setPhysicalTableOrIndexScanCostDetail(p base.PhysicalPlan, opt *coreusage.PhysicalOptimizeOp, +func setPhysicalTableOrIndexScanCostDetail(p base.PhysicalPlan, opt *optimizetrace.PhysicalOptimizeOp, rowCount, rowSize, scanFactor float64, costModelVersion int) { if opt == nil { return @@ -142,7 +142,7 @@ func setPhysicalTableOrIndexScanCostDetail(p base.PhysicalPlan, opt *coreusage.P appendPlanCostDetail4PhysicalOptimizeOp(opt, detail) } -func setPhysicalTableReaderCostDetail(p *PhysicalTableReader, opt *coreusage.PhysicalOptimizeOp, +func setPhysicalTableReaderCostDetail(p *PhysicalTableReader, opt *optimizetrace.PhysicalOptimizeOp, rowCount, rowSize, networkFactor, netSeekCost, tablePlanCost float64, scanConcurrency int, storeType kv.StoreType) { // tracer haven't support non tikv plan for now @@ -161,7 +161,7 @@ func setPhysicalTableReaderCostDetail(p *PhysicalTableReader, opt *coreusage.Phy appendPlanCostDetail4PhysicalOptimizeOp(opt, detail) } -func setPhysicalIndexReaderCostDetail(p *PhysicalIndexReader, opt *coreusage.PhysicalOptimizeOp, +func setPhysicalIndexReaderCostDetail(p *PhysicalIndexReader, opt *optimizetrace.PhysicalOptimizeOp, rowCount, rowSize, networkFactor, netSeekCost, indexPlanCost float64, scanConcurrency int) { if opt == nil { @@ -179,7 +179,7 @@ func setPhysicalIndexReaderCostDetail(p *PhysicalIndexReader, opt *coreusage.Phy appendPlanCostDetail4PhysicalOptimizeOp(opt, detail) } -func setPhysicalHashJoinCostDetail(p *PhysicalHashJoin, opt *coreusage.PhysicalOptimizeOp, spill bool, +func setPhysicalHashJoinCostDetail(p *PhysicalHashJoin, opt *optimizetrace.PhysicalOptimizeOp, spill bool, buildCnt, probeCnt, cpuFactor, rowSize, numPairs, cpuCost, probeCPUCost, memCost, diskCost, probeDiskCost, diskFactor, memoryFactor, concurrencyFactor float64, diff --git a/pkg/planner/core/plan_cost_ver1.go b/pkg/planner/core/plan_cost_ver1.go index 99ac1fdde0264..af395ce45af28 100644 --- a/pkg/planner/core/plan_cost_ver1.go +++ b/pkg/planner/core/plan_cost_ver1.go @@ -23,7 +23,8 @@ import ( "github.com/pingcap/tidb/pkg/planner/cardinality" "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/property" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/costusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/util/paging" @@ -39,9 +40,9 @@ func hasCostFlag(costFlag, flag uint64) bool { } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *basePhysicalPlan) GetPlanCostVer1(taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *basePhysicalPlan) GetPlanCostVer1(taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { // just calculate the cost once and always reuse it return p.planCost, nil } @@ -58,9 +59,9 @@ func (p *basePhysicalPlan) GetPlanCostVer1(taskType property.TaskType, option *c } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalSelection) GetPlanCostVer1(taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalSelection) GetPlanCostVer1(taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } @@ -102,9 +103,9 @@ func (p *PhysicalProjection) GetCost(count float64) float64 { } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalProjection) GetPlanCostVer1(taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalProjection) GetPlanCostVer1(taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } childCost, err := p.children[0].GetPlanCostVer1(taskType, option) @@ -164,9 +165,9 @@ func (p *PhysicalIndexLookUpReader) GetCost(costFlag uint64) (cost float64) { } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexLookUpReader) GetPlanCostVer1(_ property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalIndexLookUpReader) GetPlanCostVer1(_ property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } @@ -219,9 +220,9 @@ func (p *PhysicalIndexLookUpReader) GetPlanCostVer1(_ property.TaskType, option } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexReader) GetPlanCostVer1(_ property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalIndexReader) GetPlanCostVer1(_ property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } @@ -261,9 +262,9 @@ func (p *PhysicalIndexReader) GetNetDataSize() float64 { } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalTableReader) GetPlanCostVer1(_ property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalTableReader) GetPlanCostVer1(_ property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } @@ -324,7 +325,7 @@ func (p *PhysicalTableReader) GetPlanCostVer1(_ property.TaskType, option *coreu p.planCost /= concurrency // consider tidb_enforce_mpp if isMPP && p.SCtx().GetSessionVars().IsMPPEnforced() && - !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { // show the real cost in explain-statements + !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { // show the real cost in explain-statements p.planCost /= 1000000000 } } @@ -344,9 +345,9 @@ func (p *PhysicalTableReader) GetNetDataSize() float64 { } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexMergeReader) GetPlanCostVer1(_ property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalIndexMergeReader) GetPlanCostVer1(_ property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } @@ -413,9 +414,9 @@ func (p *PhysicalIndexMergeReader) GetPartialReaderNetDataSize(plan base.Physica } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalTableScan) GetPlanCostVer1(_ property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalTableScan) GetPlanCostVer1(_ property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } @@ -438,9 +439,9 @@ func (p *PhysicalTableScan) GetPlanCostVer1(_ property.TaskType, option *coreusa } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexScan) GetPlanCostVer1(_ property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalIndexScan) GetPlanCostVer1(_ property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } @@ -500,7 +501,7 @@ func (p *PhysicalIndexJoin) GetCost(outerCnt, innerCnt, outerCost, innerCost flo numPairs = 0 } } - if hasCostFlag(costFlag, coreusage.CostFlagUseTrueCardinality) { + if hasCostFlag(costFlag, costusage.CostFlagUseTrueCardinality) { numPairs = getOperatorActRows(p) } probeCost := numPairs * sessVars.GetCPUFactor() @@ -523,9 +524,9 @@ func (p *PhysicalIndexJoin) GetCost(outerCnt, innerCnt, outerCost, innerCost flo } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexJoin) GetPlanCostVer1(taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalIndexJoin) GetPlanCostVer1(taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } outerChild, innerChild := p.children[1-p.InnerChildIdx], p.children[p.InnerChildIdx] @@ -539,7 +540,7 @@ func (p *PhysicalIndexJoin) GetPlanCostVer1(taskType property.TaskType, option * } outerCnt := getCardinality(outerChild, costFlag) innerCnt := getCardinality(innerChild, costFlag) - if hasCostFlag(costFlag, coreusage.CostFlagUseTrueCardinality) && outerCnt > 0 { + if hasCostFlag(costFlag, costusage.CostFlagUseTrueCardinality) && outerCnt > 0 { innerCnt /= outerCnt // corresponding to one outer row when calculating IndexJoin costs innerCost /= outerCnt } @@ -587,7 +588,7 @@ func (p *PhysicalIndexHashJoin) GetCost(outerCnt, innerCnt, outerCost, innerCost numPairs = 0 } } - if hasCostFlag(costFlag, coreusage.CostFlagUseTrueCardinality) { + if hasCostFlag(costFlag, costusage.CostFlagUseTrueCardinality) { numPairs = getOperatorActRows(p) } // Inner workers do hash join in parallel, but they can only save ONE outer @@ -612,9 +613,9 @@ func (p *PhysicalIndexHashJoin) GetCost(outerCnt, innerCnt, outerCost, innerCost } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexHashJoin) GetPlanCostVer1(taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalIndexHashJoin) GetPlanCostVer1(taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } outerChild, innerChild := p.children[1-p.InnerChildIdx], p.children[p.InnerChildIdx] @@ -628,7 +629,7 @@ func (p *PhysicalIndexHashJoin) GetPlanCostVer1(taskType property.TaskType, opti } outerCnt := getCardinality(outerChild, costFlag) innerCnt := getCardinality(innerChild, costFlag) - if hasCostFlag(costFlag, coreusage.CostFlagUseTrueCardinality) && outerCnt > 0 { + if hasCostFlag(costFlag, costusage.CostFlagUseTrueCardinality) && outerCnt > 0 { innerCnt /= outerCnt // corresponding to one outer row when calculating IndexJoin costs innerCost /= outerCnt } @@ -678,7 +679,7 @@ func (p *PhysicalIndexMergeJoin) GetCost(outerCnt, innerCnt, outerCost, innerCos numPairs = 0 } } - if hasCostFlag(costFlag, coreusage.CostFlagUseTrueCardinality) { + if hasCostFlag(costFlag, costusage.CostFlagUseTrueCardinality) { numPairs = getOperatorActRows(p) } avgProbeCnt := numPairs / outerCnt @@ -703,9 +704,9 @@ func (p *PhysicalIndexMergeJoin) GetCost(outerCnt, innerCnt, outerCost, innerCos } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexMergeJoin) GetPlanCostVer1(taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalIndexMergeJoin) GetPlanCostVer1(taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } outerChild, innerChild := p.children[1-p.InnerChildIdx], p.children[p.InnerChildIdx] @@ -719,7 +720,7 @@ func (p *PhysicalIndexMergeJoin) GetPlanCostVer1(taskType property.TaskType, opt } outerCnt := getCardinality(outerChild, costFlag) innerCnt := getCardinality(innerChild, costFlag) - if hasCostFlag(costFlag, coreusage.CostFlagUseTrueCardinality) && outerCnt > 0 { + if hasCostFlag(costFlag, costusage.CostFlagUseTrueCardinality) && outerCnt > 0 { innerCnt /= outerCnt // corresponding to one outer row when calculating IndexJoin costs innerCost /= outerCnt } @@ -756,9 +757,9 @@ func (p *PhysicalApply) GetCost(lCount, rCount, lCost, rCost float64) float64 { } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalApply) GetPlanCostVer1(taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalApply) GetPlanCostVer1(taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } outerChild, innerChild := p.children[1-p.InnerChildIdx], p.children[p.InnerChildIdx] @@ -804,7 +805,7 @@ func (p *PhysicalMergeJoin) GetCost(lCnt, rCnt float64, costFlag uint64) float64 numPairs = 0 } } - if hasCostFlag(costFlag, coreusage.CostFlagUseTrueCardinality) { + if hasCostFlag(costFlag, costusage.CostFlagUseTrueCardinality) { numPairs = getOperatorActRows(p) } sessVars := p.SCtx().GetSessionVars() @@ -824,9 +825,9 @@ func (p *PhysicalMergeJoin) GetCost(lCnt, rCnt float64, costFlag uint64) float64 } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalMergeJoin) GetPlanCostVer1(taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalMergeJoin) GetPlanCostVer1(taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } p.planCost = 0 @@ -843,7 +844,7 @@ func (p *PhysicalMergeJoin) GetPlanCostVer1(taskType property.TaskType, option * } // GetCost computes cost of hash join operator itself. -func (p *PhysicalHashJoin) GetCost(lCnt, rCnt float64, _ bool, costFlag uint64, op *coreusage.PhysicalOptimizeOp) float64 { +func (p *PhysicalHashJoin) GetCost(lCnt, rCnt float64, _ bool, costFlag uint64, op *optimizetrace.PhysicalOptimizeOp) float64 { buildCnt, probeCnt := lCnt, rCnt build := p.children[0] // Taking the right as the inner for right join or using the outer to build a hash table. @@ -887,7 +888,7 @@ func (p *PhysicalHashJoin) GetCost(lCnt, rCnt float64, _ bool, costFlag uint64, numPairs = 0 } } - if hasCostFlag(costFlag, coreusage.CostFlagUseTrueCardinality) { + if hasCostFlag(costFlag, costusage.CostFlagUseTrueCardinality) { numPairs = getOperatorActRows(p) } // Cost of querying hash table is cheap actually, so we just compute the cost of @@ -931,9 +932,9 @@ func (p *PhysicalHashJoin) GetCost(lCnt, rCnt float64, _ bool, costFlag uint64, } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalHashJoin) GetPlanCostVer1(taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalHashJoin) GetPlanCostVer1(taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } p.planCost = 0 @@ -966,9 +967,9 @@ func (p *PhysicalStreamAgg) GetCost(inputRows float64, isRoot, _ bool, costFlag } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalStreamAgg) GetPlanCostVer1(taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalStreamAgg) GetPlanCostVer1(taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } childCost, err := p.children[0].GetPlanCostVer1(taskType, option) @@ -1007,9 +1008,9 @@ func (p *PhysicalHashAgg) GetCost(inputRows float64, isRoot, isMPP bool, costFla } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalHashAgg) GetPlanCostVer1(taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalHashAgg) GetPlanCostVer1(taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } childCost, err := p.children[0].GetPlanCostVer1(taskType, option) @@ -1055,9 +1056,9 @@ func (p *PhysicalSort) GetCost(count float64, schema *expression.Schema) float64 } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalSort) GetPlanCostVer1(taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalSort) GetPlanCostVer1(taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } childCost, err := p.children[0].GetPlanCostVer1(taskType, option) @@ -1094,9 +1095,9 @@ func (p *PhysicalTopN) GetCost(count float64, isRoot bool) float64 { } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalTopN) GetPlanCostVer1(taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalTopN) GetPlanCostVer1(taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } childCost, err := p.children[0].GetPlanCostVer1(taskType, option) @@ -1110,7 +1111,7 @@ func (p *PhysicalTopN) GetPlanCostVer1(taskType property.TaskType, option *coreu } // GetCost returns cost of the PointGetPlan. -func (p *BatchPointGetPlan) GetCost(opt *coreusage.PhysicalOptimizeOp) float64 { +func (p *BatchPointGetPlan) GetCost(opt *optimizetrace.PhysicalOptimizeOp) float64 { cols := p.accessCols if cols == nil { return 0 // the cost of BatchGet generated in fast plan optimization is always 0 @@ -1138,9 +1139,9 @@ func (p *BatchPointGetPlan) GetCost(opt *coreusage.PhysicalOptimizeOp) float64 { } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *BatchPointGetPlan) GetPlanCostVer1(_ property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *BatchPointGetPlan) GetPlanCostVer1(_ property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } p.planCost = p.GetCost(option.GetTracer()) @@ -1161,7 +1162,7 @@ func (p *BatchPointGetPlan) GetAvgRowSize() float64 { } // GetCost returns cost of the PointGetPlan. -func (p *PointGetPlan) GetCost(opt *coreusage.PhysicalOptimizeOp) float64 { +func (p *PointGetPlan) GetCost(opt *optimizetrace.PhysicalOptimizeOp) float64 { cols := p.accessCols if cols == nil { return 0 // the cost of PointGet generated in fast plan optimization is always 0 @@ -1186,9 +1187,9 @@ func (p *PointGetPlan) GetCost(opt *coreusage.PhysicalOptimizeOp) float64 { } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PointGetPlan) GetPlanCostVer1(_ property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PointGetPlan) GetPlanCostVer1(_ property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } p.planCost = p.GetCost(option.GetTracer()) @@ -1209,9 +1210,9 @@ func (p *PointGetPlan) GetAvgRowSize() float64 { } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalUnionAll) GetPlanCostVer1(taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalUnionAll) GetPlanCostVer1(taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } var childMaxCost float64 @@ -1228,9 +1229,9 @@ func (p *PhysicalUnionAll) GetPlanCostVer1(taskType property.TaskType, option *c } // GetPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalExchangeReceiver) GetPlanCostVer1(taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func (p *PhysicalExchangeReceiver) GetPlanCostVer1(taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { costFlag := option.CostFlag - if p.planCostInit && !hasCostFlag(costFlag, coreusage.CostFlagRecalculate) { + if p.planCostInit && !hasCostFlag(costFlag, costusage.CostFlagRecalculate) { return p.planCost, nil } childCost, err := p.children[0].GetPlanCostVer1(taskType, option) @@ -1262,7 +1263,7 @@ func getOperatorActRows(operator base.PhysicalPlan) float64 { } func getCardinality(operator base.PhysicalPlan, costFlag uint64) float64 { - if hasCostFlag(costFlag, coreusage.CostFlagUseTrueCardinality) { + if hasCostFlag(costFlag, costusage.CostFlagUseTrueCardinality) { actualProbeCnt := operator.GetActualProbeCnt(operator.SCtx().GetSessionVars().StmtCtx.RuntimeStatsColl) if actualProbeCnt == 0 { return 0 diff --git a/pkg/planner/core/plan_cost_ver2.go b/pkg/planner/core/plan_cost_ver2.go index 176d0f3577f26..dbaa4cbee358e 100644 --- a/pkg/planner/core/plan_cost_ver2.go +++ b/pkg/planner/core/plan_cost_ver2.go @@ -26,24 +26,25 @@ import ( "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/property" "github.com/pingcap/tidb/pkg/planner/util" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/costusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/util/paging" "github.com/pingcap/tipb/go-tipb" ) // GetPlanCost returns the cost of this plan. -func GetPlanCost(p base.PhysicalPlan, taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func GetPlanCost(p base.PhysicalPlan, taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { return getPlanCost(p, taskType, option) } // GenPlanCostTrace define a hook function to customize the cost calculation. -var GenPlanCostTrace func(p base.PhysicalPlan, costV *coreusage.CostVer2, taskType property.TaskType, option *coreusage.PlanCostOption) +var GenPlanCostTrace func(p base.PhysicalPlan, costV *costusage.CostVer2, taskType property.TaskType, option *optimizetrace.PlanCostOption) -func getPlanCost(p base.PhysicalPlan, taskType property.TaskType, option *coreusage.PlanCostOption) (float64, error) { +func getPlanCost(p base.PhysicalPlan, taskType property.TaskType, option *optimizetrace.PlanCostOption) (float64, error) { if p.SCtx().GetSessionVars().CostModelVersion == modelVer2 { planCost, err := p.GetPlanCostVer2(taskType, option) - if coreusage.TraceCost(option) && GenPlanCostTrace != nil { + if costusage.TraceCost(option) && GenPlanCostTrace != nil { GenPlanCostTrace(p, &planCost, taskType, option) } return planCost.GetCost(), err @@ -52,22 +53,22 @@ func getPlanCost(p base.PhysicalPlan, taskType property.TaskType, option *coreus } // GetPlanCostVer2 calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *basePhysicalPlan) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *basePhysicalPlan) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } - childCosts := make([]coreusage.CostVer2, 0, len(p.children)) + childCosts := make([]costusage.CostVer2, 0, len(p.children)) for _, child := range p.children { childCost, err := child.GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } childCosts = append(childCosts, childCost) } if len(childCosts) == 0 { - p.planCostVer2 = coreusage.NewZeroCostVer2(coreusage.TraceCost(option)) + p.planCostVer2 = costusage.NewZeroCostVer2(costusage.TraceCost(option)) } else { - p.planCostVer2 = coreusage.SumCostVer2(childCosts...) + p.planCostVer2 = costusage.SumCostVer2(childCosts...) } p.planCostInit = true return p.planCostVer2, nil @@ -75,8 +76,8 @@ func (p *basePhysicalPlan) GetPlanCostVer2(taskType property.TaskType, option *c // GetPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = child-cost + filter-cost -func (p *PhysicalSelection) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalSelection) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } @@ -87,10 +88,10 @@ func (p *PhysicalSelection) GetPlanCostVer2(taskType property.TaskType, option * childCost, err := p.children[0].GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } - p.planCostVer2 = coreusage.SumCostVer2(filterCost, childCost) + p.planCostVer2 = costusage.SumCostVer2(filterCost, childCost) p.planCostInit = true return p.planCostVer2, nil } @@ -98,8 +99,8 @@ func (p *PhysicalSelection) GetPlanCostVer2(taskType property.TaskType, option * // GetPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = child-cost + proj-cost / concurrency // proj-cost = input-rows * len(expressions) * cpu-factor -func (p *PhysicalProjection) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalProjection) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } @@ -114,10 +115,10 @@ func (p *PhysicalProjection) GetPlanCostVer2(taskType property.TaskType, option childCost, err := p.children[0].GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } - p.planCostVer2 = coreusage.SumCostVer2(childCost, coreusage.DivCostVer2(projCost, concurrency)) + p.planCostVer2 = costusage.SumCostVer2(childCost, costusage.DivCostVer2(projCost, concurrency)) p.planCostInit = true return p.planCostVer2, nil } @@ -125,8 +126,8 @@ func (p *PhysicalProjection) GetPlanCostVer2(taskType property.TaskType, option // GetPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = rows * log2(row-size) * scan-factor // log2(row-size) is from experiments. -func (p *PhysicalIndexScan) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalIndexScan) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } @@ -142,8 +143,8 @@ func (p *PhysicalIndexScan) GetPlanCostVer2(taskType property.TaskType, option * // GetPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = rows * log2(row-size) * scan-factor // log2(row-size) is from experiments. -func (p *PhysicalTableScan) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalTableScan) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } @@ -161,7 +162,7 @@ func (p *PhysicalTableScan) GetPlanCostVer2(taskType property.TaskType, option * // give TiFlash a start-up cost to let the optimizer prefers to use TiKV to process small table scans. if p.StoreType == kv.TiFlash { - p.planCostVer2 = coreusage.SumCostVer2(p.planCostVer2, scanCostVer2(option, 10000, rowSize, scanFactor)) + p.planCostVer2 = costusage.SumCostVer2(p.planCostVer2, scanCostVer2(option, 10000, rowSize, scanFactor)) } p.planCostInit = true @@ -171,8 +172,8 @@ func (p *PhysicalTableScan) GetPlanCostVer2(taskType property.TaskType, option * // GetPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = (child-cost + net-cost) / concurrency // net-cost = rows * row-size * net-factor -func (p *PhysicalIndexReader) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalIndexReader) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } @@ -185,10 +186,10 @@ func (p *PhysicalIndexReader) GetPlanCostVer2(taskType property.TaskType, option childCost, err := p.indexPlan.GetPlanCostVer2(property.CopSingleReadTaskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } - p.planCostVer2 = coreusage.DivCostVer2(coreusage.SumCostVer2(childCost, netCost), concurrency) + p.planCostVer2 = costusage.DivCostVer2(costusage.SumCostVer2(childCost, netCost), concurrency) p.planCostInit = true return p.planCostVer2, nil } @@ -196,8 +197,8 @@ func (p *PhysicalIndexReader) GetPlanCostVer2(taskType property.TaskType, option // GetPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = (child-cost + net-cost) / concurrency // net-cost = rows * row-size * net-factor -func (p *PhysicalTableReader) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalTableReader) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } @@ -214,16 +215,16 @@ func (p *PhysicalTableReader) GetPlanCostVer2(taskType property.TaskType, option childCost, err := p.tablePlan.GetPlanCostVer2(childType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } - p.planCostVer2 = coreusage.DivCostVer2(coreusage.SumCostVer2(childCost, netCost), concurrency) + p.planCostVer2 = costusage.DivCostVer2(costusage.SumCostVer2(childCost, netCost), concurrency) p.planCostInit = true // consider tidb_enforce_mpp if p.StoreType == kv.TiFlash && p.SCtx().GetSessionVars().IsMPPEnforced() && - !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { // show the real cost in explain-statements - p.planCostVer2 = coreusage.DivCostVer2(p.planCostVer2, 1000000000) + !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { // show the real cost in explain-statements + p.planCostVer2 = costusage.DivCostVer2(p.planCostVer2, 1000000000) } return p.planCostVer2, nil } @@ -236,8 +237,8 @@ func (p *PhysicalTableReader) GetPlanCostVer2(taskType property.TaskType, option // double-read-request-cost = double-read-tasks * request-factor // double-read-cpu-cost = index-rows * cpu-factor // double-read-tasks = index-rows / batch-size * task-per-batch # task-per-batch is a magic number now -func (p *PhysicalIndexLookUpReader) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalIndexLookUpReader) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } @@ -255,34 +256,34 @@ func (p *PhysicalIndexLookUpReader) GetPlanCostVer2(taskType property.TaskType, indexNetCost := netCostVer2(option, indexRows, indexRowSize, netFactor) indexChildCost, err := p.indexPlan.GetPlanCostVer2(property.CopMultiReadTaskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } - indexSideCost := coreusage.DivCostVer2(coreusage.SumCostVer2(indexNetCost, indexChildCost), distConcurrency) + indexSideCost := costusage.DivCostVer2(costusage.SumCostVer2(indexNetCost, indexChildCost), distConcurrency) // table-side tableNetCost := netCostVer2(option, tableRows, tableRowSize, netFactor) tableChildCost, err := p.tablePlan.GetPlanCostVer2(property.CopMultiReadTaskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } - tableSideCost := coreusage.DivCostVer2(coreusage.SumCostVer2(tableNetCost, tableChildCost), distConcurrency) + tableSideCost := costusage.DivCostVer2(costusage.SumCostVer2(tableNetCost, tableChildCost), distConcurrency) doubleReadRows := indexRows - doubleReadCPUCost := coreusage.NewCostVer2(option, cpuFactor, + doubleReadCPUCost := costusage.NewCostVer2(option, cpuFactor, indexRows*cpuFactor.Value, func() string { return fmt.Sprintf("double-read-cpu(%v*%v)", doubleReadRows, cpuFactor) }) batchSize := float64(p.SCtx().GetSessionVars().IndexLookupSize) taskPerBatch := 32.0 // TODO: remove this magic number doubleReadTasks := doubleReadRows / batchSize * taskPerBatch doubleReadRequestCost := doubleReadCostVer2(option, doubleReadTasks, requestFactor) - doubleReadCost := coreusage.SumCostVer2(doubleReadCPUCost, doubleReadRequestCost) + doubleReadCost := costusage.SumCostVer2(doubleReadCPUCost, doubleReadRequestCost) - p.planCostVer2 = coreusage.SumCostVer2(indexSideCost, coreusage.DivCostVer2(coreusage.SumCostVer2(tableSideCost, doubleReadCost), doubleReadConcurrency)) + p.planCostVer2 = costusage.SumCostVer2(indexSideCost, costusage.DivCostVer2(costusage.SumCostVer2(tableSideCost, doubleReadCost), doubleReadConcurrency)) if p.SCtx().GetSessionVars().EnablePaging && p.expectedCnt > 0 && p.expectedCnt <= paging.Threshold { // if the expectCnt is below the paging threshold, using paging API p.Paging = true // TODO: move this operation from cost model to physical optimization - p.planCostVer2 = coreusage.MulCostVer2(p.planCostVer2, 0.6) + p.planCostVer2 = costusage.MulCostVer2(p.planCostVer2, 0.6) } p.planCostInit = true @@ -293,15 +294,15 @@ func (p *PhysicalIndexLookUpReader) GetPlanCostVer2(taskType property.TaskType, // plan-cost = table-side-cost + sum(index-side-cost) // index-side-cost = (index-child-cost + index-net-cost) / dist-concurrency # same with IndexReader // table-side-cost = (table-child-cost + table-net-cost) / dist-concurrency # same with TableReader -func (p *PhysicalIndexMergeReader) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalIndexMergeReader) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } netFactor := getTaskNetFactorVer2(p, taskType) distConcurrency := float64(p.SCtx().GetSessionVars().DistSQLScanConcurrency()) - var tableSideCost coreusage.CostVer2 + var tableSideCost costusage.CostVer2 if tablePath := p.tablePlan; tablePath != nil { rows := getCardinality(tablePath, option.CostFlag) rowSize := getAvgRowSize(tablePath.StatsInfo(), tablePath.Schema().Columns) @@ -309,12 +310,12 @@ func (p *PhysicalIndexMergeReader) GetPlanCostVer2(taskType property.TaskType, o tableNetCost := netCostVer2(option, rows, rowSize, netFactor) tableChildCost, err := tablePath.GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } - tableSideCost = coreusage.DivCostVer2(coreusage.SumCostVer2(tableNetCost, tableChildCost), distConcurrency) + tableSideCost = costusage.DivCostVer2(costusage.SumCostVer2(tableNetCost, tableChildCost), distConcurrency) } - indexSideCost := make([]coreusage.CostVer2, 0, len(p.partialPlans)) + indexSideCost := make([]costusage.CostVer2, 0, len(p.partialPlans)) for _, indexPath := range p.partialPlans { rows := getCardinality(indexPath, option.CostFlag) rowSize := getAvgRowSize(indexPath.StatsInfo(), indexPath.Schema().Columns) @@ -322,14 +323,14 @@ func (p *PhysicalIndexMergeReader) GetPlanCostVer2(taskType property.TaskType, o indexNetCost := netCostVer2(option, rows, rowSize, netFactor) indexChildCost, err := indexPath.GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } indexSideCost = append(indexSideCost, - coreusage.DivCostVer2(coreusage.SumCostVer2(indexNetCost, indexChildCost), distConcurrency)) + costusage.DivCostVer2(costusage.SumCostVer2(indexNetCost, indexChildCost), distConcurrency)) } - sumIndexSideCost := coreusage.SumCostVer2(indexSideCost...) + sumIndexSideCost := costusage.SumCostVer2(indexSideCost...) - p.planCostVer2 = coreusage.SumCostVer2(tableSideCost, sumIndexSideCost) + p.planCostVer2 = costusage.SumCostVer2(tableSideCost, sumIndexSideCost) // give a bias to pushDown limit, since it will get the same cost with NON_PUSH_DOWN_LIMIT case via expect count. // push down limit case may reduce cop request consumption if any in some cases. // @@ -342,7 +343,7 @@ func (p *PhysicalIndexMergeReader) GetPlanCostVer2(taskType property.TaskType, o // will have the same cost, actually if limit are more close to the fetch side, the fewer rows that table plan need to read. // todo: refine the cost computation out from cost model. if p.PushedLimit != nil { - p.planCostVer2 = coreusage.MulCostVer2(p.planCostVer2, 0.99) + p.planCostVer2 = costusage.MulCostVer2(p.planCostVer2, 0.99) } p.planCostInit = true return p.planCostVer2, nil @@ -357,8 +358,8 @@ func (p *PhysicalIndexMergeReader) GetPlanCostVer2(taskType property.TaskType, o // else if spill: // 1. sort-mem-cost = mem-quota * mem-factor // 2. sort-disk-cost = rows * row-size * disk-factor -func (p *PhysicalSort) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalSort) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } @@ -376,27 +377,27 @@ func (p *PhysicalSort) GetPlanCostVer2(taskType property.TaskType, option *coreu sortCPUCost := orderCostVer2(option, rows, rows, p.ByItems, cpuFactor) - var sortMemCost, sortDiskCost coreusage.CostVer2 + var sortMemCost, sortDiskCost costusage.CostVer2 if !spill { - sortMemCost = coreusage.NewCostVer2(option, memFactor, + sortMemCost = costusage.NewCostVer2(option, memFactor, rows*rowSize*memFactor.Value, func() string { return fmt.Sprintf("sortMem(%v*%v*%v)", rows, rowSize, memFactor) }) - sortDiskCost = coreusage.ZeroCostVer2 + sortDiskCost = costusage.ZeroCostVer2 } else { - sortMemCost = coreusage.NewCostVer2(option, memFactor, + sortMemCost = costusage.NewCostVer2(option, memFactor, float64(memQuota)*memFactor.Value, func() string { return fmt.Sprintf("sortMem(%v*%v)", memQuota, memFactor) }) - sortDiskCost = coreusage.NewCostVer2(option, diskFactor, + sortDiskCost = costusage.NewCostVer2(option, diskFactor, rows*rowSize*diskFactor.Value, func() string { return fmt.Sprintf("sortDisk(%v*%v*%v)", rows, rowSize, diskFactor) }) } childCost, err := p.children[0].GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } - p.planCostVer2 = coreusage.SumCostVer2(childCost, sortCPUCost, sortMemCost, sortDiskCost) + p.planCostVer2 = costusage.SumCostVer2(childCost, sortCPUCost, sortMemCost, sortDiskCost) p.planCostInit = true return p.planCostVer2, nil } @@ -405,8 +406,8 @@ func (p *PhysicalSort) GetPlanCostVer2(taskType property.TaskType, option *coreu // plan-cost = child-cost + topn-cpu-cost + topn-mem-cost // topn-cpu-cost = rows * log2(N) * len(sort-items) * cpu-factor // topn-mem-cost = N * row-size * mem-factor -func (p *PhysicalTopN) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalTopN) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } @@ -422,24 +423,24 @@ func (p *PhysicalTopN) GetPlanCostVer2(taskType property.TaskType, option *coreu memFactor := getTaskMemFactorVer2(p, taskType) topNCPUCost := orderCostVer2(option, rows, n, p.ByItems, cpuFactor) - topNMemCost := coreusage.NewCostVer2(option, memFactor, + topNMemCost := costusage.NewCostVer2(option, memFactor, n*rowSize*memFactor.Value, func() string { return fmt.Sprintf("topMem(%v*%v*%v)", n, rowSize, memFactor) }) childCost, err := p.children[0].GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } - p.planCostVer2 = coreusage.SumCostVer2(childCost, topNCPUCost, topNMemCost) + p.planCostVer2 = costusage.SumCostVer2(childCost, topNCPUCost, topNMemCost) p.planCostInit = true return p.planCostVer2, nil } // GetPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = child-cost + agg-cost + group-cost -func (p *PhysicalStreamAgg) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalStreamAgg) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } @@ -451,18 +452,18 @@ func (p *PhysicalStreamAgg) GetPlanCostVer2(taskType property.TaskType, option * childCost, err := p.children[0].GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } - p.planCostVer2 = coreusage.SumCostVer2(childCost, aggCost, groupCost) + p.planCostVer2 = costusage.SumCostVer2(childCost, aggCost, groupCost) p.planCostInit = true return p.planCostVer2, nil } // GetPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = child-cost + (agg-cost + group-cost + hash-build-cost + hash-probe-cost) / concurrency -func (p *PhysicalHashAgg) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalHashAgg) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } @@ -477,24 +478,24 @@ func (p *PhysicalHashAgg) GetPlanCostVer2(taskType property.TaskType, option *co groupCost := groupCostVer2(option, inputRows, p.GroupByItems, cpuFactor) hashBuildCost := hashBuildCostVer2(option, outputRows, outputRowSize, float64(len(p.GroupByItems)), cpuFactor, memFactor) hashProbeCost := hashProbeCostVer2(option, inputRows, float64(len(p.GroupByItems)), cpuFactor) - startCost := coreusage.NewCostVer2(option, cpuFactor, + startCost := costusage.NewCostVer2(option, cpuFactor, 10*3*cpuFactor.Value, // 10rows * 3func * cpuFactor func() string { return fmt.Sprintf("cpu(10*3*%v)", cpuFactor) }) childCost, err := p.children[0].GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } - p.planCostVer2 = coreusage.SumCostVer2(startCost, childCost, coreusage.DivCostVer2(coreusage.SumCostVer2(aggCost, groupCost, hashBuildCost, hashProbeCost), concurrency)) + p.planCostVer2 = costusage.SumCostVer2(startCost, childCost, costusage.DivCostVer2(costusage.SumCostVer2(aggCost, groupCost, hashBuildCost, hashProbeCost), concurrency)) p.planCostInit = true return p.planCostVer2, nil } // GetPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = left-child-cost + right-child-cost + filter-cost + group-cost -func (p *PhysicalMergeJoin) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalMergeJoin) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } @@ -502,21 +503,21 @@ func (p *PhysicalMergeJoin) GetPlanCostVer2(taskType property.TaskType, option * rightRows := getCardinality(p.children[1], option.CostFlag) cpuFactor := getTaskCPUFactorVer2(p, taskType) - filterCost := coreusage.SumCostVer2(filterCostVer2(option, leftRows, p.LeftConditions, cpuFactor), + filterCost := costusage.SumCostVer2(filterCostVer2(option, leftRows, p.LeftConditions, cpuFactor), filterCostVer2(option, rightRows, p.RightConditions, cpuFactor)) - groupCost := coreusage.SumCostVer2(groupCostVer2(option, leftRows, cols2Exprs(p.LeftJoinKeys), cpuFactor), + groupCost := costusage.SumCostVer2(groupCostVer2(option, leftRows, cols2Exprs(p.LeftJoinKeys), cpuFactor), groupCostVer2(option, rightRows, cols2Exprs(p.LeftJoinKeys), cpuFactor)) leftChildCost, err := p.children[0].GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } rightChildCost, err := p.children[1].GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } - p.planCostVer2 = coreusage.SumCostVer2(leftChildCost, rightChildCost, filterCost, groupCost) + p.planCostVer2 = costusage.SumCostVer2(leftChildCost, rightChildCost, filterCost, groupCost) p.planCostInit = true return p.planCostVer2, nil } @@ -525,8 +526,8 @@ func (p *PhysicalMergeJoin) GetPlanCostVer2(taskType property.TaskType, option * // plan-cost = build-child-cost + probe-child-cost + // build-hash-cost + build-filter-cost + // (probe-filter-cost + probe-hash-cost) / concurrency -func (p *PhysicalHashJoin) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalHashJoin) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } @@ -553,29 +554,29 @@ func (p *PhysicalHashJoin) GetPlanCostVer2(taskType property.TaskType, option *c buildChildCost, err := build.GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } probeChildCost, err := probe.GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } if taskType == property.MppTaskType { // BCast or Shuffle Join, use mppConcurrency - p.planCostVer2 = coreusage.SumCostVer2(buildChildCost, probeChildCost, - coreusage.DivCostVer2(coreusage.SumCostVer2(buildHashCost, buildFilterCost, probeHashCost, probeFilterCost), mppConcurrency)) + p.planCostVer2 = costusage.SumCostVer2(buildChildCost, probeChildCost, + costusage.DivCostVer2(costusage.SumCostVer2(buildHashCost, buildFilterCost, probeHashCost, probeFilterCost), mppConcurrency)) } else { // TiDB HashJoin - startCost := coreusage.NewCostVer2(option, cpuFactor, + startCost := costusage.NewCostVer2(option, cpuFactor, 10*3*cpuFactor.Value, // 10rows * 3func * cpuFactor func() string { return fmt.Sprintf("cpu(10*3*%v)", cpuFactor) }) - p.planCostVer2 = coreusage.SumCostVer2(startCost, buildChildCost, probeChildCost, buildHashCost, buildFilterCost, - coreusage.DivCostVer2(coreusage.SumCostVer2(probeFilterCost, probeHashCost), tidbConcurrency)) + p.planCostVer2 = costusage.SumCostVer2(startCost, buildChildCost, probeChildCost, buildHashCost, buildFilterCost, + costusage.DivCostVer2(costusage.SumCostVer2(probeFilterCost, probeHashCost), tidbConcurrency)) } p.planCostInit = true return p.planCostVer2, nil } -func (p *PhysicalIndexJoin) getIndexJoinCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption, indexJoinType int) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalIndexJoin) getIndexJoinCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption, indexJoinType int) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } @@ -594,27 +595,27 @@ func (p *PhysicalIndexJoin) getIndexJoinCostVer2(taskType property.TaskType, opt buildFilterCost := filterCostVer2(option, buildRows, buildFilters, cpuFactor) buildChildCost, err := build.GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } - buildTaskCost := coreusage.NewCostVer2(option, cpuFactor, + buildTaskCost := costusage.NewCostVer2(option, cpuFactor, buildRows*10*cpuFactor.Value, func() string { return fmt.Sprintf("cpu(%v*10*%v)", buildRows, cpuFactor) }) - startCost := coreusage.NewCostVer2(option, cpuFactor, + startCost := costusage.NewCostVer2(option, cpuFactor, 10*3*cpuFactor.Value, func() string { return fmt.Sprintf("cpu(10*3*%v)", cpuFactor) }) probeFilterCost := filterCostVer2(option, probeRowsTot, probeFilters, cpuFactor) probeChildCost, err := probe.GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } - var hashTableCost coreusage.CostVer2 + var hashTableCost costusage.CostVer2 switch indexJoinType { case 1: // IndexHashJoin hashTableCost = hashBuildCostVer2(option, buildRows, buildRowSize, float64(len(p.RightJoinKeys)), cpuFactor, memFactor) case 2: // IndexMergeJoin - hashTableCost = coreusage.NewZeroCostVer2(coreusage.TraceCost(option)) + hashTableCost = costusage.NewZeroCostVer2(costusage.TraceCost(option)) default: // IndexJoin hashTableCost = hashBuildCostVer2(option, probeRowsTot, probeRowSize, float64(len(p.LeftJoinKeys)), cpuFactor, memFactor) } @@ -624,19 +625,19 @@ func (p *PhysicalIndexJoin) getIndexJoinCostVer2(taskType property.TaskType, opt // Use an empirical value batchRatio to handle this now. // TODO: remove this empirical value. batchRatio := 6.0 - probeCost := coreusage.DivCostVer2(coreusage.MulCostVer2(probeChildCost, buildRows), batchRatio) + probeCost := costusage.DivCostVer2(costusage.MulCostVer2(probeChildCost, buildRows), batchRatio) // Double Read Cost - doubleReadCost := coreusage.NewZeroCostVer2(coreusage.TraceCost(option)) + doubleReadCost := costusage.NewZeroCostVer2(costusage.TraceCost(option)) if p.SCtx().GetSessionVars().IndexJoinDoubleReadPenaltyCostRate > 0 { batchSize := float64(p.SCtx().GetSessionVars().IndexJoinBatchSize) taskPerBatch := 1024.0 // TODO: remove this magic number doubleReadTasks := buildRows / batchSize * taskPerBatch doubleReadCost = doubleReadCostVer2(option, doubleReadTasks, requestFactor) - doubleReadCost = coreusage.MulCostVer2(doubleReadCost, p.SCtx().GetSessionVars().IndexJoinDoubleReadPenaltyCostRate) + doubleReadCost = costusage.MulCostVer2(doubleReadCost, p.SCtx().GetSessionVars().IndexJoinDoubleReadPenaltyCostRate) } - p.planCostVer2 = coreusage.SumCostVer2(startCost, buildChildCost, buildFilterCost, buildTaskCost, coreusage.DivCostVer2(coreusage.SumCostVer2(doubleReadCost, probeCost, probeFilterCost, hashTableCost), probeConcurrency)) + p.planCostVer2 = costusage.SumCostVer2(startCost, buildChildCost, buildFilterCost, buildTaskCost, costusage.DivCostVer2(costusage.SumCostVer2(doubleReadCost, probeCost, probeFilterCost, hashTableCost), probeConcurrency)) p.planCostInit = true return p.planCostVer2, nil } @@ -645,25 +646,25 @@ func (p *PhysicalIndexJoin) getIndexJoinCostVer2(taskType property.TaskType, opt // plan-cost = build-child-cost + build-filter-cost + // (probe-cost + probe-filter-cost) / concurrency // probe-cost = probe-child-cost * build-rows / batchRatio -func (p *PhysicalIndexJoin) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { +func (p *PhysicalIndexJoin) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { return p.getIndexJoinCostVer2(taskType, option, 0) } // GetPlanCostVer2 implements PhysicalPlan interface. -func (p *PhysicalIndexHashJoin) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { +func (p *PhysicalIndexHashJoin) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { return p.getIndexJoinCostVer2(taskType, option, 1) } // GetPlanCostVer2 implements PhysicalPlan interface. -func (p *PhysicalIndexMergeJoin) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { +func (p *PhysicalIndexMergeJoin) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { return p.getIndexJoinCostVer2(taskType, option, 2) } // GetPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = build-child-cost + build-filter-cost + probe-cost + probe-filter-cost // probe-cost = probe-child-cost * build-rows -func (p *PhysicalApply) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalApply) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } @@ -675,46 +676,46 @@ func (p *PhysicalApply) GetPlanCostVer2(taskType property.TaskType, option *core buildFilterCost := filterCostVer2(option, buildRows, p.LeftConditions, cpuFactor) buildChildCost, err := p.children[0].GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } probeFilterCost := filterCostVer2(option, probeRowsTot, p.RightConditions, cpuFactor) probeChildCost, err := p.children[1].GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } - probeCost := coreusage.MulCostVer2(probeChildCost, buildRows) + probeCost := costusage.MulCostVer2(probeChildCost, buildRows) - p.planCostVer2 = coreusage.SumCostVer2(buildChildCost, buildFilterCost, probeCost, probeFilterCost) + p.planCostVer2 = costusage.SumCostVer2(buildChildCost, buildFilterCost, probeCost, probeFilterCost) p.planCostInit = true return p.planCostVer2, nil } // GetPlanCostVer2 calculates the cost of the plan if it has not been calculated yet and returns the cost. // plan-cost = sum(child-cost) / concurrency -func (p *PhysicalUnionAll) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalUnionAll) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } concurrency := float64(p.SCtx().GetSessionVars().UnionConcurrency()) - childCosts := make([]coreusage.CostVer2, 0, len(p.children)) + childCosts := make([]costusage.CostVer2, 0, len(p.children)) for _, child := range p.children { childCost, err := child.GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } childCosts = append(childCosts, childCost) } - p.planCostVer2 = coreusage.DivCostVer2(coreusage.SumCostVer2(childCosts...), concurrency) + p.planCostVer2 = costusage.DivCostVer2(costusage.SumCostVer2(childCosts...), concurrency) p.planCostInit = true return p.planCostVer2, nil } // GetPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = child-cost + net-cost -func (p *PhysicalExchangeReceiver) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalExchangeReceiver) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } @@ -729,28 +730,28 @@ func (p *PhysicalExchangeReceiver) GetPlanCostVer2(taskType property.TaskType, o netCost := netCostVer2(option, rows, rowSize, netFactor) if isBCast { - netCost = coreusage.MulCostVer2(netCost, numNode) + netCost = costusage.MulCostVer2(netCost, numNode) } childCost, err := p.children[0].GetPlanCostVer2(taskType, option) if err != nil { - return coreusage.ZeroCostVer2, err + return costusage.ZeroCostVer2, err } - p.planCostVer2 = coreusage.SumCostVer2(childCost, netCost) + p.planCostVer2 = costusage.SumCostVer2(childCost, netCost) p.planCostInit = true return p.planCostVer2, nil } // GetPlanCostVer2 returns the plan-cost of this sub-plan, which is: -func (p *PointGetPlan) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PointGetPlan) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } if p.accessCols == nil { // from fast plan code path - p.planCostVer2 = coreusage.ZeroCostVer2 + p.planCostVer2 = costusage.ZeroCostVer2 p.planCostInit = true - return coreusage.ZeroCostVer2, nil + return costusage.ZeroCostVer2, nil } rowSize := getAvgRowSize(p.StatsInfo(), p.schema.Columns) netFactor := getTaskNetFactorVer2(p, taskType) @@ -761,15 +762,15 @@ func (p *PointGetPlan) GetPlanCostVer2(taskType property.TaskType, option *coreu } // GetPlanCostVer2 returns the plan-cost of this sub-plan, which is: -func (p *BatchPointGetPlan) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *BatchPointGetPlan) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } if p.accessCols == nil { // from fast plan code path - p.planCostVer2 = coreusage.ZeroCostVer2 + p.planCostVer2 = costusage.ZeroCostVer2 p.planCostInit = true - return coreusage.ZeroCostVer2, nil + return costusage.ZeroCostVer2, nil } rows := getCardinality(p, option.CostFlag) rowSize := getAvgRowSize(p.StatsInfo(), p.schema.Columns) @@ -781,8 +782,8 @@ func (p *BatchPointGetPlan) GetPlanCostVer2(taskType property.TaskType, option * } // GetPlanCostVer2 implements PhysicalPlan interface. -func (p *PhysicalCTE) GetPlanCostVer2(taskType property.TaskType, option *coreusage.PlanCostOption) (coreusage.CostVer2, error) { - if p.planCostInit && !hasCostFlag(option.CostFlag, coreusage.CostFlagRecalculate) { +func (p *PhysicalCTE) GetPlanCostVer2(taskType property.TaskType, option *optimizetrace.PlanCostOption) (costusage.CostVer2, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, costusage.CostFlagRecalculate) { return p.planCostVer2, nil } @@ -796,39 +797,39 @@ func (p *PhysicalCTE) GetPlanCostVer2(taskType property.TaskType, option *coreus return p.planCostVer2, nil } -func scanCostVer2(option *coreusage.PlanCostOption, rows, rowSize float64, scanFactor coreusage.CostVer2Factor) coreusage.CostVer2 { +func scanCostVer2(option *optimizetrace.PlanCostOption, rows, rowSize float64, scanFactor costusage.CostVer2Factor) costusage.CostVer2 { if rowSize < 1 { rowSize = 1 } - return coreusage.NewCostVer2(option, scanFactor, + return costusage.NewCostVer2(option, scanFactor, // rows * log(row-size) * scanFactor, log2 from experiments rows*math.Log2(rowSize)*scanFactor.Value, func() string { return fmt.Sprintf("scan(%v*logrowsize(%v)*%v)", rows, rowSize, scanFactor) }) } -func netCostVer2(option *coreusage.PlanCostOption, rows, rowSize float64, netFactor coreusage.CostVer2Factor) coreusage.CostVer2 { - return coreusage.NewCostVer2(option, netFactor, +func netCostVer2(option *optimizetrace.PlanCostOption, rows, rowSize float64, netFactor costusage.CostVer2Factor) costusage.CostVer2 { + return costusage.NewCostVer2(option, netFactor, rows*rowSize*netFactor.Value, func() string { return fmt.Sprintf("net(%v*rowsize(%v)*%v)", rows, rowSize, netFactor) }) } -func filterCostVer2(option *coreusage.PlanCostOption, rows float64, filters []expression.Expression, cpuFactor coreusage.CostVer2Factor) coreusage.CostVer2 { +func filterCostVer2(option *optimizetrace.PlanCostOption, rows float64, filters []expression.Expression, cpuFactor costusage.CostVer2Factor) costusage.CostVer2 { numFuncs := numFunctions(filters) - return coreusage.NewCostVer2(option, cpuFactor, + return costusage.NewCostVer2(option, cpuFactor, rows*numFuncs*cpuFactor.Value, func() string { return fmt.Sprintf("cpu(%v*filters(%v)*%v)", rows, numFuncs, cpuFactor) }) } -func aggCostVer2(option *coreusage.PlanCostOption, rows float64, aggFuncs []*aggregation.AggFuncDesc, cpuFactor coreusage.CostVer2Factor) coreusage.CostVer2 { - return coreusage.NewCostVer2(option, cpuFactor, +func aggCostVer2(option *optimizetrace.PlanCostOption, rows float64, aggFuncs []*aggregation.AggFuncDesc, cpuFactor costusage.CostVer2Factor) costusage.CostVer2 { + return costusage.NewCostVer2(option, cpuFactor, // TODO: consider types of agg-funcs rows*float64(len(aggFuncs))*cpuFactor.Value, func() string { return fmt.Sprintf("agg(%v*aggs(%v)*%v)", rows, len(aggFuncs), cpuFactor) }) } -func groupCostVer2(option *coreusage.PlanCostOption, rows float64, groupItems []expression.Expression, cpuFactor coreusage.CostVer2Factor) coreusage.CostVer2 { +func groupCostVer2(option *optimizetrace.PlanCostOption, rows float64, groupItems []expression.Expression, cpuFactor costusage.CostVer2Factor) costusage.CostVer2 { numFuncs := numFunctions(groupItems) - return coreusage.NewCostVer2(option, cpuFactor, + return costusage.NewCostVer2(option, cpuFactor, rows*numFuncs*cpuFactor.Value, func() string { return fmt.Sprintf("group(%v*cols(%v)*%v)", rows, numFuncs, cpuFactor) }) } @@ -845,97 +846,97 @@ func numFunctions(exprs []expression.Expression) float64 { return num } -func orderCostVer2(option *coreusage.PlanCostOption, rows, n float64, byItems []*util.ByItems, cpuFactor coreusage.CostVer2Factor) coreusage.CostVer2 { +func orderCostVer2(option *optimizetrace.PlanCostOption, rows, n float64, byItems []*util.ByItems, cpuFactor costusage.CostVer2Factor) costusage.CostVer2 { numFuncs := 0 for _, byItem := range byItems { if _, ok := byItem.Expr.(*expression.ScalarFunction); ok { numFuncs++ } } - exprCost := coreusage.NewCostVer2(option, cpuFactor, + exprCost := costusage.NewCostVer2(option, cpuFactor, rows*float64(numFuncs)*cpuFactor.Value, func() string { return fmt.Sprintf("exprCPU(%v*%v*%v)", rows, numFuncs, cpuFactor) }) - orderCost := coreusage.NewCostVer2(option, cpuFactor, + orderCost := costusage.NewCostVer2(option, cpuFactor, rows*math.Log2(n)*cpuFactor.Value, func() string { return fmt.Sprintf("orderCPU(%v*log(%v)*%v)", rows, n, cpuFactor) }) - return coreusage.SumCostVer2(exprCost, orderCost) + return costusage.SumCostVer2(exprCost, orderCost) } -func hashBuildCostVer2(option *coreusage.PlanCostOption, buildRows, buildRowSize, nKeys float64, cpuFactor, memFactor coreusage.CostVer2Factor) coreusage.CostVer2 { +func hashBuildCostVer2(option *optimizetrace.PlanCostOption, buildRows, buildRowSize, nKeys float64, cpuFactor, memFactor costusage.CostVer2Factor) costusage.CostVer2 { // TODO: 1) consider types of keys, 2) dedicated factor for build-probe hash table - hashKeyCost := coreusage.NewCostVer2(option, cpuFactor, + hashKeyCost := costusage.NewCostVer2(option, cpuFactor, buildRows*nKeys*cpuFactor.Value, func() string { return fmt.Sprintf("hashkey(%v*%v*%v)", buildRows, nKeys, cpuFactor) }) - hashMemCost := coreusage.NewCostVer2(option, memFactor, + hashMemCost := costusage.NewCostVer2(option, memFactor, buildRows*buildRowSize*memFactor.Value, func() string { return fmt.Sprintf("hashmem(%v*%v*%v)", buildRows, buildRowSize, memFactor) }) - hashBuildCost := coreusage.NewCostVer2(option, cpuFactor, + hashBuildCost := costusage.NewCostVer2(option, cpuFactor, buildRows*cpuFactor.Value, func() string { return fmt.Sprintf("hashbuild(%v*%v)", buildRows, cpuFactor) }) - return coreusage.SumCostVer2(hashKeyCost, hashMemCost, hashBuildCost) + return costusage.SumCostVer2(hashKeyCost, hashMemCost, hashBuildCost) } -func hashProbeCostVer2(option *coreusage.PlanCostOption, probeRows, nKeys float64, cpuFactor coreusage.CostVer2Factor) coreusage.CostVer2 { +func hashProbeCostVer2(option *optimizetrace.PlanCostOption, probeRows, nKeys float64, cpuFactor costusage.CostVer2Factor) costusage.CostVer2 { // TODO: 1) consider types of keys, 2) dedicated factor for build-probe hash table - hashKeyCost := coreusage.NewCostVer2(option, cpuFactor, + hashKeyCost := costusage.NewCostVer2(option, cpuFactor, probeRows*nKeys*cpuFactor.Value, func() string { return fmt.Sprintf("hashkey(%v*%v*%v)", probeRows, nKeys, cpuFactor) }) - hashProbeCost := coreusage.NewCostVer2(option, cpuFactor, + hashProbeCost := costusage.NewCostVer2(option, cpuFactor, probeRows*cpuFactor.Value, func() string { return fmt.Sprintf("hashprobe(%v*%v)", probeRows, cpuFactor) }) - return coreusage.SumCostVer2(hashKeyCost, hashProbeCost) + return costusage.SumCostVer2(hashKeyCost, hashProbeCost) } // For simplicity and robust, only operators that need double-read like IndexLookup and IndexJoin consider this cost. -func doubleReadCostVer2(option *coreusage.PlanCostOption, numTasks float64, requestFactor coreusage.CostVer2Factor) coreusage.CostVer2 { - return coreusage.NewCostVer2(option, requestFactor, +func doubleReadCostVer2(option *optimizetrace.PlanCostOption, numTasks float64, requestFactor costusage.CostVer2Factor) costusage.CostVer2 { + return costusage.NewCostVer2(option, requestFactor, numTasks*requestFactor.Value, func() string { return fmt.Sprintf("doubleRead(tasks(%v)*%v)", numTasks, requestFactor) }) } // In Cost Ver2, we hide cost factors from users and deprecate SQL variables like `tidb_opt_scan_factor`. type costVer2Factors struct { - TiDBTemp coreusage.CostVer2Factor // operations on TiDB temporary table - TiKVScan coreusage.CostVer2Factor // per byte - TiKVDescScan coreusage.CostVer2Factor // per byte - TiFlashScan coreusage.CostVer2Factor // per byte - TiDBCPU coreusage.CostVer2Factor // per column or expression - TiKVCPU coreusage.CostVer2Factor // per column or expression - TiFlashCPU coreusage.CostVer2Factor // per column or expression - TiDB2KVNet coreusage.CostVer2Factor // per byte - TiDB2FlashNet coreusage.CostVer2Factor // per byte - TiFlashMPPNet coreusage.CostVer2Factor // per byte - TiDBMem coreusage.CostVer2Factor // per byte - TiKVMem coreusage.CostVer2Factor // per byte - TiFlashMem coreusage.CostVer2Factor // per byte - TiDBDisk coreusage.CostVer2Factor // per byte - TiDBRequest coreusage.CostVer2Factor // per net request + TiDBTemp costusage.CostVer2Factor // operations on TiDB temporary table + TiKVScan costusage.CostVer2Factor // per byte + TiKVDescScan costusage.CostVer2Factor // per byte + TiFlashScan costusage.CostVer2Factor // per byte + TiDBCPU costusage.CostVer2Factor // per column or expression + TiKVCPU costusage.CostVer2Factor // per column or expression + TiFlashCPU costusage.CostVer2Factor // per column or expression + TiDB2KVNet costusage.CostVer2Factor // per byte + TiDB2FlashNet costusage.CostVer2Factor // per byte + TiFlashMPPNet costusage.CostVer2Factor // per byte + TiDBMem costusage.CostVer2Factor // per byte + TiKVMem costusage.CostVer2Factor // per byte + TiFlashMem costusage.CostVer2Factor // per byte + TiDBDisk costusage.CostVer2Factor // per byte + TiDBRequest costusage.CostVer2Factor // per net request } -func (c costVer2Factors) tolist() (l []coreusage.CostVer2Factor) { +func (c costVer2Factors) tolist() (l []costusage.CostVer2Factor) { return append(l, c.TiDBTemp, c.TiKVScan, c.TiKVDescScan, c.TiFlashScan, c.TiDBCPU, c.TiKVCPU, c.TiFlashCPU, c.TiDB2KVNet, c.TiDB2FlashNet, c.TiFlashMPPNet, c.TiDBMem, c.TiKVMem, c.TiFlashMem, c.TiDBDisk, c.TiDBRequest) } var defaultVer2Factors = costVer2Factors{ - TiDBTemp: coreusage.CostVer2Factor{Name: "tidb_temp_table_factor", Value: 0.00}, - TiKVScan: coreusage.CostVer2Factor{Name: "tikv_scan_factor", Value: 40.70}, - TiKVDescScan: coreusage.CostVer2Factor{Name: "tikv_desc_scan_factor", Value: 61.05}, - TiFlashScan: coreusage.CostVer2Factor{Name: "tiflash_scan_factor", Value: 11.60}, - TiDBCPU: coreusage.CostVer2Factor{Name: "tidb_cpu_factor", Value: 49.90}, - TiKVCPU: coreusage.CostVer2Factor{Name: "tikv_cpu_factor", Value: 49.90}, - TiFlashCPU: coreusage.CostVer2Factor{Name: "tiflash_cpu_factor", Value: 2.40}, - TiDB2KVNet: coreusage.CostVer2Factor{Name: "tidb_kv_net_factor", Value: 3.96}, - TiDB2FlashNet: coreusage.CostVer2Factor{Name: "tidb_flash_net_factor", Value: 2.20}, - TiFlashMPPNet: coreusage.CostVer2Factor{Name: "tiflash_mpp_net_factor", Value: 1.00}, - TiDBMem: coreusage.CostVer2Factor{Name: "tidb_mem_factor", Value: 0.20}, - TiKVMem: coreusage.CostVer2Factor{Name: "tikv_mem_factor", Value: 0.20}, - TiFlashMem: coreusage.CostVer2Factor{Name: "tiflash_mem_factor", Value: 0.05}, - TiDBDisk: coreusage.CostVer2Factor{Name: "tidb_disk_factor", Value: 200.00}, - TiDBRequest: coreusage.CostVer2Factor{Name: "tidb_request_factor", Value: 6000000.00}, + TiDBTemp: costusage.CostVer2Factor{Name: "tidb_temp_table_factor", Value: 0.00}, + TiKVScan: costusage.CostVer2Factor{Name: "tikv_scan_factor", Value: 40.70}, + TiKVDescScan: costusage.CostVer2Factor{Name: "tikv_desc_scan_factor", Value: 61.05}, + TiFlashScan: costusage.CostVer2Factor{Name: "tiflash_scan_factor", Value: 11.60}, + TiDBCPU: costusage.CostVer2Factor{Name: "tidb_cpu_factor", Value: 49.90}, + TiKVCPU: costusage.CostVer2Factor{Name: "tikv_cpu_factor", Value: 49.90}, + TiFlashCPU: costusage.CostVer2Factor{Name: "tiflash_cpu_factor", Value: 2.40}, + TiDB2KVNet: costusage.CostVer2Factor{Name: "tidb_kv_net_factor", Value: 3.96}, + TiDB2FlashNet: costusage.CostVer2Factor{Name: "tidb_flash_net_factor", Value: 2.20}, + TiFlashMPPNet: costusage.CostVer2Factor{Name: "tiflash_mpp_net_factor", Value: 1.00}, + TiDBMem: costusage.CostVer2Factor{Name: "tidb_mem_factor", Value: 0.20}, + TiKVMem: costusage.CostVer2Factor{Name: "tikv_mem_factor", Value: 0.20}, + TiFlashMem: costusage.CostVer2Factor{Name: "tiflash_mem_factor", Value: 0.05}, + TiDBDisk: costusage.CostVer2Factor{Name: "tidb_disk_factor", Value: 200.00}, + TiDBRequest: costusage.CostVer2Factor{Name: "tidb_request_factor", Value: 6000000.00}, } -func getTaskCPUFactorVer2(_ base.PhysicalPlan, taskType property.TaskType) coreusage.CostVer2Factor { +func getTaskCPUFactorVer2(_ base.PhysicalPlan, taskType property.TaskType) costusage.CostVer2Factor { switch taskType { case property.RootTaskType: // TiDB return defaultVer2Factors.TiDBCPU @@ -946,7 +947,7 @@ func getTaskCPUFactorVer2(_ base.PhysicalPlan, taskType property.TaskType) coreu } } -func getTaskMemFactorVer2(_ base.PhysicalPlan, taskType property.TaskType) coreusage.CostVer2Factor { +func getTaskMemFactorVer2(_ base.PhysicalPlan, taskType property.TaskType) costusage.CostVer2Factor { switch taskType { case property.RootTaskType: // TiDB return defaultVer2Factors.TiDBMem @@ -957,7 +958,7 @@ func getTaskMemFactorVer2(_ base.PhysicalPlan, taskType property.TaskType) coreu } } -func getTaskScanFactorVer2(p base.PhysicalPlan, storeType kv.StoreType, taskType property.TaskType) coreusage.CostVer2Factor { +func getTaskScanFactorVer2(p base.PhysicalPlan, storeType kv.StoreType, taskType property.TaskType) costusage.CostVer2Factor { if isTemporaryTable(getTableInfo(p)) { return defaultVer2Factors.TiDBTemp } @@ -982,7 +983,7 @@ func getTaskScanFactorVer2(p base.PhysicalPlan, storeType kv.StoreType, taskType } } -func getTaskNetFactorVer2(p base.PhysicalPlan, _ property.TaskType) coreusage.CostVer2Factor { +func getTaskNetFactorVer2(p base.PhysicalPlan, _ property.TaskType) costusage.CostVer2Factor { if isTemporaryTable(getTableInfo(p)) { return defaultVer2Factors.TiDBTemp } @@ -997,7 +998,7 @@ func getTaskNetFactorVer2(p base.PhysicalPlan, _ property.TaskType) coreusage.Co return defaultVer2Factors.TiDB2KVNet } -func getTaskRequestFactorVer2(p base.PhysicalPlan, _ property.TaskType) coreusage.CostVer2Factor { +func getTaskRequestFactorVer2(p base.PhysicalPlan, _ property.TaskType) costusage.CostVer2Factor { if isTemporaryTable(getTableInfo(p)) { return defaultVer2Factors.TiDBTemp } diff --git a/pkg/planner/core/plan_cost_ver2_test.go b/pkg/planner/core/plan_cost_ver2_test.go index 12e718fe88ff0..84108a25b4ab7 100644 --- a/pkg/planner/core/plan_cost_ver2_test.go +++ b/pkg/planner/core/plan_cost_ver2_test.go @@ -28,7 +28,8 @@ import ( "github.com/pingcap/tidb/pkg/planner/core" "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/property" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/costusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/sessiontxn" "github.com/pingcap/tidb/pkg/testkit" "github.com/stretchr/testify/require" @@ -149,13 +150,13 @@ func BenchmarkGetPlanCost(b *testing.B) { b.Fatal(err) } phyPlan := plan.(base.PhysicalPlan) - _, err = core.GetPlanCost(phyPlan, property.RootTaskType, coreusage.NewDefaultPlanCostOption().WithCostFlag(coreusage.CostFlagRecalculate)) + _, err = core.GetPlanCost(phyPlan, property.RootTaskType, optimizetrace.NewDefaultPlanCostOption().WithCostFlag(costusage.CostFlagRecalculate)) if err != nil { b.Fatal(err) } b.ResetTimer() for i := 0; i < b.N; i++ { - _, _ = core.GetPlanCost(phyPlan, property.RootTaskType, coreusage.NewDefaultPlanCostOption().WithCostFlag(coreusage.CostFlagRecalculate)) + _, _ = core.GetPlanCost(phyPlan, property.RootTaskType, optimizetrace.NewDefaultPlanCostOption().WithCostFlag(costusage.CostFlagRecalculate)) } } diff --git a/pkg/planner/core/point_get_plan.go b/pkg/planner/core/point_get_plan.go index ebb56f91a8e35..733596788a11f 100644 --- a/pkg/planner/core/point_get_plan.go +++ b/pkg/planner/core/point_get_plan.go @@ -36,7 +36,8 @@ import ( "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/core/operator/baseimpl" "github.com/pingcap/tidb/pkg/planner/property" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/costusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/privilege" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" @@ -97,7 +98,7 @@ type PointGetPlan struct { // required by cost model planCostInit bool planCost float64 - planCostVer2 coreusage.CostVer2 + planCostVer2 costusage.CostVer2 // accessCols represents actual columns the PointGet will access, which are used to calculate row-size accessCols []*expression.Column @@ -266,7 +267,7 @@ func (p *PointGetPlan) SetOutputNames(names types.NameSlice) { } // AppendChildCandidate implements PhysicalPlan interface. -func (*PointGetPlan) AppendChildCandidate(_ *coreusage.PhysicalOptimizeOp) {} +func (*PointGetPlan) AppendChildCandidate(_ *optimizetrace.PhysicalOptimizeOp) {} const emptyPointGetPlanSize = int64(unsafe.Sizeof(PointGetPlan{})) @@ -440,7 +441,7 @@ type BatchPointGetPlan struct { // required by cost model planCostInit bool planCost float64 - planCostVer2 coreusage.CostVer2 + planCostVer2 costusage.CostVer2 // accessCols represents actual columns the PointGet will access, which are used to calculate row-size accessCols []*expression.Column @@ -582,7 +583,7 @@ func (p *BatchPointGetPlan) SetOutputNames(names types.NameSlice) { } // AppendChildCandidate implements PhysicalPlan interface. -func (*BatchPointGetPlan) AppendChildCandidate(_ *coreusage.PhysicalOptimizeOp) {} +func (*BatchPointGetPlan) AppendChildCandidate(_ *optimizetrace.PhysicalOptimizeOp) {} const emptyBatchPointGetPlanSize = int64(unsafe.Sizeof(BatchPointGetPlan{})) diff --git a/pkg/planner/core/rule_aggregation_elimination.go b/pkg/planner/core/rule_aggregation_elimination.go index 0224275ac9bcc..1ad0c3c329b12 100644 --- a/pkg/planner/core/rule_aggregation_elimination.go +++ b/pkg/planner/core/rule_aggregation_elimination.go @@ -24,7 +24,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/planner/core/base" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/types" ) @@ -49,7 +49,7 @@ type aggregationEliminateChecker struct { // e.g. select min(b) from t group by a. If a is a unique key, then this sql is equal to `select b from t group by a`. // For count(expr), sum(expr), avg(expr), count(distinct expr, [expr...]) we may need to rewrite the expr. Details are shown below. // If we can eliminate agg successful, we return a projection. Else we return a nil pointer. -func (a *aggregationEliminateChecker) tryToEliminateAggregation(agg *LogicalAggregation, opt *coreusage.LogicalOptimizeOp) *LogicalProjection { +func (a *aggregationEliminateChecker) tryToEliminateAggregation(agg *LogicalAggregation, opt *optimizetrace.LogicalOptimizeOp) *LogicalProjection { for _, af := range agg.AggFuncs { // TODO(issue #9968): Actually, we can rewrite GROUP_CONCAT when all the // arguments it accepts are promised to be NOT-NULL. @@ -90,7 +90,7 @@ func (a *aggregationEliminateChecker) tryToEliminateAggregation(agg *LogicalAggr // tryToEliminateDistinct will eliminate distinct in the aggregation function if the aggregation args // have unique key column. see detail example in https://github.com/pingcap/tidb/issues/23436 -func (*aggregationEliminateChecker) tryToEliminateDistinct(agg *LogicalAggregation, opt *coreusage.LogicalOptimizeOp) { +func (*aggregationEliminateChecker) tryToEliminateDistinct(agg *LogicalAggregation, opt *optimizetrace.LogicalOptimizeOp) { for _, af := range agg.AggFuncs { if af.HasDistinct { cols := make([]*expression.Column, 0, len(af.Args)) @@ -130,7 +130,7 @@ func (*aggregationEliminateChecker) tryToEliminateDistinct(agg *LogicalAggregati } } -func appendAggregationEliminateTraceStep(agg *LogicalAggregation, proj *LogicalProjection, uniqueKey expression.KeyInfo, opt *coreusage.LogicalOptimizeOp) { +func appendAggregationEliminateTraceStep(agg *LogicalAggregation, proj *LogicalProjection, uniqueKey expression.KeyInfo, opt *optimizetrace.LogicalOptimizeOp) { reason := func() string { return fmt.Sprintf("%s is a unique key", uniqueKey.String()) } @@ -142,7 +142,7 @@ func appendAggregationEliminateTraceStep(agg *LogicalAggregation, proj *LogicalP } func appendDistinctEliminateTraceStep(agg *LogicalAggregation, uniqueKey expression.KeyInfo, af *aggregation.AggFuncDesc, - opt *coreusage.LogicalOptimizeOp) { + opt *optimizetrace.LogicalOptimizeOp) { reason := func() string { return fmt.Sprintf("%s is a unique key", uniqueKey.String()) } @@ -255,7 +255,7 @@ func wrapCastFunction(ctx expression.BuildContext, arg expression.Expression, ta return expression.BuildCastFunction(ctx, arg, targetTp) } -func (a *aggregationEliminator) optimize(ctx context.Context, p base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (a *aggregationEliminator) optimize(ctx context.Context, p base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false newChildren := make([]base.LogicalPlan, 0, len(p.Children())) for _, child := range p.Children() { diff --git a/pkg/planner/core/rule_aggregation_push_down.go b/pkg/planner/core/rule_aggregation_push_down.go index 6c82323ed2d3c..cab6158d62196 100644 --- a/pkg/planner/core/rule_aggregation_push_down.go +++ b/pkg/planner/core/rule_aggregation_push_down.go @@ -25,7 +25,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/util" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/types" ) @@ -250,7 +250,7 @@ func (*aggregationPushDownSolver) decompose(ctx base.PlanContext, aggFunc *aggre // process it temporarily. If not, We will add additional group by columns and first row functions. We make a new aggregation operator. // If the pushed aggregation is grouped by unique key, it's no need to push it down. func (a *aggregationPushDownSolver) tryToPushDownAgg(oldAgg *LogicalAggregation, aggFuncs []*aggregation.AggFuncDesc, gbyCols []*expression.Column, - join *LogicalJoin, childIdx int, blockOffset int, opt *coreusage.LogicalOptimizeOp) (_ base.LogicalPlan, err error) { + join *LogicalJoin, childIdx int, blockOffset int, opt *optimizetrace.LogicalOptimizeOp) (_ base.LogicalPlan, err error) { child := join.children[childIdx] if aggregation.IsAllFirstRow(aggFuncs) { return child, nil @@ -435,13 +435,13 @@ func (*aggregationPushDownSolver) pushAggCrossUnion(agg *LogicalAggregation, uni return newAgg, nil } -func (a *aggregationPushDownSolver) optimize(_ context.Context, p base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (a *aggregationPushDownSolver) optimize(_ context.Context, p base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false newLogicalPlan, err := a.aggPushDown(p, opt) return newLogicalPlan, planChanged, err } -func (a *aggregationPushDownSolver) tryAggPushDownForUnion(union *LogicalUnionAll, agg *LogicalAggregation, opt *coreusage.LogicalOptimizeOp) error { +func (a *aggregationPushDownSolver) tryAggPushDownForUnion(union *LogicalUnionAll, agg *LogicalAggregation, opt *optimizetrace.LogicalOptimizeOp) error { for _, aggFunc := range agg.AggFuncs { if !a.isDecomposableWithUnion(aggFunc) { return nil @@ -476,7 +476,7 @@ func (a *aggregationPushDownSolver) tryAggPushDownForUnion(union *LogicalUnionAl } // aggPushDown tries to push down aggregate functions to join paths. -func (a *aggregationPushDownSolver) aggPushDown(p base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (_ base.LogicalPlan, err error) { +func (a *aggregationPushDownSolver) aggPushDown(p base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (_ base.LogicalPlan, err error) { if agg, ok := p.(*LogicalAggregation); ok { proj := a.tryToEliminateAggregation(agg, opt) if proj != nil { @@ -685,7 +685,7 @@ func (*aggregationPushDownSolver) name() string { } func appendAggPushDownAcrossJoinTraceStep(oldAgg, newAgg *LogicalAggregation, aggFuncs []*aggregation.AggFuncDesc, join *LogicalJoin, - childIdx int, opt *coreusage.LogicalOptimizeOp) { + childIdx int, opt *optimizetrace.LogicalOptimizeOp) { reason := func() string { buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v's functions[", oldAgg.TP(), oldAgg.ID())) for i, aggFunc := range aggFuncs { @@ -710,7 +710,7 @@ func appendAggPushDownAcrossJoinTraceStep(oldAgg, newAgg *LogicalAggregation, ag opt.AppendStepToCurrent(join.ID(), join.TP(), reason, action) } -func appendAggPushDownAcrossProjTraceStep(agg *LogicalAggregation, proj *LogicalProjection, opt *coreusage.LogicalOptimizeOp) { +func appendAggPushDownAcrossProjTraceStep(agg *LogicalAggregation, proj *LogicalProjection, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v is eliminated, and %v_%v's functions changed into[", proj.TP(), proj.ID(), agg.TP(), agg.ID())) for i, aggFunc := range agg.AggFuncs { @@ -728,7 +728,7 @@ func appendAggPushDownAcrossProjTraceStep(agg *LogicalAggregation, proj *Logical opt.AppendStepToCurrent(agg.ID(), agg.TP(), reason, action) } -func appendAggPushDownAcrossUnionTraceStep(union *LogicalUnionAll, agg *LogicalAggregation, opt *coreusage.LogicalOptimizeOp) { +func appendAggPushDownAcrossUnionTraceStep(union *LogicalUnionAll, agg *LogicalAggregation, opt *optimizetrace.LogicalOptimizeOp) { reason := func() string { buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v functions[", agg.TP(), agg.ID())) for i, aggFunc := range agg.AggFuncs { diff --git a/pkg/planner/core/rule_aggregation_skew_rewrite.go b/pkg/planner/core/rule_aggregation_skew_rewrite.go index 517204eeffba3..2300d9651db40 100644 --- a/pkg/planner/core/rule_aggregation_skew_rewrite.go +++ b/pkg/planner/core/rule_aggregation_skew_rewrite.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/tidb/pkg/expression/aggregation" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/planner/core/base" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/util/intset" ) @@ -48,7 +48,7 @@ type skewDistinctAggRewriter struct { // - The aggregate has 1 and only 1 distinct aggregate function (limited to count, avg, sum) // // This rule is disabled by default. Use tidb_opt_skew_distinct_agg to enable the rule. -func (a *skewDistinctAggRewriter) rewriteSkewDistinctAgg(agg *LogicalAggregation, opt *coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (a *skewDistinctAggRewriter) rewriteSkewDistinctAgg(agg *LogicalAggregation, opt *optimizetrace.LogicalOptimizeOp) base.LogicalPlan { // only group aggregate is applicable if len(agg.GroupByItems) == 0 { return nil @@ -264,7 +264,7 @@ func (*skewDistinctAggRewriter) isQualifiedAgg(aggFunc *aggregation.AggFuncDesc) } } -func appendSkewDistinctAggRewriteTraceStep(agg *LogicalAggregation, result base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) { +func appendSkewDistinctAggRewriteTraceStep(agg *LogicalAggregation, result base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) { reason := func() string { return fmt.Sprintf("%v_%v has a distinct agg function", agg.TP(), agg.ID()) } @@ -275,7 +275,7 @@ func appendSkewDistinctAggRewriteTraceStep(agg *LogicalAggregation, result base. opt.AppendStepToCurrent(agg.ID(), agg.TP(), reason, action) } -func (a *skewDistinctAggRewriter) optimize(ctx context.Context, p base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (a *skewDistinctAggRewriter) optimize(ctx context.Context, p base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false newChildren := make([]base.LogicalPlan, 0, len(p.Children())) for _, child := range p.Children() { diff --git a/pkg/planner/core/rule_build_key_info.go b/pkg/planner/core/rule_build_key_info.go index 1e55a2a492627..acc4d9ba725b5 100644 --- a/pkg/planner/core/rule_build_key_info.go +++ b/pkg/planner/core/rule_build_key_info.go @@ -22,12 +22,12 @@ import ( "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/planner/core/base" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" ) type buildKeySolver struct{} -func (*buildKeySolver) optimize(_ context.Context, p base.LogicalPlan, _ *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (*buildKeySolver) optimize(_ context.Context, p base.LogicalPlan, _ *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false buildKeyInfo(p) return p, planChanged, nil diff --git a/pkg/planner/core/rule_collect_plan_stats.go b/pkg/planner/core/rule_collect_plan_stats.go index af75da4073a0f..7288d97f6397c 100644 --- a/pkg/planner/core/rule_collect_plan_stats.go +++ b/pkg/planner/core/rule_collect_plan_stats.go @@ -23,7 +23,7 @@ import ( "github.com/pingcap/tidb/pkg/infoschema" "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/planner/core/base" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/table" @@ -33,7 +33,7 @@ import ( type collectPredicateColumnsPoint struct{} -func (collectPredicateColumnsPoint) optimize(_ context.Context, plan base.LogicalPlan, _ *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (collectPredicateColumnsPoint) optimize(_ context.Context, plan base.LogicalPlan, _ *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false if plan.SCtx().GetSessionVars().InRestrictedSQL { return plan, planChanged, nil @@ -79,7 +79,7 @@ func (collectPredicateColumnsPoint) name() string { type syncWaitStatsLoadPoint struct{} -func (syncWaitStatsLoadPoint) optimize(_ context.Context, plan base.LogicalPlan, _ *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (syncWaitStatsLoadPoint) optimize(_ context.Context, plan base.LogicalPlan, _ *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false if plan.SCtx().GetSessionVars().InRestrictedSQL { return plan, planChanged, nil diff --git a/pkg/planner/core/rule_column_pruning.go b/pkg/planner/core/rule_column_pruning.go index ac3dea7bfbce4..9e77a249c9152 100644 --- a/pkg/planner/core/rule_column_pruning.go +++ b/pkg/planner/core/rule_column_pruning.go @@ -29,12 +29,13 @@ import ( "github.com/pingcap/tidb/pkg/planner/util" "github.com/pingcap/tidb/pkg/planner/util/coreusage" "github.com/pingcap/tidb/pkg/planner/util/fixcontrol" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" ) type columnPruner struct { } -func (*columnPruner) optimize(_ context.Context, lp base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (*columnPruner) optimize(_ context.Context, lp base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false lp, err := lp.PruneColumns(lp.Schema().Columns, opt) if err != nil { @@ -76,7 +77,7 @@ func exprHasSetVarOrSleep(expr expression.Expression) bool { // the level projection expressions construction is left to the last logical optimize rule) // // so when do the rule_column_pruning here, we just prune the schema is enough. -func (p *LogicalExpand) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (p *LogicalExpand) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { // Expand need those extra redundant distinct group by columns projected from underlying projection. // distinct GroupByCol must be used by aggregate above, to make sure this, append distinctGroupByCol again. parentUsedCols = append(parentUsedCols, p.distinctGroupByCol...) @@ -101,7 +102,7 @@ func (p *LogicalExpand) PruneColumns(parentUsedCols []*expression.Column, opt *c // PruneColumns implements base.LogicalPlan interface. // If any expression has SetVar function or Sleep function, we do not prune it. -func (p *LogicalProjection) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (p *LogicalProjection) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { used := expression.GetUsedList(p.SCtx().GetExprCtx().GetEvalCtx(), parentUsedCols, p.schema) prunedColumns := make([]*expression.Column, 0) @@ -125,7 +126,7 @@ func (p *LogicalProjection) PruneColumns(parentUsedCols []*expression.Column, op } // PruneColumns implements base.LogicalPlan interface. -func (p *LogicalSelection) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (p *LogicalSelection) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { child := p.children[0] parentUsedCols = expression.ExtractColumnsFromExpressions(parentUsedCols, p.Conditions, nil) var err error @@ -137,7 +138,7 @@ func (p *LogicalSelection) PruneColumns(parentUsedCols []*expression.Column, opt } // PruneColumns implements base.LogicalPlan interface. -func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { child := la.children[0] used := expression.GetUsedList(la.SCtx().GetExprCtx().GetEvalCtx(), parentUsedCols, la.Schema()) prunedColumns := make([]*expression.Column, 0) @@ -230,7 +231,7 @@ func (la *LogicalAggregation) PruneColumns(parentUsedCols []*expression.Column, return la, nil } -func pruneByItems(p base.LogicalPlan, old []*util.ByItems, opt *coreusage.LogicalOptimizeOp) (byItems []*util.ByItems, +func pruneByItems(p base.LogicalPlan, old []*util.ByItems, opt *optimizetrace.LogicalOptimizeOp) (byItems []*util.ByItems, parentUsedCols []*expression.Column) { prunedByItems := make([]*util.ByItems, 0) byItems = make([]*util.ByItems, 0, len(old)) @@ -264,7 +265,7 @@ func pruneByItems(p base.LogicalPlan, old []*util.ByItems, opt *coreusage.Logica // PruneColumns implements base.LogicalPlan interface. // If any expression can view as a constant in execution stage, such as correlated column, constant, // we do prune them. Note that we can't prune the expressions contain non-deterministic functions, such as rand(). -func (ls *LogicalSort) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (ls *LogicalSort) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { var cols []*expression.Column ls.ByItems, cols = pruneByItems(ls, ls.ByItems, opt) parentUsedCols = append(parentUsedCols, cols...) @@ -279,7 +280,7 @@ func (ls *LogicalSort) PruneColumns(parentUsedCols []*expression.Column, opt *co // PruneColumns implements base.LogicalPlan interface. // If any expression can view as a constant in execution stage, such as correlated column, constant, // we do prune them. Note that we can't prune the expressions contain non-deterministic functions, such as rand(). -func (lt *LogicalTopN) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (lt *LogicalTopN) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { child := lt.children[0] var cols []*expression.Column lt.ByItems, cols = pruneByItems(lt, lt.ByItems, opt) @@ -293,7 +294,7 @@ func (lt *LogicalTopN) PruneColumns(parentUsedCols []*expression.Column, opt *co } // PruneColumns implements base.LogicalPlan interface. -func (p *LogicalUnionAll) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (p *LogicalUnionAll) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { used := expression.GetUsedList(p.SCtx().GetExprCtx().GetEvalCtx(), parentUsedCols, p.schema) hasBeenUsed := false for i := range used { @@ -348,7 +349,7 @@ func (p *LogicalUnionAll) PruneColumns(parentUsedCols []*expression.Column, opt } // PruneColumns implements base.LogicalPlan interface. -func (p *LogicalUnionScan) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (p *LogicalUnionScan) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { for i := 0; i < p.handleCols.NumCols(); i++ { parentUsedCols = append(parentUsedCols, p.handleCols.GetCol(i)) } @@ -368,7 +369,7 @@ func (p *LogicalUnionScan) PruneColumns(parentUsedCols []*expression.Column, opt } // PruneColumns implements base.LogicalPlan interface. -func (ds *DataSource) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (ds *DataSource) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { used := expression.GetUsedList(ds.SCtx().GetExprCtx().GetEvalCtx(), parentUsedCols, ds.schema) exprCols := expression.ExtractColumnsFromExpressions(nil, ds.allConds, nil) @@ -434,7 +435,7 @@ func (ds *DataSource) PruneColumns(parentUsedCols []*expression.Column, opt *cor } // PruneColumns implements base.LogicalPlan interface. -func (p *LogicalMemTable) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (p *LogicalMemTable) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { switch p.TableInfo.Name.O { case infoschema.TableStatementsSummary, infoschema.TableStatementsSummaryHistory, @@ -466,7 +467,7 @@ func (p *LogicalMemTable) PruneColumns(parentUsedCols []*expression.Column, opt } // PruneColumns implements base.LogicalPlan interface. -func (p *LogicalTableDual) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (p *LogicalTableDual) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { used := expression.GetUsedList(p.SCtx().GetExprCtx().GetEvalCtx(), parentUsedCols, p.Schema()) prunedColumns := make([]*expression.Column, 0) for i := len(used) - 1; i >= 0; i-- { @@ -512,7 +513,7 @@ func (p *LogicalJoin) mergeSchema() { } // PruneColumns implements base.LogicalPlan interface. -func (p *LogicalJoin) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (p *LogicalJoin) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { leftCols, rightCols := p.extractUsedCols(parentUsedCols) var err error @@ -538,7 +539,7 @@ func (p *LogicalJoin) PruneColumns(parentUsedCols []*expression.Column, opt *cor } // PruneColumns implements base.LogicalPlan interface. -func (la *LogicalApply) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (la *LogicalApply) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { leftCols, rightCols := la.extractUsedCols(parentUsedCols) allowEliminateApply := fixcontrol.GetBoolWithDefault(la.SCtx().GetSessionVars().GetOptimizerFixControlMap(), fixcontrol.Fix45822, true) var err error @@ -556,7 +557,7 @@ func (la *LogicalApply) PruneColumns(parentUsedCols []*expression.Column, opt *c } addConstOneForEmptyProjection(la.children[1]) - la.CorCols = extractCorColumnsBySchema4LogicalPlan(la.children[1], la.children[0].Schema()) + la.CorCols = coreusage.ExtractCorColumnsBySchema4LogicalPlan(la.children[1], la.children[0].Schema()) for _, col := range la.CorCols { leftCols = append(leftCols, &col.Column) } @@ -572,7 +573,7 @@ func (la *LogicalApply) PruneColumns(parentUsedCols []*expression.Column, opt *c } // PruneColumns implements base.LogicalPlan interface. -func (p *LogicalLock) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (p *LogicalLock) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { var err error if !IsSelectForUpdateLockType(p.Lock.LockType) { // when use .baseLogicalPlan to call the PruneColumns, it means current plan itself has @@ -605,7 +606,7 @@ func (p *LogicalLock) PruneColumns(parentUsedCols []*expression.Column, opt *cor } // PruneColumns implements base.LogicalPlan interface. -func (p *LogicalWindow) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (p *LogicalWindow) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { windowColumns := p.GetWindowResultColumns() cnt := 0 for _, col := range parentUsedCols { @@ -650,7 +651,7 @@ func (p *LogicalWindow) extractUsedCols(parentUsedCols []*expression.Column) []* } // PruneColumns implements base.LogicalPlan interface. -func (p *LogicalLimit) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (p *LogicalLimit) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { if len(parentUsedCols) == 0 { // happens when LIMIT appears in UPDATE. return p, nil } @@ -692,7 +693,7 @@ func addConstOneForEmptyProjection(p base.LogicalPlan) { }) } -func appendColumnPruneTraceStep(p base.LogicalPlan, prunedColumns []*expression.Column, opt *coreusage.LogicalOptimizeOp) { +func appendColumnPruneTraceStep(p base.LogicalPlan, prunedColumns []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) { if len(prunedColumns) < 1 { return } @@ -703,7 +704,7 @@ func appendColumnPruneTraceStep(p base.LogicalPlan, prunedColumns []*expression. appendItemPruneTraceStep(p, "columns", s, opt) } -func appendFunctionPruneTraceStep(p base.LogicalPlan, prunedFunctions []*aggregation.AggFuncDesc, opt *coreusage.LogicalOptimizeOp) { +func appendFunctionPruneTraceStep(p base.LogicalPlan, prunedFunctions []*aggregation.AggFuncDesc, opt *optimizetrace.LogicalOptimizeOp) { if len(prunedFunctions) < 1 { return } @@ -714,7 +715,7 @@ func appendFunctionPruneTraceStep(p base.LogicalPlan, prunedFunctions []*aggrega appendItemPruneTraceStep(p, "aggregation functions", s, opt) } -func appendByItemsPruneTraceStep(p base.LogicalPlan, prunedByItems []*util.ByItems, opt *coreusage.LogicalOptimizeOp) { +func appendByItemsPruneTraceStep(p base.LogicalPlan, prunedByItems []*util.ByItems, opt *optimizetrace.LogicalOptimizeOp) { if len(prunedByItems) < 1 { return } @@ -725,7 +726,7 @@ func appendByItemsPruneTraceStep(p base.LogicalPlan, prunedByItems []*util.ByIte appendItemPruneTraceStep(p, "byItems", s, opt) } -func appendGroupByItemsPruneTraceStep(p base.LogicalPlan, prunedGroupByItems []expression.Expression, opt *coreusage.LogicalOptimizeOp) { +func appendGroupByItemsPruneTraceStep(p base.LogicalPlan, prunedGroupByItems []expression.Expression, opt *optimizetrace.LogicalOptimizeOp) { if len(prunedGroupByItems) < 1 { return } @@ -736,7 +737,7 @@ func appendGroupByItemsPruneTraceStep(p base.LogicalPlan, prunedGroupByItems []e appendItemPruneTraceStep(p, "groupByItems", s, opt) } -func appendItemPruneTraceStep(p base.LogicalPlan, itemType string, prunedObjects []fmt.Stringer, opt *coreusage.LogicalOptimizeOp) { +func appendItemPruneTraceStep(p base.LogicalPlan, itemType string, prunedObjects []fmt.Stringer, opt *optimizetrace.LogicalOptimizeOp) { if len(prunedObjects) < 1 { return } @@ -784,12 +785,12 @@ func preferKeyColumnFromTable(dataSource *DataSource, originColumns []*expressio // PruneColumns implements the interface of base.LogicalPlan. // LogicalCTE just do a empty function call. It's logical optimize is indivisual phase. -func (p *LogicalCTE) PruneColumns(_ []*expression.Column, _ *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (p *LogicalCTE) PruneColumns(_ []*expression.Column, _ *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { return p, nil } // PruneColumns implements the interface of base.LogicalPlan. -func (p *LogicalSequence) PruneColumns(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (p *LogicalSequence) PruneColumns(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { var err error p.children[len(p.children)-1], err = p.children[len(p.children)-1].PruneColumns(parentUsedCols, opt) if err != nil { @@ -798,7 +799,7 @@ func (p *LogicalSequence) PruneColumns(parentUsedCols []*expression.Column, opt return p, nil } -func applyEliminateTraceStep(lp base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) { +func applyEliminateTraceStep(lp base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { buffer := bytes.NewBufferString( fmt.Sprintf("%v_%v is eliminated.", lp.TP(), lp.ID())) diff --git a/pkg/planner/core/rule_constant_propagation.go b/pkg/planner/core/rule_constant_propagation.go index 426d25b7b1078..6603e982b87ac 100644 --- a/pkg/planner/core/rule_constant_propagation.go +++ b/pkg/planner/core/rule_constant_propagation.go @@ -20,7 +20,7 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/planner/core/base" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" ) // constantPropagationSolver can support constant propagated cross-query block. @@ -52,7 +52,7 @@ type constantPropagationSolver struct { // which is mainly implemented in the interface "constantPropagation" of LogicalPlan. // Currently only the Logical Join implements this function. (Used for the subquery in FROM List) // In the future, the Logical Apply will implements this function. (Used for the subquery in WHERE or SELECT list) -func (cp *constantPropagationSolver) optimize(_ context.Context, p base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (cp *constantPropagationSolver) optimize(_ context.Context, p base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false // constant propagation root plan newRoot := p.ConstantPropagation(nil, 0, opt) @@ -69,7 +69,7 @@ func (cp *constantPropagationSolver) optimize(_ context.Context, p base.LogicalP } // execOptimize optimize constant propagation exclude root plan node -func (cp *constantPropagationSolver) execOptimize(currentPlan base.LogicalPlan, parentPlan base.LogicalPlan, currentChildIdx int, opt *coreusage.LogicalOptimizeOp) { +func (cp *constantPropagationSolver) execOptimize(currentPlan base.LogicalPlan, parentPlan base.LogicalPlan, currentChildIdx int, opt *optimizetrace.LogicalOptimizeOp) { if parentPlan == nil { // Attention: The function 'execOptimize' could not handle the root plan, so the parent plan could not be nil. return @@ -87,7 +87,7 @@ func (*constantPropagationSolver) name() string { } // ConstantPropagation implements the LogicalPlan interface. -func (*baseLogicalPlan) ConstantPropagation(_ base.LogicalPlan, _ int, _ *coreusage.LogicalOptimizeOp) (newRoot base.LogicalPlan) { +func (*baseLogicalPlan) ConstantPropagation(_ base.LogicalPlan, _ int, _ *optimizetrace.LogicalOptimizeOp) (newRoot base.LogicalPlan) { // Only LogicalJoin can apply constant propagation // Other Logical plan do nothing return nil @@ -145,7 +145,7 @@ func (*baseLogicalPlan) ConstantPropagation(_ base.LogicalPlan, _ int, _ *coreus */ // Return nil if the root of plan has not been changed // Return new root if the root of plan is changed to selection -func (logicalJoin *LogicalJoin) ConstantPropagation(parentPlan base.LogicalPlan, currentChildIdx int, opt *coreusage.LogicalOptimizeOp) (newRoot base.LogicalPlan) { +func (logicalJoin *LogicalJoin) ConstantPropagation(parentPlan base.LogicalPlan, currentChildIdx int, opt *optimizetrace.LogicalOptimizeOp) (newRoot base.LogicalPlan) { // step1: get constant predicate from left or right according to the JoinType var getConstantPredicateFromLeft bool var getConstantPredicateFromRight bool @@ -273,7 +273,7 @@ func validCompareConstantPredicate(candidatePredicate expression.Expression) boo // If the currentPlan at the top of query plan, return new root plan (selection) // Else return nil func addCandidateSelection(currentPlan base.LogicalPlan, currentChildIdx int, parentPlan base.LogicalPlan, - candidatePredicates []expression.Expression, opt *coreusage.LogicalOptimizeOp) (newRoot base.LogicalPlan) { + candidatePredicates []expression.Expression, opt *optimizetrace.LogicalOptimizeOp) (newRoot base.LogicalPlan) { // generate a new selection for candidatePredicates selection := LogicalSelection{Conditions: candidatePredicates}.Init(currentPlan.SCtx(), currentPlan.QueryBlockOffset()) // add selection above of p diff --git a/pkg/planner/core/rule_decorrelate.go b/pkg/planner/core/rule_decorrelate.go index 5916c2b846584..128a1f435f85d 100644 --- a/pkg/planner/core/rule_decorrelate.go +++ b/pkg/planner/core/rule_decorrelate.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/plancodec" ) @@ -87,26 +88,6 @@ func (la *LogicalApply) deCorColFromEqExpr(expr expression.Expression) expressio return nil } -// ExtractCorrelatedCols4LogicalPlan recursively extracts all of the correlated columns -// from a plan tree by calling base.LogicalPlan.ExtractCorrelatedCols. -func ExtractCorrelatedCols4LogicalPlan(p base.LogicalPlan) []*expression.CorrelatedColumn { - corCols := p.ExtractCorrelatedCols() - for _, child := range p.Children() { - corCols = append(corCols, ExtractCorrelatedCols4LogicalPlan(child)...) - } - return corCols -} - -// ExtractCorrelatedCols4PhysicalPlan recursively extracts all of the correlated columns -// from a plan tree by calling PhysicalPlan.ExtractCorrelatedCols. -func ExtractCorrelatedCols4PhysicalPlan(p base.PhysicalPlan) []*expression.CorrelatedColumn { - corCols := p.ExtractCorrelatedCols() - for _, child := range p.Children() { - corCols = append(corCols, ExtractCorrelatedCols4PhysicalPlan(child)...) - } - return corCols -} - // ExtractOuterApplyCorrelatedCols only extract the correlated columns whose corresponding Apply operator is outside the plan. // For Plan-1, ExtractOuterApplyCorrelatedCols(CTE-1) will return cor_col_1. // Plan-1: @@ -195,12 +176,12 @@ func (*decorrelateSolver) aggDefaultValueMap(agg *LogicalAggregation) map[int]*e } // optimize implements logicalOptRule interface. -func (s *decorrelateSolver) optimize(ctx context.Context, p base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (s *decorrelateSolver) optimize(ctx context.Context, p base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false if apply, ok := p.(*LogicalApply); ok { outerPlan := apply.children[0] innerPlan := apply.children[1] - apply.CorCols = extractCorColumnsBySchema4LogicalPlan(apply.children[1], apply.children[0].Schema()) + apply.CorCols = coreusage.ExtractCorColumnsBySchema4LogicalPlan(apply.children[1], apply.children[0].Schema()) if len(apply.CorCols) == 0 { // If the inner plan is non-correlated, the apply will be simplified to join. join := &apply.LogicalJoin @@ -379,7 +360,7 @@ func (s *decorrelateSolver) optimize(ctx context.Context, p base.LogicalPlan, op if len(eqCondWithCorCol) > 0 { originalExpr := sel.Conditions sel.Conditions = remainedExpr - apply.CorCols = extractCorColumnsBySchema4LogicalPlan(apply.children[1], apply.children[0].Schema()) + apply.CorCols = coreusage.ExtractCorColumnsBySchema4LogicalPlan(apply.children[1], apply.children[0].Schema()) // There's no other correlated column. groupByCols := expression.NewSchema(agg.GetGroupByCols()...) if len(apply.CorCols) == 0 { @@ -432,7 +413,7 @@ func (s *decorrelateSolver) optimize(ctx context.Context, p base.LogicalPlan, op return s.optimize(ctx, p, opt) } sel.Conditions = originalExpr - apply.CorCols = extractCorColumnsBySchema4LogicalPlan(apply.children[1], apply.children[0].Schema()) + apply.CorCols = coreusage.ExtractCorColumnsBySchema4LogicalPlan(apply.children[1], apply.children[0].Schema()) } } } else if sort, ok := innerPlan.(*LogicalSort); ok { @@ -465,7 +446,7 @@ func (*decorrelateSolver) name() string { return "decorrelate" } -func appendApplySimplifiedTraceStep(p *LogicalApply, j *LogicalJoin, opt *coreusage.LogicalOptimizeOp) { +func appendApplySimplifiedTraceStep(p *LogicalApply, j *LogicalJoin, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v simplified into %v_%v", plancodec.TypeApply, p.ID(), plancodec.TypeJoin, j.ID()) } @@ -475,7 +456,7 @@ func appendApplySimplifiedTraceStep(p *LogicalApply, j *LogicalJoin, opt *coreus opt.AppendStepToCurrent(p.ID(), p.TP(), reason, action) } -func appendRemoveSelectionTraceStep(p base.LogicalPlan, s *LogicalSelection, opt *coreusage.LogicalOptimizeOp) { +func appendRemoveSelectionTraceStep(p base.LogicalPlan, s *LogicalSelection, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v removed from plan tree", s.TP(), s.ID()) } @@ -485,7 +466,7 @@ func appendRemoveSelectionTraceStep(p base.LogicalPlan, s *LogicalSelection, opt opt.AppendStepToCurrent(s.ID(), s.TP(), reason, action) } -func appendRemoveMaxOneRowTraceStep(m *LogicalMaxOneRow, opt *coreusage.LogicalOptimizeOp) { +func appendRemoveMaxOneRowTraceStep(m *LogicalMaxOneRow, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v removed from plan tree", m.TP(), m.ID()) } @@ -495,7 +476,7 @@ func appendRemoveMaxOneRowTraceStep(m *LogicalMaxOneRow, opt *coreusage.LogicalO opt.AppendStepToCurrent(m.ID(), m.TP(), reason, action) } -func appendRemoveLimitTraceStep(limit *LogicalLimit, opt *coreusage.LogicalOptimizeOp) { +func appendRemoveLimitTraceStep(limit *LogicalLimit, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v removed from plan tree", limit.TP(), limit.ID()) } @@ -505,7 +486,7 @@ func appendRemoveLimitTraceStep(limit *LogicalLimit, opt *coreusage.LogicalOptim opt.AppendStepToCurrent(limit.ID(), limit.TP(), reason, action) } -func appendRemoveProjTraceStep(p *LogicalApply, proj *LogicalProjection, opt *coreusage.LogicalOptimizeOp) { +func appendRemoveProjTraceStep(p *LogicalApply, proj *LogicalProjection, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v removed from plan tree", proj.TP(), proj.ID()) } @@ -515,7 +496,7 @@ func appendRemoveProjTraceStep(p *LogicalApply, proj *LogicalProjection, opt *co opt.AppendStepToCurrent(proj.ID(), proj.TP(), reason, action) } -func appendMoveProjTraceStep(p *LogicalApply, np base.LogicalPlan, proj *LogicalProjection, opt *coreusage.LogicalOptimizeOp) { +func appendMoveProjTraceStep(p *LogicalApply, np base.LogicalPlan, proj *LogicalProjection, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v is moved as %v_%v's parent", proj.TP(), proj.ID(), np.TP(), np.ID()) } @@ -525,7 +506,7 @@ func appendMoveProjTraceStep(p *LogicalApply, np base.LogicalPlan, proj *Logical opt.AppendStepToCurrent(proj.ID(), proj.TP(), reason, action) } -func appendRemoveSortTraceStep(sort *LogicalSort, opt *coreusage.LogicalOptimizeOp) { +func appendRemoveSortTraceStep(sort *LogicalSort, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v removed from plan tree", sort.TP(), sort.ID()) } @@ -535,7 +516,7 @@ func appendRemoveSortTraceStep(sort *LogicalSort, opt *coreusage.LogicalOptimize opt.AppendStepToCurrent(sort.ID(), sort.TP(), reason, action) } -func appendPullUpAggTraceStep(p *LogicalApply, np base.LogicalPlan, agg *LogicalAggregation, opt *coreusage.LogicalOptimizeOp) { +func appendPullUpAggTraceStep(p *LogicalApply, np base.LogicalPlan, agg *LogicalAggregation, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v pulled up as %v_%v's parent, and %v_%v's join type becomes %v", agg.TP(), agg.ID(), np.TP(), np.ID(), p.TP(), p.ID(), p.JoinType.String()) @@ -547,7 +528,7 @@ func appendPullUpAggTraceStep(p *LogicalApply, np base.LogicalPlan, agg *Logical opt.AppendStepToCurrent(agg.ID(), agg.TP(), reason, action) } -func appendAddProjTraceStep(p *LogicalApply, proj *LogicalProjection, opt *coreusage.LogicalOptimizeOp) { +func appendAddProjTraceStep(p *LogicalApply, proj *LogicalProjection, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v is added as %v_%v's parent", proj.TP(), proj.ID(), p.TP(), p.ID()) } @@ -559,7 +540,7 @@ func appendAddProjTraceStep(p *LogicalApply, proj *LogicalProjection, opt *coreu func appendModifyAggTraceStep(outerPlan base.LogicalPlan, p *LogicalApply, agg *LogicalAggregation, sel *LogicalSelection, appendedGroupByCols *expression.Schema, appendedAggFuncs []*aggregation.AggFuncDesc, - eqCondWithCorCol []*expression.ScalarFunction, opt *coreusage.LogicalOptimizeOp) { + eqCondWithCorCol []*expression.ScalarFunction, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v's groupby items added [", agg.TP(), agg.ID())) for i, col := range appendedGroupByCols.Columns { diff --git a/pkg/planner/core/rule_derive_topn_from_window.go b/pkg/planner/core/rule_derive_topn_from_window.go index 87b2cb816a59e..582e917beadef 100644 --- a/pkg/planner/core/rule_derive_topn_from_window.go +++ b/pkg/planner/core/rule_derive_topn_from_window.go @@ -23,14 +23,14 @@ import ( "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/util" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" ) // deriveTopNFromWindow pushes down the topN or limit. In the future we will remove the limit from `requiredProperty` in CBO phase. type deriveTopNFromWindow struct { } -func appendDerivedTopNTrace(topN base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) { +func appendDerivedTopNTrace(topN base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) { child := topN.Children()[0] action := func() string { return fmt.Sprintf("%v_%v top N added below %v_%v ", topN.TP(), topN.ID(), child.TP(), child.ID()) @@ -118,13 +118,13 @@ func windowIsTopN(p *LogicalSelection) (bool, uint64) { return false, 0 } -func (*deriveTopNFromWindow) optimize(_ context.Context, p base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (*deriveTopNFromWindow) optimize(_ context.Context, p base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false return p.DeriveTopN(opt), planChanged, nil } // DeriveTopN implements the LogicalPlan interface. -func (s *baseLogicalPlan) DeriveTopN(opt *coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (s *baseLogicalPlan) DeriveTopN(opt *optimizetrace.LogicalOptimizeOp) base.LogicalPlan { p := s.self if p.SCtx().GetSessionVars().AllowDeriveTopN { for i, child := range p.Children() { @@ -136,7 +136,7 @@ func (s *baseLogicalPlan) DeriveTopN(opt *coreusage.LogicalOptimizeOp) base.Logi } // DeriveTopN implements the LogicalPlan interface. -func (s *LogicalSelection) DeriveTopN(opt *coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (s *LogicalSelection) DeriveTopN(opt *optimizetrace.LogicalOptimizeOp) base.LogicalPlan { p := s.self.(*LogicalSelection) windowIsTopN, limitValue := windowIsTopN(p) if windowIsTopN { diff --git a/pkg/planner/core/rule_eliminate_projection.go b/pkg/planner/core/rule_eliminate_projection.go index 5cbf5b9de08cf..cbee379d3fb19 100644 --- a/pkg/planner/core/rule_eliminate_projection.go +++ b/pkg/planner/core/rule_eliminate_projection.go @@ -24,7 +24,7 @@ import ( "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/planner/core/base" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" ) // canProjectionBeEliminatedLoose checks whether a projection can be eliminated, @@ -169,14 +169,14 @@ type projectionEliminator struct { } // optimize implements the logicalOptRule interface. -func (pe *projectionEliminator) optimize(_ context.Context, lp base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (pe *projectionEliminator) optimize(_ context.Context, lp base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false root := pe.eliminate(lp, make(map[string]*expression.Column), false, opt) return root, planChanged, nil } // eliminate eliminates the redundant projection in a logical plan. -func (pe *projectionEliminator) eliminate(p base.LogicalPlan, replace map[string]*expression.Column, canEliminate bool, opt *coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (pe *projectionEliminator) eliminate(p base.LogicalPlan, replace map[string]*expression.Column, canEliminate bool, opt *optimizetrace.LogicalOptimizeOp) base.LogicalPlan { // LogicalCTE's logical optimization is independent. if _, ok := p.(*LogicalCTE); ok { return p @@ -340,7 +340,7 @@ func (*projectionEliminator) name() string { return "projection_eliminate" } -func appendDupProjEliminateTraceStep(parent, child *LogicalProjection, opt *coreusage.LogicalOptimizeOp) { +func appendDupProjEliminateTraceStep(parent, child *LogicalProjection, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { buffer := bytes.NewBufferString( fmt.Sprintf("%v_%v is eliminated, %v_%v's expressions changed into[", child.TP(), child.ID(), parent.TP(), parent.ID())) @@ -359,7 +359,7 @@ func appendDupProjEliminateTraceStep(parent, child *LogicalProjection, opt *core opt.AppendStepToCurrent(child.ID(), child.TP(), reason, action) } -func appendProjEliminateTraceStep(proj *LogicalProjection, opt *coreusage.LogicalOptimizeOp) { +func appendProjEliminateTraceStep(proj *LogicalProjection, opt *optimizetrace.LogicalOptimizeOp) { reason := func() string { return fmt.Sprintf("%v_%v's Exprs are all Columns", proj.TP(), proj.ID()) } diff --git a/pkg/planner/core/rule_generate_column_substitute.go b/pkg/planner/core/rule_generate_column_substitute.go index 0f6a46714a34b..6f2f45eaff821 100644 --- a/pkg/planner/core/rule_generate_column_substitute.go +++ b/pkg/planner/core/rule_generate_column_substitute.go @@ -21,7 +21,7 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/planner/core/base" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/types" h "github.com/pingcap/tidb/pkg/util/hint" ) @@ -39,7 +39,7 @@ type ExprColumnMap map[expression.Expression]*expression.Column // For example: select a+1 from t order by a+1, with a virtual generate column c as (a+1) and // an index on c. We need to replace a+1 with c so that we can use the index on c. // See also https://dev.mysql.com/doc/refman/8.0/en/generated-column-index-optimizations.html -func (gc *gcSubstituter) optimize(ctx context.Context, lp base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (gc *gcSubstituter) optimize(ctx context.Context, lp base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false exprToColumn := make(ExprColumnMap) collectGenerateColumn(lp, exprToColumn) @@ -86,7 +86,7 @@ func collectGenerateColumn(lp base.LogicalPlan, exprToColumn ExprColumnMap) { } } -func tryToSubstituteExpr(expr *expression.Expression, lp base.LogicalPlan, candidateExpr expression.Expression, tp types.EvalType, schema *expression.Schema, col *expression.Column, opt *coreusage.LogicalOptimizeOp) bool { +func tryToSubstituteExpr(expr *expression.Expression, lp base.LogicalPlan, candidateExpr expression.Expression, tp types.EvalType, schema *expression.Schema, col *expression.Column, opt *optimizetrace.LogicalOptimizeOp) bool { changed := false if (*expr).Equal(lp.SCtx().GetExprCtx().GetEvalCtx(), candidateExpr) && candidateExpr.GetType().EvalType() == tp && schema.ColumnIndex(col) != -1 { @@ -97,7 +97,7 @@ func tryToSubstituteExpr(expr *expression.Expression, lp base.LogicalPlan, candi return changed } -func appendSubstituteColumnStep(lp base.LogicalPlan, candidateExpr expression.Expression, col *expression.Column, opt *coreusage.LogicalOptimizeOp) { +func appendSubstituteColumnStep(lp base.LogicalPlan, candidateExpr expression.Expression, col *expression.Column, opt *optimizetrace.LogicalOptimizeOp) { reason := func() string { return "" } action := func() string { buffer := bytes.NewBufferString("expression:") @@ -111,11 +111,11 @@ func appendSubstituteColumnStep(lp base.LogicalPlan, candidateExpr expression.Ex } // SubstituteExpression is Exported for bench -func SubstituteExpression(cond expression.Expression, lp base.LogicalPlan, exprToColumn ExprColumnMap, schema *expression.Schema, opt *coreusage.LogicalOptimizeOp) bool { +func SubstituteExpression(cond expression.Expression, lp base.LogicalPlan, exprToColumn ExprColumnMap, schema *expression.Schema, opt *optimizetrace.LogicalOptimizeOp) bool { return substituteExpression(cond, lp, exprToColumn, schema, opt) } -func substituteExpression(cond expression.Expression, lp base.LogicalPlan, exprToColumn ExprColumnMap, schema *expression.Schema, opt *coreusage.LogicalOptimizeOp) bool { +func substituteExpression(cond expression.Expression, lp base.LogicalPlan, exprToColumn ExprColumnMap, schema *expression.Schema, opt *optimizetrace.LogicalOptimizeOp) bool { sf, ok := cond.(*expression.ScalarFunction) if !ok { return false @@ -174,7 +174,7 @@ func substituteExpression(cond expression.Expression, lp base.LogicalPlan, exprT return changed } -func (gc *gcSubstituter) substitute(ctx context.Context, lp base.LogicalPlan, exprToColumn ExprColumnMap, opt *coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (gc *gcSubstituter) substitute(ctx context.Context, lp base.LogicalPlan, exprToColumn ExprColumnMap, opt *optimizetrace.LogicalOptimizeOp) base.LogicalPlan { var tp types.EvalType switch x := lp.(type) { case *LogicalSelection: diff --git a/pkg/planner/core/rule_join_elimination.go b/pkg/planner/core/rule_join_elimination.go index 772aebb744893..b4fb19b69fdc5 100644 --- a/pkg/planner/core/rule_join_elimination.go +++ b/pkg/planner/core/rule_join_elimination.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/planner/core/base" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/util/set" ) @@ -36,7 +36,7 @@ type outerJoinEliminator struct { // 2. outer join elimination with duplicate agnostic aggregate functions: For example left outer join. // If the parent only use the columns from left table with 'distinct' label. The left outer join can // be eliminated. -func (o *outerJoinEliminator) tryToEliminateOuterJoin(p *LogicalJoin, aggCols []*expression.Column, parentCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (o *outerJoinEliminator) tryToEliminateOuterJoin(p *LogicalJoin, aggCols []*expression.Column, parentCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { var innerChildIdx int switch p.JoinType { case LeftOuterJoin: @@ -193,7 +193,7 @@ func GetDupAgnosticAggCols( return true, newAggCols } -func (o *outerJoinEliminator) doOptimize(p base.LogicalPlan, aggCols []*expression.Column, parentCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (o *outerJoinEliminator) doOptimize(p base.LogicalPlan, aggCols []*expression.Column, parentCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { // CTE's logical optimization is independent. if _, ok := p.(*LogicalCTE); ok { return p, nil @@ -247,7 +247,7 @@ func (o *outerJoinEliminator) doOptimize(p base.LogicalPlan, aggCols []*expressi return p, nil } -func (o *outerJoinEliminator) optimize(_ context.Context, p base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (o *outerJoinEliminator) optimize(_ context.Context, p base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false p, err := o.doOptimize(p, nil, nil, opt) return p, planChanged, err @@ -258,7 +258,7 @@ func (*outerJoinEliminator) name() string { } func appendOuterJoinEliminateTraceStep(join *LogicalJoin, outerPlan base.LogicalPlan, parentCols []*expression.Column, - innerJoinKeys *expression.Schema, opt *coreusage.LogicalOptimizeOp) { + innerJoinKeys *expression.Schema, opt *optimizetrace.LogicalOptimizeOp) { reason := func() string { buffer := bytes.NewBufferString("The columns[") for i, col := range parentCols { @@ -283,7 +283,7 @@ func appendOuterJoinEliminateTraceStep(join *LogicalJoin, outerPlan base.Logical opt.AppendStepToCurrent(join.ID(), join.TP(), reason, action) } -func appendOuterJoinEliminateAggregationTraceStep(join *LogicalJoin, outerPlan base.LogicalPlan, aggCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) { +func appendOuterJoinEliminateAggregationTraceStep(join *LogicalJoin, outerPlan base.LogicalPlan, aggCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) { reason := func() string { buffer := bytes.NewBufferString("The columns[") for i, col := range aggCols { diff --git a/pkg/planner/core/rule_join_reorder.go b/pkg/planner/core/rule_join_reorder.go index 79acb2130d6d4..d052cf87a2749 100644 --- a/pkg/planner/core/rule_join_reorder.go +++ b/pkg/planner/core/rule_join_reorder.go @@ -23,7 +23,7 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/planner/core/base" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" h "github.com/pingcap/tidb/pkg/util/hint" "github.com/pingcap/tidb/pkg/util/plancodec" "github.com/pingcap/tidb/pkg/util/tracing" @@ -224,7 +224,7 @@ type joinTypeWithExtMsg struct { outerBindCondition []expression.Expression } -func (s *joinReOrderSolver) optimize(_ context.Context, p base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (s *joinReOrderSolver) optimize(_ context.Context, p base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false tracer := &joinReorderTrace{cost: map[string]float64{}, opt: opt} tracer.traceJoinReorder(p) @@ -664,7 +664,7 @@ func (*joinReOrderSolver) name() string { return "join_reorder" } -func appendJoinReorderTraceStep(tracer *joinReorderTrace, plan base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) { +func appendJoinReorderTraceStep(tracer *joinReorderTrace, plan base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) { if len(tracer.initial) < 1 || len(tracer.final) < 1 { return } @@ -774,7 +774,7 @@ func findRoots(t *tracing.PlanTrace) []*tracing.PlanTrace { } type joinReorderTrace struct { - opt *coreusage.LogicalOptimizeOp + opt *optimizetrace.LogicalOptimizeOp initial string final string cost map[string]float64 diff --git a/pkg/planner/core/rule_max_min_eliminate.go b/pkg/planner/core/rule_max_min_eliminate.go index 7457d730cc42b..36ab3106156cd 100644 --- a/pkg/planner/core/rule_max_min_eliminate.go +++ b/pkg/planner/core/rule_max_min_eliminate.go @@ -26,7 +26,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/util" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/ranger" ) @@ -38,13 +38,13 @@ import ( type maxMinEliminator struct { } -func (a *maxMinEliminator) optimize(_ context.Context, p base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (a *maxMinEliminator) optimize(_ context.Context, p base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false return a.eliminateMaxMin(p, opt), planChanged, nil } // composeAggsByInnerJoin composes the scalar aggregations by cartesianJoin. -func (*maxMinEliminator) composeAggsByInnerJoin(originAgg *LogicalAggregation, aggs []*LogicalAggregation, opt *coreusage.LogicalOptimizeOp) (plan base.LogicalPlan) { +func (*maxMinEliminator) composeAggsByInnerJoin(originAgg *LogicalAggregation, aggs []*LogicalAggregation, opt *optimizetrace.LogicalOptimizeOp) (plan base.LogicalPlan) { plan = aggs[0] sctx := plan.SCtx() joins := make([]*LogicalJoin, 0) @@ -140,7 +140,7 @@ func (a *maxMinEliminator) cloneSubPlans(plan base.LogicalPlan) base.LogicalPlan // `select max(a) from t` + `select min(a) from t` + `select max(b) from t`. // Then we check whether `a` and `b` have indices. If any of the used column has no index, we cannot eliminate // this aggregation. -func (a *maxMinEliminator) splitAggFuncAndCheckIndices(agg *LogicalAggregation, opt *coreusage.LogicalOptimizeOp) (aggs []*LogicalAggregation, canEliminate bool) { +func (a *maxMinEliminator) splitAggFuncAndCheckIndices(agg *LogicalAggregation, opt *optimizetrace.LogicalOptimizeOp) (aggs []*LogicalAggregation, canEliminate bool) { for _, f := range agg.AggFuncs { // We must make sure the args of max/min is a simple single column. col, ok := f.Args[0].(*expression.Column) @@ -172,7 +172,7 @@ func (a *maxMinEliminator) splitAggFuncAndCheckIndices(agg *LogicalAggregation, } // eliminateSingleMaxMin tries to convert a single max/min to Limit+Sort operators. -func (*maxMinEliminator) eliminateSingleMaxMin(agg *LogicalAggregation, opt *coreusage.LogicalOptimizeOp) *LogicalAggregation { +func (*maxMinEliminator) eliminateSingleMaxMin(agg *LogicalAggregation, opt *optimizetrace.LogicalOptimizeOp) *LogicalAggregation { f := agg.AggFuncs[0] child := agg.Children()[0] ctx := agg.SCtx() @@ -213,7 +213,7 @@ func (*maxMinEliminator) eliminateSingleMaxMin(agg *LogicalAggregation, opt *cor } // eliminateMaxMin tries to convert max/min to Limit+Sort operators. -func (a *maxMinEliminator) eliminateMaxMin(p base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (a *maxMinEliminator) eliminateMaxMin(p base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) base.LogicalPlan { // CTE's logical optimization is indenpent. if _, ok := p.(*LogicalCTE); ok { return p @@ -263,7 +263,7 @@ func (*maxMinEliminator) name() string { return "max_min_eliminate" } -func appendEliminateSingleMaxMinTrace(agg *LogicalAggregation, sel *LogicalSelection, sort *LogicalSort, limit *LogicalLimit, opt *coreusage.LogicalOptimizeOp) { +func appendEliminateSingleMaxMinTrace(agg *LogicalAggregation, sel *LogicalSelection, sort *LogicalSort, limit *LogicalLimit, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { buffer := bytes.NewBufferString("") if sel != nil { @@ -288,7 +288,7 @@ func appendEliminateSingleMaxMinTrace(agg *LogicalAggregation, sel *LogicalSelec opt.AppendStepToCurrent(agg.ID(), agg.TP(), reason, action) } -func appendEliminateMultiMinMaxTraceStep(originAgg *LogicalAggregation, aggs []*LogicalAggregation, joins []*LogicalJoin, opt *coreusage.LogicalOptimizeOp) { +func appendEliminateMultiMinMaxTraceStep(originAgg *LogicalAggregation, aggs []*LogicalAggregation, joins []*LogicalJoin, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v splited into [", originAgg.TP(), originAgg.ID())) for i, agg := range aggs { diff --git a/pkg/planner/core/rule_partition_processor.go b/pkg/planner/core/rule_partition_processor.go index a0cc3031fddc8..b29210b19cc8c 100644 --- a/pkg/planner/core/rule_partition_processor.go +++ b/pkg/planner/core/rule_partition_processor.go @@ -30,7 +30,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/planner/core/base" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/table/tables" "github.com/pingcap/tidb/pkg/types" @@ -63,13 +63,13 @@ const FullRange = -1 // partitionProcessor is here because it's easier to prune partition after predicate push down. type partitionProcessor struct{} -func (s *partitionProcessor) optimize(_ context.Context, lp base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (s *partitionProcessor) optimize(_ context.Context, lp base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false p, err := s.rewriteDataSource(lp, opt) return p, planChanged, err } -func (s *partitionProcessor) rewriteDataSource(lp base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (s *partitionProcessor) rewriteDataSource(lp base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { // Assert there will not be sel -> sel in the ast. switch p := lp.(type) { case *DataSource: @@ -503,7 +503,7 @@ func (*partitionProcessor) reconstructTableColNames(ds *DataSource) ([]*types.Fi return names, nil } -func (s *partitionProcessor) processHashOrKeyPartition(ds *DataSource, pi *model.PartitionInfo, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (s *partitionProcessor) processHashOrKeyPartition(ds *DataSource, pi *model.PartitionInfo, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { names, err := s.reconstructTableColNames(ds) if err != nil { return nil, err @@ -825,7 +825,7 @@ func (s *partitionProcessor) pruneListPartition(ctx base.PlanContext, tbl table. return used, nil } -func (s *partitionProcessor) prune(ds *DataSource, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (s *partitionProcessor) prune(ds *DataSource, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { pi := ds.tableInfo.GetPartitionInfo() if pi == nil { return ds, nil @@ -1038,7 +1038,7 @@ func (s *partitionProcessor) pruneRangePartition(ctx base.PlanContext, pi *model return result, nil } -func (s *partitionProcessor) processRangePartition(ds *DataSource, pi *model.PartitionInfo, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (s *partitionProcessor) processRangePartition(ds *DataSource, pi *model.PartitionInfo, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { used, err := s.pruneRangePartition(ds.SCtx(), pi, ds.table.(table.PartitionedTable), ds.allConds, ds.TblCols, ds.names) if err != nil { return nil, err @@ -1046,7 +1046,7 @@ func (s *partitionProcessor) processRangePartition(ds *DataSource, pi *model.Par return s.makeUnionAllChildren(ds, pi, used, opt) } -func (s *partitionProcessor) processListPartition(ds *DataSource, pi *model.PartitionInfo, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (s *partitionProcessor) processListPartition(ds *DataSource, pi *model.PartitionInfo, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { used, err := s.pruneListPartition(ds.SCtx(), ds.table, ds.partitionNames, ds.allConds, ds.TblCols) if err != nil { return nil, err @@ -1768,7 +1768,7 @@ func (*partitionProcessor) checkHintsApplicable(ds *DataSource, partitionSet set appendWarnForUnknownPartitions(ds.SCtx(), h.HintReadFromStorage, unknownPartitions) } -func (s *partitionProcessor) makeUnionAllChildren(ds *DataSource, pi *model.PartitionInfo, or partitionRangeOR, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func (s *partitionProcessor) makeUnionAllChildren(ds *DataSource, pi *model.PartitionInfo, or partitionRangeOR, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { children := make([]base.LogicalPlan, 0, len(pi.Definitions)) partitionNameSet := make(set.StringSet) usedDefinition := make(map[int64]model.PartitionDefinition) @@ -2004,7 +2004,7 @@ func (p *rangeColumnsPruner) pruneUseBinarySearch(sctx base.PlanContext, op stri return start, end } -func appendMakeUnionAllChildrenTranceStep(origin *DataSource, usedMap map[int64]model.PartitionDefinition, plan base.LogicalPlan, children []base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) { +func appendMakeUnionAllChildrenTranceStep(origin *DataSource, usedMap map[int64]model.PartitionDefinition, plan base.LogicalPlan, children []base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) { if opt.TracerIsNil() { return } @@ -2060,7 +2060,7 @@ func appendMakeUnionAllChildrenTranceStep(origin *DataSource, usedMap map[int64] opt.AppendStepToCurrent(origin.ID(), origin.TP(), reason, action) } -func appendNoPartitionChildTraceStep(ds *DataSource, dual base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) { +func appendNoPartitionChildTraceStep(ds *DataSource, dual base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v becomes %v_%v", ds.TP(), ds.ID(), dual.TP(), dual.ID()) } diff --git a/pkg/planner/core/rule_predicate_push_down.go b/pkg/planner/core/rule_predicate_push_down.go index 6241f3f47f35c..25921c8fcc667 100644 --- a/pkg/planner/core/rule_predicate_push_down.go +++ b/pkg/planner/core/rule_predicate_push_down.go @@ -25,7 +25,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/util" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/ranger" @@ -43,13 +43,13 @@ type exprPrefixAdder struct { lengths []int } -func (*ppdSolver) optimize(_ context.Context, lp base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (*ppdSolver) optimize(_ context.Context, lp base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false _, p := lp.PredicatePushDown(nil, opt) return p, planChanged, nil } -func addSelection(p base.LogicalPlan, child base.LogicalPlan, conditions []expression.Expression, chIdx int, opt *coreusage.LogicalOptimizeOp) { +func addSelection(p base.LogicalPlan, child base.LogicalPlan, conditions []expression.Expression, chIdx int, opt *optimizetrace.LogicalOptimizeOp) { if len(conditions) == 0 { p.Children()[chIdx] = child return @@ -75,7 +75,7 @@ func addSelection(p base.LogicalPlan, child base.LogicalPlan, conditions []expre } // PredicatePushDown implements base.LogicalPlan interface. -func (p *baseLogicalPlan) PredicatePushDown(predicates []expression.Expression, opt *coreusage.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { +func (p *baseLogicalPlan) PredicatePushDown(predicates []expression.Expression, opt *optimizetrace.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { if len(p.children) == 0 { return predicates, p.self } @@ -99,7 +99,7 @@ func splitSetGetVarFunc(filters []expression.Expression) ([]expression.Expressio } // PredicatePushDown implements base.LogicalPlan PredicatePushDown interface. -func (p *LogicalSelection) PredicatePushDown(predicates []expression.Expression, opt *coreusage.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { +func (p *LogicalSelection) PredicatePushDown(predicates []expression.Expression, opt *optimizetrace.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { predicates = DeleteTrueExprs(p, predicates) p.Conditions = DeleteTrueExprs(p, p.Conditions) var child base.LogicalPlan @@ -125,7 +125,7 @@ func (p *LogicalSelection) PredicatePushDown(predicates []expression.Expression, } // PredicatePushDown implements base.LogicalPlan PredicatePushDown interface. -func (p *LogicalUnionScan) PredicatePushDown(predicates []expression.Expression, opt *coreusage.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { +func (p *LogicalUnionScan) PredicatePushDown(predicates []expression.Expression, opt *optimizetrace.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { retainedPredicates, _ := p.children[0].PredicatePushDown(predicates, opt) p.conditions = make([]expression.Expression, 0, len(predicates)) p.conditions = append(p.conditions, predicates...) @@ -134,7 +134,7 @@ func (p *LogicalUnionScan) PredicatePushDown(predicates []expression.Expression, } // PredicatePushDown implements base.LogicalPlan PredicatePushDown interface. -func (ds *DataSource) PredicatePushDown(predicates []expression.Expression, opt *coreusage.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { +func (ds *DataSource) PredicatePushDown(predicates []expression.Expression, opt *optimizetrace.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { predicates = expression.PropagateConstant(ds.SCtx().GetExprCtx(), predicates) predicates = DeleteTrueExprs(ds, predicates) // Add tidb_shard() prefix to the condtion for shard index in some scenarios @@ -147,12 +147,12 @@ func (ds *DataSource) PredicatePushDown(predicates []expression.Expression, opt } // PredicatePushDown implements base.LogicalPlan PredicatePushDown interface. -func (p *LogicalTableDual) PredicatePushDown(predicates []expression.Expression, _ *coreusage.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { +func (p *LogicalTableDual) PredicatePushDown(predicates []expression.Expression, _ *optimizetrace.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { return predicates, p } // PredicatePushDown implements base.LogicalPlan PredicatePushDown interface. -func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression, opt *coreusage.LogicalOptimizeOp) (ret []expression.Expression, retPlan base.LogicalPlan) { +func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression, opt *optimizetrace.LogicalOptimizeOp) (ret []expression.Expression, retPlan base.LogicalPlan) { simplifyOuterJoin(p, predicates) var equalCond []*expression.ScalarFunction var leftPushCond, rightPushCond, otherCond, leftCond, rightCond []expression.Expression @@ -419,7 +419,7 @@ func simplifyOuterJoin(p *LogicalJoin, predicates []expression.Expression) { } // PredicatePushDown implements base.LogicalPlan PredicatePushDown interface. -func (p *LogicalExpand) PredicatePushDown(predicates []expression.Expression, opt *coreusage.LogicalOptimizeOp) (ret []expression.Expression, retPlan base.LogicalPlan) { +func (p *LogicalExpand) PredicatePushDown(predicates []expression.Expression, opt *optimizetrace.LogicalOptimizeOp) (ret []expression.Expression, retPlan base.LogicalPlan) { // Note that, grouping column related predicates can't be pushed down, since grouping column has nullability change after Expand OP itself. // condition related with grouping column shouldn't be pushed down through it. // currently, since expand is adjacent to aggregate, any filter above aggregate wanted to be push down through expand only have two cases: @@ -431,7 +431,7 @@ func (p *LogicalExpand) PredicatePushDown(predicates []expression.Expression, op } // PredicatePushDown implements base.LogicalPlan PredicatePushDown interface. -func (p *LogicalProjection) PredicatePushDown(predicates []expression.Expression, opt *coreusage.LogicalOptimizeOp) (ret []expression.Expression, retPlan base.LogicalPlan) { +func (p *LogicalProjection) PredicatePushDown(predicates []expression.Expression, opt *optimizetrace.LogicalOptimizeOp) (ret []expression.Expression, retPlan base.LogicalPlan) { canBePushed := make([]expression.Expression, 0, len(predicates)) canNotBePushed := make([]expression.Expression, 0, len(predicates)) for _, expr := range p.Exprs { @@ -454,7 +454,7 @@ func (p *LogicalProjection) PredicatePushDown(predicates []expression.Expression } // PredicatePushDown implements base.LogicalPlan PredicatePushDown interface. -func (p *LogicalUnionAll) PredicatePushDown(predicates []expression.Expression, opt *coreusage.LogicalOptimizeOp) (ret []expression.Expression, retPlan base.LogicalPlan) { +func (p *LogicalUnionAll) PredicatePushDown(predicates []expression.Expression, opt *optimizetrace.LogicalOptimizeOp) (ret []expression.Expression, retPlan base.LogicalPlan) { for i, proj := range p.children { newExprs := make([]expression.Expression, 0, len(predicates)) newExprs = append(newExprs, predicates...) @@ -555,7 +555,7 @@ func (la *LogicalAggregation) pushDownDNFPredicatesForAggregation(cond expressio } // PredicatePushDown implements base.LogicalPlan PredicatePushDown interface. -func (la *LogicalAggregation) PredicatePushDown(predicates []expression.Expression, opt *coreusage.LogicalOptimizeOp) (ret []expression.Expression, retPlan base.LogicalPlan) { +func (la *LogicalAggregation) PredicatePushDown(predicates []expression.Expression, opt *optimizetrace.LogicalOptimizeOp) (ret []expression.Expression, retPlan base.LogicalPlan) { var condsToPush []expression.Expression exprsOriginal := make([]expression.Expression, 0, len(la.AggFuncs)) for _, fun := range la.AggFuncs { @@ -577,14 +577,14 @@ func (la *LogicalAggregation) PredicatePushDown(predicates []expression.Expressi } // PredicatePushDown implements base.LogicalPlan PredicatePushDown interface. -func (p *LogicalLimit) PredicatePushDown(predicates []expression.Expression, opt *coreusage.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { +func (p *LogicalLimit) PredicatePushDown(predicates []expression.Expression, opt *optimizetrace.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { // Limit forbids any condition to push down. p.baseLogicalPlan.PredicatePushDown(nil, opt) return predicates, p } // PredicatePushDown implements base.LogicalPlan PredicatePushDown interface. -func (p *LogicalMaxOneRow) PredicatePushDown(predicates []expression.Expression, opt *coreusage.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { +func (p *LogicalMaxOneRow) PredicatePushDown(predicates []expression.Expression, opt *optimizetrace.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { // MaxOneRow forbids any condition to push down. p.baseLogicalPlan.PredicatePushDown(nil, opt) return predicates, p @@ -735,7 +735,7 @@ func (p *LogicalWindow) GetPartitionByCols() []*expression.Column { } // PredicatePushDown implements base.LogicalPlan PredicatePushDown interface. -func (p *LogicalWindow) PredicatePushDown(predicates []expression.Expression, opt *coreusage.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { +func (p *LogicalWindow) PredicatePushDown(predicates []expression.Expression, opt *optimizetrace.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { canBePushed := make([]expression.Expression, 0, len(predicates)) canNotBePushed := make([]expression.Expression, 0, len(predicates)) partitionCols := expression.NewSchema(p.GetPartitionByCols()...) @@ -753,7 +753,7 @@ func (p *LogicalWindow) PredicatePushDown(predicates []expression.Expression, op } // PredicatePushDown implements base.LogicalPlan PredicatePushDown interface. -func (p *LogicalMemTable) PredicatePushDown(predicates []expression.Expression, _ *coreusage.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { +func (p *LogicalMemTable) PredicatePushDown(predicates []expression.Expression, _ *optimizetrace.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { if p.Extractor != nil { predicates = p.Extractor.Extract(p.SCtx(), p.schema, p.names, predicates) } @@ -764,7 +764,7 @@ func (*ppdSolver) name() string { return "predicate_push_down" } -func appendTableDualTraceStep(replaced base.LogicalPlan, dual base.LogicalPlan, conditions []expression.Expression, opt *coreusage.LogicalOptimizeOp) { +func appendTableDualTraceStep(replaced base.LogicalPlan, dual base.LogicalPlan, conditions []expression.Expression, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v is replaced by %v_%v", replaced.TP(), replaced.ID(), dual.TP(), dual.ID()) } @@ -782,7 +782,7 @@ func appendTableDualTraceStep(replaced base.LogicalPlan, dual base.LogicalPlan, opt.AppendStepToCurrent(dual.ID(), dual.TP(), reason, action) } -func appendSelectionPredicatePushDownTraceStep(p *LogicalSelection, conditions []expression.Expression, opt *coreusage.LogicalOptimizeOp) { +func appendSelectionPredicatePushDownTraceStep(p *LogicalSelection, conditions []expression.Expression, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v is removed", p.TP(), p.ID()) } @@ -805,7 +805,7 @@ func appendSelectionPredicatePushDownTraceStep(p *LogicalSelection, conditions [ opt.AppendStepToCurrent(p.ID(), p.TP(), reason, action) } -func appendDataSourcePredicatePushDownTraceStep(ds *DataSource, opt *coreusage.LogicalOptimizeOp) { +func appendDataSourcePredicatePushDownTraceStep(ds *DataSource, opt *optimizetrace.LogicalOptimizeOp) { if len(ds.pushedDownConds) < 1 { return } @@ -826,7 +826,7 @@ func appendDataSourcePredicatePushDownTraceStep(ds *DataSource, opt *coreusage.L opt.AppendStepToCurrent(ds.ID(), ds.TP(), reason, action) } -func appendAddSelectionTraceStep(p base.LogicalPlan, child base.LogicalPlan, sel *LogicalSelection, opt *coreusage.LogicalOptimizeOp) { +func appendAddSelectionTraceStep(p base.LogicalPlan, child base.LogicalPlan, sel *LogicalSelection, opt *optimizetrace.LogicalOptimizeOp) { reason := func() string { return "" } @@ -957,7 +957,7 @@ func (adder *exprPrefixAdder) addExprPrefix4DNFCond(condition *expression.Scalar } // PredicatePushDown implements base.LogicalPlan PredicatePushDown interface. -func (p *LogicalCTE) PredicatePushDown(predicates []expression.Expression, _ *coreusage.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { +func (p *LogicalCTE) PredicatePushDown(predicates []expression.Expression, _ *optimizetrace.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { if p.cte.recursivePartLogicalPlan != nil { // Doesn't support recursive CTE yet. return predicates, p.self @@ -993,7 +993,7 @@ func (p *LogicalCTE) PredicatePushDown(predicates []expression.Expression, _ *co // PredicatePushDown implements the base.LogicalPlan interface. // Currently, we only maintain the main query tree. -func (p *LogicalSequence) PredicatePushDown(predicates []expression.Expression, op *coreusage.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { +func (p *LogicalSequence) PredicatePushDown(predicates []expression.Expression, op *optimizetrace.LogicalOptimizeOp) ([]expression.Expression, base.LogicalPlan) { lastIdx := len(p.children) - 1 remained, newLastChild := p.children[lastIdx].PredicatePushDown(predicates, op) p.SetChild(lastIdx, newLastChild) diff --git a/pkg/planner/core/rule_predicate_simplification.go b/pkg/planner/core/rule_predicate_simplification.go index 32279a2a3b4d5..a5fc6c740a7f3 100644 --- a/pkg/planner/core/rule_predicate_simplification.go +++ b/pkg/planner/core/rule_predicate_simplification.go @@ -21,7 +21,7 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/planner/core/base" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" ) // predicateSimplification consolidates different predcicates on a column and its equivalence classes. Initial out is for @@ -65,13 +65,13 @@ func findPredicateType(expr expression.Expression) (*expression.Column, predicat return nil, otherPredicate } -func (*predicateSimplification) optimize(_ context.Context, p base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (*predicateSimplification) optimize(_ context.Context, p base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false return p.PredicateSimplification(opt), planChanged, nil } // PredicateSimplification implements the LogicalPlan interface. -func (s *baseLogicalPlan) PredicateSimplification(opt *coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (s *baseLogicalPlan) PredicateSimplification(opt *optimizetrace.LogicalOptimizeOp) base.LogicalPlan { p := s.self for i, child := range p.Children() { newChild := child.PredicateSimplification(opt) @@ -157,7 +157,7 @@ func applyPredicateSimplification(sctx base.PlanContext, predicates []expression } // PredicateSimplification implements the LogicalPlan interface. -func (ds *DataSource) PredicateSimplification(*coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (ds *DataSource) PredicateSimplification(*optimizetrace.LogicalOptimizeOp) base.LogicalPlan { p := ds.self.(*DataSource) p.pushedDownConds = applyPredicateSimplification(p.SCtx(), p.pushedDownConds) p.allConds = applyPredicateSimplification(p.SCtx(), p.allConds) diff --git a/pkg/planner/core/rule_push_down_sequence.go b/pkg/planner/core/rule_push_down_sequence.go index 0435299491906..464ca3fa2dd2c 100644 --- a/pkg/planner/core/rule_push_down_sequence.go +++ b/pkg/planner/core/rule_push_down_sequence.go @@ -18,7 +18,7 @@ import ( "context" "github.com/pingcap/tidb/pkg/planner/core/base" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" ) type pushDownSequenceSolver struct { @@ -28,7 +28,7 @@ func (*pushDownSequenceSolver) name() string { return "push_down_sequence" } -func (pdss *pushDownSequenceSolver) optimize(_ context.Context, lp base.LogicalPlan, _ *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (pdss *pushDownSequenceSolver) optimize(_ context.Context, lp base.LogicalPlan, _ *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false return pdss.recursiveOptimize(nil, lp), planChanged, nil } diff --git a/pkg/planner/core/rule_resolve_grouping_expand.go b/pkg/planner/core/rule_resolve_grouping_expand.go index c548387d9f483..27d886583d2ec 100644 --- a/pkg/planner/core/rule_resolve_grouping_expand.go +++ b/pkg/planner/core/rule_resolve_grouping_expand.go @@ -18,7 +18,7 @@ import ( "context" "github.com/pingcap/tidb/pkg/planner/core/base" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" ) // For normal rollup Expand construction, its logical Expand should be bound @@ -75,7 +75,7 @@ type resolveExpand struct { // (upper required) (grouping sets columns appended) // // Expand operator itself is kind like a projection, while difference is that it has a multi projection list, named as leveled projection. -func (*resolveExpand) optimize(_ context.Context, p base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (*resolveExpand) optimize(_ context.Context, p base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false // As you see, Expand's leveled projection should be built after all column-prune is done. So we just make generating-leveled-projection // as the last rule of logical optimization, which is more clear. (spark has column prune action before building expand) @@ -87,7 +87,7 @@ func (*resolveExpand) name() string { return "resolve_expand" } -func genExpand(p base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, error) { +func genExpand(p base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, error) { for i, child := range p.Children() { np, err := genExpand(child, opt) if err != nil { diff --git a/pkg/planner/core/rule_result_reorder.go b/pkg/planner/core/rule_result_reorder.go index 8590ac6408a6f..d1165d7d9740b 100644 --- a/pkg/planner/core/rule_result_reorder.go +++ b/pkg/planner/core/rule_result_reorder.go @@ -20,7 +20,7 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/util" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" ) /* @@ -41,7 +41,7 @@ This rule reorders results by modifying or injecting a Sort operator: type resultReorder struct { } -func (rs *resultReorder) optimize(_ context.Context, lp base.LogicalPlan, _ *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (rs *resultReorder) optimize(_ context.Context, lp base.LogicalPlan, _ *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false ordered := rs.completeSort(lp) if !ordered { diff --git a/pkg/planner/core/rule_semi_join_rewrite.go b/pkg/planner/core/rule_semi_join_rewrite.go index 1a294e2b5c5fb..87d0292fa351e 100644 --- a/pkg/planner/core/rule_semi_join_rewrite.go +++ b/pkg/planner/core/rule_semi_join_rewrite.go @@ -21,7 +21,7 @@ import ( "github.com/pingcap/tidb/pkg/expression/aggregation" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/planner/core/base" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" h "github.com/pingcap/tidb/pkg/util/hint" ) @@ -38,7 +38,7 @@ import ( type semiJoinRewriter struct { } -func (smj *semiJoinRewriter) optimize(_ context.Context, p base.LogicalPlan, _ *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (smj *semiJoinRewriter) optimize(_ context.Context, p base.LogicalPlan, _ *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false newLogicalPlan, err := smj.recursivePlan(p) return newLogicalPlan, planChanged, err diff --git a/pkg/planner/core/rule_topn_push_down.go b/pkg/planner/core/rule_topn_push_down.go index 0343e9bda108c..36acff2330e85 100644 --- a/pkg/planner/core/rule_topn_push_down.go +++ b/pkg/planner/core/rule_topn_push_down.go @@ -22,20 +22,20 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/util" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" ) // pushDownTopNOptimizer pushes down the topN or limit. In the future we will remove the limit from `requiredProperty` in CBO phase. type pushDownTopNOptimizer struct { } -func (*pushDownTopNOptimizer) optimize(_ context.Context, p base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { +func (*pushDownTopNOptimizer) optimize(_ context.Context, p base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) (base.LogicalPlan, bool, error) { planChanged := false return p.PushDownTopN(nil, opt), planChanged, nil } // PushDownTopN implements the LogicalPlan interface. -func (s *baseLogicalPlan) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (s *baseLogicalPlan) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) base.LogicalPlan { var topN *LogicalTopN if topNLogicalPlan != nil { topN = topNLogicalPlan.(*LogicalTopN) @@ -51,7 +51,7 @@ func (s *baseLogicalPlan) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *co } // PushDownTopN implements the LogicalPlan interface. -func (p *LogicalCTE) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (p *LogicalCTE) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) base.LogicalPlan { var topN *LogicalTopN if topNLogicalPlan != nil { topN = topNLogicalPlan.(*LogicalTopN) @@ -63,7 +63,7 @@ func (p *LogicalCTE) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *coreusa } // setChild set p as topn's child. -func (lt *LogicalTopN) setChild(p base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (lt *LogicalTopN) setChild(p base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) base.LogicalPlan { // Remove this TopN if its child is a TableDual. dual, isDual := p.(*LogicalTableDual) if isDual { @@ -94,7 +94,7 @@ func (lt *LogicalTopN) setChild(p base.LogicalPlan, opt *coreusage.LogicalOptimi } // PushDownTopN implements LogicalPlan interface. -func (ls *LogicalSort) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (ls *LogicalSort) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) base.LogicalPlan { var topN *LogicalTopN if topNLogicalPlan != nil { topN = topNLogicalPlan.(*LogicalTopN) @@ -110,14 +110,14 @@ func (ls *LogicalSort) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *coreu return ls.children[0].PushDownTopN(topN, opt) } -func (p *LogicalLimit) convertToTopN(opt *coreusage.LogicalOptimizeOp) *LogicalTopN { +func (p *LogicalLimit) convertToTopN(opt *optimizetrace.LogicalOptimizeOp) *LogicalTopN { topn := LogicalTopN{Offset: p.Offset, Count: p.Count, PreferLimitToCop: p.PreferLimitToCop}.Init(p.SCtx(), p.QueryBlockOffset()) appendConvertTopNTraceStep(p, topn, opt) return topn } // PushDownTopN implements the LogicalPlan interface. -func (p *LogicalLimit) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (p *LogicalLimit) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) base.LogicalPlan { var topN *LogicalTopN if topNLogicalPlan != nil { topN = topNLogicalPlan.(*LogicalTopN) @@ -130,7 +130,7 @@ func (p *LogicalLimit) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *coreu } // PushDownTopN implements the LogicalPlan interface. -func (p *LogicalUnionAll) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (p *LogicalUnionAll) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) base.LogicalPlan { var topN *LogicalTopN if topNLogicalPlan != nil { topN = topNLogicalPlan.(*LogicalTopN) @@ -154,7 +154,7 @@ func (p *LogicalUnionAll) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *co } // PushDownTopN implements LogicalPlan interface. -func (p *LogicalProjection) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (p *LogicalProjection) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) base.LogicalPlan { var topN *LogicalTopN if topNLogicalPlan != nil { topN = topNLogicalPlan.(*LogicalTopN) @@ -197,7 +197,7 @@ func (p *LogicalProjection) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt * } // PushDownTopN implements the LogicalPlan interface. -func (p *LogicalLock) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (p *LogicalLock) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) base.LogicalPlan { var topN *LogicalTopN if topNLogicalPlan != nil { topN = topNLogicalPlan.(*LogicalTopN) @@ -209,7 +209,7 @@ func (p *LogicalLock) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *coreus } // pushDownTopNToChild will push a topN to one child of join. The idx stands for join child index. 0 is for left child. -func (p *LogicalJoin) pushDownTopNToChild(topN *LogicalTopN, idx int, opt *coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (p *LogicalJoin) pushDownTopNToChild(topN *LogicalTopN, idx int, opt *optimizetrace.LogicalOptimizeOp) base.LogicalPlan { if topN == nil { return p.children[idx].PushDownTopN(nil, opt) } @@ -236,7 +236,7 @@ func (p *LogicalJoin) pushDownTopNToChild(topN *LogicalTopN, idx int, opt *coreu } // PushDownTopN implements the LogicalPlan interface. -func (p *LogicalJoin) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) base.LogicalPlan { +func (p *LogicalJoin) PushDownTopN(topNLogicalPlan base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) base.LogicalPlan { var topN *LogicalTopN if topNLogicalPlan != nil { topN = topNLogicalPlan.(*LogicalTopN) @@ -263,7 +263,7 @@ func (*pushDownTopNOptimizer) name() string { return "topn_push_down" } -func appendTopNPushDownTraceStep(parent base.LogicalPlan, child base.LogicalPlan, opt *coreusage.LogicalOptimizeOp) { +func appendTopNPushDownTraceStep(parent base.LogicalPlan, child base.LogicalPlan, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { return fmt.Sprintf("%v_%v is added as %v_%v's parent", parent.TP(), parent.ID(), child.TP(), child.ID()) } @@ -273,7 +273,7 @@ func appendTopNPushDownTraceStep(parent base.LogicalPlan, child base.LogicalPlan opt.AppendStepToCurrent(parent.ID(), parent.TP(), reason, action) } -func appendTopNPushDownJoinTraceStep(p *LogicalJoin, topN *LogicalTopN, idx int, opt *coreusage.LogicalOptimizeOp) { +func appendTopNPushDownJoinTraceStep(p *LogicalJoin, topN *LogicalTopN, idx int, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v is added and pushed into %v_%v's ", topN.TP(), topN.ID(), p.TP(), p.ID())) @@ -305,7 +305,7 @@ func appendTopNPushDownJoinTraceStep(p *LogicalJoin, topN *LogicalTopN, idx int, opt.AppendStepToCurrent(p.ID(), p.TP(), reason, action) } -func appendSortPassByItemsTraceStep(sort *LogicalSort, topN *LogicalTopN, opt *coreusage.LogicalOptimizeOp) { +func appendSortPassByItemsTraceStep(sort *LogicalSort, topN *LogicalTopN, opt *optimizetrace.LogicalOptimizeOp) { action := func() string { buffer := bytes.NewBufferString(fmt.Sprintf("%v_%v passes ByItems[", sort.TP(), sort.ID())) for i, item := range sort.ByItems { @@ -323,7 +323,7 @@ func appendSortPassByItemsTraceStep(sort *LogicalSort, topN *LogicalTopN, opt *c opt.AppendStepToCurrent(sort.ID(), sort.TP(), reason, action) } -func appendNewTopNTraceStep(topN *LogicalTopN, union *LogicalUnionAll, opt *coreusage.LogicalOptimizeOp) { +func appendNewTopNTraceStep(topN *LogicalTopN, union *LogicalUnionAll, opt *optimizetrace.LogicalOptimizeOp) { reason := func() string { return "" } @@ -333,7 +333,7 @@ func appendNewTopNTraceStep(topN *LogicalTopN, union *LogicalUnionAll, opt *core opt.AppendStepToCurrent(topN.ID(), topN.TP(), reason, action) } -func appendConvertTopNTraceStep(p base.LogicalPlan, topN *LogicalTopN, opt *coreusage.LogicalOptimizeOp) { +func appendConvertTopNTraceStep(p base.LogicalPlan, topN *LogicalTopN, opt *optimizetrace.LogicalOptimizeOp) { reason := func() string { return "" } diff --git a/pkg/planner/core/util.go b/pkg/planner/core/util.go index 58f331a29f174..bbe8d76c254a6 100644 --- a/pkg/planner/core/util.go +++ b/pkg/planner/core/util.go @@ -26,7 +26,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/core/operator/baseimpl" - "github.com/pingcap/tidb/pkg/planner/util/coreusage" + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/tablecodec" @@ -138,7 +138,7 @@ func (s *logicalSchemaProducer) setSchemaAndNames(schema *expression.Schema, nam } // inlineProjection prunes unneeded columns inline a executor. -func (s *logicalSchemaProducer) inlineProjection(parentUsedCols []*expression.Column, opt *coreusage.LogicalOptimizeOp) { +func (s *logicalSchemaProducer) inlineProjection(parentUsedCols []*expression.Column, opt *optimizetrace.LogicalOptimizeOp) { prunedColumns := make([]*expression.Column, 0) used := expression.GetUsedList(s.SCtx().GetExprCtx().GetEvalCtx(), parentUsedCols, s.Schema()) for i := len(used) - 1; i >= 0; i-- { diff --git a/pkg/planner/util/coreusage/BUILD.bazel b/pkg/planner/util/coreusage/BUILD.bazel index ccf6c58c1638e..6f083eb7f78f5 100644 --- a/pkg/planner/util/coreusage/BUILD.bazel +++ b/pkg/planner/util/coreusage/BUILD.bazel @@ -4,14 +4,14 @@ go_library( name = "coreusage", srcs = [ "cast_misc.go", - "cost_misc.go", - "opt_tracer.go", + "correlated_misc.go", ], importpath = "github.com/pingcap/tidb/pkg/planner/util/coreusage", visibility = ["//visibility:public"], deps = [ "//pkg/expression", "//pkg/expression/aggregation", - "//pkg/util/tracing", + "//pkg/planner/core/base", + "//pkg/types", ], ) diff --git a/pkg/planner/util/coreusage/correlated_misc.go b/pkg/planner/util/coreusage/correlated_misc.go new file mode 100644 index 0000000000000..2d6c1dcb7ed05 --- /dev/null +++ b/pkg/planner/util/coreusage/correlated_misc.go @@ -0,0 +1,95 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package coreusage + +import ( + "github.com/pingcap/tidb/pkg/expression" + "github.com/pingcap/tidb/pkg/planner/core/base" + "github.com/pingcap/tidb/pkg/types" +) + +// ExtractCorrelatedCols4LogicalPlan recursively extracts all of the correlated columns +// from a plan tree by calling base.LogicalPlan.ExtractCorrelatedCols. +func ExtractCorrelatedCols4LogicalPlan(p base.LogicalPlan) []*expression.CorrelatedColumn { + corCols := p.ExtractCorrelatedCols() + for _, child := range p.Children() { + corCols = append(corCols, ExtractCorrelatedCols4LogicalPlan(child)...) + } + return corCols +} + +// ExtractCorrelatedCols4PhysicalPlan recursively extracts all of the correlated columns +// from a plan tree by calling PhysicalPlan.ExtractCorrelatedCols. +func ExtractCorrelatedCols4PhysicalPlan(p base.PhysicalPlan) []*expression.CorrelatedColumn { + corCols := p.ExtractCorrelatedCols() + for _, child := range p.Children() { + corCols = append(corCols, ExtractCorrelatedCols4PhysicalPlan(child)...) + } + return corCols +} + +// ExtractCorColumnsBySchema4LogicalPlan only extracts the correlated columns that match the specified schema. +// e.g. If the correlated columns from plan are [t1.a, t2.a, t3.a] and specified schema is [t2.a, t2.b, t2.c], +// only [t2.a] is returned. +func ExtractCorColumnsBySchema4LogicalPlan(p base.LogicalPlan, + schema *expression.Schema) []*expression.CorrelatedColumn { + corCols := ExtractCorrelatedCols4LogicalPlan(p) + return ExtractCorColumnsBySchema(corCols, schema, false) +} + +// ExtractCorColumnsBySchema4PhysicalPlan only extracts the correlated columns that match the specified schema. +// e.g. If the correlated columns from plan are [t1.a, t2.a, t3.a] and specified schema is [t2.a, t2.b, t2.c], +// only [t2.a] is returned. +func ExtractCorColumnsBySchema4PhysicalPlan(p base.PhysicalPlan, + schema *expression.Schema) []*expression.CorrelatedColumn { + corCols := ExtractCorrelatedCols4PhysicalPlan(p) + return ExtractCorColumnsBySchema(corCols, schema, true) +} + +// ExtractCorColumnsBySchema only extracts the correlated columns that match the specified schema. +// e.g. If the correlated columns from plan are [t1.a, t2.a, t3.a] and specified schema is [t2.a, t2.b, t2.c], +// only [t2.a] is returned. +func ExtractCorColumnsBySchema(corCols []*expression.CorrelatedColumn, + schema *expression.Schema, resolveIndex bool) []*expression.CorrelatedColumn { + resultCorCols := make([]*expression.CorrelatedColumn, schema.Len()) + for _, corCol := range corCols { + idx := schema.ColumnIndex(&corCol.Column) + if idx != -1 { + if resultCorCols[idx] == nil { + resultCorCols[idx] = &expression.CorrelatedColumn{ + Column: *schema.Columns[idx], + Data: new(types.Datum), + } + } + corCol.Data = resultCorCols[idx].Data + } + } + // Shrink slice. e.g. [col1, nil, col2, nil] will be changed to [col1, col2]. + length := 0 + for _, col := range resultCorCols { + if col != nil { + resultCorCols[length] = col + length++ + } + } + resultCorCols = resultCorCols[:length] + + if resolveIndex { + for _, corCol := range resultCorCols { + corCol.Index = schema.ColumnIndex(&corCol.Column) + } + } + return resultCorCols +} diff --git a/pkg/planner/util/costusage/BUILD.bazel b/pkg/planner/util/costusage/BUILD.bazel new file mode 100644 index 0000000000000..4eab5d18c1ef8 --- /dev/null +++ b/pkg/planner/util/costusage/BUILD.bazel @@ -0,0 +1,9 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "costusage", + srcs = ["cost_misc.go"], + importpath = "github.com/pingcap/tidb/pkg/planner/util/costusage", + visibility = ["//visibility:public"], + deps = ["//pkg/planner/util/optimizetrace"], +) diff --git a/pkg/planner/util/coreusage/cost_misc.go b/pkg/planner/util/costusage/cost_misc.go similarity index 80% rename from pkg/planner/util/coreusage/cost_misc.go rename to pkg/planner/util/costusage/cost_misc.go index 1ed4b9ad64721..ba3a5cc6d1761 100644 --- a/pkg/planner/util/coreusage/cost_misc.go +++ b/pkg/planner/util/costusage/cost_misc.go @@ -12,13 +12,30 @@ // See the License for the specific language governing permissions and // limitations under the License. -package coreusage +package costusage import ( "fmt" "strconv" + + "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" ) +// optimizetrace and costusage is isolated from `util` because `core/base` depended on them +// for interface definition. Ideally, the dependency chain should be: +// +// `base` <- `util`/`util.coreusage` <- `core` +// ^ +---------------^ | +// +------------------------------------+ +// +// since `base` depended on optimizetrace and costusage for definition, we should separate +// them out of `util`/`util.coreusage` to avoid import cycle. +// +// util.optimizetrace/util.costusage <- `base` <- `util`/`util.coreusage` <- `core` +// ^ ^ || +// | +------------------------------------+| +// +-------------------------------------------------------------+ + const ( // CostFlagRecalculate indicates the optimizer to ignore cached cost and recalculate it again. CostFlagRecalculate uint64 = 1 << iota @@ -30,6 +47,10 @@ const ( CostFlagTrace ) +func init() { + optimizetrace.CostFlagTrace = CostFlagTrace +} + // CostVer2 is a structure of cost basic of version2 type CostVer2 struct { cost float64 @@ -75,7 +96,7 @@ func hasCostFlag(costFlag, flag uint64) bool { } // TraceCost indicates whether to trace cost. -func TraceCost(option *PlanCostOption) bool { +func TraceCost(option *optimizetrace.PlanCostOption) bool { if option != nil && hasCostFlag(option.CostFlag, CostFlagTrace) { return true } @@ -83,7 +104,7 @@ func TraceCost(option *PlanCostOption) bool { } // NewCostVer2 is the constructor of CostVer2. -func NewCostVer2(option *PlanCostOption, factor CostVer2Factor, cost float64, +func NewCostVer2(option *optimizetrace.PlanCostOption, factor CostVer2Factor, cost float64, lazyFormula func() string) (ret CostVer2) { ret.cost = cost if TraceCost(option) { diff --git a/pkg/planner/util/optimizetrace/BUILD.bazel b/pkg/planner/util/optimizetrace/BUILD.bazel new file mode 100644 index 0000000000000..ca91edc06035b --- /dev/null +++ b/pkg/planner/util/optimizetrace/BUILD.bazel @@ -0,0 +1,9 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "optimizetrace", + srcs = ["opt_tracer.go"], + importpath = "github.com/pingcap/tidb/pkg/planner/util/optimizetrace", + visibility = ["//visibility:public"], + deps = ["//pkg/util/tracing"], +) diff --git a/pkg/planner/util/coreusage/opt_tracer.go b/pkg/planner/util/optimizetrace/opt_tracer.go similarity index 81% rename from pkg/planner/util/coreusage/opt_tracer.go rename to pkg/planner/util/optimizetrace/opt_tracer.go index d7a6a0cd38a13..3008a438392e1 100644 --- a/pkg/planner/util/coreusage/opt_tracer.go +++ b/pkg/planner/util/optimizetrace/opt_tracer.go @@ -12,14 +12,29 @@ // See the License for the specific language governing permissions and // limitations under the License. -package coreusage +package optimizetrace -import "github.com/pingcap/tidb/pkg/util/tracing" +import ( + "github.com/pingcap/tidb/pkg/util/tracing" +) +// optimizetrace and costusage is isolated from util because core/base depended on them for +// interface definition. Ideally, the dependency chain should be: +// +// `base` <- `util`/`util.coreusage` <- `core` +// ^ +---------------^ | +// +------------------------------------+ +// +// since `base` depended on optimizetrace and costusage for definition, we should separate +// them out of `util`/`util.coreusage` to avoid import cycle. +// +// util.optimizetrace/util.costusage <- `base` <- `util`/`util.coreusage` <- `core` +// ^ ^ || +// | +------------------------------------+| +// +-------------------------------------------------------------+ +// // optTracer define those basic element for logical optimizing trace and physical optimizing trace. -// logicalOptRule inside the accommodated pkg `util` should only be depended on by logical `rule` pkg. // -// rule related -----> core/util //********************** below logical optimize trace related ************************* // LogicalOptimizeOp is logical optimizing option for tracing. @@ -122,6 +137,10 @@ func (op *PlanCostOption) WithCostFlag(flag uint64) *PlanCostOption { return op } +// CostFlagTrace is a mirror of costusage.CostFlagTrace, leveraging +// initialization assigment to avoid import cycle. +var CostFlagTrace uint64 + // WithOptimizeTracer set tracer func (op *PlanCostOption) WithOptimizeTracer(v *PhysicalOptimizeOp) *PlanCostOption { if op == nil { From 6ed307ec244cb8823f3e1df9c76b862db901c216 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Thu, 25 Apr 2024 20:21:11 +0800 Subject: [PATCH 08/20] testkit: fix a previous mistake that causes infoschema v2 not run (#52880) close pingcap/tidb#52753 --- pkg/infoschema/test/infoschemav2test/v2_test.go | 1 - pkg/testkit/testkit.go | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/infoschema/test/infoschemav2test/v2_test.go b/pkg/infoschema/test/infoschemav2test/v2_test.go index aa1e12cab6f4d..936f948cc29e0 100644 --- a/pkg/infoschema/test/infoschemav2test/v2_test.go +++ b/pkg/infoschema/test/infoschemav2test/v2_test.go @@ -32,7 +32,6 @@ func TestSpecialSchemas(t *testing.T) { require.NoError(t, tk.Session().Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil, nil)) tk.MustExec("use test") - tk.MustQuery("select @@global.tidb_schema_cache_size;").Check(testkit.Rows("0")) tk.MustExec("set @@global.tidb_schema_cache_size = 1024;") tk.MustQuery("select @@global.tidb_schema_cache_size;").Check(testkit.Rows("1024")) tk.MustExec("create table t (id int);") diff --git a/pkg/testkit/testkit.go b/pkg/testkit/testkit.go index c435b0b762044..41aadd32573f5 100644 --- a/pkg/testkit/testkit.go +++ b/pkg/testkit/testkit.go @@ -122,8 +122,8 @@ func (tk *TestKit) RefreshSession() { eb, ok := tk.store.(kv.EtcdBackend) if ok { // Only for unit test now addrs, err := eb.EtcdAddrs() - if err == nil && len(addrs) > 0 { - if rand.Intn(10) > 3 { // 70% chance to run infoschema v2 + if err == nil && len(addrs) == 0 { + if rand.Intn(10) >= 3 { // 70% chance to run infoschema v2 tk.MustExec("set @@global.tidb_schema_cache_size = 1024") } } From 54ba0ed68407878abcd2a8f538cb87be2fbfdf23 Mon Sep 17 00:00:00 2001 From: Luo Yangzhixin Date: Thu, 25 Apr 2024 22:21:12 +0800 Subject: [PATCH 09/20] lightning: fix delete additionally inserted rows for conflict detection 'replace' mode failed (#52888) close pingcap/tidb#52886 --- pkg/lightning/errormanager/errormanager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/lightning/errormanager/errormanager.go b/pkg/lightning/errormanager/errormanager.go index ec0e27582dffc..bb86304dc2f49 100644 --- a/pkg/lightning/errormanager/errormanager.go +++ b/pkg/lightning/errormanager/errormanager.go @@ -164,7 +164,7 @@ const ( deleteNullDataRow = ` DELETE FROM %s.` + ConflictErrorTableName + ` - WHERE key_data = "" and row_data = ""; + WHERE key_data = '' and row_data = ''; ` insertIntoDupRecord = ` From 545b4a3a44323094608ea90585dc3a5ba5c0ae2a Mon Sep 17 00:00:00 2001 From: 3pointer Date: Fri, 26 Apr 2024 00:25:41 +0800 Subject: [PATCH 10/20] advancer: fix the incorrect gc safepoint behaviours (#52835) ref pingcap/tidb#52082 --- br/pkg/streamhelper/advancer.go | 2 +- br/pkg/streamhelper/advancer_env.go | 18 +++++++++++++++++- br/pkg/streamhelper/advancer_test.go | 2 +- br/pkg/streamhelper/basic_lib_for_test.go | 20 ++++++++++++-------- br/pkg/streamhelper/regioniter.go | 3 +++ br/pkg/streamhelper/regioniter_test.go | 4 ++++ 6 files changed, 38 insertions(+), 11 deletions(-) diff --git a/br/pkg/streamhelper/advancer.go b/br/pkg/streamhelper/advancer.go index aaa4231010dc9..0ed0c79b70d01 100644 --- a/br/pkg/streamhelper/advancer.go +++ b/br/pkg/streamhelper/advancer.go @@ -444,7 +444,7 @@ func (c *CheckpointAdvancer) onTaskEvent(ctx context.Context, e TaskEvent) error if err := c.env.ClearV3GlobalCheckpointForTask(ctx, e.Name); err != nil { log.Warn("failed to clear global checkpoint", logutil.ShortError(err)) } - if _, err := c.env.BlockGCUntil(ctx, 0); err != nil { + if err := c.env.UnblockGC(ctx); err != nil { log.Warn("failed to remove service GC safepoint", logutil.ShortError(err)) } metrics.LastCheckpoint.DeleteLabelValues(e.Name) diff --git a/br/pkg/streamhelper/advancer_env.go b/br/pkg/streamhelper/advancer_env.go index 7092edb3686ea..d1f83d24bff93 100644 --- a/br/pkg/streamhelper/advancer_env.go +++ b/br/pkg/streamhelper/advancer_env.go @@ -4,8 +4,10 @@ package streamhelper import ( "context" + "math" "time" + "github.com/pingcap/errors" logbackup "github.com/pingcap/kvproto/pkg/logbackuppb" "github.com/pingcap/tidb/br/pkg/utils" "github.com/pingcap/tidb/pkg/config" @@ -46,7 +48,21 @@ type PDRegionScanner struct { // Returns the minimal service GC safe point across all services. // If the arguments is `0`, this would remove the service safe point. func (c PDRegionScanner) BlockGCUntil(ctx context.Context, at uint64) (uint64, error) { - return c.UpdateServiceGCSafePoint(ctx, logBackupServiceID, int64(logBackupSafePointTTL.Seconds()), at) + minimalSafePoint, err := c.UpdateServiceGCSafePoint( + ctx, logBackupServiceID, int64(logBackupSafePointTTL.Seconds()), at) + if err != nil { + return 0, errors.Annotate(err, "failed to block gc until") + } + if minimalSafePoint > at { + return 0, errors.Errorf("minimal safe point %d is greater than the target %d", minimalSafePoint, at) + } + return at, nil +} + +func (c PDRegionScanner) UnblockGC(ctx context.Context) error { + // set ttl to 0, means remove the safe point. + _, err := c.UpdateServiceGCSafePoint(ctx, logBackupServiceID, 0, math.MaxUint64) + return err } // TODO: It should be able to synchoronize the current TS with the PD. diff --git a/br/pkg/streamhelper/advancer_test.go b/br/pkg/streamhelper/advancer_test.go index ef617a736d132..f8889c4f4e8a2 100644 --- a/br/pkg/streamhelper/advancer_test.go +++ b/br/pkg/streamhelper/advancer_test.go @@ -208,7 +208,7 @@ func TestGCServiceSafePoint(t *testing.T) { req.Eventually(func() bool { env.fakeCluster.mu.Lock() defer env.fakeCluster.mu.Unlock() - return env.serviceGCSafePoint == 0 + return env.serviceGCSafePoint != 0 && env.serviceGCSafePointDeleted }, 3*time.Second, 100*time.Millisecond) } diff --git a/br/pkg/streamhelper/basic_lib_for_test.go b/br/pkg/streamhelper/basic_lib_for_test.go index 4d26ebdb14b62..82cdb709f11ca 100644 --- a/br/pkg/streamhelper/basic_lib_for_test.go +++ b/br/pkg/streamhelper/basic_lib_for_test.go @@ -100,10 +100,11 @@ type fakeCluster struct { regions []*region testCtx *testing.T - onGetClient func(uint64) error - onClearCache func(uint64) error - serviceGCSafePoint uint64 - currentTS uint64 + onGetClient func(uint64) error + onClearCache func(uint64) error + serviceGCSafePoint uint64 + serviceGCSafePointDeleted bool + currentTS uint64 } func (r *region) splitAt(newID uint64, k string) *region { @@ -264,10 +265,6 @@ func (f *fakeStore) GetLastFlushTSOfRegion(ctx context.Context, in *logbackup.Ge func (f *fakeCluster) BlockGCUntil(ctx context.Context, at uint64) (uint64, error) { f.mu.Lock() defer f.mu.Unlock() - if at == 0 { - f.serviceGCSafePoint = at - return at, nil - } if f.serviceGCSafePoint > at { return f.serviceGCSafePoint, nil } @@ -275,6 +272,13 @@ func (f *fakeCluster) BlockGCUntil(ctx context.Context, at uint64) (uint64, erro return at, nil } +func (f *fakeCluster) UnblockGC(ctx context.Context) error { + f.mu.Lock() + defer f.mu.Unlock() + f.serviceGCSafePointDeleted = true + return nil +} + func (f *fakeCluster) FetchCurrentTS(ctx context.Context) (uint64, error) { return f.currentTS, nil } diff --git a/br/pkg/streamhelper/regioniter.go b/br/pkg/streamhelper/regioniter.go index 80e75f1895b50..b6f77b45d574f 100644 --- a/br/pkg/streamhelper/regioniter.go +++ b/br/pkg/streamhelper/regioniter.go @@ -43,6 +43,9 @@ type TiKVClusterMeta interface { // For now, all tasks (exactly one task in fact) use the same checkpoint. BlockGCUntil(ctx context.Context, at uint64) (uint64, error) + // UnblockGC used to remove the service GC safe point in PD. + UnblockGC(ctx context.Context) error + FetchCurrentTS(ctx context.Context) (uint64, error) } diff --git a/br/pkg/streamhelper/regioniter_test.go b/br/pkg/streamhelper/regioniter_test.go index 967b24b71f2a4..367f3ff35884c 100644 --- a/br/pkg/streamhelper/regioniter_test.go +++ b/br/pkg/streamhelper/regioniter_test.go @@ -83,6 +83,10 @@ func (c constantRegions) BlockGCUntil(ctx context.Context, at uint64) (uint64, e return 0, status.Error(codes.Unimplemented, "Unsupported operation") } +func (c constantRegions) UnblockGC(ctx context.Context) error { + return status.Error(codes.Unimplemented, "Unsupported operation") +} + // TODO: It should be able to synchoronize the current TS with the PD. func (c constantRegions) FetchCurrentTS(ctx context.Context) (uint64, error) { return oracle.ComposeTS(time.Now().UnixMilli(), 0), nil From 9fffc2f5ee4df96929300b082d190dd85b108491 Mon Sep 17 00:00:00 2001 From: yibin Date: Fri, 26 Apr 2024 00:25:48 +0800 Subject: [PATCH 11/20] planner: make sure mpp join task's hashCols are all contained of its plan's schema (#52836) close pingcap/tidb#52828 --- pkg/planner/core/casetest/mpp/BUILD.bazel | 2 +- pkg/planner/core/casetest/mpp/mpp_test.go | 39 +++ .../mpp/testdata/integration_suite_out.json | 249 +++++++++--------- .../testdata/plan_suite_out.json | 35 +-- pkg/planner/core/rule_eliminate_projection.go | 6 +- pkg/planner/core/task.go | 51 +++- 6 files changed, 228 insertions(+), 154 deletions(-) diff --git a/pkg/planner/core/casetest/mpp/BUILD.bazel b/pkg/planner/core/casetest/mpp/BUILD.bazel index 5a32c6bebf92e..8e5eb92d33939 100644 --- a/pkg/planner/core/casetest/mpp/BUILD.bazel +++ b/pkg/planner/core/casetest/mpp/BUILD.bazel @@ -9,7 +9,7 @@ go_test( ], data = glob(["testdata/**"]), flaky = True, - shard_count = 20, + shard_count = 21, deps = [ "//pkg/config", "//pkg/domain", diff --git a/pkg/planner/core/casetest/mpp/mpp_test.go b/pkg/planner/core/casetest/mpp/mpp_test.go index 631a31f0e6558..73c32ff2a04cf 100644 --- a/pkg/planner/core/casetest/mpp/mpp_test.go +++ b/pkg/planner/core/casetest/mpp/mpp_test.go @@ -993,3 +993,42 @@ func TestMppVersion(t *testing.T) { } } } + +func TestIssue52828(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("DROP TABLE IF EXISTS b") + tk.MustExec("CREATE TABLE b (col_int int(11) DEFAULT NULL, col_datetime_not_null datetime NOT NULL, col_int_not_null int(11) NOT NULL, col_decimal_not_null decimal(10,0) NOT NULL)") + tk.MustExec("INSERT INTO b VALUES (0,'2001-09-16 00:00:00',1,1622212608)") + tk.MustExec("DROP TABLE IF EXISTS c") + tk.MustExec("CREATE TABLE c (pk int(11) NOT NULL, col_decimal_not_null decimal(10,0) NOT NULL)") + tk.MustExec("INSERT INTO C VALUES (30,-636485632); INSERT INTO c VALUES (50,-1094713344)") + tk.MustExec("DROP TABLE IF EXISTS dd") + tk.MustExec("CREATE TABLE dd (col_varchar_10 varchar(10) DEFAULT NULL, col_varchar_10_not_null varchar(10) NOT NULL, pk int(11), col_int_not_null int(11) NOT NULL)") + tk.MustExec("INSERT INTO dd VALUES ('','t',1,-1823473664), ('for','p',2,1150025728), ('p','',3,2014511104), ('y','this',4,0), ('y','w',5,-510132224)") + tk.MustExec("analyze table b") + tk.MustExec("analyze table c") + tk.MustExec("analyze table dd") + + // Create virtual tiflash replica info. + is := dom.InfoSchema() + db, exists := is.SchemaByName(model.NewCIStr("test")) + require.True(t, exists) + for _, tbl := range is.SchemaTables(db.Name) { + tblInfo := tbl.Meta() + if tblInfo.Name.L == "b" || tblInfo.Name.L == "c" || tblInfo.Name.L == "dd" { + tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{ + Count: 1, + Available: true, + } + } + } + + tk.MustExec("set @@tidb_allow_mpp=1; set @@tidb_enforce_mpp=1") + tk.MustExec("set @@tidb_isolation_read_engines = 'tiflash'") + tk.MustExec("set @@tidb_broadcast_join_threshold_size = 0") + tk.MustExec("set @@tidb_broadcast_join_threshold_count = 0") + tk.MustQuery("explain SELECT MAX( OUTR . col_int ) AS X FROM C AS OUTR2 INNER JOIN B AS OUTR ON ( OUTR2 . col_decimal_not_null = OUTR . col_decimal_not_null AND OUTR2 . pk = OUTR . col_int_not_null ) " + + "WHERE OUTR . col_decimal_not_null IN ( SELECT INNR . col_int_not_null + 1 AS Y FROM DD AS INNR WHERE INNR . pk > INNR . pk OR INNR . col_varchar_10_not_null >= INNR . col_varchar_10 ) GROUP BY OUTR . col_datetime_not_null") +} diff --git a/pkg/planner/core/casetest/mpp/testdata/integration_suite_out.json b/pkg/planner/core/casetest/mpp/testdata/integration_suite_out.json index 68da07965345c..0c339b9b06b6c 100644 --- a/pkg/planner/core/casetest/mpp/testdata/integration_suite_out.json +++ b/pkg/planner/core/casetest/mpp/testdata/integration_suite_out.json @@ -608,7 +608,7 @@ "StreamAgg 1.00 root funcs:count(1)->Column#7", "└─TableReader 2.00 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 2.00 mpp[tiflash] ExchangeType: PassThrough", - " └─Projection 2.00 mpp[tiflash] 1->Column#11", + " └─Projection 2.00 mpp[tiflash] test.a.id", " └─HashJoin 2.00 mpp[tiflash] left outer join, equal:[eq(test.a.id, test.b.id)]", " ├─ExchangeReceiver(Build) 3.00 mpp[tiflash] ", " │ └─ExchangeSender 3.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.b.id, collate: binary]", @@ -625,7 +625,7 @@ "StreamAgg 1.00 root funcs:count(1)->Column#7", "└─TableReader 2.00 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 2.00 mpp[tiflash] ExchangeType: PassThrough", - " └─Projection 2.00 mpp[tiflash] 1->Column#11", + " └─Projection 2.00 mpp[tiflash] test.a.id", " └─HashJoin 2.00 mpp[tiflash] right outer join, equal:[eq(test.b.id, test.a.id)]", " ├─ExchangeReceiver(Build) 3.00 mpp[tiflash] ", " │ └─ExchangeSender 3.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.b.id, collate: binary]", @@ -647,7 +647,7 @@ "StreamAgg 1.00 root funcs:count(1)->Column#7", "└─TableReader 2.00 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 2.00 mpp[tiflash] ExchangeType: PassThrough", - " └─Projection 2.00 mpp[tiflash] 1->Column#11", + " └─Projection 2.00 mpp[tiflash] test.a.id", " └─HashJoin 2.00 mpp[tiflash] left outer join, equal:[eq(test.a.id, test.b.id)]", " ├─ExchangeReceiver(Build) 2.00 mpp[tiflash] ", " │ └─ExchangeSender 2.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.a.id, collate: binary]", @@ -664,7 +664,7 @@ "StreamAgg 1.00 root funcs:count(1)->Column#7", "└─TableReader 2.00 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 2.00 mpp[tiflash] ExchangeType: PassThrough", - " └─Projection 2.00 mpp[tiflash] 1->Column#11", + " └─Projection 2.00 mpp[tiflash] test.a.id", " └─HashJoin 2.00 mpp[tiflash] right outer join, equal:[eq(test.b.id, test.a.id)]", " ├─ExchangeReceiver(Build) 2.00 mpp[tiflash] ", " │ └─ExchangeSender 2.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.a.id, collate: binary]", @@ -683,11 +683,11 @@ { "SQL": "explain format = 'brief' select count(*) from fact_t, d1_t where fact_t.d1_k = d1_t.d1_k", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#14)->Column#11", + "HashAgg 1.00 root funcs:count(Column#12)->Column#11", "└─TableReader 1.00 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#14", - " └─Projection 32.00 mpp[tiflash] 1->Column#13", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#12", + " └─Projection 32.00 mpp[tiflash] test.fact_t.d1_k", " └─HashJoin 32.00 mpp[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d1_t.d1_k, collate: binary]", @@ -702,11 +702,11 @@ { "SQL": "explain format = 'brief' select count(*) from fact_t, d1_t, d2_t, d3_t where fact_t.d1_k = d1_t.d1_k and fact_t.d2_k = d2_t.d2_k and fact_t.d3_k = d3_t.d3_k", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#20)->Column#17", + "HashAgg 1.00 root funcs:count(Column#18)->Column#17", "└─TableReader 1.00 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#20", - " └─Projection 128.00 mpp[tiflash] 1->Column#19", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#18", + " └─Projection 128.00 mpp[tiflash] test.fact_t.d3_k", " └─HashJoin 128.00 mpp[tiflash] inner join, equal:[eq(test.fact_t.d3_k, test.d3_t.d3_k)]", " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d3_t.d3_k, collate: binary]", @@ -737,11 +737,11 @@ { "SQL": "explain format = 'brief' select count(*) from fact_t, d1_t where fact_t.d1_k = d1_t.d1_k", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#14)->Column#11", + "HashAgg 1.00 root funcs:count(Column#12)->Column#11", "└─TableReader 1.00 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#14", - " └─Projection 32.00 mpp[tiflash] 1->Column#13", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#12", + " └─Projection 32.00 mpp[tiflash] test.fact_t.d1_k", " └─HashJoin 32.00 mpp[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d1_t.d1_k, collate: binary]", @@ -756,11 +756,11 @@ { "SQL": "explain format = 'brief' select count(*) from fact_t, d1_t, d2_t, d3_t where fact_t.d1_k = d1_t.d1_k and fact_t.d1_k = d2_t.value and fact_t.d1_k = d3_t.value", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#20)->Column#17", + "HashAgg 1.00 root funcs:count(Column#18)->Column#17", "└─TableReader 1.00 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#20", - " └─Projection 128.00 mpp[tiflash] 1->Column#19", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#18", + " └─Projection 128.00 mpp[tiflash] test.fact_t.d1_k", " └─HashJoin 128.00 mpp[tiflash] inner join, equal:[eq(test.fact_t.d1_k, test.d3_t.value)]", " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d3_t.value, collate: binary]", @@ -787,11 +787,11 @@ { "SQL": "explain format = 'brief' select count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#14)->Column#11", + "HashAgg 1.00 root funcs:count(Column#12)->Column#11", "└─TableReader 1.00 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#14", - " └─Projection 32.00 mpp[tiflash] 1->Column#13", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#12", + " └─Projection 32.00 mpp[tiflash] test.fact_t.d1_k", " └─HashJoin 32.00 mpp[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d1_t.d1_k, collate: binary]", @@ -805,11 +805,11 @@ { "SQL": "explain format = 'brief' select count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#14)->Column#11", + "HashAgg 1.00 root funcs:count(Column#12)->Column#11", "└─TableReader 1.00 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#14", - " └─Projection 32.00 mpp[tiflash] 1->Column#13", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#12", + " └─Projection 32.00 mpp[tiflash] test.d1_t.d1_k", " └─HashJoin 32.00 mpp[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d1_t.d1_k, collate: binary]", @@ -823,11 +823,11 @@ { "SQL": "explain format = 'brief' select count(*) from fact_t join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col1 > d1_t.value", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#14)->Column#11", + "HashAgg 1.00 root funcs:count(Column#12)->Column#11", "└─TableReader 1.00 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#14", - " └─Projection 32.00 mpp[tiflash] 1->Column#13", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#12", + " └─Projection 32.00 mpp[tiflash] test.fact_t.d1_k", " └─HashJoin 32.00 mpp[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)], other cond:gt(test.fact_t.col1, test.d1_t.value)", " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d1_t.d1_k, collate: binary]", @@ -842,11 +842,11 @@ { "SQL": "explain format = 'brief' select count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col1 > 10", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#14)->Column#11", + "HashAgg 1.00 root funcs:count(Column#12)->Column#11", "└─TableReader 1.00 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#14", - " └─Projection 32.00 mpp[tiflash] 1->Column#13", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#12", + " └─Projection 32.00 mpp[tiflash] test.fact_t.d1_k", " └─HashJoin 32.00 mpp[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], left cond:[gt(test.fact_t.col1, 10)]", " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d1_t.d1_k, collate: binary]", @@ -860,11 +860,11 @@ { "SQL": "explain format = 'brief' select count(*) from (select case when t1.col1 is null then t2.col1 + 5 else 10 end as col1, t2.d1_k as d1_k from fact_t t1 right join fact_t t2 on t1.d1_k = t2.d1_k) fact_t join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col1 > 5", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#24)->Column#19", + "HashAgg 1.00 root funcs:count(Column#22)->Column#19", "└─TableReader 1.00 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#24", - " └─Projection 204.80 mpp[tiflash] 1->Column#23", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#22", + " └─Projection 204.80 mpp[tiflash] test.fact_t.d1_k", " └─HashJoin 204.80 mpp[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d1_t.d1_k, collate: binary]", @@ -887,11 +887,11 @@ { "SQL": "explain format = 'brief' select count(*) from fact_t left join d1_t on fact_t.d1_k = d1_t.d1_k and fact_t.col2 > 10 and fact_t.col1 > d1_t.value", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#14)->Column#11", + "HashAgg 1.00 root funcs:count(Column#12)->Column#11", "└─TableReader 1.00 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#14", - " └─Projection 32.00 mpp[tiflash] 1->Column#13", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#12", + " └─Projection 32.00 mpp[tiflash] test.fact_t.d1_k", " └─HashJoin 32.00 mpp[tiflash] left outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], left cond:[gt(test.fact_t.col2, 10)], other cond:gt(test.fact_t.col1, test.d1_t.value)", " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d1_t.d1_k, collate: binary]", @@ -905,11 +905,11 @@ { "SQL": "explain format = 'brief' select count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k and d1_t.value > 10", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#14)->Column#11", + "HashAgg 1.00 root funcs:count(Column#12)->Column#11", "└─TableReader 1.00 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#14", - " └─Projection 32.00 mpp[tiflash] 1->Column#13", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#12", + " └─Projection 32.00 mpp[tiflash] test.d1_t.d1_k", " └─HashJoin 32.00 mpp[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], right cond:gt(test.d1_t.value, 10)", " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d1_t.d1_k, collate: binary]", @@ -923,11 +923,11 @@ { "SQL": "explain format = 'brief' select count(*) from fact_t right join d1_t on fact_t.d1_k = d1_t.d1_k and d1_t.value > 10 and fact_t.col1 > d1_t.value", "Plan": [ - "HashAgg 1.00 root funcs:count(Column#14)->Column#11", + "HashAgg 1.00 root funcs:count(Column#12)->Column#11", "└─TableReader 1.00 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", - " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#14", - " └─Projection 32.00 mpp[tiflash] 1->Column#13", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#12", + " └─Projection 32.00 mpp[tiflash] test.d1_t.d1_k", " └─HashJoin 32.00 mpp[tiflash] right outer join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], right cond:gt(test.d1_t.value, 10), other cond:gt(test.fact_t.col1, test.d1_t.value)", " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d1_t.d1_k, collate: binary]", @@ -944,16 +944,15 @@ "StreamAgg 1.00 root funcs:count(1)->Column#12", "└─TableReader 12.80 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 12.80 mpp[tiflash] ExchangeType: PassThrough", - " └─Projection 12.80 mpp[tiflash] 1->Column#16", - " └─HashJoin 12.80 mpp[tiflash] semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", - " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d1_t.d1_k, collate: binary]", - " │ └─Selection 4.00 mpp[tiflash] not(isnull(test.d1_t.d1_k))", - " │ └─TableFullScan 4.00 mpp[tiflash] table:d1_t pushed down filter:empty, keep order:false", - " └─ExchangeReceiver(Probe) 16.00 mpp[tiflash] ", - " └─ExchangeSender 16.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.fact_t.d1_k, collate: binary]", - " └─Selection 16.00 mpp[tiflash] not(isnull(test.fact_t.d1_k))", - " └─TableFullScan 16.00 mpp[tiflash] table:fact_t pushed down filter:empty, keep order:false" + " └─HashJoin 12.80 mpp[tiflash] semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", + " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d1_t.d1_k, collate: binary]", + " │ └─Selection 4.00 mpp[tiflash] not(isnull(test.d1_t.d1_k))", + " │ └─TableFullScan 4.00 mpp[tiflash] table:d1_t pushed down filter:empty, keep order:false", + " └─ExchangeReceiver(Probe) 16.00 mpp[tiflash] ", + " └─ExchangeSender 16.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.fact_t.d1_k, collate: binary]", + " └─Selection 16.00 mpp[tiflash] not(isnull(test.fact_t.d1_k))", + " └─TableFullScan 16.00 mpp[tiflash] table:fact_t pushed down filter:empty, keep order:false" ] }, { @@ -962,7 +961,7 @@ "StreamAgg 1.00 root funcs:count(1)->Column#12", "└─TableReader 12.80 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 12.80 mpp[tiflash] ExchangeType: PassThrough", - " └─Projection 12.80 mpp[tiflash] 1->Column#16", + " └─Projection 12.80 mpp[tiflash] test.fact_t.d1_k", " └─HashJoin 12.80 mpp[tiflash] semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.d1_t.value, test.fact_t.col1)", " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d1_t.d1_k, collate: binary]", @@ -980,14 +979,13 @@ "StreamAgg 1.00 root funcs:count(1)->Column#12", "└─TableReader 12.80 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 12.80 mpp[tiflash] ExchangeType: PassThrough", - " └─Projection 12.80 mpp[tiflash] 1->Column#16", - " └─HashJoin 12.80 mpp[tiflash] anti semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", - " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", - " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d1_t.d1_k, collate: binary]", - " │ └─TableFullScan 4.00 mpp[tiflash] table:d1_t keep order:false", - " └─ExchangeReceiver(Probe) 16.00 mpp[tiflash] ", - " └─ExchangeSender 16.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.fact_t.d1_k, collate: binary]", - " └─TableFullScan 16.00 mpp[tiflash] table:fact_t keep order:false" + " └─HashJoin 12.80 mpp[tiflash] anti semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)]", + " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", + " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d1_t.d1_k, collate: binary]", + " │ └─TableFullScan 4.00 mpp[tiflash] table:d1_t keep order:false", + " └─ExchangeReceiver(Probe) 16.00 mpp[tiflash] ", + " └─ExchangeSender 16.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.fact_t.d1_k, collate: binary]", + " └─TableFullScan 16.00 mpp[tiflash] table:fact_t keep order:false" ] }, { @@ -996,7 +994,7 @@ "StreamAgg 1.00 root funcs:count(1)->Column#12", "└─TableReader 12.80 root MppVersion: 2, data:ExchangeSender", " └─ExchangeSender 12.80 mpp[tiflash] ExchangeType: PassThrough", - " └─Projection 12.80 mpp[tiflash] 1->Column#16", + " └─Projection 12.80 mpp[tiflash] test.fact_t.d1_k", " └─HashJoin 12.80 mpp[tiflash] anti semi join, equal:[eq(test.fact_t.d1_k, test.d1_t.d1_k)], other cond:gt(test.d1_t.value, test.fact_t.col1)", " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d1_t.d1_k, collate: binary]", @@ -1017,23 +1015,24 @@ "TableReader 1.00 root MppVersion: 2, data:ExchangeSender", "└─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", " └─Projection 1.00 mpp[tiflash] Column#13", - " └─HashJoin 1.00 mpp[tiflash] left outer join, equal:[eq(test.t3.v1, test.t1.v1) eq(test.t3.v2, test.t1.v2)]", - " ├─ExchangeReceiver(Build) 1.00 mpp[tiflash] ", - " │ └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#23, collate: binary], [name: Column#24, collate: binary]", - " │ └─Projection 1.00 mpp[tiflash] test.t3.v1, test.t3.v2, cast(test.t3.v1, decimal(20,2))->Column#23, cast(test.t3.v2, decimal(20,2))->Column#24", - " │ └─TableFullScan 1.00 mpp[tiflash] table:a keep order:false", - " └─Projection(Probe) 2.00 mpp[tiflash] test.t1.v1, test.t1.v2, plus(test.t1.v1, test.t1.v2)->Column#13", - " └─Projection 2.00 mpp[tiflash] test.t1.v1, test.t1.v2", - " └─HashJoin 2.00 mpp[tiflash] left outer join, equal:[eq(test.t1.v1, test.t2.v1) eq(test.t1.v2, test.t2.v2)]", - " ├─ExchangeReceiver(Build) 2.00 mpp[tiflash] ", - " │ └─ExchangeSender 2.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.t1.v1, collate: binary], [name: test.t1.v2, collate: binary]", - " │ └─Selection 2.00 mpp[tiflash] not(isnull(test.t1.v1)), not(isnull(test.t1.v2))", - " │ └─TableFullScan 2.00 mpp[tiflash] table:t1 pushed down filter:empty, keep order:false", - " └─ExchangeReceiver(Probe) 8.00 mpp[tiflash] ", - " └─ExchangeSender 8.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#15, collate: binary], [name: Column#16, collate: binary]", - " └─Projection 8.00 mpp[tiflash] test.t2.v1, test.t2.v2, cast(test.t2.v1, decimal(20,2))->Column#15, cast(test.t2.v2, decimal(20,2))->Column#16", - " └─Selection 8.00 mpp[tiflash] not(isnull(test.t2.v1)), not(isnull(test.t2.v2))", - " └─TableFullScan 8.00 mpp[tiflash] table:t2 pushed down filter:empty, keep order:false" + " └─Projection 1.00 mpp[tiflash] Column#13", + " └─HashJoin 1.00 mpp[tiflash] left outer join, equal:[eq(test.t3.v1, test.t1.v1) eq(test.t3.v2, test.t1.v2)]", + " ├─ExchangeReceiver(Build) 1.00 mpp[tiflash] ", + " │ └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#23, collate: binary], [name: Column#24, collate: binary]", + " │ └─Projection 1.00 mpp[tiflash] test.t3.v1, test.t3.v2, cast(test.t3.v1, decimal(20,2))->Column#23, cast(test.t3.v2, decimal(20,2))->Column#24", + " │ └─TableFullScan 1.00 mpp[tiflash] table:a keep order:false", + " └─Projection(Probe) 2.00 mpp[tiflash] test.t1.v1, test.t1.v2, plus(test.t1.v1, test.t1.v2)->Column#13", + " └─Projection 2.00 mpp[tiflash] test.t1.v1, test.t1.v2", + " └─HashJoin 2.00 mpp[tiflash] left outer join, equal:[eq(test.t1.v1, test.t2.v1) eq(test.t1.v2, test.t2.v2)]", + " ├─ExchangeReceiver(Build) 2.00 mpp[tiflash] ", + " │ └─ExchangeSender 2.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.t1.v1, collate: binary], [name: test.t1.v2, collate: binary]", + " │ └─Selection 2.00 mpp[tiflash] not(isnull(test.t1.v1)), not(isnull(test.t1.v2))", + " │ └─TableFullScan 2.00 mpp[tiflash] table:t1 pushed down filter:empty, keep order:false", + " └─ExchangeReceiver(Probe) 8.00 mpp[tiflash] ", + " └─ExchangeSender 8.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#15, collate: binary], [name: Column#16, collate: binary]", + " └─Projection 8.00 mpp[tiflash] test.t2.v1, test.t2.v2, cast(test.t2.v1, decimal(20,2))->Column#15, cast(test.t2.v2, decimal(20,2))->Column#16", + " └─Selection 8.00 mpp[tiflash] not(isnull(test.t2.v1)), not(isnull(test.t2.v2))", + " └─TableFullScan 8.00 mpp[tiflash] table:t2 pushed down filter:empty, keep order:false" ] }, { @@ -1042,19 +1041,20 @@ "TableReader 2.00 root MppVersion: 2, data:ExchangeSender", "└─ExchangeSender 2.00 mpp[tiflash] ExchangeType: PassThrough", " └─Projection 2.00 mpp[tiflash] Column#9, test.t2.v1, test.t2.v2", - " └─HashAgg 2.00 mpp[tiflash] group by:test.t2.v1, test.t2.v2, funcs:count(1)->Column#9, funcs:firstrow(test.t2.v1)->test.t2.v1, funcs:firstrow(test.t2.v2)->test.t2.v2", + " └─HashAgg 2.00 mpp[tiflash] group by:test.t2.v1, test.t2.v2, funcs:sum(Column#22)->Column#9, funcs:firstrow(test.t2.v1)->test.t2.v1, funcs:firstrow(test.t2.v2)->test.t2.v2", " └─ExchangeReceiver 2.00 mpp[tiflash] ", " └─ExchangeSender 2.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.t2.v1, collate: binary], [name: test.t2.v2, collate: binary]", - " └─Projection 2.00 mpp[tiflash] test.t2.v1, test.t2.v2", - " └─HashJoin 2.00 mpp[tiflash] left outer join, equal:[eq(test.t1.v1, test.t2.v1) eq(test.t1.v2, test.t2.v2)]", - " ├─ExchangeReceiver(Build) 2.00 mpp[tiflash] ", - " │ └─ExchangeSender 2.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.t1.v1, collate: binary], [name: test.t1.v2, collate: binary]", - " │ └─TableFullScan 2.00 mpp[tiflash] table:t1 keep order:false", - " └─ExchangeReceiver(Probe) 8.00 mpp[tiflash] ", - " └─ExchangeSender 8.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#12, collate: binary], [name: Column#13, collate: binary]", - " └─Projection 8.00 mpp[tiflash] test.t2.v1, test.t2.v2, cast(test.t2.v1, decimal(20,2))->Column#12, cast(test.t2.v2, decimal(20,2))->Column#13", - " └─Selection 8.00 mpp[tiflash] not(isnull(test.t2.v1)), not(isnull(test.t2.v2))", - " └─TableFullScan 8.00 mpp[tiflash] table:t2 pushed down filter:empty, keep order:false" + " └─HashAgg 2.00 mpp[tiflash] group by:test.t2.v1, test.t2.v2, funcs:count(1)->Column#22", + " └─Projection 2.00 mpp[tiflash] test.t2.v1, test.t2.v2", + " └─HashJoin 2.00 mpp[tiflash] left outer join, equal:[eq(test.t1.v1, test.t2.v1) eq(test.t1.v2, test.t2.v2)]", + " ├─ExchangeReceiver(Build) 2.00 mpp[tiflash] ", + " │ └─ExchangeSender 2.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.t1.v1, collate: binary], [name: test.t1.v2, collate: binary]", + " │ └─TableFullScan 2.00 mpp[tiflash] table:t1 keep order:false", + " └─ExchangeReceiver(Probe) 8.00 mpp[tiflash] ", + " └─ExchangeSender 8.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#14, collate: binary], [name: Column#15, collate: binary]", + " └─Projection 8.00 mpp[tiflash] test.t2.v1, test.t2.v2, cast(test.t2.v1, decimal(20,2))->Column#14, cast(test.t2.v2, decimal(20,2))->Column#15", + " └─Selection 8.00 mpp[tiflash] not(isnull(test.t2.v1)), not(isnull(test.t2.v2))", + " └─TableFullScan 8.00 mpp[tiflash] table:t2 pushed down filter:empty, keep order:false" ] }, { @@ -1593,16 +1593,17 @@ "TableReader 12500.00 root MppVersion: 2, data:ExchangeSender", "└─ExchangeSender 12500.00 mpp[tiflash] ExchangeType: PassThrough", " └─Projection 12500.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c1, test.t.c2, test.t.c3", - " └─HashJoin 12500.00 mpp[tiflash] inner join, equal:[eq(Column#13, Column#14) eq(Column#15, Column#16)]", - " ├─ExchangeReceiver(Build) 10000.00 mpp[tiflash] ", - " │ └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#23, collate: binary], [name: Column#24, collate: binary]", - " │ └─Projection 10000.00 mpp[tiflash] test.t.c1, test.t.c2, Column#13, Column#15, cast(Column#13, decimal(13,8) BINARY)->Column#23, cast(Column#15, decimal(10,5) BINARY)->Column#24", - " │ └─Projection 10000.00 mpp[tiflash] test.t.c1, test.t.c2, mul(test.t.c1, 3)->Column#13, plus(test.t.c1, 1)->Column#15", - " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", - " └─ExchangeReceiver(Probe) 10000.00 mpp[tiflash] ", - " └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#14, collate: binary], [name: Column#16, collate: binary]", - " └─Projection 10000.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c3, div(test.t.c3, 2)->Column#14, minus(test.t.c2, 10)->Column#16", - " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" + " └─Projection 12500.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c1, test.t.c2, test.t.c3", + " └─HashJoin 12500.00 mpp[tiflash] inner join, equal:[eq(Column#13, Column#14) eq(Column#15, Column#16)]", + " ├─ExchangeReceiver(Build) 10000.00 mpp[tiflash] ", + " │ └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#23, collate: binary], [name: Column#24, collate: binary]", + " │ └─Projection 10000.00 mpp[tiflash] test.t.c1, test.t.c2, Column#13, Column#15, cast(Column#13, decimal(13,8) BINARY)->Column#23, cast(Column#15, decimal(10,5) BINARY)->Column#24", + " │ └─Projection 10000.00 mpp[tiflash] test.t.c1, test.t.c2, mul(test.t.c1, 3)->Column#13, plus(test.t.c1, 1)->Column#15", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─ExchangeReceiver(Probe) 10000.00 mpp[tiflash] ", + " └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#14, collate: binary], [name: Column#16, collate: binary]", + " └─Projection 10000.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c3, div(test.t.c3, 2)->Column#14, minus(test.t.c2, 10)->Column#16", + " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" ] }, { @@ -1677,16 +1678,17 @@ "TableReader 12500.00 root MppVersion: 2, data:ExchangeSender", "└─ExchangeSender 12500.00 mpp[tiflash] ExchangeType: PassThrough", " └─Projection 12500.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5", - " └─HashJoin 12500.00 mpp[tiflash] inner join, equal:[eq(Column#13, Column#14)]", - " ├─ExchangeReceiver(Build) 10000.00 mpp[tiflash] ", - " │ └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#18, collate: binary]", - " │ └─Projection 10000.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, Column#13, cast(Column#13, decimal(17,9) BINARY)->Column#18", - " │ └─Projection 10000.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, plus(test.t.c1, test.t.c2)->Column#13", - " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", - " └─ExchangeReceiver(Probe) 10000.00 mpp[tiflash] ", - " └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#14, collate: binary]", - " └─Projection 10000.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, div(test.t.c2, test.t.c3)->Column#14", - " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" + " └─Projection 12500.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5", + " └─HashJoin 12500.00 mpp[tiflash] inner join, equal:[eq(Column#13, Column#14)]", + " ├─ExchangeReceiver(Build) 10000.00 mpp[tiflash] ", + " │ └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#18, collate: binary]", + " │ └─Projection 10000.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, Column#13, cast(Column#13, decimal(17,9) BINARY)->Column#18", + " │ └─Projection 10000.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, plus(test.t.c1, test.t.c2)->Column#13", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─ExchangeReceiver(Probe) 10000.00 mpp[tiflash] ", + " └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#14, collate: binary]", + " └─Projection 10000.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, div(test.t.c2, test.t.c3)->Column#14", + " └─TableFullScan 10000.00 mpp[tiflash] table:t2 keep order:false, stats:pseudo" ] }, { @@ -1749,22 +1751,21 @@ " └─ExchangeReceiver 8000.00 mpp[tiflash] ", " └─ExchangeSender 8000.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.tt.col_varchar_64, collate: utf8mb4_bin], [name: test.tt.col_char_64_not_null, collate: utf8mb4_bin]", " └─HashAgg 8000.00 mpp[tiflash] group by:test.tt.col_char_64_not_null, test.tt.col_varchar_64, ", - " └─Projection 15609.38 mpp[tiflash] test.tt.col_varchar_64, test.tt.col_char_64_not_null", - " └─HashJoin 15609.38 mpp[tiflash] inner join, equal:[eq(test.tt.col_char_64_not_null, test.tt.col_varchar_64)]", - " ├─ExchangeReceiver(Build) 10000.00 mpp[tiflash] ", - " │ └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.tt.col_char_64_not_null, collate: utf8mb4_bin]", - " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", - " └─Projection(Probe) 12487.50 mpp[tiflash] test.tt.col_varchar_64", - " └─HashJoin 12487.50 mpp[tiflash] inner join, equal:[eq(test.tt.col_varchar_key, test.tt.col_varchar_64) eq(Column#19, test.tt.col_decimal_30_10_key)]", - " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", - " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.tt.col_varchar_key, collate: utf8mb4_bin]", - " │ └─Projection 9990.00 mpp[tiflash] test.tt.col_varchar_key, cast(test.tt.col_tinyint, decimal(3,0) BINARY)->Column#19", - " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.tt.col_varchar_key))", - " │ └─TableFullScan 10000.00 mpp[tiflash] table:t2 pushed down filter:empty, keep order:false, stats:pseudo", - " └─ExchangeReceiver(Probe) 9990.00 mpp[tiflash] ", - " └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.tt.col_varchar_64, collate: utf8mb4_bin]", - " └─Selection 9990.00 mpp[tiflash] not(isnull(test.tt.col_varchar_64))", - " └─TableFullScan 10000.00 mpp[tiflash] table:t3 pushed down filter:empty, keep order:false, stats:pseudo" + " └─HashJoin 15609.38 mpp[tiflash] inner join, equal:[eq(test.tt.col_char_64_not_null, test.tt.col_varchar_64)]", + " ├─ExchangeReceiver(Build) 10000.00 mpp[tiflash] ", + " │ └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.tt.col_char_64_not_null, collate: utf8mb4_bin]", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Projection(Probe) 12487.50 mpp[tiflash] test.tt.col_varchar_64", + " └─HashJoin 12487.50 mpp[tiflash] inner join, equal:[eq(test.tt.col_varchar_key, test.tt.col_varchar_64) eq(Column#19, test.tt.col_decimal_30_10_key)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.tt.col_varchar_key, collate: utf8mb4_bin]", + " │ └─Projection 9990.00 mpp[tiflash] test.tt.col_varchar_key, cast(test.tt.col_tinyint, decimal(3,0) BINARY)->Column#19", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.tt.col_varchar_key))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t2 pushed down filter:empty, keep order:false, stats:pseudo", + " └─ExchangeReceiver(Probe) 9990.00 mpp[tiflash] ", + " └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.tt.col_varchar_64, collate: utf8mb4_bin]", + " └─Selection 9990.00 mpp[tiflash] not(isnull(test.tt.col_varchar_64))", + " └─TableFullScan 10000.00 mpp[tiflash] table:t3 pushed down filter:empty, keep order:false, stats:pseudo" ] } ] diff --git a/pkg/planner/core/casetest/physicalplantest/testdata/plan_suite_out.json b/pkg/planner/core/casetest/physicalplantest/testdata/plan_suite_out.json index c1c5cce4711c9..4b106ec137efe 100644 --- a/pkg/planner/core/casetest/physicalplantest/testdata/plan_suite_out.json +++ b/pkg/planner/core/casetest/physicalplantest/testdata/plan_suite_out.json @@ -3114,23 +3114,24 @@ "TableReader 8000.00 root MppVersion: 2, data:ExchangeSender", "└─ExchangeSender 8000.00 mpp[tiflash] ExchangeType: PassThrough", " └─Projection 8000.00 mpp[tiflash] test.employee.empid, test.employee.deptid, test.employee.salary, Column#10, Column#9", - " └─HashJoin 8000.00 mpp[tiflash] inner join, equal:[eq(test.employee.deptid, Column#10)]", - " ├─ExchangeReceiver(Build) 6400.00 mpp[tiflash] ", - " │ └─ExchangeSender 6400.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#10, collate: binary]", - " │ └─Projection 6400.00 mpp[tiflash] plus(test.employee.deptid, 1)->Column#10, Column#9", - " │ └─Selection 6400.00 mpp[tiflash] not(isnull(plus(test.employee.deptid, 1)))", - " │ └─Projection 8000.00 mpp[tiflash] Column#9, test.employee.deptid", - " │ └─HashAgg 8000.00 mpp[tiflash] group by:Column#13, funcs:sum(Column#14)->Column#9, funcs:firstrow(Column#15)->test.employee.deptid", - " │ └─ExchangeReceiver 8000.00 mpp[tiflash] ", - " │ └─ExchangeSender 8000.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#13, collate: binary]", - " │ └─HashAgg 8000.00 mpp[tiflash] group by:Column#29, funcs:count(Column#27)->Column#14, funcs:firstrow(Column#28)->Column#15", - " │ └─Projection 10000.00 mpp[tiflash] test.employee.empid->Column#27, test.employee.deptid->Column#28, plus(test.employee.deptid, 1)->Column#29", - " │ └─TableFullScan 10000.00 mpp[tiflash] table:employee keep order:false, stats:pseudo", - " └─ExchangeReceiver(Probe) 9990.00 mpp[tiflash] ", - " └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#26, collate: binary]", - " └─Projection 9990.00 mpp[tiflash] test.employee.empid, test.employee.deptid, test.employee.salary, cast(test.employee.deptid, bigint(20))->Column#26", - " └─Selection 9990.00 mpp[tiflash] not(isnull(test.employee.deptid))", - " └─TableFullScan 10000.00 mpp[tiflash] table:e1 pushed down filter:empty, keep order:false, stats:pseudo" + " └─Projection 8000.00 mpp[tiflash] test.employee.empid, test.employee.deptid, test.employee.salary, Column#10, Column#9", + " └─HashJoin 8000.00 mpp[tiflash] inner join, equal:[eq(test.employee.deptid, Column#10)]", + " ├─ExchangeReceiver(Build) 6400.00 mpp[tiflash] ", + " │ └─ExchangeSender 6400.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#10, collate: binary]", + " │ └─Projection 6400.00 mpp[tiflash] plus(test.employee.deptid, 1)->Column#10, Column#9", + " │ └─Selection 6400.00 mpp[tiflash] not(isnull(plus(test.employee.deptid, 1)))", + " │ └─Projection 8000.00 mpp[tiflash] Column#9, test.employee.deptid", + " │ └─HashAgg 8000.00 mpp[tiflash] group by:Column#13, funcs:sum(Column#14)->Column#9, funcs:firstrow(Column#15)->test.employee.deptid", + " │ └─ExchangeReceiver 8000.00 mpp[tiflash] ", + " │ └─ExchangeSender 8000.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#13, collate: binary]", + " │ └─HashAgg 8000.00 mpp[tiflash] group by:Column#29, funcs:count(Column#27)->Column#14, funcs:firstrow(Column#28)->Column#15", + " │ └─Projection 10000.00 mpp[tiflash] test.employee.empid->Column#27, test.employee.deptid->Column#28, plus(test.employee.deptid, 1)->Column#29", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:employee keep order:false, stats:pseudo", + " └─ExchangeReceiver(Probe) 9990.00 mpp[tiflash] ", + " └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#26, collate: binary]", + " └─Projection 9990.00 mpp[tiflash] test.employee.empid, test.employee.deptid, test.employee.salary, cast(test.employee.deptid, bigint(20))->Column#26", + " └─Selection 9990.00 mpp[tiflash] not(isnull(test.employee.deptid))", + " └─TableFullScan 10000.00 mpp[tiflash] table:e1 pushed down filter:empty, keep order:false, stats:pseudo" ] }, { diff --git a/pkg/planner/core/rule_eliminate_projection.go b/pkg/planner/core/rule_eliminate_projection.go index cbee379d3fb19..00e65a0446e2b 100644 --- a/pkg/planner/core/rule_eliminate_projection.go +++ b/pkg/planner/core/rule_eliminate_projection.go @@ -140,7 +140,11 @@ func doPhysicalProjectionElimination(p base.PhysicalPlan) base.PhysicalPlan { } child := p.Children()[0] if childProj, ok := child.(*PhysicalProjection); ok { - childProj.SetSchema(p.Schema()) + // when current projection is an empty projection(schema pruned by column pruner), no need to reset child's schema + // TODO: avoid producing empty projection in column pruner. + if p.Schema().Len() != 0 { + childProj.SetSchema(p.Schema()) + } } return child } diff --git a/pkg/planner/core/task.go b/pkg/planner/core/task.go index 9cff55046a0e0..19bc02024b877 100644 --- a/pkg/planner/core/task.go +++ b/pkg/planner/core/task.go @@ -427,6 +427,17 @@ func (p *PhysicalHashJoin) attach2TaskForMpp(tasks ...base.Task) base.Task { // To avoid the performance issue, add a projection here above the Join operator to prune useless columns explicitly. // TODO(hyb): transfer Join executors' schema to TiFlash through DagRequest, and use it directly in TiFlash. defaultSchema := BuildPhysicalJoinSchema(p.JoinType, p) + hashColArray := make([]*expression.Column, 0, len(task.hashCols)) + // For task.hashCols, these columns may not be contained in pruned columns: + // select A.id from A join B on A.id = B.id; Suppose B is probe side, and it's hash inner join. + // After column prune, the output schema of A join B will be A.id only; while the task's hashCols will be B.id. + // To make matters worse, the hashCols may be used to check if extra cast projection needs to be added, then the newly + // added projection will expect B.id as input schema. So make sure hashCols are included in task.p's schema. + // TODO: planner should takes the hashCols attribute into consideration when perform column pruning; Or provide mechanism + // to constraint hashCols are always chosen inside Join's pruned schema + for _, hashCol := range task.hashCols { + hashColArray = append(hashColArray, hashCol.Col) + } if p.schema.Len() < defaultSchema.Len() { if p.schema.Len() > 0 { proj := PhysicalProjection{ @@ -434,20 +445,38 @@ func (p *PhysicalHashJoin) attach2TaskForMpp(tasks ...base.Task) base.Task { }.Init(p.SCtx(), p.StatsInfo(), p.QueryBlockOffset()) proj.SetSchema(p.Schema().Clone()) + for _, hashCol := range hashColArray { + if !proj.Schema().Contains(hashCol) { + proj.Schema().Append(hashCol) + } + } attachPlan2Task(proj, task) } else { - constOne := expression.NewOne() - expr := make([]expression.Expression, 0, 1) - expr = append(expr, constOne) - proj := PhysicalProjection{ - Exprs: expr, - }.Init(p.SCtx(), p.StatsInfo(), p.QueryBlockOffset()) + if len(hashColArray) == 0 { + constOne := expression.NewOne() + expr := make([]expression.Expression, 0, 1) + expr = append(expr, constOne) + proj := PhysicalProjection{ + Exprs: expr, + }.Init(p.SCtx(), p.StatsInfo(), p.QueryBlockOffset()) + + proj.schema = expression.NewSchema(&expression.Column{ + UniqueID: proj.SCtx().GetSessionVars().AllocPlanColumnID(), + RetType: constOne.GetType(), + }) + attachPlan2Task(proj, task) + } else { + proj := PhysicalProjection{ + Exprs: make([]expression.Expression, 0, len(hashColArray)), + }.Init(p.SCtx(), p.StatsInfo(), p.QueryBlockOffset()) - proj.schema = expression.NewSchema(&expression.Column{ - UniqueID: proj.SCtx().GetSessionVars().AllocPlanColumnID(), - RetType: constOne.GetType(), - }) - attachPlan2Task(proj, task) + for _, hashCol := range hashColArray { + proj.Exprs = append(proj.Exprs, hashCol) + } + + proj.SetSchema(expression.NewSchema(hashColArray...)) + attachPlan2Task(proj, task) + } } } p.schema = defaultSchema From 4cd8e94516e711d956f3c1e710bd9e01d27c130b Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Fri, 26 Apr 2024 10:07:41 +0800 Subject: [PATCH 12/20] executor: remove unnecessary stack for insert and load data (#52784) close pingcap/tidb#52783 --- pkg/executor/benchmark_test.go | 24 ++++++++++++++++++++++++ pkg/executor/insert_common.go | 14 +++++++------- pkg/executor/union_scan_test.go | 2 ++ pkg/executor/update.go | 3 ++- pkg/executor/write.go | 2 +- 5 files changed, 36 insertions(+), 9 deletions(-) diff --git a/pkg/executor/benchmark_test.go b/pkg/executor/benchmark_test.go index b9b4cf4e1f17e..6af51f7fa2340 100644 --- a/pkg/executor/benchmark_test.go +++ b/pkg/executor/benchmark_test.go @@ -35,6 +35,7 @@ import ( "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/expression/aggregation" "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/planner/core" "github.com/pingcap/tidb/pkg/planner/core/base" @@ -1838,3 +1839,26 @@ func BenchmarkAggPartialResultMapperMemoryUsage(b *testing.B) { func BenchmarkPipelinedRowNumberWindowFunctionExecution(b *testing.B) { b.ReportAllocs() } + +func BenchmarkCompleteInsertErr(b *testing.B) { + b.ReportAllocs() + col := &model.ColumnInfo{ + Name: model.NewCIStr("a"), + FieldType: *types.NewFieldType(mysql.TypeBlob), + } + err := types.ErrWarnDataOutOfRange + for n := 0; n < b.N; n++ { + completeInsertErr(col, nil, 0, err) + } +} + +func BenchmarkCompleteLoadErr(b *testing.B) { + b.ReportAllocs() + col := &model.ColumnInfo{ + Name: model.NewCIStr("a"), + } + err := types.ErrDataTooLong + for n := 0; n < b.N; n++ { + completeLoadErr(col, 0, err) + } +} diff --git a/pkg/executor/insert_common.go b/pkg/executor/insert_common.go index b4f24759afc73..4915286e920fa 100644 --- a/pkg/executor/insert_common.go +++ b/pkg/executor/insert_common.go @@ -270,23 +270,23 @@ func completeInsertErr(col *model.ColumnInfo, val *types.Datum, rowIdx int, err if types.ErrDataTooLong.Equal(err) { err = resetErrDataTooLong(colName, rowIdx+1, err) } else if types.ErrOverflow.Equal(err) { - err = types.ErrWarnDataOutOfRange.GenWithStackByArgs(colName, rowIdx+1) + err = types.ErrWarnDataOutOfRange.FastGenByArgs(colName, rowIdx+1) } else if types.ErrTruncated.Equal(err) { - err = types.ErrTruncated.GenWithStackByArgs(colName, rowIdx+1) + err = types.ErrTruncated.FastGenByArgs(colName, rowIdx+1) } else if types.ErrTruncatedWrongVal.Equal(err) && (colTp == mysql.TypeDuration || colTp == mysql.TypeDatetime || colTp == mysql.TypeDate || colTp == mysql.TypeTimestamp) { valStr, err1 := val.ToString() if err1 != nil { logutil.BgLogger().Debug("time truncated error", zap.Error(err1)) } - err = exeerrors.ErrTruncateWrongInsertValue.GenWithStackByArgs(types.TypeStr(colTp), valStr, colName, rowIdx+1) + err = exeerrors.ErrTruncateWrongInsertValue.FastGenByArgs(types.TypeStr(colTp), valStr, colName, rowIdx+1) } else if types.ErrTruncatedWrongVal.Equal(err) || types.ErrWrongValue.Equal(err) { valStr, err1 := val.ToString() if err1 != nil { logutil.BgLogger().Debug("truncated/wrong value error", zap.Error(err1)) } - err = table.ErrTruncatedWrongValueForField.GenWithStackByArgs(types.TypeStr(colTp), valStr, colName, rowIdx+1) + err = table.ErrTruncatedWrongValueForField.FastGenByArgs(types.TypeStr(colTp), valStr, colName, rowIdx+1) } else if types.ErrWarnDataOutOfRange.Equal(err) { - err = types.ErrWarnDataOutOfRange.GenWithStackByArgs(colName, rowIdx+1) + err = types.ErrWarnDataOutOfRange.FastGenByArgs(colName, rowIdx+1) } return err } @@ -300,7 +300,7 @@ func completeLoadErr(col *model.ColumnInfo, rowIdx int, err error) error { } if types.ErrDataTooLong.Equal(err) { - err = types.ErrTruncated.GenWithStack("Data truncated for column '%v' at row %v", colName, rowIdx) + err = types.ErrTruncated.FastGen("Data truncated for column '%v' at row %v", colName, rowIdx) } return err } @@ -324,7 +324,7 @@ func (e *InsertValues) handleErr(col *table.Column, val *types.Datum, rowIdx int // TODO: should not filter all types of errors here. if err != nil { ec := e.Ctx().GetSessionVars().StmtCtx.ErrCtx() - return ec.HandleErrorWithAlias(kv.ErrKeyExists, err, err) + return errors.AddStack(ec.HandleErrorWithAlias(kv.ErrKeyExists, err, err)) } return nil } diff --git a/pkg/executor/union_scan_test.go b/pkg/executor/union_scan_test.go index 08478e87b5aee..52997c16eff59 100644 --- a/pkg/executor/union_scan_test.go +++ b/pkg/executor/union_scan_test.go @@ -393,6 +393,8 @@ func BenchmarkUnionScanIndexLookUpDescRead(b *testing.B) { func TestBenchDaily(t *testing.T) { benchdaily.Run( executor.BenchmarkReadLastLinesOfHugeLine, + executor.BenchmarkCompleteInsertErr, + executor.BenchmarkCompleteLoadErr, BenchmarkUnionScanRead, BenchmarkUnionScanIndexReadDescRead, BenchmarkUnionScanTableReadDescRead, diff --git a/pkg/executor/update.go b/pkg/executor/update.go index 724a33cb27fd6..4a50ccd75b909 100644 --- a/pkg/executor/update.go +++ b/pkg/executor/update.go @@ -20,6 +20,7 @@ import ( "fmt" "runtime/trace" + "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/executor/internal/exec" "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/kv" @@ -338,7 +339,7 @@ func (*UpdateExec) handleErr(colName model.CIStr, rowIdx int, err error) error { } if types.ErrDataTooLong.Equal(err) { - return resetErrDataTooLong(colName.O, rowIdx+1, err) + return errors.AddStack(resetErrDataTooLong(colName.O, rowIdx+1, err)) } if types.ErrOverflow.Equal(err) { diff --git a/pkg/executor/write.go b/pkg/executor/write.go index a0b5d4129a503..9004a22f28861 100644 --- a/pkg/executor/write.go +++ b/pkg/executor/write.go @@ -313,7 +313,7 @@ func rebaseAutoRandomValue( // types.ErrDataTooLong is produced in types.ProduceStrWithSpecifiedTp, there is no column info in there, // so we reset the error msg here, and wrap old err with errors.Wrap. func resetErrDataTooLong(colName string, rowIdx int, _ error) error { - newErr := types.ErrDataTooLong.GenWithStack("Data too long for column '%v' at row %v", colName, rowIdx) + newErr := types.ErrDataTooLong.FastGen("Data too long for column '%v' at row %v", colName, rowIdx) return newErr } From 9036169d2be5465ad3046bfd6126aa87a6c38eb3 Mon Sep 17 00:00:00 2001 From: Jack Yu Date: Fri, 26 Apr 2024 11:54:11 +0800 Subject: [PATCH 13/20] variable: avoid generating stack for warning (#52782) close pingcap/tidb#49291 --- pkg/sessionctx/variable/sysvar.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go index 30d02a0a17fac..0f72635f0361e 100644 --- a/pkg/sessionctx/variable/sysvar.go +++ b/pkg/sessionctx/variable/sysvar.go @@ -891,7 +891,7 @@ var defaultSysVars = []*SysVar{ {Scope: ScopeGlobal, Name: TiDBEnableTelemetry, Value: BoolToOnOff(DefTiDBEnableTelemetry), Type: TypeBool, GetGlobal: func(_ context.Context, s *SessionVars) (string, error) { return "OFF", nil }, SetGlobal: func(_ context.Context, s *SessionVars, val string) error { - s.StmtCtx.AppendWarning(ErrWarnDeprecatedSyntaxSimpleMsg.GenWithStack("tidb_enable_telemetry is deprecated since Telemetry has been removed, this variable is 'OFF' always.")) + s.StmtCtx.AppendWarning(ErrWarnDeprecatedSyntaxSimpleMsg.FastGen("tidb_enable_telemetry is deprecated since Telemetry has been removed, this variable is 'OFF' always.")) return nil }}, {Scope: ScopeGlobal, Name: TiDBEnableHistoricalStats, Value: On, Type: TypeBool, Depended: true}, From 58bfd1cb919cd40514066b4331dd6c9e561e207c Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Fri, 26 Apr 2024 12:42:11 +0800 Subject: [PATCH 14/20] flashback: fix `realtikvtest/flashbacktest` test cases (#52899) close pingcap/tidb#52898 --- Makefile | 8 +++++ pkg/statistics/handle/ddl/ddl.go | 33 +++++++++++-------- .../flashbacktest/flashback_test.go | 1 + 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index f0812409c8a07..55e310d304feb 100644 --- a/Makefile +++ b/Makefile @@ -761,6 +761,14 @@ bazel_pipelineddmltest: failpoint-enable bazel_ci_simple_prepare -- //tests/realtikvtest/pipelineddmltest/... ./build/jenkins_collect_coverage.sh +# on timeout, bazel won't print log sometimes, so we use --test_output=all to print log always +.PHONY: bazel_flashbacktest +bazel_flashbacktest: failpoint-enable bazel_ci_simple_prepare + bazel $(BAZEL_GLOBAL_CONFIG) coverage $(BAZEL_CMD_CONFIG) $(BAZEL_INSTRUMENTATION_FILTER) --test_output=all --test_arg=-with-real-tikv --define gotags=deadlock,intest \ + --@io_bazel_rules_go//go/config:cover_format=go_cover \ + -- //tests/realtikvtest/flashbacktest/... + ./build/jenkins_collect_coverage.sh + .PHONY: bazel_lint bazel_lint: bazel_prepare bazel build //... --//build:with_nogo_flag=true diff --git a/pkg/statistics/handle/ddl/ddl.go b/pkg/statistics/handle/ddl/ddl.go index 4b7e6f3eb4567..3c32194df0498 100644 --- a/pkg/statistics/handle/ddl/ddl.go +++ b/pkg/statistics/handle/ddl/ddl.go @@ -55,20 +55,25 @@ func (h *ddlHandlerImpl) HandleDDLEvent(t *util.DDLEvent) error { return err } defer h.statsHandler.SPool().Put(sctx) - if isSysDB, err := t.IsMemOrSysDB(sctx.(sessionctx.Context)); err != nil { - return err - } else if isSysDB { - // EXCHANGE PARTITION EVENT NOTES: - // 1. When a partition is exchanged with a system table, we need to adjust the global statistics - // based on the count delta and modify count delta. However, due to the involvement of the system table, - // a complete update of the global statistics is not feasible. Therefore, we bypass the statistics update - // for the table in this scenario. Despite this, the table id still changes, so the statistics for the - // system table will still be visible. - // 2. If the system table is a partitioned table, we will update the global statistics for the partitioned table. - // It is rare to exchange a partition from a system table, so we can ignore this case. In this case, - // the system table will have statistics, but this is not a significant issue. - logutil.StatsLogger().Info("Skip handle system database ddl event", zap.Stringer("event", t)) - return nil + + // ActionFlashbackCluster will not create any new stats info + // and it's SchemaID alwayws equals to 0, so skip check it. + if t.GetType() != model.ActionFlashbackCluster { + if isSysDB, err := t.IsMemOrSysDB(sctx.(sessionctx.Context)); err != nil { + return err + } else if isSysDB { + // EXCHANGE PARTITION EVENT NOTES: + // 1. When a partition is exchanged with a system table, we need to adjust the global statistics + // based on the count delta and modify count delta. However, due to the involvement of the system table, + // a complete update of the global statistics is not feasible. Therefore, we bypass the statistics update + // for the table in this scenario. Despite this, the table id still changes, so the statistics for the + // system table will still be visible. + // 2. If the system table is a partitioned table, we will update the global statistics for the partitioned table. + // It is rare to exchange a partition from a system table, so we can ignore this case. In this case, + // the system table will have statistics, but this is not a significant issue. + logutil.StatsLogger().Info("Skip handle system database ddl event", zap.Stringer("event", t)) + return nil + } } logutil.StatsLogger().Info("Handle ddl event", zap.Stringer("event", t)) diff --git a/tests/realtikvtest/flashbacktest/flashback_test.go b/tests/realtikvtest/flashbacktest/flashback_test.go index dfe3b4bc92e22..4b4ebd13d36e6 100644 --- a/tests/realtikvtest/flashbacktest/flashback_test.go +++ b/tests/realtikvtest/flashbacktest/flashback_test.go @@ -429,6 +429,7 @@ func TestFlashbackSequence(t *testing.T) { require.Equal(t, res[0][0], "101") require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/injectSafeTS")) + tk.MustExec("drop sequence seq") } } From d626b6adf0933571c85b4d5e577e94c017609cb7 Mon Sep 17 00:00:00 2001 From: EasonBall <592838129@qq.com> Date: Fri, 26 Apr 2024 15:19:42 +0800 Subject: [PATCH 15/20] disttask: fix scope can't set different value (#52627) close pingcap/tidb#52441 --- br/pkg/restore/db_test.go | 2 +- cmd/tidb-server/BUILD.bazel | 2 +- cmd/tidb-server/main.go | 13 +- pkg/ddl/backfilling_dist_scheduler_test.go | 2 +- pkg/ddl/ddl_api.go | 2 + pkg/ddl/index.go | 8 +- pkg/disttask/framework/handle/handle.go | 10 +- pkg/disttask/framework/handle/handle_test.go | 10 +- .../framework/integrationtests/BUILD.bazel | 5 +- .../framework/integrationtests/bench_test.go | 2 +- .../framework_err_handling_test.go | 4 +- .../integrationtests/framework_ha_test.go | 2 +- .../framework_pause_and_resume_test.go | 4 +- .../integrationtests/framework_role_test.go | 94 ---------- .../framework_rollback_test.go | 2 +- .../integrationtests/framework_scope_test.go | 174 ++++++++++++++++++ .../integrationtests/framework_test.go | 23 +-- .../integrationtests/resource_control_test.go | 2 +- pkg/disttask/framework/mock/scheduler_mock.go | 15 -- pkg/disttask/framework/planner/BUILD.bazel | 1 + pkg/disttask/framework/planner/planner.go | 6 +- pkg/disttask/framework/proto/task.go | 5 + pkg/disttask/framework/scheduler/BUILD.bazel | 2 +- pkg/disttask/framework/scheduler/balancer.go | 7 +- .../framework/scheduler/balancer_test.go | 2 +- pkg/disttask/framework/scheduler/interface.go | 5 - pkg/disttask/framework/scheduler/nodes.go | 62 +++++-- .../framework/scheduler/nodes_test.go | 91 ++++++++- pkg/disttask/framework/scheduler/scheduler.go | 6 +- .../framework/scheduler/scheduler_manager.go | 4 +- .../scheduler/scheduler_manager_test.go | 2 +- .../framework/scheduler/scheduler_test.go | 10 +- pkg/disttask/framework/scheduler/slots.go | 4 +- .../framework/scheduler/slots_test.go | 4 +- pkg/disttask/framework/storage/converter.go | 17 +- pkg/disttask/framework/storage/nodes.go | 40 +--- pkg/disttask/framework/storage/table_test.go | 58 +++--- .../framework/storage/task_state_test.go | 12 +- pkg/disttask/framework/storage/task_table.go | 19 +- .../task_executor_testkit_test.go | 2 +- pkg/disttask/framework/testutil/context.go | 7 + .../framework/testutil/disttest_util.go | 4 +- pkg/disttask/importinto/job_testkit_test.go | 4 +- .../importinto/scheduler_testkit_test.go | 4 +- pkg/executor/importer/import.go | 2 +- pkg/parser/model/reorg.go | 1 + pkg/session/bootstrap.go | 19 +- pkg/sessionctx/variable/BUILD.bazel | 2 +- pkg/sessionctx/variable/sysvar.go | 11 +- pkg/util/distrole/BUILD.bazel | 8 - pkg/util/distrole/role.go | 37 ---- pkg/util/servicescope/BUILD.bazel | 17 ++ pkg/util/servicescope/scope.go | 29 +++ pkg/util/servicescope/scope_test.go | 30 +++ 54 files changed, 555 insertions(+), 355 deletions(-) delete mode 100644 pkg/disttask/framework/integrationtests/framework_role_test.go create mode 100644 pkg/disttask/framework/integrationtests/framework_scope_test.go delete mode 100644 pkg/util/distrole/BUILD.bazel delete mode 100644 pkg/util/distrole/role.go create mode 100644 pkg/util/servicescope/BUILD.bazel create mode 100644 pkg/util/servicescope/scope.go create mode 100644 pkg/util/servicescope/scope_test.go diff --git a/br/pkg/restore/db_test.go b/br/pkg/restore/db_test.go index 21a468a399a69..800058e63b367 100644 --- a/br/pkg/restore/db_test.go +++ b/br/pkg/restore/db_test.go @@ -425,5 +425,5 @@ func TestGetExistedUserDBs(t *testing.T) { // // The above variables are in the file br/pkg/restore/systable_restore.go func TestMonitorTheSystemTableIncremental(t *testing.T) { - require.Equal(t, int64(195), session.CurrentBootstrapVersion) + require.Equal(t, int64(196), session.CurrentBootstrapVersion) } diff --git a/cmd/tidb-server/BUILD.bazel b/cmd/tidb-server/BUILD.bazel index 778b62f58f789..7e498cf2ee8e5 100644 --- a/cmd/tidb-server/BUILD.bazel +++ b/cmd/tidb-server/BUILD.bazel @@ -41,7 +41,6 @@ go_library( "//pkg/util/cpuprofile", "//pkg/util/deadlockhistory", "//pkg/util/disk", - "//pkg/util/distrole", "//pkg/util/domainutil", "//pkg/util/kvcache", "//pkg/util/logutil", @@ -50,6 +49,7 @@ go_library( "//pkg/util/printer", "//pkg/util/redact", "//pkg/util/sem", + "//pkg/util/servicescope", "//pkg/util/signal", "//pkg/util/stmtsummary/v2:stmtsummary", "//pkg/util/sys/linux", diff --git a/cmd/tidb-server/main.go b/cmd/tidb-server/main.go index 0fae4d6773864..e2fec3206ce12 100644 --- a/cmd/tidb-server/main.go +++ b/cmd/tidb-server/main.go @@ -65,7 +65,6 @@ import ( "github.com/pingcap/tidb/pkg/util/cpuprofile" "github.com/pingcap/tidb/pkg/util/deadlockhistory" "github.com/pingcap/tidb/pkg/util/disk" - distroleutil "github.com/pingcap/tidb/pkg/util/distrole" "github.com/pingcap/tidb/pkg/util/domainutil" "github.com/pingcap/tidb/pkg/util/kvcache" "github.com/pingcap/tidb/pkg/util/logutil" @@ -74,6 +73,7 @@ import ( "github.com/pingcap/tidb/pkg/util/printer" "github.com/pingcap/tidb/pkg/util/redact" "github.com/pingcap/tidb/pkg/util/sem" + "github.com/pingcap/tidb/pkg/util/servicescope" "github.com/pingcap/tidb/pkg/util/signal" stmtsummaryv2 "github.com/pingcap/tidb/pkg/util/stmtsummary/v2" "github.com/pingcap/tidb/pkg/util/sys/linux" @@ -677,14 +677,9 @@ func overrideConfig(cfg *config.Config, fset *flag.FlagSet) { } if actualFlags[nmTiDBServiceScope] { - scope, ok := distroleutil.ToTiDBServiceScope(*serviceScope) - if !ok { - err := fmt.Errorf("incorrect value: `%s`. %s options: %s", - *serviceScope, - nmTiDBServiceScope, `"", background`) - terror.MustNil(err) - } - cfg.Instance.TiDBServiceScope = scope + err = servicescope.CheckServiceScope(*serviceScope) + terror.MustNil(err) + cfg.Instance.TiDBServiceScope = *serviceScope } } diff --git a/pkg/ddl/backfilling_dist_scheduler_test.go b/pkg/ddl/backfilling_dist_scheduler_test.go index efbdc79a6bd96..b73bf3da310c8 100644 --- a/pkg/ddl/backfilling_dist_scheduler_test.go +++ b/pkg/ddl/backfilling_dist_scheduler_test.go @@ -156,7 +156,7 @@ func TestBackfillingSchedulerGlobalSortMode(t *testing.T) { ext.(*ddl.BackfillingSchedulerExt).GlobalSort = true sch.Extension = ext - taskID, err := mgr.CreateTask(ctx, task.Key, proto.Backfill, 1, task.Meta) + taskID, err := mgr.CreateTask(ctx, task.Key, proto.Backfill, 1, "", task.Meta) require.NoError(t, err) task.ID = taskID execIDs := []string{":4000"} diff --git a/pkg/ddl/ddl_api.go b/pkg/ddl/ddl_api.go index b7b527b28ee77..63a8051ba04c6 100644 --- a/pkg/ddl/ddl_api.go +++ b/pkg/ddl/ddl_api.go @@ -7720,6 +7720,8 @@ func newReorgMetaFromVariables(job *model.Job, sctx sessionctx.Context) (*model. reorgMeta := NewDDLReorgMeta(sctx) reorgMeta.IsDistReorg = variable.EnableDistTask.Load() reorgMeta.IsFastReorg = variable.EnableFastReorg.Load() + reorgMeta.TargetScope = variable.ServiceScope.Load() + if reorgMeta.IsDistReorg && !reorgMeta.IsFastReorg { return nil, dbterror.ErrUnsupportedDistTask } diff --git a/pkg/ddl/index.go b/pkg/ddl/index.go index 99b735d6afdd4..a48b01ab6646a 100644 --- a/pkg/ddl/index.go +++ b/pkg/ddl/index.go @@ -2079,7 +2079,7 @@ func (w *worker) executeDistTask(t table.Table, reorgInfo *reorgInfo) error { } else { job := reorgInfo.Job workerCntLimit := int(variable.GetDDLReorgWorkerCounter()) - cpuCount, err := handle.GetCPUCountOfManagedNode(ctx) + cpuCount, err := handle.GetCPUCountOfNode(ctx) if err != nil { return err } @@ -2103,7 +2103,7 @@ func (w *worker) executeDistTask(t table.Table, reorgInfo *reorgInfo) error { g.Go(func() error { defer close(done) - err := submitAndWaitTask(ctx, taskKey, taskType, concurrency, metaData) + err := submitAndWaitTask(ctx, taskKey, taskType, concurrency, reorgInfo.ReorgMeta.TargetScope, metaData) failpoint.Inject("pauseAfterDistTaskFinished", func() { MockDMLExecutionOnTaskFinished() }) @@ -2265,8 +2265,8 @@ func (w *worker) updateJobRowCount(taskKey string, jobID int64) { } // submitAndWaitTask submits a task and wait for it to finish. -func submitAndWaitTask(ctx context.Context, taskKey string, taskType proto.TaskType, concurrency int, taskMeta []byte) error { - task, err := handle.SubmitTask(ctx, taskKey, taskType, concurrency, taskMeta) +func submitAndWaitTask(ctx context.Context, taskKey string, taskType proto.TaskType, concurrency int, targetScope string, taskMeta []byte) error { + task, err := handle.SubmitTask(ctx, taskKey, taskType, concurrency, targetScope, taskMeta) if err != nil { return err } diff --git a/pkg/disttask/framework/handle/handle.go b/pkg/disttask/framework/handle/handle.go index 4ab9ff6e4313e..bb42c10b27003 100644 --- a/pkg/disttask/framework/handle/handle.go +++ b/pkg/disttask/framework/handle/handle.go @@ -45,17 +45,17 @@ func NotifyTaskChange() { } } -// GetCPUCountOfManagedNode gets the CPU count of the managed node. -func GetCPUCountOfManagedNode(ctx context.Context) (int, error) { +// GetCPUCountOfNode gets the CPU count of the managed node. +func GetCPUCountOfNode(ctx context.Context) (int, error) { manager, err := storage.GetTaskManager() if err != nil { return 0, err } - return manager.GetCPUCountOfManagedNode(ctx) + return manager.GetCPUCountOfNode(ctx) } // SubmitTask submits a task. -func SubmitTask(ctx context.Context, taskKey string, taskType proto.TaskType, concurrency int, taskMeta []byte) (*proto.Task, error) { +func SubmitTask(ctx context.Context, taskKey string, taskType proto.TaskType, concurrency int, targetScope string, taskMeta []byte) (*proto.Task, error) { taskManager, err := storage.GetTaskManager() if err != nil { return nil, err @@ -68,7 +68,7 @@ func SubmitTask(ctx context.Context, taskKey string, taskType proto.TaskType, co return nil, storage.ErrTaskAlreadyExists } - taskID, err := taskManager.CreateTask(ctx, taskKey, taskType, concurrency, taskMeta) + taskID, err := taskManager.CreateTask(ctx, taskKey, taskType, concurrency, targetScope, taskMeta) if err != nil { return nil, err } diff --git a/pkg/disttask/framework/handle/handle_test.go b/pkg/disttask/framework/handle/handle_test.go index 26d9d57f287cb..4cb11f4b00a09 100644 --- a/pkg/disttask/framework/handle/handle_test.go +++ b/pkg/disttask/framework/handle/handle_test.go @@ -48,7 +48,7 @@ func TestHandle(t *testing.T) { storage.SetTaskManager(mgr) // no scheduler registered - task, err := handle.SubmitTask(ctx, "1", proto.TaskTypeExample, 2, proto.EmptyMeta) + task, err := handle.SubmitTask(ctx, "1", proto.TaskTypeExample, 2, "", proto.EmptyMeta) require.NoError(t, err) waitedTaskBase, err := handle.WaitTask(ctx, task.ID, func(task *proto.TaskBase) bool { return task.IsDone() @@ -72,13 +72,13 @@ func TestHandle(t *testing.T) { require.NoError(t, handle.CancelTask(ctx, "1")) - task, err = handle.SubmitTask(ctx, "2", proto.TaskTypeExample, 2, proto.EmptyMeta) + task, err = handle.SubmitTask(ctx, "2", proto.TaskTypeExample, 2, "", proto.EmptyMeta) require.NoError(t, err) require.Equal(t, int64(2), task.ID) require.Equal(t, "2", task.Key) // submit same task. - task, err = handle.SubmitTask(ctx, "2", proto.TaskTypeExample, 2, proto.EmptyMeta) + task, err = handle.SubmitTask(ctx, "2", proto.TaskTypeExample, 2, "", proto.EmptyMeta) require.Nil(t, task) require.Error(t, storage.ErrTaskAlreadyExists, err) // pause and resume task. @@ -86,11 +86,11 @@ func TestHandle(t *testing.T) { require.NoError(t, handle.ResumeTask(ctx, "2")) // submit task with same key - task, err = handle.SubmitTask(ctx, "3", proto.TaskTypeExample, 2, proto.EmptyMeta) + task, err = handle.SubmitTask(ctx, "3", proto.TaskTypeExample, 2, "", proto.EmptyMeta) require.NoError(t, err) require.Equal(t, int64(3), task.ID) require.NoError(t, mgr.TransferTasks2History(ctx, []*proto.Task{task})) - task, err = handle.SubmitTask(ctx, "3", proto.TaskTypeExample, 2, proto.EmptyMeta) + task, err = handle.SubmitTask(ctx, "3", proto.TaskTypeExample, 2, "", proto.EmptyMeta) require.Nil(t, task) require.Error(t, storage.ErrTaskAlreadyExists, err) } diff --git a/pkg/disttask/framework/integrationtests/BUILD.bazel b/pkg/disttask/framework/integrationtests/BUILD.bazel index aad66119a3bc1..b633e56cbde8f 100644 --- a/pkg/disttask/framework/integrationtests/BUILD.bazel +++ b/pkg/disttask/framework/integrationtests/BUILD.bazel @@ -8,15 +8,15 @@ go_test( "framework_err_handling_test.go", "framework_ha_test.go", "framework_pause_and_resume_test.go", - "framework_role_test.go", "framework_rollback_test.go", + "framework_scope_test.go", "framework_test.go", "main_test.go", "resource_control_test.go", ], flaky = True, race = "off", - shard_count = 22, + shard_count = 23, deps = [ "//pkg/disttask/framework/handle", "//pkg/disttask/framework/proto", @@ -34,6 +34,7 @@ go_test( "@com_github_pingcap_failpoint//:failpoint", "@com_github_stretchr_testify//require", "@io_opencensus_go//stats/view", + "@org_golang_x_exp//rand", "@org_uber_go_goleak//:goleak", "@org_uber_go_mock//gomock", ], diff --git a/pkg/disttask/framework/integrationtests/bench_test.go b/pkg/disttask/framework/integrationtests/bench_test.go index 69fb60fe36477..75150f0a91037 100644 --- a/pkg/disttask/framework/integrationtests/bench_test.go +++ b/pkg/disttask/framework/integrationtests/bench_test.go @@ -92,7 +92,7 @@ func BenchmarkSchedulerOverhead(b *testing.B) { for i := 0; i < 4*proto.MaxConcurrentTask; i++ { taskKey := fmt.Sprintf("task-%03d", i) taskMeta := make([]byte, *taskMetaSize) - _, err := handle.SubmitTask(c.Ctx, taskKey, proto.TaskTypeExample, 1, taskMeta) + _, err := handle.SubmitTask(c.Ctx, taskKey, proto.TaskTypeExample, 1, "", taskMeta) require.NoError(c.T, err) } // task has 2 steps, each step has 1 subtask，wait in serial to reduce WaitTask check overhead. diff --git a/pkg/disttask/framework/integrationtests/framework_err_handling_test.go b/pkg/disttask/framework/integrationtests/framework_err_handling_test.go index de6dc30b5154e..090794766319c 100644 --- a/pkg/disttask/framework/integrationtests/framework_err_handling_test.go +++ b/pkg/disttask/framework/integrationtests/framework_err_handling_test.go @@ -32,9 +32,9 @@ func TestPlanNotRetryableOnNextSubtasksBatchErr(t *testing.T) { c := testutil.NewTestDXFContext(t, 2, 16, true) testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetPlanNotRetryableErrSchedulerExt(c.MockCtrl), c.TestContext, nil) - task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", 1) + task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", "", 1) require.Equal(t, proto.TaskStateReverted, task.State) testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetStepTwoPlanNotRetryableErrSchedulerExt(c.MockCtrl), c.TestContext, nil) - task = testutil.SubmitAndWaitTask(c.Ctx, t, "key2", 1) + task = testutil.SubmitAndWaitTask(c.Ctx, t, "key2", "", 1) require.Equal(t, proto.TaskStateReverted, task.State) } diff --git a/pkg/disttask/framework/integrationtests/framework_ha_test.go b/pkg/disttask/framework/integrationtests/framework_ha_test.go index af1dec723e8fc..872656d4f0f22 100644 --- a/pkg/disttask/framework/integrationtests/framework_ha_test.go +++ b/pkg/disttask/framework/integrationtests/framework_ha_test.go @@ -29,7 +29,7 @@ import ( ) func submitTaskAndCheckSuccessForHA(ctx context.Context, t *testing.T, taskKey string, testContext *testutil.TestContext) { - submitTaskAndCheckSuccess(ctx, t, taskKey, testContext, map[proto.Step]int{ + submitTaskAndCheckSuccess(ctx, t, taskKey, "", testContext, map[proto.Step]int{ proto.StepOne: 10, proto.StepTwo: 5, }) diff --git a/pkg/disttask/framework/integrationtests/framework_pause_and_resume_test.go b/pkg/disttask/framework/integrationtests/framework_pause_and_resume_test.go index fde32079a0df7..c23c98d405fd2 100644 --- a/pkg/disttask/framework/integrationtests/framework_pause_and_resume_test.go +++ b/pkg/disttask/framework/integrationtests/framework_pause_and_resume_test.go @@ -51,7 +51,7 @@ func TestFrameworkPauseAndResume(t *testing.T) { testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil) // 1. schedule and pause one running task. testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/pauseTaskAfterRefreshTask", "2*return(true)") - task1 := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", 1) + task1 := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", "", 1) require.Equal(t, proto.TaskStatePaused, task1.State) require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/disttask/framework/scheduler/pauseTaskAfterRefreshTask")) // 4 subtask scheduled. @@ -68,7 +68,7 @@ func TestFrameworkPauseAndResume(t *testing.T) { // 2. pause pending task. testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/pausePendingTask", "2*return(true)") - task2 := testutil.SubmitAndWaitTask(c.Ctx, t, "key2", 1) + task2 := testutil.SubmitAndWaitTask(c.Ctx, t, "key2", "", 1) require.Equal(t, proto.TaskStatePaused, task2.State) require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/disttask/framework/scheduler/pausePendingTask")) // 4 subtask scheduled. diff --git a/pkg/disttask/framework/integrationtests/framework_role_test.go b/pkg/disttask/framework/integrationtests/framework_role_test.go deleted file mode 100644 index 156328e88bd37..0000000000000 --- a/pkg/disttask/framework/integrationtests/framework_role_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2024 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package integrationtests - -import ( - "context" - "slices" - "testing" - "time" - - "github.com/pingcap/failpoint" - "github.com/pingcap/tidb/pkg/disttask/framework/scheduler" - "github.com/pingcap/tidb/pkg/disttask/framework/storage" - "github.com/pingcap/tidb/pkg/disttask/framework/testutil" - "github.com/pingcap/tidb/pkg/testkit" - "github.com/stretchr/testify/require" -) - -func checkSubtaskOnNodes(ctx context.Context, t *testing.T, taskID int64, expectedNodes []string) { - mgr, err := storage.GetTaskManager() - require.NoError(t, err) - nodes, err := testutil.GetSubtaskNodes(ctx, mgr, taskID) - require.NoError(t, err) - slices.Sort(nodes) - slices.Sort(expectedNodes) - require.EqualValues(t, expectedNodes, nodes) -} - -func TestRoleBasic(t *testing.T) { - c := testutil.NewTestDXFContext(t, 3, 16, true) - - testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil) - tk := testkit.NewTestKit(t, c.Store) - tk.MustExec("set @@global.tidb_schema_cache_size = default") - - // 1. all "" role. - submitTaskAndCheckSuccessForBasic(c.Ctx, t, "😁", c.TestContext) - - checkSubtaskOnNodes(c.Ctx, t, 1, []string{":4000", ":4001", ":4002"}) - tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4000"`).Check(testkit.Rows("")) - tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4001"`).Check(testkit.Rows("")) - tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4002"`).Check(testkit.Rows("")) - - // 2. one "background" role. - tk.MustExec("set global tidb_service_scope=background") - tk.MustQuery("select @@global.tidb_service_scope").Check(testkit.Rows("background")) - tk.MustQuery("select @@tidb_service_scope").Check(testkit.Rows("background")) - - testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh", "1*return()") - <-scheduler.TestRefreshedChan - submitTaskAndCheckSuccessForBasic(c.Ctx, t, "😊", c.TestContext) - require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh")) - - tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4000"`).Check(testkit.Rows("background")) - tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4001"`).Check(testkit.Rows("")) - tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4002"`).Check(testkit.Rows("")) - - checkSubtaskOnNodes(c.Ctx, t, 2, []string{":4000"}) - - // 3. 2 "background" role. - tk.MustExec("update mysql.dist_framework_meta set role = \"background\" where host = \":4001\"") - time.Sleep(5 * time.Second) - testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh", "1*return()") - <-scheduler.TestRefreshedChan - submitTaskAndCheckSuccessForBasic(c.Ctx, t, "😆", c.TestContext) - require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh")) - - checkSubtaskOnNodes(c.Ctx, t, 3, []string{":4000", ":4001"}) - tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4000"`).Check(testkit.Rows("background")) - tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4001"`).Check(testkit.Rows("background")) - tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4002"`).Check(testkit.Rows("")) -} - -func TestSetRole(t *testing.T) { - store := testkit.CreateMockStore(t) - tk := testkit.NewTestKit(t, store) - // 1. set wrong sys var. - tk.MustMatchErrMsg("set global tidb_service_scope=wrong", `incorrect value: .*. tidb_service_scope options: "", background`) - // 2. set keyspace id. - tk.MustExec("update mysql.dist_framework_meta set keyspace_id = 16777216 where host = \":4000\"") - tk.MustQuery("select keyspace_id from mysql.dist_framework_meta where host = \":4000\"").Check(testkit.Rows("16777216")) -} diff --git a/pkg/disttask/framework/integrationtests/framework_rollback_test.go b/pkg/disttask/framework/integrationtests/framework_rollback_test.go index a1467bd191cef..59847b40871d8 100644 --- a/pkg/disttask/framework/integrationtests/framework_rollback_test.go +++ b/pkg/disttask/framework/integrationtests/framework_rollback_test.go @@ -28,6 +28,6 @@ func TestFrameworkRollback(t *testing.T) { testutil.RegisterRollbackTaskMeta(t, c.MockCtrl, testutil.GetMockRollbackSchedulerExt(c.MockCtrl), c.TestContext) testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/cancelTaskAfterRefreshTask", "2*return(true)") - task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", 1) + task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", "", 1) require.Equal(t, proto.TaskStateReverted, task.State) } diff --git a/pkg/disttask/framework/integrationtests/framework_scope_test.go b/pkg/disttask/framework/integrationtests/framework_scope_test.go new file mode 100644 index 0000000000000..c3a63f5589867 --- /dev/null +++ b/pkg/disttask/framework/integrationtests/framework_scope_test.go @@ -0,0 +1,174 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package integrationtests + +import ( + "context" + "fmt" + "slices" + "strconv" + "testing" + "time" + + "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/pkg/disttask/framework/proto" + "github.com/pingcap/tidb/pkg/disttask/framework/scheduler" + "github.com/pingcap/tidb/pkg/disttask/framework/storage" + "github.com/pingcap/tidb/pkg/disttask/framework/testutil" + "github.com/pingcap/tidb/pkg/testkit" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "golang.org/x/exp/rand" +) + +func getMockBasicSchedulerExtForScope(ctrl *gomock.Controller, subtaskCnt int) scheduler.Extension { + return testutil.GetMockSchedulerExt(ctrl, testutil.SchedulerInfo{ + AllErrorRetryable: true, + StepInfos: []testutil.StepInfo{ + {Step: proto.StepOne, SubtaskCnt: subtaskCnt}, + {Step: proto.StepTwo, SubtaskCnt: 1}, + }, + }) +} + +func submitTaskAndCheckSuccessForScope(ctx context.Context, t *testing.T, taskKey string, nodeCnt int, targetScope string, testContext *testutil.TestContext) int64 { + return submitTaskAndCheckSuccess(ctx, t, taskKey, targetScope, testContext, map[proto.Step]int{ + proto.StepOne: nodeCnt, + proto.StepTwo: 1, + }) +} + +func checkSubtaskOnNodes(ctx context.Context, t *testing.T, taskID int64, expectedNodes []string) { + mgr, err := storage.GetTaskManager() + require.NoError(t, err) + nodes, err := testutil.GetSubtaskNodes(ctx, mgr, taskID) + require.NoError(t, err) + slices.Sort(nodes) + slices.Sort(expectedNodes) + require.EqualValues(t, expectedNodes, nodes) +} + +func TestScopeBasic(t *testing.T) { + nodeCnt := 3 + c := testutil.NewTestDXFContext(t, nodeCnt, 16, true) + + testutil.RegisterTaskMeta(t, c.MockCtrl, getMockBasicSchedulerExtForScope(c.MockCtrl, 3), c.TestContext, nil) + tk := testkit.NewTestKit(t, c.Store) + + // 1. all "" role. + taskID := submitTaskAndCheckSuccessForScope(c.Ctx, t, "😁", nodeCnt, "", c.TestContext) + + checkSubtaskOnNodes(c.Ctx, t, taskID, []string{":4000", ":4001", ":4002"}) + tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4000"`).Check(testkit.Rows("")) + tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4001"`).Check(testkit.Rows("")) + tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4002"`).Check(testkit.Rows("")) + + // 2. one "background" role. + tk.MustExec("set global tidb_service_scope=background") + tk.MustQuery("select @@global.tidb_service_scope").Check(testkit.Rows("background")) + tk.MustQuery("select @@tidb_service_scope").Check(testkit.Rows("background")) + + testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh", "1*return()") + <-scheduler.TestRefreshedChan + taskID = submitTaskAndCheckSuccessForScope(c.Ctx, t, "😊", nodeCnt, "background", c.TestContext) + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh")) + + tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4000"`).Check(testkit.Rows("background")) + tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4001"`).Check(testkit.Rows("")) + tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4002"`).Check(testkit.Rows("")) + checkSubtaskOnNodes(c.Ctx, t, taskID, []string{":4000"}) + + // 3. 2 "background" role. + tk.MustExec("update mysql.dist_framework_meta set role = \"background\" where host = \":4001\"") + time.Sleep(5 * time.Second) + testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh", "1*return()") + <-scheduler.TestRefreshedChan + taskID = submitTaskAndCheckSuccessForScope(c.Ctx, t, "😆", nodeCnt, "background", c.TestContext) + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh")) + checkSubtaskOnNodes(c.Ctx, t, taskID, []string{":4000", ":4001"}) + tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4000"`).Check(testkit.Rows("background")) + tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4001"`).Check(testkit.Rows("background")) + tk.MustQuery(`select role from mysql.dist_framework_meta where host=":4002"`).Check(testkit.Rows("")) +} + +func TestSetScope(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + + defer func() { + tk.MustExec(`set global tidb_service_scope=""`) + }() + // 1. set rand sys var. + tk.MustExec("set global tidb_service_scope=rand") + // 2. set keyspace id. + tk.MustExec("update mysql.dist_framework_meta set keyspace_id = 16777216 where host = \":4000\"") + tk.MustQuery("select keyspace_id from mysql.dist_framework_meta where host = \":4000\"").Check(testkit.Rows("16777216")) +} + +type targetScopeCase struct { + scope string + nodeScopes []string +} + +func generateScopeCase(nodeCnt int, scopeCnt int) targetScopeCase { + seed := time.Now().Unix() + rand.Seed(uint64(seed)) + scope := fmt.Sprintf("scope-%d", rand.Intn(100)) + + nodeScopes := make([]string, nodeCnt) + for i := 0; i < nodeCnt-scopeCnt; i++ { + nodeScopes[i] = fmt.Sprintf("scope-%d", rand.Intn(100)) + } + for i := 0; i < scopeCnt; i++ { + nodeScopes[nodeCnt-scopeCnt+i] = scope + } + + return targetScopeCase{ + scope: scope, + nodeScopes: nodeScopes, + } +} + +func runTargetScopeCase(t *testing.T, c *testutil.TestDXFContext, tk *testkit.TestKit, testCase targetScopeCase, idx int, nodeCnt int) { + for i := 0; i < len(testCase.nodeScopes); i++ { + tk.MustExec(fmt.Sprintf("update mysql.dist_framework_meta set role = \"%s\" where host = \"%s\"", testCase.nodeScopes[i], c.GetNodeIDByIdx(i))) + } + testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh", "3*return()") + <-scheduler.TestRefreshedChan + <-scheduler.TestRefreshedChan + <-scheduler.TestRefreshedChan + taskID := submitTaskAndCheckSuccessForScope(c.Ctx, t, "task"+strconv.Itoa(idx), nodeCnt, testCase.scope, c.TestContext) + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh")) + expected := make([]string, 0) + for i, scope := range testCase.nodeScopes { + if scope == testCase.scope { + expected = append(expected, c.GetNodeIDByIdx(i)) + } + } + checkSubtaskOnNodes(c.Ctx, t, taskID, expected) +} + +func TestTargetScope(t *testing.T) { + nodeCnt := 10 + c := testutil.NewTestDXFContext(t, nodeCnt, 16, true) + testutil.RegisterTaskMeta(t, c.MockCtrl, getMockBasicSchedulerExtForScope(c.MockCtrl, nodeCnt), c.TestContext, nil) + tk := testkit.NewTestKit(t, c.Store) + caseNum := 10 + for i := 0; i < caseNum; i++ { + t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) { + runTargetScopeCase(t, c, tk, generateScopeCase(nodeCnt, 5), i, nodeCnt) + }) + } +} diff --git a/pkg/disttask/framework/integrationtests/framework_test.go b/pkg/disttask/framework/integrationtests/framework_test.go index c612340392da3..a68acb1138852 100644 --- a/pkg/disttask/framework/integrationtests/framework_test.go +++ b/pkg/disttask/framework/integrationtests/framework_test.go @@ -30,20 +30,21 @@ import ( "github.com/stretchr/testify/require" ) -func submitTaskAndCheckSuccessForBasic(ctx context.Context, t *testing.T, taskKey string, testContext *testutil.TestContext) { - submitTaskAndCheckSuccess(ctx, t, taskKey, testContext, map[proto.Step]int{ +func submitTaskAndCheckSuccessForBasic(ctx context.Context, t *testing.T, taskKey string, testContext *testutil.TestContext) int64 { + return submitTaskAndCheckSuccess(ctx, t, taskKey, "", testContext, map[proto.Step]int{ proto.StepOne: 3, proto.StepTwo: 1, }) } -func submitTaskAndCheckSuccess(ctx context.Context, t *testing.T, taskKey string, - testContext *testutil.TestContext, subtaskCnts map[proto.Step]int) { - task := testutil.SubmitAndWaitTask(ctx, t, taskKey, 1) +func submitTaskAndCheckSuccess(ctx context.Context, t *testing.T, taskKey string, targetScope string, + testContext *testutil.TestContext, subtaskCnts map[proto.Step]int) int64 { + task := testutil.SubmitAndWaitTask(ctx, t, taskKey, targetScope, 1) require.Equal(t, proto.TaskStateSucceed, task.State) for step, cnt := range subtaskCnts { require.Equal(t, cnt, testContext.CollectedSubtaskCnt(task.ID, step)) } + return task.ID } func TestRandomOwnerChangeWithMultipleTasks(t *testing.T) { @@ -126,7 +127,7 @@ func TestFrameworkCancelTask(t *testing.T) { testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil) testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/MockExecutorRunCancel", "1*return(1)") - task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", 1) + task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", "", 1) require.Equal(t, proto.TaskStateReverted, task.State) } @@ -135,7 +136,7 @@ func TestFrameworkSubTaskFailed(t *testing.T) { testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil) testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/MockExecutorRunErr", "1*return(true)") - task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", 1) + task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", "", 1) require.Equal(t, proto.TaskStateReverted, task.State) } @@ -143,7 +144,7 @@ func TestFrameworkSubTaskInitEnvFailed(t *testing.T) { c := testutil.NewTestDXFContext(t, 1, 16, true) testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil) testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/mockExecSubtaskInitEnvErr", "return()") - task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", 1) + task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", "", 1) require.Equal(t, proto.TaskStateReverted, task.State) } @@ -194,7 +195,7 @@ func TestFrameworkSubtaskFinishedCancel(t *testing.T) { testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil) testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/MockSubtaskFinishedCancel", "1*return(true)") - task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", 1) + task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", "", 1) require.Equal(t, proto.TaskStateReverted, task.State) } @@ -203,7 +204,7 @@ func TestFrameworkRunSubtaskCancel(t *testing.T) { testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil) testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/MockRunSubtaskCancel", "1*return(true)") - task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", 1) + task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", "", 1) require.Equal(t, proto.TaskStateReverted, task.State) } @@ -248,6 +249,6 @@ func TestTaskCancelledBeforeUpdateTask(t *testing.T) { testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil) testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/cancelBeforeUpdateTask", "1*return(true)") - task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", 1) + task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", "", 1) require.Equal(t, proto.TaskStateReverted, task.State) } diff --git a/pkg/disttask/framework/integrationtests/resource_control_test.go b/pkg/disttask/framework/integrationtests/resource_control_test.go index 4dda3399e407d..4a419cc4dd912 100644 --- a/pkg/disttask/framework/integrationtests/resource_control_test.go +++ b/pkg/disttask/framework/integrationtests/resource_control_test.go @@ -96,7 +96,7 @@ func (c *resourceCtrlCaseContext) init(subtaskCntMap map[int64]map[proto.Step]in func (c *resourceCtrlCaseContext) runTaskAsync(prefix string, concurrencies []int) { for i, concurrency := range concurrencies { taskKey := fmt.Sprintf("%s-%d", prefix, i) - _, err := handle.SubmitTask(c.Ctx, taskKey, proto.TaskTypeExample, concurrency, nil) + _, err := handle.SubmitTask(c.Ctx, taskKey, proto.TaskTypeExample, concurrency, "", nil) require.NoError(c.T, err) c.taskWG.RunWithLog(func() { task := testutil.WaitTaskDoneOrPaused(c.Ctx, c.T, taskKey) diff --git a/pkg/disttask/framework/mock/scheduler_mock.go b/pkg/disttask/framework/mock/scheduler_mock.go index a37a16b894be7..313a87ff121b7 100644 --- a/pkg/disttask/framework/mock/scheduler_mock.go +++ b/pkg/disttask/framework/mock/scheduler_mock.go @@ -369,21 +369,6 @@ func (mr *MockTaskManagerMockRecorder) GetAllSubtasksByStepAndState(arg0, arg1, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllSubtasksByStepAndState", reflect.TypeOf((*MockTaskManager)(nil).GetAllSubtasksByStepAndState), arg0, arg1, arg2, arg3) } -// GetManagedNodes mocks base method. -func (m *MockTaskManager) GetManagedNodes(arg0 context.Context) ([]proto.ManagedNode, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetManagedNodes", arg0) - ret0, _ := ret[0].([]proto.ManagedNode) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetManagedNodes indicates an expected call of GetManagedNodes. -func (mr *MockTaskManagerMockRecorder) GetManagedNodes(arg0 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetManagedNodes", reflect.TypeOf((*MockTaskManager)(nil).GetManagedNodes), arg0) -} - // GetSubtaskCntGroupByStates mocks base method. func (m *MockTaskManager) GetSubtaskCntGroupByStates(arg0 context.Context, arg1 int64, arg2 proto.Step) (map[proto.SubtaskState]int64, error) { m.ctrl.T.Helper() diff --git a/pkg/disttask/framework/planner/BUILD.bazel b/pkg/disttask/framework/planner/BUILD.bazel index 3f28cede80197..4876832aee915 100644 --- a/pkg/disttask/framework/planner/BUILD.bazel +++ b/pkg/disttask/framework/planner/BUILD.bazel @@ -9,6 +9,7 @@ go_library( importpath = "github.com/pingcap/tidb/pkg/disttask/framework/planner", visibility = ["//visibility:public"], deps = [ + "//pkg/config", "//pkg/disttask/framework/proto", "//pkg/disttask/framework/storage", "//pkg/sessionctx", diff --git a/pkg/disttask/framework/planner/planner.go b/pkg/disttask/framework/planner/planner.go index e4b0ceaa4fa16..6b5c6f4a35684 100644 --- a/pkg/disttask/framework/planner/planner.go +++ b/pkg/disttask/framework/planner/planner.go @@ -14,7 +14,10 @@ package planner -import "github.com/pingcap/tidb/pkg/disttask/framework/storage" +import ( + "github.com/pingcap/tidb/pkg/config" + "github.com/pingcap/tidb/pkg/disttask/framework/storage" +) // Planner represents a distribute plan planner. type Planner struct{} @@ -42,6 +45,7 @@ func (*Planner) Run(planCtx PlanCtx, plan LogicalPlan) (int64, error) { planCtx.TaskKey, planCtx.TaskType, planCtx.ThreadCnt, + config.GetGlobalConfig().Instance.TiDBServiceScope, taskMeta, ) } diff --git a/pkg/disttask/framework/proto/task.go b/pkg/disttask/framework/proto/task.go index ba08dc78d5114..c9378d42c7dd9 100644 --- a/pkg/disttask/framework/proto/task.go +++ b/pkg/disttask/framework/proto/task.go @@ -93,6 +93,11 @@ type TaskBase struct { // Concurrency controls the max resource usage of the task, i.e. the max number // of slots the task can use on each node. Concurrency int + // TargetScope indicates that the task should be running on tidb nodes which + // contain the tidb_service_scope=TargetScope label. + // To be compatible with previous version, if it's "" or "background", the task try run on nodes of "background" scope, + // if there is no such nodes, will try nodes of "" scope. + TargetScope string CreateTime time.Time } diff --git a/pkg/disttask/framework/scheduler/BUILD.bazel b/pkg/disttask/framework/scheduler/BUILD.bazel index 7415ee015b389..5abf7dd96f547 100644 --- a/pkg/disttask/framework/scheduler/BUILD.bazel +++ b/pkg/disttask/framework/scheduler/BUILD.bazel @@ -57,7 +57,7 @@ go_test( embed = [":scheduler"], flaky = True, race = "off", - shard_count = 33, + shard_count = 34, deps = [ "//pkg/config", "//pkg/disttask/framework/mock", diff --git a/pkg/disttask/framework/scheduler/balancer.go b/pkg/disttask/framework/scheduler/balancer.go index 9af8f3d18cc3e..29dda4ffb1a99 100644 --- a/pkg/disttask/framework/scheduler/balancer.go +++ b/pkg/disttask/framework/scheduler/balancer.go @@ -74,15 +74,16 @@ func (b *balancer) balance(ctx context.Context, sm *Manager) { // we will use currUsedSlots to calculate adjusted eligible nodes during balance, // it's initial value depends on the managed nodes, to have a consistent view, // DO NOT call getManagedNodes twice during 1 balance. - managedNodes := b.nodeMgr.getManagedNodes() + managedNodes := b.nodeMgr.getNodes() b.currUsedSlots = make(map[string]int, len(managedNodes)) for _, n := range managedNodes { - b.currUsedSlots[n] = 0 + b.currUsedSlots[n.ID] = 0 } schedulers := sm.getSchedulers() for _, sch := range schedulers { - if err := b.balanceSubtasks(ctx, sch, managedNodes); err != nil { + nodeIDs := filterByScope(managedNodes, sch.GetTask().TargetScope) + if err := b.balanceSubtasks(ctx, sch, nodeIDs); err != nil { b.logger.Warn("failed to balance subtasks", zap.Int64("task-id", sch.GetTask().ID), llog.ShortError(err)) return diff --git a/pkg/disttask/framework/scheduler/balancer_test.go b/pkg/disttask/framework/scheduler/balancer_test.go index c1802ca3e42f9..984b8cab53f54 100644 --- a/pkg/disttask/framework/scheduler/balancer_test.go +++ b/pkg/disttask/framework/scheduler/balancer_test.go @@ -370,7 +370,7 @@ func TestBalanceMultipleTasks(t *testing.T) { manager := NewManager(ctx, mockTaskMgr, "1") manager.slotMgr.updateCapacity(16) - manager.nodeMgr.managedNodes.Store(&[]string{"tidb1", "tidb2", "tidb3"}) + manager.nodeMgr.nodes.Store(&[]proto.ManagedNode{{ID: "tidb1", Role: ""}, {ID: "tidb2", Role: ""}, {ID: "tidb3", Role: ""}}) b := newBalancer(Param{ taskMgr: manager.taskMgr, nodeMgr: manager.nodeMgr, diff --git a/pkg/disttask/framework/scheduler/interface.go b/pkg/disttask/framework/scheduler/interface.go index 4a61273eb2c97..23a3c1f1d9758 100644 --- a/pkg/disttask/framework/scheduler/interface.go +++ b/pkg/disttask/framework/scheduler/interface.go @@ -79,11 +79,6 @@ type TaskManager interface { ResumeSubtasks(ctx context.Context, taskID int64) error GetSubtaskErrors(ctx context.Context, taskID int64) ([]error, error) UpdateSubtasksExecIDs(ctx context.Context, subtasks []*proto.SubtaskBase) error - // GetManagedNodes returns the nodes managed by dist framework and can be used - // to execute tasks. If there are any nodes with background role, we use them, - // else we use nodes without role. - // returned nodes are sorted by node id(host:port). - GetManagedNodes(ctx context.Context) ([]proto.ManagedNode, error) // GetAllSubtasksByStepAndState gets all subtasks by given states for one step. GetAllSubtasksByStepAndState(ctx context.Context, taskID int64, step proto.Step, state proto.SubtaskState) ([]*proto.Subtask, error) diff --git a/pkg/disttask/framework/scheduler/nodes.go b/pkg/disttask/framework/scheduler/nodes.go index 6ecf52babdc8a..7a8dc8cb401c4 100644 --- a/pkg/disttask/framework/scheduler/nodes.go +++ b/pkg/disttask/framework/scheduler/nodes.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/log" + "github.com/pingcap/tidb/pkg/disttask/framework/proto" llog "github.com/pingcap/tidb/pkg/lightning/log" "github.com/pingcap/tidb/pkg/util/intest" "go.uber.org/zap" @@ -37,9 +38,9 @@ type NodeManager struct { logger *zap.Logger // prevLiveNodes is used to record the live nodes in last checking. prevLiveNodes map[string]struct{} - // managedNodes is the cached nodes managed by the framework. - // see TaskManager.GetManagedNodes for more details. - managedNodes atomic.Pointer[[]string] + // nodes is the cached nodes managed by the framework. + // see TaskManager.GetNodes for more details. + nodes atomic.Pointer[[]proto.ManagedNode] } func newNodeManager(serverID string) *NodeManager { @@ -51,8 +52,8 @@ func newNodeManager(serverID string) *NodeManager { logger: logger, prevLiveNodes: make(map[string]struct{}), } - managedNodes := make([]string, 0, 10) - nm.managedNodes.Store(&managedNodes) + nodes := make([]proto.ManagedNode, 0, 10) + nm.nodes.Store(&nodes) return nm } @@ -116,7 +117,7 @@ func (nm *NodeManager) maintainLiveNodes(ctx context.Context, taskMgr TaskManage nm.prevLiveNodes = currLiveNodes } -func (nm *NodeManager) refreshManagedNodesLoop(ctx context.Context, taskMgr TaskManager, slotMgr *SlotManager) { +func (nm *NodeManager) refreshNodesLoop(ctx context.Context, taskMgr TaskManager, slotMgr *SlotManager) { ticker := time.NewTicker(nodesCheckInterval) defer ticker.Stop() for { @@ -124,7 +125,7 @@ func (nm *NodeManager) refreshManagedNodesLoop(ctx context.Context, taskMgr Task case <-ctx.Done(): return case <-ticker.C: - nm.refreshManagedNodes(ctx, taskMgr, slotMgr) + nm.refreshNodes(ctx, taskMgr, slotMgr) } } } @@ -132,34 +133,59 @@ func (nm *NodeManager) refreshManagedNodesLoop(ctx context.Context, taskMgr Task // TestRefreshedChan is used to sync the test. var TestRefreshedChan = make(chan struct{}) -// refreshManagedNodes maintains the nodes managed by the framework. -func (nm *NodeManager) refreshManagedNodes(ctx context.Context, taskMgr TaskManager, slotMgr *SlotManager) { - newNodes, err := taskMgr.GetManagedNodes(ctx) +// refreshNodes maintains the nodes managed by the framework. +func (nm *NodeManager) refreshNodes(ctx context.Context, taskMgr TaskManager, slotMgr *SlotManager) { + newNodes, err := taskMgr.GetAllNodes(ctx) if err != nil { nm.logger.Warn("get managed nodes met error", llog.ShortError(err)) return } - nodeIDs := make([]string, 0, len(newNodes)) + var cpuCount int for _, node := range newNodes { - nodeIDs = append(nodeIDs, node.ID) if node.CPUCount > 0 { cpuCount = node.CPUCount } } slotMgr.updateCapacity(cpuCount) - nm.managedNodes.Store(&nodeIDs) + nm.nodes.Store(&newNodes) failpoint.Inject("syncRefresh", func() { TestRefreshedChan <- struct{}{} }) } -// GetManagedNodes returns the nodes managed by the framework. -// return a copy of the managed nodes. -func (nm *NodeManager) getManagedNodes() []string { - nodes := *nm.managedNodes.Load() - res := make([]string, len(nodes)) +// GetNodes returns the nodes managed by the framework. +// return a copy of the nodes. +func (nm *NodeManager) getNodes() []proto.ManagedNode { + nodes := *nm.nodes.Load() + res := make([]proto.ManagedNode, len(nodes)) copy(res, nodes) return res } + +func filterByScope(nodes []proto.ManagedNode, targetScope string) []string { + var nodeIDs []string + haveBackground := false + for _, node := range nodes { + if node.Role == "background" { + haveBackground = true + } + } + // prefer to use "background" node instead of "" node. + if targetScope == "" && haveBackground { + for _, node := range nodes { + if node.Role == "background" { + nodeIDs = append(nodeIDs, node.ID) + } + } + return nodeIDs + } + + for _, node := range nodes { + if node.Role == targetScope { + nodeIDs = append(nodeIDs, node.ID) + } + } + return nodeIDs +} diff --git a/pkg/disttask/framework/scheduler/nodes_test.go b/pkg/disttask/framework/scheduler/nodes_test.go index 4979d3ffed054..e136f44cac1cf 100644 --- a/pkg/disttask/framework/scheduler/nodes_test.go +++ b/pkg/disttask/framework/scheduler/nodes_test.go @@ -16,6 +16,7 @@ package scheduler import ( "context" + "fmt" "testing" "github.com/pingcap/errors" @@ -97,24 +98,94 @@ func TestMaintainManagedNodes(t *testing.T) { nodeMgr := newNodeManager("") slotMgr := newSlotManager() - mockTaskMgr.EXPECT().GetManagedNodes(gomock.Any()).Return(nil, errors.New("mock error")) - nodeMgr.refreshManagedNodes(ctx, mockTaskMgr, slotMgr) + mockTaskMgr.EXPECT().GetAllNodes(gomock.Any()).Return(nil, errors.New("mock error")) + nodeMgr.refreshNodes(ctx, mockTaskMgr, slotMgr) require.Equal(t, cpu.GetCPUCount(), int(slotMgr.capacity.Load())) - require.Empty(t, nodeMgr.getManagedNodes()) + require.Empty(t, nodeMgr.getNodes()) require.True(t, ctrl.Satisfied()) - mockTaskMgr.EXPECT().GetManagedNodes(gomock.Any()).Return([]proto.ManagedNode{ + mockTaskMgr.EXPECT().GetAllNodes(gomock.Any()).Return([]proto.ManagedNode{ {ID: ":4000", CPUCount: 100}, {ID: ":4001", CPUCount: 100}, }, nil) - nodeMgr.refreshManagedNodes(ctx, mockTaskMgr, slotMgr) - require.Equal(t, []string{":4000", ":4001"}, nodeMgr.getManagedNodes()) + nodeMgr.refreshNodes(ctx, mockTaskMgr, slotMgr) + require.Equal(t, []proto.ManagedNode{{ID: ":4000", Role: "", CPUCount: 100}, {ID: ":4001", Role: "", CPUCount: 100}}, nodeMgr.getNodes()) require.Equal(t, 100, int(slotMgr.capacity.Load())) require.True(t, ctrl.Satisfied()) - mockTaskMgr.EXPECT().GetManagedNodes(gomock.Any()).Return(nil, nil) - nodeMgr.refreshManagedNodes(ctx, mockTaskMgr, slotMgr) - require.NotNil(t, nodeMgr.getManagedNodes()) - require.Empty(t, nodeMgr.getManagedNodes()) + mockTaskMgr.EXPECT().GetAllNodes(gomock.Any()).Return(nil, nil) + nodeMgr.refreshNodes(ctx, mockTaskMgr, slotMgr) + require.NotNil(t, nodeMgr.getNodes()) + require.Empty(t, nodeMgr.getNodes()) require.Equal(t, 100, int(slotMgr.capacity.Load())) require.True(t, ctrl.Satisfied()) } + +type filterCase struct { + nodes []proto.ManagedNode + targetScope string + expectedNodes []string +} + +func mockManagedNode(id string, role string) proto.ManagedNode { + return proto.ManagedNode{ + ID: id, + Role: role, + CPUCount: 100, + } +} + +func TestFilterByScope(t *testing.T) { + cases := []filterCase{ + { + nodes: []proto.ManagedNode{mockManagedNode("1", "background"), mockManagedNode("2", "background"), mockManagedNode("3", "")}, + targetScope: "", + expectedNodes: []string{"1", "2"}, + }, + { + nodes: []proto.ManagedNode{mockManagedNode("1", ""), mockManagedNode("2", ""), mockManagedNode("3", "")}, + targetScope: "", + expectedNodes: []string{"1", "2", "3"}, + }, + { + nodes: []proto.ManagedNode{mockManagedNode("1", "1"), mockManagedNode("2", ""), mockManagedNode("3", "")}, + targetScope: "", + expectedNodes: []string{"2", "3"}, + }, + { + nodes: []proto.ManagedNode{mockManagedNode("1", "1"), mockManagedNode("2", ""), mockManagedNode("3", "")}, + targetScope: "1", + expectedNodes: []string{"1"}, + }, + { + nodes: []proto.ManagedNode{mockManagedNode("1", "1"), mockManagedNode("2", "2"), mockManagedNode("3", "2")}, + targetScope: "1", + expectedNodes: []string{"1"}, + }, + { + nodes: []proto.ManagedNode{mockManagedNode("1", "1"), mockManagedNode("2", "2"), mockManagedNode("3", "3")}, + targetScope: "2", + expectedNodes: []string{"2"}, + }, + { + nodes: []proto.ManagedNode{mockManagedNode("1", "1"), mockManagedNode("2", "2"), mockManagedNode("3", "background")}, + targetScope: "background", + expectedNodes: []string{"3"}, + }, + { + nodes: []proto.ManagedNode{mockManagedNode("1", "1"), mockManagedNode("2", ""), mockManagedNode("3", "background")}, + targetScope: "", + expectedNodes: []string{"3"}, + }, + { + nodes: []proto.ManagedNode{mockManagedNode("1", ""), mockManagedNode("2", ""), mockManagedNode("3", "")}, + targetScope: "background", + expectedNodes: nil, + }, + } + + for i, cas := range cases { + t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) { + require.Equal(t, cas.expectedNodes, filterByScope(cas.nodes, cas.targetScope)) + }) + } +} diff --git a/pkg/disttask/framework/scheduler/scheduler.go b/pkg/disttask/framework/scheduler/scheduler.go index ea43e4d59092c..3ac08f319136a 100644 --- a/pkg/disttask/framework/scheduler/scheduler.go +++ b/pkg/disttask/framework/scheduler/scheduler.go @@ -436,10 +436,13 @@ func (s *BaseScheduler) switch2NextStep() error { return nil } - eligibleNodes, err := getEligibleNodes(s.ctx, s, s.nodeMgr.getManagedNodes()) + nodes := s.nodeMgr.getNodes() + nodeIDs := filterByScope(nodes, task.TargetScope) + eligibleNodes, err := getEligibleNodes(s.ctx, s, nodeIDs) if err != nil { return err } + s.logger.Info("eligible instances", zap.Int("num", len(eligibleNodes))) if len(eligibleNodes) == 0 { return errors.New("no available TiDB node to dispatch subtasks") @@ -629,5 +632,6 @@ func getEligibleNodes(ctx context.Context, sch Scheduler, managedNodes []string) if len(serverNodes) == 0 { serverNodes = managedNodes } + return serverNodes, nil } diff --git a/pkg/disttask/framework/scheduler/scheduler_manager.go b/pkg/disttask/framework/scheduler/scheduler_manager.go index b556cf64356d1..a2f8f8789a708 100644 --- a/pkg/disttask/framework/scheduler/scheduler_manager.go +++ b/pkg/disttask/framework/scheduler/scheduler_manager.go @@ -156,7 +156,7 @@ func NewManager(ctx context.Context, taskMgr TaskManager, serverID string) *Mana // Start the schedulerManager, start the scheduleTaskLoop to start multiple schedulers. func (sm *Manager) Start() { // init cached managed nodes - sm.nodeMgr.refreshManagedNodes(sm.ctx, sm.taskMgr, sm.slotMgr) + sm.nodeMgr.refreshNodes(sm.ctx, sm.taskMgr, sm.slotMgr) sm.wg.Run(sm.scheduleTaskLoop) sm.wg.Run(sm.gcSubtaskHistoryTableLoop) @@ -166,7 +166,7 @@ func (sm *Manager) Start() { sm.nodeMgr.maintainLiveNodesLoop(sm.ctx, sm.taskMgr) }) sm.wg.Run(func() { - sm.nodeMgr.refreshManagedNodesLoop(sm.ctx, sm.taskMgr, sm.slotMgr) + sm.nodeMgr.refreshNodesLoop(sm.ctx, sm.taskMgr, sm.slotMgr) }) sm.wg.Run(func() { sm.balancer.balanceLoop(sm.ctx, sm) diff --git a/pkg/disttask/framework/scheduler/scheduler_manager_test.go b/pkg/disttask/framework/scheduler/scheduler_manager_test.go index 66d17091606b2..daa1029741f4f 100644 --- a/pkg/disttask/framework/scheduler/scheduler_manager_test.go +++ b/pkg/disttask/framework/scheduler/scheduler_manager_test.go @@ -46,7 +46,7 @@ func TestCleanUpRoutine(t *testing.T) { mockCleanupRoutine.EXPECT().CleanUp(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() sch.Start() defer sch.Stop() - taskID, err := mgr.CreateTask(ctx, "test", proto.TaskTypeExample, 1, nil) + taskID, err := mgr.CreateTask(ctx, "test", proto.TaskTypeExample, 1, "", nil) require.NoError(t, err) checkTaskRunningCnt := func() []*proto.Task { diff --git a/pkg/disttask/framework/scheduler/scheduler_test.go b/pkg/disttask/framework/scheduler/scheduler_test.go index 22c489b32cd4c..7820c56fea6d7 100644 --- a/pkg/disttask/framework/scheduler/scheduler_test.go +++ b/pkg/disttask/framework/scheduler/scheduler_test.go @@ -129,7 +129,7 @@ func TestTaskFailInManager(t *testing.T) { defer schManager.Stop() // unknown task type - taskID, err := mgr.CreateTask(ctx, "test", "test-type", 1, nil) + taskID, err := mgr.CreateTask(ctx, "test", "test-type", 1, "", nil) require.NoError(t, err) require.Eventually(t, func() bool { task, err := mgr.GetTaskByID(ctx, taskID) @@ -139,7 +139,7 @@ func TestTaskFailInManager(t *testing.T) { }, time.Second*10, time.Millisecond*300) // scheduler init error - taskID, err = mgr.CreateTask(ctx, "test2", proto.TaskTypeExample, 1, nil) + taskID, err = mgr.CreateTask(ctx, "test2", proto.TaskTypeExample, 1, "", nil) require.NoError(t, err) require.Eventually(t, func() bool { task, err := mgr.GetTaskByID(ctx, taskID) @@ -215,7 +215,7 @@ func checkSchedule(t *testing.T, taskCnt int, isSucc, isCancel, isSubtaskCancel, // Mock add tasks. taskIDs := make([]int64, 0, taskCnt) for i := 0; i < taskCnt; i++ { - taskID, err := mgr.CreateTask(ctx, fmt.Sprintf("%d", i), proto.TaskTypeExample, 0, nil) + taskID, err := mgr.CreateTask(ctx, fmt.Sprintf("%d", i), proto.TaskTypeExample, 0, "background", nil) require.NoError(t, err) taskIDs = append(taskIDs, taskID) } @@ -225,7 +225,7 @@ func checkSchedule(t *testing.T, taskCnt int, isSucc, isCancel, isSubtaskCancel, checkSubtaskCnt(tasks, taskIDs) // test parallelism control if taskCnt == 1 { - taskID, err := mgr.CreateTask(ctx, fmt.Sprintf("%d", taskCnt), proto.TaskTypeExample, 0, nil) + taskID, err := mgr.CreateTask(ctx, fmt.Sprintf("%d", taskCnt), proto.TaskTypeExample, 0, "background", nil) require.NoError(t, err) checkGetRunningTaskCnt(taskCnt) // Clean the task. @@ -454,7 +454,7 @@ func TestManagerScheduleLoop(t *testing.T) { }, ) for i := 0; i < len(concurrencies); i++ { - _, err := taskMgr.CreateTask(ctx, fmt.Sprintf("key/%d", i), proto.TaskTypeExample, concurrencies[i], []byte("{}")) + _, err := taskMgr.CreateTask(ctx, fmt.Sprintf("key/%d", i), proto.TaskTypeExample, concurrencies[i], "", []byte("{}")) require.NoError(t, err) } getRunningTaskKeys := func() []string { diff --git a/pkg/disttask/framework/scheduler/slots.go b/pkg/disttask/framework/scheduler/slots.go index 76bfb3bd1d662..3d0fcd4ace848 100644 --- a/pkg/disttask/framework/scheduler/slots.go +++ b/pkg/disttask/framework/scheduler/slots.go @@ -91,14 +91,14 @@ func newSlotManager() *SlotManager { // Update updates the used slots on each node. // TODO: on concurrent call, update once. func (sm *SlotManager) update(ctx context.Context, nodeMgr *NodeManager, taskMgr TaskManager) error { - nodes := nodeMgr.getManagedNodes() + nodes := nodeMgr.getNodes() slotsOnNodes, err := taskMgr.GetUsedSlotsOnNodes(ctx) if err != nil { return err } newUsedSlots := make(map[string]int, len(nodes)) for _, node := range nodes { - newUsedSlots[node] = slotsOnNodes[node] + newUsedSlots[node.ID] = slotsOnNodes[node.ID] } sm.usedSlots.Store(&newUsedSlots) diff --git a/pkg/disttask/framework/scheduler/slots_test.go b/pkg/disttask/framework/scheduler/slots_test.go index aeff1db7abf2a..4327e7e8d9e4e 100644 --- a/pkg/disttask/framework/scheduler/slots_test.go +++ b/pkg/disttask/framework/scheduler/slots_test.go @@ -183,7 +183,7 @@ func TestSlotManagerUpdate(t *testing.T) { nodeMgr := newNodeManager("") taskMgr := mock.NewMockTaskManager(ctrl) - nodeMgr.managedNodes.Store(&[]string{"tidb-1", "tidb-2", "tidb-3"}) + nodeMgr.nodes.Store(&[]proto.ManagedNode{{ID: "tidb-1", Role: ""}, {ID: "tidb-2", Role: ""}, {ID: "tidb-3", Role: ""}}) taskMgr.EXPECT().GetUsedSlotsOnNodes(gomock.Any()).Return(map[string]int{ "tidb-1": 12, "tidb-2": 8, @@ -202,7 +202,7 @@ func TestSlotManagerUpdate(t *testing.T) { require.True(t, ctrl.Satisfied()) // some node scaled in, should be reflected - nodeMgr.managedNodes.Store(&[]string{"tidb-1"}) + nodeMgr.nodes.Store(&[]proto.ManagedNode{{ID: "tidb-1", Role: ""}}) taskMgr.EXPECT().GetUsedSlotsOnNodes(gomock.Any()).Return(map[string]int{ "tidb-1": 12, "tidb-2": 8, diff --git a/pkg/disttask/framework/storage/converter.go b/pkg/disttask/framework/storage/converter.go index 990ee64e78792..b18a196c041b9 100644 --- a/pkg/disttask/framework/storage/converter.go +++ b/pkg/disttask/framework/storage/converter.go @@ -34,6 +34,7 @@ func row2TaskBasic(r chunk.Row) *proto.TaskBase { Step: proto.Step(r.GetInt64(4)), Priority: int(r.GetInt64(5)), Concurrency: int(r.GetInt64(6)), + TargetScope: r.GetString(8), } task.CreateTime, _ = r.GetTime(7).GoTime(time.Local) return task @@ -44,18 +45,18 @@ func Row2Task(r chunk.Row) *proto.Task { taskBase := row2TaskBasic(r) task := &proto.Task{TaskBase: *taskBase} var startTime, updateTime time.Time - if !r.IsNull(8) { - startTime, _ = r.GetTime(8).GoTime(time.Local) - } if !r.IsNull(9) { - updateTime, _ = r.GetTime(9).GoTime(time.Local) + startTime, _ = r.GetTime(9).GoTime(time.Local) + } + if !r.IsNull(10) { + updateTime, _ = r.GetTime(10).GoTime(time.Local) } task.StartTime = startTime task.StateUpdateTime = updateTime - task.Meta = r.GetBytes(10) - task.SchedulerID = r.GetString(11) - if !r.IsNull(12) { - errBytes := r.GetBytes(12) + task.Meta = r.GetBytes(11) + task.SchedulerID = r.GetString(12) + if !r.IsNull(13) { + errBytes := r.GetBytes(13) stdErr := errors.Normalize("") err := stdErr.UnmarshalJSON(errBytes) if err != nil { diff --git a/pkg/disttask/framework/storage/nodes.go b/pkg/disttask/framework/storage/nodes.go index e6075c5952f98..b8c2c1b3af1ae 100644 --- a/pkg/disttask/framework/storage/nodes.go +++ b/pkg/disttask/framework/storage/nodes.go @@ -84,32 +84,6 @@ func (mgr *TaskManager) DeleteDeadNodes(ctx context.Context, nodes []string) err }) } -// GetManagedNodes implements scheduler.TaskManager interface. -func (mgr *TaskManager) GetManagedNodes(ctx context.Context) ([]proto.ManagedNode, error) { - var nodes []proto.ManagedNode - err := mgr.WithNewSession(func(se sessionctx.Context) error { - var err2 error - nodes, err2 = mgr.getManagedNodesWithSession(ctx, se) - return err2 - }) - return nodes, err -} - -func (mgr *TaskManager) getManagedNodesWithSession(ctx context.Context, se sessionctx.Context) ([]proto.ManagedNode, error) { - nodes, err := mgr.getAllNodesWithSession(ctx, se) - if err != nil { - return nil, err - } - nodeMap := make(map[string][]proto.ManagedNode, 2) - for _, node := range nodes { - nodeMap[node.Role] = append(nodeMap[node.Role], node) - } - if len(nodeMap["background"]) == 0 { - return nodeMap[""], nil - } - return nodeMap["background"], nil -} - // GetAllNodes gets nodes in dist_framework_meta. func (mgr *TaskManager) GetAllNodes(ctx context.Context) ([]proto.ManagedNode, error) { var nodes []proto.ManagedNode @@ -168,21 +142,21 @@ func (mgr *TaskManager) GetUsedSlotsOnNodes(ctx context.Context) (map[string]int return slots, nil } -// GetCPUCountOfManagedNode gets the cpu count of managed node. -func (mgr *TaskManager) GetCPUCountOfManagedNode(ctx context.Context) (int, error) { +// GetCPUCountOfNode gets the cpu count of node. +func (mgr *TaskManager) GetCPUCountOfNode(ctx context.Context) (int, error) { var cnt int err := mgr.WithNewSession(func(se sessionctx.Context) error { var err2 error - cnt, err2 = mgr.getCPUCountOfManagedNode(ctx, se) + cnt, err2 = mgr.getCPUCountOfNode(ctx, se) return err2 }) return cnt, err } -// getCPUCountOfManagedNode gets the cpu count of managed node. -// returns error when there's no managed node or no node has valid cpu count. -func (mgr *TaskManager) getCPUCountOfManagedNode(ctx context.Context, se sessionctx.Context) (int, error) { - nodes, err := mgr.getManagedNodesWithSession(ctx, se) +// getCPUCountOfNode gets the cpu count of managed node. +// returns error when there's no node or no node has valid cpu count. +func (mgr *TaskManager) getCPUCountOfNode(ctx context.Context, se sessionctx.Context) (int, error) { + nodes, err := mgr.getAllNodesWithSession(ctx, se) if err != nil { return 0, err } diff --git a/pkg/disttask/framework/storage/table_test.go b/pkg/disttask/framework/storage/table_test.go index 583218edec601..73067b79a398d 100644 --- a/pkg/disttask/framework/storage/table_test.go +++ b/pkg/disttask/framework/storage/table_test.go @@ -45,11 +45,11 @@ func TestTaskTable(t *testing.T) { require.NoError(t, gm.InitMeta(ctx, ":4000", "")) - _, err := gm.CreateTask(ctx, "key1", "test", 999, []byte("test")) + _, err := gm.CreateTask(ctx, "key1", "test", 999, "", []byte("test")) require.ErrorContains(t, err, "task concurrency(999) larger than cpu count") timeBeforeCreate := time.Unix(time.Now().Unix(), 0) - id, err := gm.CreateTask(ctx, "key1", "test", 4, []byte("test")) + id, err := gm.CreateTask(ctx, "key1", "test", 4, "", []byte("test")) require.NoError(t, err) require.Equal(t, int64(1), id) @@ -98,11 +98,11 @@ func TestTaskTable(t *testing.T) { require.Equal(t, task.State, task6.State) // test cannot insert task with dup key - _, err = gm.CreateTask(ctx, "key1", "test2", 4, []byte("test2")) + _, err = gm.CreateTask(ctx, "key1", "test2", 4, "", []byte("test2")) require.EqualError(t, err, "[kv:1062]Duplicate entry 'key1' for key 'tidb_global_task.task_key'") // test cancel task - id, err = gm.CreateTask(ctx, "key2", "test", 4, []byte("test")) + id, err = gm.CreateTask(ctx, "key2", "test", 4, "", []byte("test")) require.NoError(t, err) cancelling, err := testutil.IsTaskCancelling(ctx, gm, id) @@ -114,7 +114,7 @@ func TestTaskTable(t *testing.T) { require.NoError(t, err) require.True(t, cancelling) - id, err = gm.CreateTask(ctx, "key-fail", "test2", 4, []byte("test2")) + id, err = gm.CreateTask(ctx, "key-fail", "test2", 4, "", []byte("test2")) require.NoError(t, err) // state not right, update nothing require.NoError(t, gm.FailTask(ctx, id, proto.TaskStateRunning, errors.New("test error"))) @@ -134,7 +134,7 @@ func TestTaskTable(t *testing.T) { require.GreaterOrEqual(t, endTime, curTime) // succeed a pending task, no effect - id, err = gm.CreateTask(ctx, "key-success", "test", 4, []byte("test")) + id, err = gm.CreateTask(ctx, "key-success", "test", 4, "", []byte("test")) require.NoError(t, err) require.NoError(t, gm.SucceedTask(ctx, id)) task, err = gm.GetTaskByID(ctx, id) @@ -153,7 +153,7 @@ func TestTaskTable(t *testing.T) { require.GreaterOrEqual(t, task.StateUpdateTime, startTime) // reverted a pending task, no effect - id, err = gm.CreateTask(ctx, "key-reverted", "test", 4, []byte("test")) + id, err = gm.CreateTask(ctx, "key-reverted", "test", 4, "", []byte("test")) require.NoError(t, err) require.NoError(t, gm.RevertedTask(ctx, id)) task, err = gm.GetTaskByID(ctx, id) @@ -177,7 +177,7 @@ func TestTaskTable(t *testing.T) { require.Equal(t, proto.TaskStateReverted, task.State) // paused - id, err = gm.CreateTask(ctx, "key-paused", "test", 4, []byte("test")) + id, err = gm.CreateTask(ctx, "key-paused", "test", 4, "", []byte("test")) require.NoError(t, err) require.NoError(t, gm.PausedTask(ctx, id)) task, err = gm.GetTaskByID(ctx, id) @@ -228,7 +228,7 @@ func TestSwitchTaskStep(t *testing.T) { tk := testkit.NewTestKit(t, store) require.NoError(t, tm.InitMeta(ctx, ":4000", "")) - taskID, err := tm.CreateTask(ctx, "key1", "test", 4, []byte("test")) + taskID, err := tm.CreateTask(ctx, "key1", "test", 4, "", []byte("test")) require.NoError(t, err) task, err := tm.GetTaskByID(ctx, taskID) require.NoError(t, err) @@ -280,7 +280,7 @@ func TestSwitchTaskStepInBatch(t *testing.T) { require.NoError(t, tm.InitMeta(ctx, ":4000", "")) // normal flow prepare := func(taskKey string) (*proto.Task, []*proto.Subtask) { - taskID, err := tm.CreateTask(ctx, taskKey, "test", 4, []byte("test")) + taskID, err := tm.CreateTask(ctx, taskKey, "test", 4, "", []byte("test")) require.NoError(t, err) task, err := tm.GetTaskByID(ctx, taskID) require.NoError(t, err) @@ -371,7 +371,7 @@ func TestGetTopUnfinishedTasks(t *testing.T) { } for i, state := range taskStates { taskKey := fmt.Sprintf("key/%d", i) - _, err := gm.CreateTask(ctx, taskKey, "test", 4, []byte("test")) + _, err := gm.CreateTask(ctx, taskKey, "test", 4, "", []byte("test")) require.NoError(t, err) require.NoError(t, gm.WithNewSession(func(se sessionctx.Context) error { _, err := se.GetSQLExecutor().ExecuteInternal(ctx, ` @@ -435,7 +435,7 @@ func TestGetUsedSlotsOnNodes(t *testing.T) { func TestGetActiveSubtasks(t *testing.T) { _, tm, ctx := testutil.InitTableTest(t) require.NoError(t, tm.InitMeta(ctx, ":4000", "")) - id, err := tm.CreateTask(ctx, "key1", "test", 4, []byte("test")) + id, err := tm.CreateTask(ctx, "key1", "test", 4, "", []byte("test")) require.NoError(t, err) require.Equal(t, int64(1), id) task, err := tm.GetTaskByID(ctx, id) @@ -467,7 +467,7 @@ func TestSubTaskTable(t *testing.T) { _, sm, ctx := testutil.InitTableTest(t) timeBeforeCreate := time.Unix(time.Now().Unix(), 0) require.NoError(t, sm.InitMeta(ctx, ":4000", "")) - id, err := sm.CreateTask(ctx, "key1", "test", 4, []byte("test")) + id, err := sm.CreateTask(ctx, "key1", "test", 4, "", []byte("test")) require.NoError(t, err) require.Equal(t, int64(1), id) err = sm.SwitchTaskStep( @@ -634,7 +634,7 @@ func TestSubTaskTable(t *testing.T) { func TestBothTaskAndSubTaskTable(t *testing.T) { _, sm, ctx := testutil.InitTableTest(t) require.NoError(t, sm.InitMeta(ctx, ":4000", "")) - id, err := sm.CreateTask(ctx, "key1", "test", 4, []byte("test")) + id, err := sm.CreateTask(ctx, "key1", "test", 4, "", []byte("test")) require.NoError(t, err) require.Equal(t, int64(1), id) @@ -697,11 +697,11 @@ func TestDistFrameworkMeta(t *testing.T) { _, sm, ctx := testutil.InitTableTest(t) // when no node - _, err := sm.GetCPUCountOfManagedNode(ctx) + _, err := sm.GetCPUCountOfNode(ctx) require.ErrorContains(t, err, "no managed nodes") testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", "return(0)") require.NoError(t, sm.InitMeta(ctx, ":4000", "background")) - _, err = sm.GetCPUCountOfManagedNode(ctx) + _, err = sm.GetCPUCountOfNode(ctx) require.ErrorContains(t, err, "no managed node have enough resource") testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", "return(100)") @@ -729,48 +729,52 @@ func TestDistFrameworkMeta(t *testing.T) { {ID: ":4002", Role: "", CPUCount: 100}, {ID: ":4003", Role: "background", CPUCount: 100}, }, nodes) - cpuCount, err := sm.GetCPUCountOfManagedNode(ctx) + cpuCount, err := sm.GetCPUCountOfNode(ctx) require.NoError(t, err) require.Equal(t, 100, cpuCount) require.NoError(t, sm.InitMeta(ctx, ":4002", "background")) require.NoError(t, sm.DeleteDeadNodes(ctx, []string{":4000"})) - nodes, err = sm.GetManagedNodes(ctx) + nodes, err = sm.GetAllNodes(ctx) require.NoError(t, err) require.Equal(t, []proto.ManagedNode{ + {ID: ":4001", Role: "", CPUCount: 8}, {ID: ":4002", Role: "background", CPUCount: 100}, {ID: ":4003", Role: "background", CPUCount: 100}, }, nodes) require.NoError(t, sm.DeleteDeadNodes(ctx, []string{":4003"})) - nodes, err = sm.GetManagedNodes(ctx) + nodes, err = sm.GetAllNodes(ctx) require.NoError(t, err) require.Equal(t, []proto.ManagedNode{ + {ID: ":4001", Role: "", CPUCount: 8}, {ID: ":4002", Role: "background", CPUCount: 100}, }, nodes) require.NoError(t, sm.DeleteDeadNodes(ctx, []string{":4002"})) - nodes, err = sm.GetManagedNodes(ctx) + nodes, err = sm.GetAllNodes(ctx) require.NoError(t, err) require.Equal(t, []proto.ManagedNode{ {ID: ":4001", Role: "", CPUCount: 8}, }, nodes) - cpuCount, err = sm.GetCPUCountOfManagedNode(ctx) + cpuCount, err = sm.GetCPUCountOfNode(ctx) require.NoError(t, err) require.Equal(t, 8, cpuCount) require.NoError(t, sm.RecoverMeta(ctx, ":4002", "background")) - nodes, err = sm.GetManagedNodes(ctx) + nodes, err = sm.GetAllNodes(ctx) require.NoError(t, err) require.Equal(t, []proto.ManagedNode{ + {ID: ":4001", Role: "", CPUCount: 8}, {ID: ":4002", Role: "background", CPUCount: 100}, }, nodes) // should not reset role require.NoError(t, sm.RecoverMeta(ctx, ":4002", "")) - nodes, err = sm.GetManagedNodes(ctx) + nodes, err = sm.GetAllNodes(ctx) require.NoError(t, err) require.Equal(t, []proto.ManagedNode{ + {ID: ":4001", Role: "", CPUCount: 8}, {ID: ":4002", Role: "background", CPUCount: 100}, }, nodes) } @@ -841,9 +845,9 @@ func TestTaskHistoryTable(t *testing.T) { _, gm, ctx := testutil.InitTableTest(t) require.NoError(t, gm.InitMeta(ctx, ":4000", "")) - _, err := gm.CreateTask(ctx, "1", proto.TaskTypeExample, 1, nil) + _, err := gm.CreateTask(ctx, "1", proto.TaskTypeExample, 1, "", nil) require.NoError(t, err) - taskID, err := gm.CreateTask(ctx, "2", proto.TaskTypeExample, 1, nil) + taskID, err := gm.CreateTask(ctx, "2", proto.TaskTypeExample, 1, "", nil) require.NoError(t, err) tasks, err := gm.GetTasksInStates(ctx, proto.TaskStatePending) @@ -876,7 +880,7 @@ func TestTaskHistoryTable(t *testing.T) { require.NotNil(t, task) // task with fail transfer - _, err = gm.CreateTask(ctx, "3", proto.TaskTypeExample, 1, nil) + _, err = gm.CreateTask(ctx, "3", proto.TaskTypeExample, 1, "", nil) require.NoError(t, err) tasks, err = gm.GetTasksInStates(ctx, proto.TaskStatePending) require.NoError(t, err) @@ -1133,7 +1137,7 @@ func TestGetActiveTaskExecInfo(t *testing.T) { taskStates := []proto.TaskState{proto.TaskStateRunning, proto.TaskStateReverting, proto.TaskStateReverting, proto.TaskStatePausing} tasks := make([]*proto.Task, 0, len(taskStates)) for i, expectedState := range taskStates { - taskID, err := tm.CreateTask(ctx, fmt.Sprintf("key-%d", i), proto.TaskTypeExample, 8, []byte("")) + taskID, err := tm.CreateTask(ctx, fmt.Sprintf("key-%d", i), proto.TaskTypeExample, 8, "", []byte("")) require.NoError(t, err) task, err := tm.GetTaskByID(ctx, taskID) require.NoError(t, err) diff --git a/pkg/disttask/framework/storage/task_state_test.go b/pkg/disttask/framework/storage/task_state_test.go index d61e85e41506f..196baddc6b460 100644 --- a/pkg/disttask/framework/storage/task_state_test.go +++ b/pkg/disttask/framework/storage/task_state_test.go @@ -32,7 +32,7 @@ func TestTaskState(t *testing.T) { require.NoError(t, gm.InitMeta(ctx, ":4000", "")) // 1. cancel task - id, err := gm.CreateTask(ctx, "key1", "test", 4, []byte("test")) + id, err := gm.CreateTask(ctx, "key1", "test", 4, "", []byte("test")) require.NoError(t, err) // require.Equal(t, int64(1), id) TODO: unstable for infoschema v2 require.NoError(t, gm.CancelTask(ctx, id)) @@ -41,7 +41,7 @@ func TestTaskState(t *testing.T) { checkTaskStateStep(t, task, proto.TaskStateCancelling, proto.StepInit) // 2. cancel task by key session - id, err = gm.CreateTask(ctx, "key2", "test", 4, []byte("test")) + id, err = gm.CreateTask(ctx, "key2", "test", 4, "", []byte("test")) require.NoError(t, err) // require.Equal(t, int64(2), id) TODO: unstable for infoschema v2 require.NoError(t, gm.WithNewTxn(ctx, func(se sessionctx.Context) error { @@ -53,7 +53,7 @@ func TestTaskState(t *testing.T) { checkTaskStateStep(t, task, proto.TaskStateCancelling, proto.StepInit) // 3. fail task - id, err = gm.CreateTask(ctx, "key3", "test", 4, []byte("test")) + id, err = gm.CreateTask(ctx, "key3", "test", 4, "", []byte("test")) require.NoError(t, err) // require.Equal(t, int64(3), id) TODO: unstable for infoschema v2 failedErr := errors.New("test err") @@ -64,7 +64,7 @@ func TestTaskState(t *testing.T) { require.ErrorContains(t, task.Error, "test err") // 4. Reverted task - id, err = gm.CreateTask(ctx, "key4", "test", 4, []byte("test")) + id, err = gm.CreateTask(ctx, "key4", "test", 4, "", []byte("test")) require.NoError(t, err) // require.Equal(t, int64(4), id) TODO: unstable for infoschema v2 task, err = gm.GetTaskByID(ctx, 4) @@ -82,7 +82,7 @@ func TestTaskState(t *testing.T) { checkTaskStateStep(t, task, proto.TaskStateReverted, proto.StepInit) // 5. pause task - id, err = gm.CreateTask(ctx, "key5", "test", 4, []byte("test")) + id, err = gm.CreateTask(ctx, "key5", "test", 4, "", []byte("test")) require.NoError(t, err) // require.Equal(t, int64(5), id) TODO: unstable for infoschema v2 found, err := gm.PauseTask(ctx, "key5") @@ -111,7 +111,7 @@ func TestTaskState(t *testing.T) { require.Equal(t, proto.TaskStateRunning, task.State) // 8. succeed task - id, err = gm.CreateTask(ctx, "key6", "test", 4, []byte("test")) + id, err = gm.CreateTask(ctx, "key6", "test", 4, "", []byte("test")) require.NoError(t, err) // require.Equal(t, int64(6), id) TODO: unstable for infoschema v2 task, err = gm.GetTaskByID(ctx, id) diff --git a/pkg/disttask/framework/storage/task_table.go b/pkg/disttask/framework/storage/task_table.go index 9144ddf527450..aa8f82262faff 100644 --- a/pkg/disttask/framework/storage/task_table.go +++ b/pkg/disttask/framework/storage/task_table.go @@ -36,12 +36,12 @@ import ( const ( defaultSubtaskKeepDays = 14 - basicTaskColumns = `t.id, t.task_key, t.type, t.state, t.step, t.priority, t.concurrency, t.create_time` + basicTaskColumns = `t.id, t.task_key, t.type, t.state, t.step, t.priority, t.concurrency, t.create_time, t.target_scope` // TaskColumns is the columns for task. // TODO: dispatcher_id will update to scheduler_id later TaskColumns = basicTaskColumns + `, t.start_time, t.state_update_time, t.meta, t.dispatcher_id, t.error` // InsertTaskColumns is the columns used in insert task. - InsertTaskColumns = `task_key, type, state, priority, concurrency, step, meta, create_time` + InsertTaskColumns = `task_key, type, state, priority, concurrency, step, meta, create_time, target_scope` basicSubtaskColumns = `id, step, task_key, type, exec_id, state, concurrency, create_time, ordinal, start_time` // SubtaskColumns is the columns for subtask. SubtaskColumns = basicSubtaskColumns + `, state_update_time, meta, summary` @@ -192,10 +192,10 @@ func (mgr *TaskManager) ExecuteSQLWithNewSession(ctx context.Context, sql string } // CreateTask adds a new task to task table. -func (mgr *TaskManager) CreateTask(ctx context.Context, key string, tp proto.TaskType, concurrency int, meta []byte) (taskID int64, err error) { +func (mgr *TaskManager) CreateTask(ctx context.Context, key string, tp proto.TaskType, concurrency int, targetScope string, meta []byte) (taskID int64, err error) { err = mgr.WithNewSession(func(se sessionctx.Context) error { var err2 error - taskID, err2 = mgr.CreateTaskWithSession(ctx, se, key, tp, concurrency, meta) + taskID, err2 = mgr.CreateTaskWithSession(ctx, se, key, tp, concurrency, targetScope, meta) return err2 }) return @@ -208,9 +208,10 @@ func (mgr *TaskManager) CreateTaskWithSession( key string, tp proto.TaskType, concurrency int, + targetScope string, meta []byte, ) (taskID int64, err error) { - cpuCount, err := mgr.getCPUCountOfManagedNode(ctx, se) + cpuCount, err := mgr.getCPUCountOfNode(ctx, se) if err != nil { return 0, err } @@ -219,8 +220,8 @@ func (mgr *TaskManager) CreateTaskWithSession( } _, err = sqlexec.ExecSQL(ctx, se.GetSQLExecutor(), ` insert into mysql.tidb_global_task(`+InsertTaskColumns+`) - values (%?, %?, %?, %?, %?, %?, %?, CURRENT_TIMESTAMP())`, - key, tp, proto.TaskStatePending, proto.NormalPriority, concurrency, proto.StepInit, meta) + values (%?, %?, %?, %?, %?, %?, %?, CURRENT_TIMESTAMP(), %?)`, + key, tp, proto.TaskStatePending, proto.NormalPriority, concurrency, proto.StepInit, meta, targetScope) if err != nil { return 0, err } @@ -281,7 +282,7 @@ func (mgr *TaskManager) GetTaskExecInfoByExecID(ctx context.Context, execID stri for _, r := range rs { res = append(res, &TaskExecInfo{ TaskBase: row2TaskBasic(r), - SubtaskConcurrency: int(r.GetInt64(8)), + SubtaskConcurrency: int(r.GetInt64(9)), }) } return res, nil @@ -803,7 +804,7 @@ func (mgr *TaskManager) GetAllSubtasks(ctx context.Context) ([]*proto.SubtaskBas // For details, see https://github.com/pingcap/tidb/issues/50894. // For the following versions, there is a check when submiting a new task. This function should be a no-op. func (mgr *TaskManager) AdjustTaskOverflowConcurrency(ctx context.Context, se sessionctx.Context) error { - cpuCount, err := mgr.getCPUCountOfManagedNode(ctx, se) + cpuCount, err := mgr.getCPUCountOfNode(ctx, se) if err != nil { return err } diff --git a/pkg/disttask/framework/taskexecutor/task_executor_testkit_test.go b/pkg/disttask/framework/taskexecutor/task_executor_testkit_test.go index 7b8b0ba25d7cb..56b786d308e6e 100644 --- a/pkg/disttask/framework/taskexecutor/task_executor_testkit_test.go +++ b/pkg/disttask/framework/taskexecutor/task_executor_testkit_test.go @@ -34,7 +34,7 @@ import ( ) func runOneTask(ctx context.Context, t *testing.T, mgr *storage.TaskManager, taskKey string, subtaskCnt int) { - taskID, err := mgr.CreateTask(ctx, taskKey, proto.TaskTypeExample, 1, nil) + taskID, err := mgr.CreateTask(ctx, taskKey, proto.TaskTypeExample, 1, "", nil) require.NoError(t, err) task, err := mgr.GetTaskByID(ctx, taskID) require.NoError(t, err) diff --git a/pkg/disttask/framework/testutil/context.go b/pkg/disttask/framework/testutil/context.go index 469015d45672a..a8ee04aee6147 100644 --- a/pkg/disttask/framework/testutil/context.go +++ b/pkg/disttask/framework/testutil/context.go @@ -317,6 +317,13 @@ func (c *TestDXFContext) GetRandNodeIDs(limit int) map[string]struct{} { return ids } +// GetNodeIDByIdx returns nodeID by idx in nodes. +func (c *TestDXFContext) GetNodeIDByIdx(idx int) string { + c.mu.Lock() + defer c.mu.Unlock() + return c.mu.nodes[idx].id +} + // NodeCount returns the number of nodes. func (c *TestDXFContext) NodeCount() int { c.mu.RLock() diff --git a/pkg/disttask/framework/testutil/disttest_util.go b/pkg/disttask/framework/testutil/disttest_util.go index fbf288dcad01f..d741657b2aeee 100644 --- a/pkg/disttask/framework/testutil/disttest_util.go +++ b/pkg/disttask/framework/testutil/disttest_util.go @@ -111,8 +111,8 @@ func RegisterRollbackTaskMeta(t testing.TB, ctrl *gomock.Controller, schedulerEx } // SubmitAndWaitTask schedule one task. -func SubmitAndWaitTask(ctx context.Context, t testing.TB, taskKey string, concurrency int) *proto.TaskBase { - _, err := handle.SubmitTask(ctx, taskKey, proto.TaskTypeExample, concurrency, nil) +func SubmitAndWaitTask(ctx context.Context, t testing.TB, taskKey string, targetScope string, concurrency int) *proto.TaskBase { + _, err := handle.SubmitTask(ctx, taskKey, proto.TaskTypeExample, concurrency, targetScope, nil) require.NoError(t, err) return WaitTaskDoneOrPaused(ctx, t, taskKey) } diff --git a/pkg/disttask/importinto/job_testkit_test.go b/pkg/disttask/importinto/job_testkit_test.go index f165afa6d8a54..14dd732d9052c 100644 --- a/pkg/disttask/importinto/job_testkit_test.go +++ b/pkg/disttask/importinto/job_testkit_test.go @@ -53,7 +53,7 @@ func TestGetTaskImportedRows(t *testing.T) { } bytes, err := json.Marshal(taskMeta) require.NoError(t, err) - taskID, err := manager.CreateTask(ctx, importinto.TaskKey(111), proto.ImportInto, 1, bytes) + taskID, err := manager.CreateTask(ctx, importinto.TaskKey(111), proto.ImportInto, 1, "", bytes) require.NoError(t, err) importStepMetas := []*importinto.ImportStepMeta{ { @@ -85,7 +85,7 @@ func TestGetTaskImportedRows(t *testing.T) { } bytes, err = json.Marshal(taskMeta) require.NoError(t, err) - taskID, err = manager.CreateTask(ctx, importinto.TaskKey(222), proto.ImportInto, 1, bytes) + taskID, err = manager.CreateTask(ctx, importinto.TaskKey(222), proto.ImportInto, 1, "", bytes) require.NoError(t, err) ingestStepMetas := []*importinto.WriteIngestStepMeta{ { diff --git a/pkg/disttask/importinto/scheduler_testkit_test.go b/pkg/disttask/importinto/scheduler_testkit_test.go index 7f7f2cf3773cd..5efcfa01e0600 100644 --- a/pkg/disttask/importinto/scheduler_testkit_test.go +++ b/pkg/disttask/importinto/scheduler_testkit_test.go @@ -84,7 +84,7 @@ func TestSchedulerExtLocalSort(t *testing.T) { require.NoError(t, err) taskMeta, err := json.Marshal(task) require.NoError(t, err) - taskID, err := manager.CreateTask(ctx, importinto.TaskKey(jobID), proto.ImportInto, 1, taskMeta) + taskID, err := manager.CreateTask(ctx, importinto.TaskKey(jobID), proto.ImportInto, 1, "", taskMeta) require.NoError(t, err) task.ID = taskID @@ -227,7 +227,7 @@ func TestSchedulerExtGlobalSort(t *testing.T) { require.NoError(t, err) taskMeta, err := json.Marshal(task) require.NoError(t, err) - taskID, err := manager.CreateTask(ctx, importinto.TaskKey(jobID), proto.ImportInto, 1, taskMeta) + taskID, err := manager.CreateTask(ctx, importinto.TaskKey(jobID), proto.ImportInto, 1, "", taskMeta) require.NoError(t, err) task.ID = taskID diff --git a/pkg/executor/importer/import.go b/pkg/executor/importer/import.go index 0e78fba71d34a..2b526953cff8a 100644 --- a/pkg/executor/importer/import.go +++ b/pkg/executor/importer/import.go @@ -1395,7 +1395,7 @@ func GetTargetNodeCPUCnt(ctx context.Context, sourceType DataSourceType, path st if serverDiskImport || !variable.EnableDistTask.Load() { return cpu.GetCPUCount(), nil } - return handle.GetCPUCountOfManagedNode(ctx) + return handle.GetCPUCountOfNode(ctx) } // TestSyncCh is used in unit test to synchronize the execution. diff --git a/pkg/parser/model/reorg.go b/pkg/parser/model/reorg.go index b54e30f2574d0..9355d796271eb 100644 --- a/pkg/parser/model/reorg.go +++ b/pkg/parser/model/reorg.go @@ -33,6 +33,7 @@ type DDLReorgMeta struct { UseCloudStorage bool `json:"use_cloud_storage"` ResourceGroupName string `json:"resource_group_name"` Version int64 `json:"version"` + TargetScope string `json:"target_scope"` } const ( diff --git a/pkg/session/bootstrap.go b/pkg/session/bootstrap.go index 398ed9cd5de22..e53976cd88ed5 100644 --- a/pkg/session/bootstrap.go +++ b/pkg/session/bootstrap.go @@ -583,6 +583,7 @@ const ( meta LONGBLOB, concurrency INT(11), step INT(11), + target_scope VARCHAR(256) DEFAULT "", error BLOB, key(state), UNIQUE KEY task_key(task_key) @@ -603,6 +604,7 @@ const ( meta LONGBLOB, concurrency INT(11), step INT(11), + target_scope VARCHAR(256) DEFAULT "", error BLOB, key(state), UNIQUE KEY task_key(task_key) @@ -1082,11 +1084,16 @@ const ( // create `sys` schema // create `sys.schema_unused_indexes` table version195 = 195 + + // version 196 + // add column `target_scope` for 'mysql.tidb_global_task` table + // add column `target_scope` for 'mysql.tidb_global_task_history` table + version196 = 196 ) // currentBootstrapVersion is defined as a variable, so we can modify its value for testing. // please make sure this is the largest version -var currentBootstrapVersion int64 = version195 +var currentBootstrapVersion int64 = version196 // DDL owner key's expired time is ManagerSessionTTL seconds, we should wait the time and give more time to have a chance to finish it. var internalSQLTimeout = owner.ManagerSessionTTL + 15 @@ -1247,6 +1254,7 @@ var ( upgradeToVer193, upgradeToVer194, upgradeToVer195, + upgradeToVer196, } ) @@ -3073,6 +3081,15 @@ func upgradeToVer195(s sessiontypes.Session, ver int64) { doReentrantDDL(s, DropMySQLIndexUsageTable) } +func upgradeToVer196(s sessiontypes.Session, ver int64) { + if ver >= version196 { + return + } + + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task ADD COLUMN target_scope VARCHAR(256) DEFAULT '' AFTER `step`;", infoschema.ErrColumnExists) + doReentrantDDL(s, "ALTER TABLE mysql.tidb_global_task_history ADD COLUMN target_scope VARCHAR(256) DEFAULT '' AFTER `step`;", infoschema.ErrColumnExists) +} + func writeOOMAction(s sessiontypes.Session) { comment := "oom-action is `log` by default in v3.0.x, `cancel` by default in v4.0.11+" mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, %?) ON DUPLICATE KEY UPDATE VARIABLE_VALUE= %?`, diff --git a/pkg/sessionctx/variable/BUILD.bazel b/pkg/sessionctx/variable/BUILD.bazel index e2086195f21c3..edb40510ab231 100644 --- a/pkg/sessionctx/variable/BUILD.bazel +++ b/pkg/sessionctx/variable/BUILD.bazel @@ -46,7 +46,6 @@ go_library( "//pkg/util/dbterror", "//pkg/util/dbterror/plannererrors", "//pkg/util/disk", - "//pkg/util/distrole", "//pkg/util/execdetails", "//pkg/util/gctuner", "//pkg/util/intest", @@ -57,6 +56,7 @@ go_library( "//pkg/util/paging", "//pkg/util/replayer", "//pkg/util/rowcodec", + "//pkg/util/servicescope", "//pkg/util/size", "//pkg/util/sqlkiller", "//pkg/util/stmtsummary/v2:stmtsummary", diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go index 0f72635f0361e..c8922a86afa9e 100644 --- a/pkg/sessionctx/variable/sysvar.go +++ b/pkg/sessionctx/variable/sysvar.go @@ -41,12 +41,12 @@ import ( _ "github.com/pingcap/tidb/pkg/types/parser_driver" // for parser driver "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/collate" - distroleutil "github.com/pingcap/tidb/pkg/util/distrole" "github.com/pingcap/tidb/pkg/util/gctuner" "github.com/pingcap/tidb/pkg/util/intest" "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/mathutil" "github.com/pingcap/tidb/pkg/util/memory" + "github.com/pingcap/tidb/pkg/util/servicescope" stmtsummaryv2 "github.com/pingcap/tidb/pkg/util/stmtsummary/v2" "github.com/pingcap/tidb/pkg/util/tiflash" "github.com/pingcap/tidb/pkg/util/tiflashcompute" @@ -3136,14 +3136,7 @@ var defaultSysVars = []*SysVar{ }, {Scope: ScopeInstance, Name: TiDBServiceScope, Value: "", Type: TypeStr, Validation: func(_ *SessionVars, normalizedValue string, originalValue string, _ ScopeFlag) (string, error) { - _, ok := distroleutil.ToTiDBServiceScope(originalValue) - if !ok { - err := fmt.Errorf("incorrect value: `%s`. %s options: %s", - originalValue, - TiDBServiceScope, `"", background`) - return normalizedValue, err - } - return normalizedValue, nil + return normalizedValue, servicescope.CheckServiceScope(originalValue) }, SetGlobal: func(ctx context.Context, vars *SessionVars, s string) error { newValue := strings.ToLower(s) diff --git a/pkg/util/distrole/BUILD.bazel b/pkg/util/distrole/BUILD.bazel deleted file mode 100644 index 3f3f8e208504a..0000000000000 --- a/pkg/util/distrole/BUILD.bazel +++ /dev/null @@ -1,8 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "distrole", - srcs = ["role.go"], - importpath = "github.com/pingcap/tidb/pkg/util/distrole", - visibility = ["//visibility:public"], -) diff --git a/pkg/util/distrole/role.go b/pkg/util/distrole/role.go deleted file mode 100644 index 643580220ca57..0000000000000 --- a/pkg/util/distrole/role.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package distroleutil - -import "strings" - -const ( - // TiDBServiceScopeDefault indicates no role. - TiDBServiceScopeDefault = "" - // TiDBServiceScopeBackground indicates node can run background tasks in distributed execution framework. - TiDBServiceScopeBackground = "background" -) - -// ToTiDBServiceScope returns the TiDBServiceScope from name -func ToTiDBServiceScope(name string) (string, bool) { - name = strings.ToLower(name) - switch name { - case TiDBServiceScopeDefault: - return TiDBServiceScopeDefault, true - case TiDBServiceScopeBackground: - return TiDBServiceScopeBackground, true - default: - return "", false - } -} diff --git a/pkg/util/servicescope/BUILD.bazel b/pkg/util/servicescope/BUILD.bazel new file mode 100644 index 0000000000000..3caba03b0e251 --- /dev/null +++ b/pkg/util/servicescope/BUILD.bazel @@ -0,0 +1,17 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "servicescope", + srcs = ["scope.go"], + importpath = "github.com/pingcap/tidb/pkg/util/servicescope", + visibility = ["//visibility:public"], +) + +go_test( + name = "servicescope_test", + timeout = "short", + srcs = ["scope_test.go"], + embed = [":servicescope"], + flaky = True, + deps = ["@com_github_stretchr_testify//require"], +) diff --git a/pkg/util/servicescope/scope.go b/pkg/util/servicescope/scope.go new file mode 100644 index 0000000000000..ba1e28b295fd0 --- /dev/null +++ b/pkg/util/servicescope/scope.go @@ -0,0 +1,29 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package servicescope + +import ( + "fmt" + "regexp" +) + +// CheckServiceScope check if the tidb-service-scope set by users is valid. +func CheckServiceScope(scope string) error { + re := regexp.MustCompile(`^[a-zA-Z0-9_-]{0,64}$`) + if !re.MatchString(scope) { + return fmt.Errorf("the tidb-service-scope value '%s' is invalid. It must be 64 characters or fewer and consist only of letters (a-z, A-Z), numbers (0-9), hyphens (-), and underscores (_)", scope) + } + return nil +} diff --git a/pkg/util/servicescope/scope_test.go b/pkg/util/servicescope/scope_test.go new file mode 100644 index 0000000000000..2640ff9625f11 --- /dev/null +++ b/pkg/util/servicescope/scope_test.go @@ -0,0 +1,30 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package servicescope + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestScope(t *testing.T) { + require.NoError(t, CheckServiceScope("789z-_")) + require.Error(t, CheckServiceScope("789z-_)")) + require.Error(t, CheckServiceScope("78912345678982u7389217897238917389127893781278937128973812728397281378932179837")) + require.NoError(t, CheckServiceScope("scope1")) + require.NoError(t, CheckServiceScope("")) + require.NoError(t, CheckServiceScope("-----")) +} From 10971ea5b0d0ca48792751801954c301e2f9fe08 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Fri, 26 Apr 2024 15:19:48 +0800 Subject: [PATCH 16/20] ddl: use package-level logger which preset "category" field (#52885) close pingcap/tidb#52883 --- pkg/ddl/BUILD.bazel | 3 +- pkg/ddl/backfilling.go | 31 ++++---- pkg/ddl/backfilling_dist_executor.go | 7 +- pkg/ddl/backfilling_dist_scheduler.go | 11 ++- pkg/ddl/backfilling_read_index.go | 19 +++-- pkg/ddl/backfilling_scheduler.go | 3 +- pkg/ddl/callback.go | 18 ++--- pkg/ddl/cluster.go | 18 ++--- pkg/ddl/column.go | 33 +++++---- pkg/ddl/db_rename_test.go | 4 +- pkg/ddl/ddl.go | 72 +++++++++---------- pkg/ddl/ddl_api.go | 63 ++++++++++------- pkg/ddl/ddl_history.go | 4 +- pkg/ddl/ddl_tiflash_api.go | 46 ++++++------ pkg/ddl/ddl_worker.go | 49 ++++++++----- pkg/ddl/ddl_workerpool.go | 5 +- pkg/ddl/delete_range.go | 26 +++---- pkg/ddl/index.go | 74 ++++++++++---------- pkg/ddl/index_merge_tmp.go | 4 +- pkg/ddl/ingest/BUILD.bazel | 1 + pkg/ddl/ingest/backend_mgr.go | 7 +- pkg/ddl/ingest/checkpoint.go | 19 ++--- pkg/ddl/ingest/disk_root.go | 9 +-- pkg/ddl/ingest/env.go | 20 +++--- pkg/ddl/ingest/mock.go | 20 +++--- pkg/ddl/internal/session/BUILD.bazel | 3 +- pkg/ddl/internal/session/session_pool.go | 5 +- pkg/ddl/job_table.go | 45 ++++++------ pkg/ddl/logutil/BUILD.bazel | 12 ++++ pkg/ddl/logutil/logutil.go | 40 +++++++++++ pkg/ddl/partition.go | 37 +++++----- pkg/ddl/reorg.go | 33 +++++---- pkg/ddl/reorg_partition_test.go | 4 +- pkg/ddl/resource_group.go | 10 +-- pkg/ddl/sanity_check.go | 6 +- pkg/ddl/schema_test.go | 6 +- pkg/ddl/split_region.go | 10 +-- pkg/ddl/syncer/BUILD.bazel | 2 +- pkg/ddl/syncer/state_syncer.go | 6 +- pkg/ddl/syncer/syncer.go | 19 ++--- pkg/ddl/table.go | 24 +++---- pkg/ddl/tests/adminpause/BUILD.bazel | 2 +- pkg/ddl/tests/adminpause/global.go | 4 +- pkg/ddl/tests/partition/BUILD.bazel | 2 +- pkg/ddl/tests/partition/db_partition_test.go | 8 +-- pkg/ddl/tests/tiflash/BUILD.bazel | 2 +- pkg/ddl/tests/tiflash/ddl_tiflash_test.go | 8 +-- pkg/ddl/testutil/BUILD.bazel | 2 +- pkg/ddl/testutil/testutil.go | 4 +- pkg/ddl/util/BUILD.bazel | 2 +- pkg/ddl/util/callback/BUILD.bazel | 2 +- pkg/ddl/util/callback/callback.go | 14 ++-- pkg/ddl/util/dead_table_lock_checker.go | 4 +- pkg/ddl/util/util.go | 10 +-- pkg/util/logutil/log.go | 4 +- 55 files changed, 490 insertions(+), 406 deletions(-) create mode 100644 pkg/ddl/logutil/BUILD.bazel create mode 100644 pkg/ddl/logutil/logutil.go diff --git a/pkg/ddl/BUILD.bazel b/pkg/ddl/BUILD.bazel index 1f504132bb680..df643deea7e08 100644 --- a/pkg/ddl/BUILD.bazel +++ b/pkg/ddl/BUILD.bazel @@ -72,6 +72,7 @@ go_library( "//pkg/ddl/ingest", "//pkg/ddl/internal/session", "//pkg/ddl/label", + "//pkg/ddl/logutil", "//pkg/ddl/placement", "//pkg/ddl/resourcegroup", "//pkg/ddl/syncer", @@ -259,6 +260,7 @@ go_test( "//pkg/ddl/copr", "//pkg/ddl/ingest", "//pkg/ddl/internal/session", + "//pkg/ddl/logutil", "//pkg/ddl/placement", "//pkg/ddl/schematracker", "//pkg/ddl/syncer", @@ -313,7 +315,6 @@ go_test( "//pkg/util/dbterror/plannererrors", "//pkg/util/domainutil", "//pkg/util/gcutil", - "//pkg/util/logutil", "//pkg/util/mathutil", "//pkg/util/mock", "//pkg/util/sem", diff --git a/pkg/ddl/backfilling.go b/pkg/ddl/backfilling.go index 8a176c96b7f6f..d18506d4aee85 100644 --- a/pkg/ddl/backfilling.go +++ b/pkg/ddl/backfilling.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" sess "github.com/pingcap/tidb/pkg/ddl/internal/session" + "github.com/pingcap/tidb/pkg/ddl/logutil" ddlutil "github.com/pingcap/tidb/pkg/ddl/util" "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/kv" @@ -40,7 +41,6 @@ import ( "github.com/pingcap/tidb/pkg/tablecodec" "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/dbterror" - "github.com/pingcap/tidb/pkg/util/logutil" decoder "github.com/pingcap/tidb/pkg/util/rowDecoder" "github.com/pingcap/tidb/pkg/util/timeutil" "github.com/pingcap/tidb/pkg/util/topsql" @@ -315,7 +315,7 @@ func (w *backfillWorker) handleBackfillTask(d *ddlCtx, task *reorgBackfillTask, if num := result.scanCount - lastLogCount; num >= 90000 { lastLogCount = result.scanCount - logutil.BgLogger().Info("backfill worker back fill index", zap.String("category", "ddl"), zap.Stringer("worker", w), + logutil.DDLLogger().Info("backfill worker back fill index", zap.Stringer("worker", w), zap.Int("addedCount", result.addedCount), zap.Int("scanCount", result.scanCount), zap.String("next key", hex.EncodeToString(taskCtx.nextKey)), zap.Float64("speed(rows/s)", float64(num)/time.Since(lastLogTime).Seconds())) @@ -327,7 +327,7 @@ func (w *backfillWorker) handleBackfillTask(d *ddlCtx, task *reorgBackfillTask, break } } - logutil.BgLogger().Info("backfill worker finish task", zap.String("category", "ddl"), + logutil.DDLLogger().Info("backfill worker finish task", zap.Stringer("worker", w), zap.Stringer("task", task), zap.Int("added count", result.addedCount), zap.Int("scan count", result.scanCount), @@ -347,7 +347,7 @@ func (w *backfillWorker) sendResult(result *backfillResult) { } func (w *backfillWorker) run(d *ddlCtx, bf backfiller, job *model.Job) { - logger := ddlLogger.With(zap.Stringer("worker", w), zap.Int64("jobID", job.ID)) + logger := logutil.DDLLogger().With(zap.Stringer("worker", w), zap.Int64("jobID", job.ID)) var ( curTaskID int task *reorgBackfillTask @@ -413,12 +413,13 @@ func splitTableRanges( startKey, endKey kv.Key, limit int, ) ([]kv.KeyRange, error) { - logutil.BgLogger().Info("split table range from PD", zap.String("category", "ddl"), + logutil.DDLLogger().Info("split table range from PD", zap.Int64("physicalTableID", t.GetPhysicalID()), zap.String("start key", hex.EncodeToString(startKey)), zap.String("end key", hex.EncodeToString(endKey))) if len(startKey) == 0 && len(endKey) == 0 { - logutil.BgLogger().Info("split table range from PD, get noop table range", zap.String("category", "ddl"), zap.Int64("physicalTableID", t.GetPhysicalID())) + logutil.DDLLogger().Info("split table range from PD, get noop table range", + zap.Int64("physicalTableID", t.GetPhysicalID())) return []kv.KeyRange{}, nil } @@ -469,9 +470,9 @@ func getBatchTasks(t table.Table, reorgInfo *reorgInfo, kvRanges []kv.KeyRange, } endK, err := GetRangeEndKey(jobCtx, reorgInfo.d.store, job.Priority, prefix, startKey, endKey) if err != nil { - logutil.BgLogger().Info("get backfill range task, get reverse key failed", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Info("get backfill range task, get reverse key failed", zap.Error(err)) } else { - logutil.BgLogger().Info("get backfill range task, change end key", zap.String("category", "ddl"), + logutil.DDLLogger().Info("get backfill range task, change end key", zap.Int("id", taskID), zap.Int64("pTbl", phyTbl.GetPhysicalID()), zap.String("end key", hex.EncodeToString(endKey)), zap.String("current end key", hex.EncodeToString(endK))) endKey = endK @@ -628,7 +629,7 @@ func (dc *ddlCtx) writePhysicalTableRecord( return egCtx.Err() case result, ok := <-scheduler.resultChan(): if !ok { - ddlLogger.Info("backfill workers successfully processed", + logutil.DDLLogger().Info("backfill workers successfully processed", zap.Stringer("element", reorgInfo.currElement), zap.Int64("total added count", totalAddedCount), zap.String("start key", hex.EncodeToString(startKey))) @@ -637,7 +638,7 @@ func (dc *ddlCtx) writePhysicalTableRecord( cnt++ if result.err != nil { - ddlLogger.Warn("backfill worker failed", + logutil.DDLLogger().Warn("backfill worker failed", zap.Int64("job ID", reorgInfo.ID), zap.Int64("total added count", totalAddedCount), zap.String("start key", hex.EncodeToString(startKey)), @@ -658,7 +659,7 @@ func (dc *ddlCtx) writePhysicalTableRecord( if cnt%(scheduler.currentWorkerSize()*4) == 0 { err2 := reorgInfo.UpdateReorgMeta(keeper.nextKey, sessPool) if err2 != nil { - ddlLogger.Warn("update reorg meta failed", + logutil.DDLLogger().Warn("update reorg meta failed", zap.Int64("job ID", reorgInfo.ID), zap.Error(err2)) } @@ -666,7 +667,7 @@ func (dc *ddlCtx) writePhysicalTableRecord( // the overhead of loading the DDL related global variables. err2 = scheduler.adjustWorkerSize() if err2 != nil { - ddlLogger.Warn("cannot adjust backfill worker size", + logutil.DDLLogger().Warn("cannot adjust backfill worker size", zap.Int64("job ID", reorgInfo.ID), zap.Error(err2)) } @@ -688,7 +689,7 @@ func (dc *ddlCtx) writePhysicalTableRecord( if len(kvRanges) == 0 { break } - ddlLogger.Info("start backfill workers to reorg record", + logutil.DDLLogger().Info("start backfill workers to reorg record", zap.Stringer("type", bfWorkerType), zap.Int("workerCnt", scheduler.currentWorkerSize()), zap.Int("regionCnt", len(kvRanges)), @@ -847,7 +848,9 @@ func logSlowOperations(elapsed time.Duration, slowMsg string, threshold uint32) } if elapsed >= time.Duration(threshold)*time.Millisecond { - logutil.BgLogger().Info("slow operations", zap.String("category", "ddl"), zap.Duration("takeTimes", elapsed), zap.String("msg", slowMsg)) + logutil.DDLLogger().Info("slow operations", + zap.Duration("takeTimes", elapsed), + zap.String("msg", slowMsg)) } } diff --git a/pkg/ddl/backfilling_dist_executor.go b/pkg/ddl/backfilling_dist_executor.go index ac1024279931d..afb440a2161a4 100644 --- a/pkg/ddl/backfilling_dist_executor.go +++ b/pkg/ddl/backfilling_dist_executor.go @@ -20,6 +20,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/ddl/ingest" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/disttask/framework/proto" "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor" "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/execute" @@ -29,7 +30,6 @@ import ( "github.com/pingcap/tidb/pkg/parser/terror" "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/util/dbterror" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/tikv/client-go/v2/tikv" "go.uber.org/zap" ) @@ -101,8 +101,9 @@ func (s *backfillDistExecutor) newBackfillSubtaskExecutor( for _, eid := range eleIDs { indexInfo := model.FindIndexInfoByID(tbl.Meta().Indices, eid) if indexInfo == nil { - logutil.BgLogger().Warn("index info not found", zap.String("category", "ddl-ingest"), - zap.Int64("table ID", tbl.Meta().ID), zap.Int64("index ID", eid)) + logutil.DDLIngestLogger().Warn("index info not found", + zap.Int64("table ID", tbl.Meta().ID), + zap.Int64("index ID", eid)) return nil, errors.Errorf("index info not found: %d", eid) } indexInfos = append(indexInfos, indexInfo) diff --git a/pkg/ddl/backfilling_dist_scheduler.go b/pkg/ddl/backfilling_dist_scheduler.go index d8f005deebbcb..f36a510478eaa 100644 --- a/pkg/ddl/backfilling_dist_scheduler.go +++ b/pkg/ddl/backfilling_dist_scheduler.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/br/pkg/storage" "github.com/pingcap/tidb/pkg/ddl/ingest" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/disttask/framework/handle" "github.com/pingcap/tidb/pkg/disttask/framework/proto" "github.com/pingcap/tidb/pkg/disttask/framework/scheduler" @@ -40,7 +41,7 @@ import ( "github.com/pingcap/tidb/pkg/store/helper" "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/util/backoff" - "github.com/pingcap/tidb/pkg/util/logutil" + tidblogutil "github.com/pingcap/tidb/pkg/util/logutil" "github.com/tikv/client-go/v2/tikv" "go.uber.org/zap" ) @@ -76,7 +77,7 @@ func (sch *BackfillingSchedulerExt) OnNextSubtasksBatch( execIDs []string, nextStep proto.Step, ) (taskMeta [][]byte, err error) { - logger := logutil.BgLogger().With( + logger := logutil.DDLLogger().With( zap.Stringer("type", task.Type), zap.Int64("task-id", task.ID), zap.String("curr-step", proto.Step2Str(task.Type, task.Step)), @@ -265,7 +266,7 @@ func generateNonPartitionPlan( subTaskMetas := make([][]byte, 0, 4) backoffer := backoff.NewExponential(scanRegionBackoffBase, 2, scanRegionBackoffMax) - err = handle.RunWithRetry(d.ctx, 8, backoffer, logutil.Logger(d.ctx), func(_ context.Context) (bool, error) { + err = handle.RunWithRetry(d.ctx, 8, backoffer, tidblogutil.Logger(d.ctx), func(_ context.Context) (bool, error) { regionCache := d.store.(helper.Storage).GetRegionCache() recordRegionMetas, err := regionCache.LoadRegionsInKeyRange(tikv.NewBackofferWithVars(context.Background(), 20000, nil), startKey, endKey) if err != nil { @@ -590,9 +591,7 @@ func forEachBackfillSubtaskMeta( for _, subTaskMeta := range subTaskMetas { subtask, err := decodeBackfillSubTaskMeta(subTaskMeta) if err != nil { - logutil.BgLogger().Error("unmarshal error", - zap.String("category", "ddl"), - zap.Error(err)) + logutil.DDLLogger().Error("unmarshal error", zap.Error(err)) return errors.Trace(err) } fn(subtask) diff --git a/pkg/ddl/backfilling_read_index.go b/pkg/ddl/backfilling_read_index.go index 3876f53a24e28..fddfd946c229d 100644 --- a/pkg/ddl/backfilling_read_index.go +++ b/pkg/ddl/backfilling_read_index.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/ddl/ingest" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/disttask/framework/proto" "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/execute" "github.com/pingcap/tidb/pkg/disttask/operator" @@ -33,7 +34,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/table" - "github.com/pingcap/tidb/pkg/util/logutil" + tidblogutil "github.com/pingcap/tidb/pkg/util/logutil" "go.uber.org/zap" ) @@ -87,14 +88,12 @@ func newReadIndexExecutor( } func (*readIndexExecutor) Init(_ context.Context) error { - logutil.BgLogger().Info("read index executor init subtask exec env", - zap.String("category", "ddl")) + logutil.DDLLogger().Info("read index executor init subtask exec env") return nil } func (r *readIndexExecutor) RunSubtask(ctx context.Context, subtask *proto.Subtask) error { - logutil.BgLogger().Info("read index executor run subtask", - zap.String("category", "ddl"), + logutil.DDLLogger().Info("read index executor run subtask", zap.Bool("use cloud", len(r.cloudStorageURI) > 0)) r.subtaskSummary.Store(subtask.ID, &readIndexSummary{ @@ -149,8 +148,7 @@ func (r *readIndexExecutor) RealtimeSummary() *execute.SubtaskSummary { } func (r *readIndexExecutor) Cleanup(ctx context.Context) error { - logutil.Logger(ctx).Info("read index executor cleanup subtask exec env", - zap.String("category", "ddl")) + tidblogutil.Logger(ctx).Info("read index executor cleanup subtask exec env") // cleanup backend context ingest.LitBackCtxMgr.Unregister(r.job.ID) return nil @@ -182,7 +180,7 @@ func (r *readIndexExecutor) OnFinished(ctx context.Context, subtask *proto.Subta } sm.MetaGroups = s.metaGroups - logutil.Logger(ctx).Info("get key boundary on subtask finished", + tidblogutil.Logger(ctx).Info("get key boundary on subtask finished", zap.String("start", hex.EncodeToString(all.StartKey)), zap.String("end", hex.EncodeToString(all.EndKey)), zap.Int("fileCount", len(all.MultipleFilesStats)), @@ -205,8 +203,7 @@ func (r *readIndexExecutor) getTableStartEndKey(sm *BackfillSubTaskMeta) ( pid := sm.PhysicalTableID start, end, err = getTableRange(r.jc, r.d.ddlCtx, parTbl.GetPartition(pid), currentVer.Ver, r.job.Priority) if err != nil { - logutil.BgLogger().Error("get table range error", - zap.String("category", "ddl"), + logutil.DDLLogger().Error("get table range error", zap.Error(err)) return nil, nil, nil, err } @@ -233,7 +230,7 @@ func (r *readIndexExecutor) buildLocalStorePipeline( for _, index := range r.indexes { ei, err := r.bc.Register(r.job.ID, index.ID, r.job.SchemaName, r.job.TableName) if err != nil { - logutil.Logger(opCtx).Warn("cannot register new engine", zap.Error(err), + tidblogutil.Logger(opCtx).Warn("cannot register new engine", zap.Error(err), zap.Int64("job ID", r.job.ID), zap.Int64("index ID", index.ID)) return nil, err } diff --git a/pkg/ddl/backfilling_scheduler.go b/pkg/ddl/backfilling_scheduler.go index 57fbb997e8b01..6e3a0ee76fbda 100644 --- a/pkg/ddl/backfilling_scheduler.go +++ b/pkg/ddl/backfilling_scheduler.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/tidb/pkg/ddl/copr" "github.com/pingcap/tidb/pkg/ddl/ingest" sess "github.com/pingcap/tidb/pkg/ddl/internal/session" + ddllogutil "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/errctx" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/metrics" @@ -230,7 +231,7 @@ func (b *txnBackfillScheduler) adjustWorkerSize() error { job := reorgInfo.Job jc := b.jobCtx if err := loadDDLReorgVars(b.ctx, b.sessPool); err != nil { - logutil.BgLogger().Error("load DDL reorganization variable failed", zap.String("category", "ddl"), zap.Error(err)) + ddllogutil.DDLLogger().Error("load DDL reorganization variable failed", zap.Error(err)) } workerCnt := b.expectedWorkerSize() // Increase the worker. diff --git a/pkg/ddl/callback.go b/pkg/ddl/callback.go index 8b89ed42a7776..0f42cf9e34590 100644 --- a/pkg/ddl/callback.go +++ b/pkg/ddl/callback.go @@ -22,10 +22,10 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/infoschema" "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/sessionctx" - "github.com/pingcap/tidb/pkg/util/logutil" "go.uber.org/zap" ) @@ -137,11 +137,11 @@ func (c *DefaultCallback) OnChanged(err error) error { if err != nil { return err } - logutil.BgLogger().Info("performing DDL change, must reload") + logutil.DDLLogger().Info("performing DDL change, must reload") err = c.do.Reload() if err != nil { - logutil.BgLogger().Error("performing DDL change failed", zap.Error(err)) + logutil.DDLLogger().Error("performing DDL change failed", zap.Error(err)) } return nil @@ -151,7 +151,7 @@ func (c *DefaultCallback) OnChanged(err error) error { func (c *DefaultCallback) OnSchemaStateChanged(_ int64) { err := c.do.Reload() if err != nil { - logutil.BgLogger().Error("domain callback failed on schema state changed", zap.Error(err)) + logutil.DDLLogger().Error("domain callback failed on schema state changed", zap.Error(err)) } } @@ -175,11 +175,11 @@ func (c *ctcCallback) OnChanged(err error) error { if err != nil { return err } - logutil.BgLogger().Info("performing DDL change, must reload") + logutil.DDLLogger().Info("performing DDL change, must reload") err = c.do.Reload() if err != nil { - logutil.BgLogger().Error("performing DDL change failed", zap.Error(err)) + logutil.DDLLogger().Error("performing DDL change failed", zap.Error(err)) } return nil } @@ -188,7 +188,7 @@ func (c *ctcCallback) OnChanged(err error) error { func (c *ctcCallback) OnSchemaStateChanged(_ int64) { err := c.do.Reload() if err != nil { - logutil.BgLogger().Error("domain callback failed on schema state changed", zap.Error(err)) + logutil.DDLLogger().Error("domain callback failed on schema state changed", zap.Error(err)) } } @@ -201,7 +201,7 @@ func (*ctcCallback) OnJobRunBefore(job *model.Job) { } switch job.SchemaState { case model.StateDeleteOnly, model.StateWriteOnly, model.StateWriteReorganization: - logutil.BgLogger().Warn(fmt.Sprintf("[DDL_HOOK] Hang for 0.5 seconds on %s state triggered", job.SchemaState.String())) + logutil.DDLLogger().Warn(fmt.Sprintf("[DDL_HOOK] Hang for 0.5 seconds on %s state triggered", job.SchemaState.String())) time.Sleep(500 * time.Millisecond) } } @@ -228,7 +228,7 @@ func GetCustomizedHook(s string) (func(do DomainReloader) Callback, error) { s = strings.TrimSpace(s) fact, ok := customizedCallBackRegisterMap[s] if !ok { - logutil.BgLogger().Error("bad ddl hook " + s) + logutil.DDLLogger().Error("bad ddl hook " + s) return nil, errors.Errorf("ddl hook `%s` is not found in hook registered map", s) } return fact, nil diff --git a/pkg/ddl/cluster.go b/pkg/ddl/cluster.go index 1aa3cd82060b9..5a6aabd5b65f0 100644 --- a/pkg/ddl/cluster.go +++ b/pkg/ddl/cluster.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/kvproto/pkg/errorpb" "github.com/pingcap/kvproto/pkg/kvrpcpb" sess "github.com/pingcap/tidb/pkg/ddl/internal/session" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/infoschema" "github.com/pingcap/tidb/pkg/kv" @@ -42,7 +43,6 @@ import ( "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/filter" "github.com/pingcap/tidb/pkg/util/gcutil" - "github.com/pingcap/tidb/pkg/util/logutil" tikvstore "github.com/tikv/client-go/v2/kv" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/tikv" @@ -445,7 +445,7 @@ func SendPrepareFlashbackToVersionRPC( endKey = rangeEndKey } - logutil.BgLogger().Info("send prepare flashback request", zap.String("category", "ddl"), zap.Uint64("region_id", loc.Region.GetID()), + logutil.DDLLogger().Info("send prepare flashback request", zap.Uint64("region_id", loc.Region.GetID()), zap.String("start_key", hex.EncodeToString(startKey)), zap.String("end_key", hex.EncodeToString(endKey))) req := tikvrpc.NewRequest(tikvrpc.CmdPrepareFlashbackToVersion, &kvrpcpb.PrepareFlashbackToVersionRequest{ @@ -479,7 +479,7 @@ func SendPrepareFlashbackToVersionRPC( continue } if resp.Resp == nil { - logutil.BgLogger().Warn("prepare flashback miss resp body", zap.Uint64("region_id", loc.Region.GetID())) + logutil.DDLLogger().Warn("prepare flashback miss resp body", zap.Uint64("region_id", loc.Region.GetID())) err = bo.Backoff(tikv.BoTiKVRPC(), errors.New("prepare flashback rpc miss resp body")) if err != nil { return taskStat, err @@ -539,7 +539,7 @@ func SendFlashbackToVersionRPC( endKey = rangeEndKey } - logutil.BgLogger().Info("send flashback request", zap.String("category", "ddl"), zap.Uint64("region_id", loc.Region.GetID()), + logutil.DDLLogger().Info("send flashback request", zap.Uint64("region_id", loc.Region.GetID()), zap.String("start_key", hex.EncodeToString(startKey)), zap.String("end_key", hex.EncodeToString(endKey))) req := tikvrpc.NewRequest(tikvrpc.CmdFlashbackToVersion, &kvrpcpb.FlashbackToVersionRequest{ @@ -552,7 +552,7 @@ func SendFlashbackToVersionRPC( resp, err := s.SendReq(bo, req, loc.Region, flashbackTimeout) if err != nil { - logutil.BgLogger().Warn("send request meets error", zap.Uint64("region_id", loc.Region.GetID()), zap.Error(err)) + logutil.DDLLogger().Warn("send request meets error", zap.Uint64("region_id", loc.Region.GetID()), zap.Error(err)) if err.Error() != fmt.Sprintf("region %d is not prepared for the flashback", loc.Region.GetID()) { return taskStat, err } @@ -569,7 +569,7 @@ func SendFlashbackToVersionRPC( continue } if resp.Resp == nil { - logutil.BgLogger().Warn("flashback miss resp body", zap.Uint64("region_id", loc.Region.GetID())) + logutil.DDLLogger().Warn("flashback miss resp body", zap.Uint64("region_id", loc.Region.GetID())) err = bo.Backoff(tikv.BoTiKVRPC(), errors.New("flashback rpc miss resp body")) if err != nil { return taskStat, err @@ -731,7 +731,7 @@ func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ve totalRegions.Add(uint64(stats.CompletedRegions)) return stats, err }, r.StartKey, r.EndKey); err != nil { - logutil.BgLogger().Warn("Get error when do flashback", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Warn("Get error when do flashback", zap.Error(err)) return ver, err } } @@ -761,13 +761,13 @@ func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ve // Use same startTS as prepare phase to simulate 1PC txn. stats, err := SendFlashbackToVersionRPC(ctx, d.store.(tikv.Storage), flashbackTS, startTS, commitTS, r) completedRegions.Add(uint64(stats.CompletedRegions)) - logutil.BgLogger().Info("flashback cluster stats", zap.String("category", "ddl"), + logutil.DDLLogger().Info("flashback cluster stats", zap.Uint64("complete regions", completedRegions.Load()), zap.Uint64("total regions", totalRegions.Load()), zap.Error(err)) return stats, err }, r.StartKey, r.EndKey); err != nil { - logutil.BgLogger().Warn("Get error when do flashback", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Warn("Get error when do flashback", zap.Error(err)) return ver, errors.Trace(err) } } diff --git a/pkg/ddl/column.go b/pkg/ddl/column.go index ef79ca5a34675..16c4a1664b81b 100644 --- a/pkg/ddl/column.go +++ b/pkg/ddl/column.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/config" sess "github.com/pingcap/tidb/pkg/ddl/internal/session" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/infoschema" "github.com/pingcap/tidb/pkg/kv" @@ -48,7 +49,6 @@ import ( "github.com/pingcap/tidb/pkg/util" contextutil "github.com/pingcap/tidb/pkg/util/context" "github.com/pingcap/tidb/pkg/util/dbterror" - "github.com/pingcap/tidb/pkg/util/logutil" decoder "github.com/pingcap/tidb/pkg/util/rowDecoder" "github.com/pingcap/tidb/pkg/util/rowcodec" kvutil "github.com/tikv/client-go/v2/util" @@ -132,7 +132,7 @@ func onAddColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) } if columnInfo == nil { columnInfo = InitAndAddColumnToTable(tblInfo, colFromArgs) - logutil.BgLogger().Info("run add column job", zap.String("category", "ddl"), zap.String("job", job.String()), zap.Reflect("columnInfo", *columnInfo)) + logutil.DDLLogger().Info("run add column job", zap.Stringer("job", job), zap.Reflect("columnInfo", *columnInfo)) if err = checkAddColumnTooManyColumns(len(tblInfo.Columns)); err != nil { job.State = model.JobStateCancelled return ver, errors.Trace(err) @@ -497,12 +497,12 @@ func GetOriginDefaultValueForModifyColumn(sessCtx sessionctx.Context, changingCo if originDefVal != nil { odv, err := table.CastValue(sessCtx, types.NewDatum(originDefVal), changingCol, false, false) if err != nil { - logutil.BgLogger().Info("cast origin default value failed", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Info("cast origin default value failed", zap.Error(err)) } if !odv.IsNull() { if originDefVal, err = odv.ToString(); err != nil { originDefVal = nil - logutil.BgLogger().Info("convert default value to string failed", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Info("convert default value to string failed", zap.Error(err)) } } } @@ -612,7 +612,7 @@ func (w *worker) onModifyColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in } else { changingCol = model.FindColumnInfoByID(tblInfo.Columns, modifyInfo.changingCol.ID) if changingCol == nil { - logutil.BgLogger().Error("the changing column has been removed", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Error("the changing column has been removed", zap.Error(err)) job.State = model.JobStateCancelled return ver, errors.Trace(infoschema.ErrColumnNotExists.GenWithStackByArgs(oldCol.Name, tblInfo.Name)) } @@ -872,10 +872,10 @@ func doReorgWorkForModifyColumn(w *worker, d *ddlCtx, t *meta.Meta, job *model.J return false, ver, errors.Trace(err) } if err1 := rh.RemoveDDLReorgHandle(job, reorgInfo.elements); err1 != nil { - logutil.BgLogger().Warn("run modify column job failed, RemoveDDLReorgHandle failed, can't convert job to rollback", zap.String("category", "ddl"), + logutil.DDLLogger().Warn("run modify column job failed, RemoveDDLReorgHandle failed, can't convert job to rollback", zap.String("job", job.String()), zap.Error(err1)) } - logutil.BgLogger().Warn("run modify column job failed, convert job to rollback", zap.String("category", "ddl"), zap.String("job", job.String()), zap.Error(err)) + logutil.DDLLogger().Warn("run modify column job failed, convert job to rollback", zap.Stringer("job", job), zap.Error(err)) job.State = model.JobStateRollingback return false, ver, errors.Trace(err) } @@ -1073,7 +1073,7 @@ func BuildElements(changingCol *model.ColumnInfo, changingIdxs []*model.IndexInf } func (w *worker) updatePhysicalTableRow(t table.Table, reorgInfo *reorgInfo) error { - logutil.BgLogger().Info("start to update table row", zap.String("category", "ddl"), zap.String("job", reorgInfo.Job.String()), zap.String("reorgInfo", reorgInfo.String())) + logutil.DDLLogger().Info("start to update table row", zap.Stringer("job", reorgInfo.Job), zap.Stringer("reorgInfo", reorgInfo)) if tbl, ok := t.(table.PartitionedTable); ok { done := false for !done { @@ -1178,7 +1178,7 @@ func (w *worker) updateCurrentElement(t table.Table, reorgInfo *reorgInfo) error reorgInfo.currElement = reorgInfo.elements[i+1] // Write the reorg info to store so the whole reorganize process can recover from panic. err := reorgInfo.UpdateReorgMeta(reorgInfo.StartKey, w.sessPool) - logutil.BgLogger().Info("update column and indexes", zap.String("category", "ddl"), + logutil.DDLLogger().Info("update column and indexes", zap.Int64("job ID", reorgInfo.Job.ID), zap.Stringer("element", reorgInfo.currElement), zap.String("start key", hex.EncodeToString(reorgInfo.StartKey)), @@ -1211,8 +1211,8 @@ type updateColumnWorker struct { func newUpdateColumnWorker(sessCtx sessionctx.Context, id int, t table.PhysicalTable, decodeColMap map[int64]decoder.Column, reorgInfo *reorgInfo, jc *JobContext) *updateColumnWorker { if !bytes.Equal(reorgInfo.currElement.TypeKey, meta.ColumnElementKey) { - logutil.BgLogger().Error("Element type for updateColumnWorker incorrect", zap.String("jobQuery", reorgInfo.Query), - zap.String("reorgInfo", reorgInfo.String())) + logutil.DDLLogger().Error("Element type for updateColumnWorker incorrect", zap.String("jobQuery", reorgInfo.Query), + zap.Stringer("reorgInfo", reorgInfo)) return nil } var oldCol, newCol *model.ColumnInfo @@ -1238,8 +1238,8 @@ func newUpdateColumnWorker(sessCtx sessionctx.Context, id int, t table.PhysicalT for i, col := range t.DeletableCols() { cols[i] = col.ToInfo() } - logutil.BgLogger().Warn("skip checksum in update-column backfill since the number of non-public columns is greater than 1", - zap.String("jobQuery", reorgInfo.Query), zap.String("reorgInfo", reorgInfo.String()), zap.Any("cols", cols)) + logutil.DDLLogger().Warn("skip checksum in update-column backfill since the number of non-public columns is greater than 1", + zap.String("jobQuery", reorgInfo.Query), zap.Stringer("reorgInfo", reorgInfo), zap.Any("cols", cols)) } else { checksumNeeded = true } @@ -1318,7 +1318,10 @@ func (w *updateColumnWorker) fetchRowColVals(txn kv.Transaction, taskRange reorg taskDone = true } - logutil.BgLogger().Debug("txn fetches handle info", zap.String("category", "ddl"), zap.Uint64("txnStartTS", txn.StartTS()), zap.String("taskRange", taskRange.String()), zap.Duration("takeTime", time.Since(startTime))) + logutil.DDLLogger().Debug("txn fetches handle info", + zap.Uint64("txnStartTS", txn.StartTS()), + zap.String("taskRange", taskRange.String()), + zap.Duration("takeTime", time.Since(startTime))) return w.rowRecords, getNextHandleKey(taskRange, taskDone, lastAccessedHandle), taskDone, errors.Trace(err) } @@ -1419,7 +1422,7 @@ func (w *updateColumnWorker) calcChecksums() []uint32 { } checksum, err := w.checksumBuffer.Checksum(w.sessCtx.GetSessionVars().StmtCtx.TimeZone()) if err != nil { - logutil.BgLogger().Warn("skip checksum in update-column backfill due to encode error", zap.Error(err)) + logutil.DDLLogger().Warn("skip checksum in update-column backfill due to encode error", zap.Error(err)) return nil } checksums[i] = checksum diff --git a/pkg/ddl/db_rename_test.go b/pkg/ddl/db_rename_test.go index d9550c7f6c202..0c04e5a3677d9 100644 --- a/pkg/ddl/db_rename_test.go +++ b/pkg/ddl/db_rename_test.go @@ -20,12 +20,12 @@ import ( gotime "time" "github.com/pingcap/tidb/pkg/config" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/domain" "github.com/pingcap/tidb/pkg/errno" "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/store/mockstore" "github.com/pingcap/tidb/pkg/testkit" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/stretchr/testify/require" "go.uber.org/zap" ) @@ -334,7 +334,7 @@ func TestRenameConcurrentAutoID(t *testing.T) { } res := tk3.MustQuery(`admin show ddl jobs where table_name = '` + tableName + `' and job_type = 'rename table'`).Rows() if len(res) == 1 && res[0][pos] == s { - logutil.BgLogger().Info("Got state", zap.String("State", s)) + logutil.DDLLogger().Info("Got state", zap.String("State", s)) break } gotime.Sleep(50 * gotime.Millisecond) diff --git a/pkg/ddl/ddl.go b/pkg/ddl/ddl.go index faad5a126f207..8e643fa99d046 100644 --- a/pkg/ddl/ddl.go +++ b/pkg/ddl/ddl.go @@ -36,6 +36,7 @@ import ( "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/ddl/ingest" sess "github.com/pingcap/tidb/pkg/ddl/internal/session" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/ddl/syncer" "github.com/pingcap/tidb/pkg/ddl/util" "github.com/pingcap/tidb/pkg/disttask/framework/proto" @@ -63,7 +64,6 @@ import ( "github.com/pingcap/tidb/pkg/util/dbterror" "github.com/pingcap/tidb/pkg/util/dbterror/exeerrors" "github.com/pingcap/tidb/pkg/util/gcutil" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/syncutil" "github.com/tikv/client-go/v2/tikvrpc" clientv3 "go.etcd.io/etcd/client/v3" @@ -101,8 +101,6 @@ const ( recoverCheckFlagDisableGC ) -var ddlLogger = logutil.BgLogger().With(zap.String("category", "ddl")) - // OnExist specifies what to do when a new object has a name collision. type OnExist uint8 @@ -485,7 +483,7 @@ func (sv *schemaVersionManager) unlockSchemaVersion(jobID int64) { err := lockInfo.mu.Unlock(sv.ctx) outer: for err != nil { - logutil.BgLogger().Error("unlock schema version", zap.Error(err)) + logutil.DDLLogger().Error("unlock schema version", zap.Error(err)) select { case <-sv.ctx.Done(): break outer @@ -495,7 +493,7 @@ func (sv *schemaVersionManager) unlockSchemaVersion(jobID int64) { err = lockInfo.mu.Unlock(sv.ctx) } if err := lockInfo.se.Close(); err != nil { - logutil.BgLogger().Error("close etcd session", zap.Error(err)) + logutil.DDLLogger().Error("close etcd session", zap.Error(err)) } } sv.lockOwner.Store(0) @@ -505,7 +503,7 @@ func (sv *schemaVersionManager) unlockSchemaVersion(jobID int64) { func (dc *ddlCtx) isOwner() bool { isOwner := dc.ownerManager.IsOwner() - logutil.BgLogger().Debug("check whether is the DDL owner", zap.String("category", "ddl"), zap.Bool("isOwner", isOwner), zap.String("selfID", dc.uuid)) + logutil.DDLLogger().Debug("check whether is the DDL owner", zap.Bool("isOwner", isOwner), zap.String("selfID", dc.uuid)) if isOwner { metrics.DDLCounter.WithLabelValues(metrics.DDLOwner + "_" + mysql.TiDBReleaseVersion).Inc() } @@ -611,12 +609,12 @@ func (dc *ddlCtx) removeReorgCtx(jobID int64) { func (dc *ddlCtx) notifyReorgWorkerJobStateChange(job *model.Job) { rc := dc.getReorgCtx(job.ID) if rc == nil { - logutil.BgLogger().Warn("cannot find reorgCtx", zap.Int64("Job ID", job.ID)) + logutil.DDLLogger().Warn("cannot find reorgCtx", zap.Int64("Job ID", job.ID)) return } - logutil.BgLogger().Info("notify reorg worker the job's state", - zap.Int64("Job ID", job.ID), zap.String("Job State", job.State.String()), - zap.String("Schema State", job.SchemaState.String()), zap.String("category", "ddl")) + logutil.DDLLogger().Info("notify reorg worker the job's state", + zap.Int64("Job ID", job.ID), zap.Stringer("Job State", job.State), + zap.Stringer("Schema State", job.SchemaState)) rc.notifyJobState(job.State) } @@ -665,7 +663,7 @@ func asyncNotifyEvent(d *ddlCtx, e *statsutil.DDLEvent) { time.Sleep(time.Microsecond * 10) } } - logutil.BgLogger().Warn("fail to notify DDL event", zap.String("category", "ddl"), zap.String("event", e.String())) + logutil.DDLLogger().Warn("fail to notify DDL event", zap.Stringer("event", e)) } } @@ -768,7 +766,7 @@ func (d *ddl) Stop() error { defer d.m.Unlock() d.close() - logutil.BgLogger().Info("stop DDL", zap.String("category", "ddl"), zap.String("ID", d.uuid)) + logutil.DDLLogger().Info("stop DDL", zap.String("ID", d.uuid)) return nil } @@ -776,7 +774,7 @@ func (d *ddl) newDeleteRangeManager(mock bool) delRangeManager { var delRangeMgr delRangeManager if !mock { delRangeMgr = newDelRangeManager(d.store, d.sessPool) - logutil.BgLogger().Info("start delRangeManager OK", zap.String("category", "ddl"), zap.Bool("is a emulator", !d.store.SupportDeleteRange())) + logutil.DDLLogger().Info("start delRangeManager OK", zap.Bool("is a emulator", !d.store.SupportDeleteRange())) } else { delRangeMgr = newMockDelRangeManager() } @@ -817,7 +815,7 @@ func (d *ddl) prepareWorkers4ConcurrencyDDL() { // Start implements DDL.Start interface. func (d *ddl) Start(ctxPool *pools.ResourcePool) error { - logutil.BgLogger().Info("start DDL", zap.String("category", "ddl"), zap.String("ID", d.uuid), zap.Bool("runWorker", config.GetGlobalConfig().Instance.TiDBEnableDDL.Load())) + logutil.DDLLogger().Info("start DDL", zap.String("ID", d.uuid), zap.Bool("runWorker", config.GetGlobalConfig().Instance.TiDBEnableDDL.Load())) d.wg.Run(func() { d.limitDDLJobs(d.limitJobCh, d.addBatchDDLJobsV1) @@ -832,7 +830,7 @@ func (d *ddl) Start(ctxPool *pools.ResourcePool) error { defer d.ddlSeqNumMu.Unlock() d.ddlSeqNumMu.seqNum, err = d.GetNextDDLSeqNum() if err != nil { - logutil.BgLogger().Error("error when getting the ddl history count", zap.Error(err)) + logutil.DDLLogger().Error("error when getting the ddl history count", zap.Error(err)) } d.runningJobs.clear() }) @@ -840,7 +838,7 @@ func (d *ddl) Start(ctxPool *pools.ResourcePool) error { d.delRangeMgr = d.newDeleteRangeManager(ctxPool == nil) if err := d.stateSyncer.Init(d.ctx); err != nil { - logutil.BgLogger().Warn("start DDL init state syncer failed", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Warn("start DDL init state syncer failed", zap.Error(err)) return errors.Trace(err) } @@ -893,7 +891,7 @@ func (d *ddl) DisableDDL() error { // If there is only one node, we should NOT disable ddl. serverInfo, err := infosync.GetAllServerInfo(d.ctx) if err != nil { - logutil.BgLogger().Error("error when GetAllServerInfo", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Error("error when GetAllServerInfo", zap.Error(err)) return err } if len(serverInfo) <= 1 { @@ -950,7 +948,7 @@ func (d *ddl) close() { } variable.UnregisterStatistics(d) - logutil.BgLogger().Info("DDL closed", zap.String("category", "ddl"), zap.String("ID", d.uuid), zap.Duration("take time", time.Since(startTime))) + logutil.DDLLogger().Info("DDL closed", zap.String("ID", d.uuid), zap.Duration("take time", time.Since(startTime))) } // GetLease implements DDL.GetLease interface. @@ -1181,7 +1179,7 @@ func (d *ddl) DoDDLJob(ctx sessionctx.Context, job *model.Job) error { // Notice worker that we push a new job and wait the job done. d.asyncNotifyWorker(d.ddlJobCh, addingDDLJobConcurrent, job.ID, job.Type.String()) - logutil.BgLogger().Info("start DDL job", zap.String("category", "ddl"), zap.String("job", job.String()), zap.String("query", job.Query)) + logutil.DDLLogger().Info("start DDL job", zap.Stringer("job", job), zap.String("query", job.Query)) if !d.shouldCheckHistoryJob(job) { return nil } @@ -1218,30 +1216,30 @@ func (d *ddl) DoDDLJob(ctx sessionctx.Context, job *model.Job) error { i++ ticker = updateTickerInterval(ticker, 10*d.lease, job, i) case <-d.ctx.Done(): - logutil.BgLogger().Info("DoDDLJob will quit because context done", zap.String("category", "ddl")) + logutil.DDLLogger().Info("DoDDLJob will quit because context done") return context.Canceled } // If the connection being killed, we need to CANCEL the DDL job. if sessVars.SQLKiller.HandleSignal() == exeerrors.ErrQueryInterrupted { if atomic.LoadInt32(&sessVars.ConnectionStatus) == variable.ConnStatusShutdown { - logutil.BgLogger().Info("DoDDLJob will quit because context done", zap.String("category", "ddl")) + logutil.DDLLogger().Info("DoDDLJob will quit because context done") return context.Canceled } if sessVars.StmtCtx.DDLJobID != 0 { se, err := d.sessPool.Get() if err != nil { - logutil.BgLogger().Error("get session failed, check again", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Error("get session failed, check again", zap.Error(err)) continue } sessVars.StmtCtx.DDLJobID = 0 // Avoid repeat. errs, err := CancelJobsBySystem(se, []int64{jobID}) d.sessPool.Put(se) if len(errs) > 0 { - logutil.BgLogger().Warn("error canceling DDL job", zap.Error(errs[0])) + logutil.DDLLogger().Warn("error canceling DDL job", zap.Error(errs[0])) } if err != nil { - logutil.BgLogger().Warn("Kill command could not cancel DDL job", zap.Error(err)) + logutil.DDLLogger().Warn("Kill command could not cancel DDL job", zap.Error(err)) continue } } @@ -1249,17 +1247,17 @@ func (d *ddl) DoDDLJob(ctx sessionctx.Context, job *model.Job) error { se, err := d.sessPool.Get() if err != nil { - logutil.BgLogger().Error("get session failed, check again", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Error("get session failed, check again", zap.Error(err)) continue } historyJob, err = GetHistoryJobByID(se, jobID) d.sessPool.Put(se) if err != nil { - logutil.BgLogger().Error("get history DDL job failed, check again", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Error("get history DDL job failed, check again", zap.Error(err)) continue } if historyJob == nil { - logutil.BgLogger().Debug("DDL job is not in history, maybe not run", zap.String("category", "ddl"), zap.Int64("jobID", jobID)) + logutil.DDLLogger().Debug("DDL job is not in history, maybe not run", zap.Int64("jobID", jobID)) continue } @@ -1270,7 +1268,7 @@ func (d *ddl) DoDDLJob(ctx sessionctx.Context, job *model.Job) error { // Judge whether there are some warnings when executing DDL under the certain SQL mode. if historyJob.ReorgMeta != nil && len(historyJob.ReorgMeta.Warnings) != 0 { if len(historyJob.ReorgMeta.Warnings) != len(historyJob.ReorgMeta.WarningsCount) { - logutil.BgLogger().Info("DDL warnings doesn't match the warnings count", zap.String("category", "ddl"), zap.Int64("jobID", jobID)) + logutil.DDLLogger().Info("DDL warnings doesn't match the warnings count", zap.Int64("jobID", jobID)) } else { for key, warning := range historyJob.ReorgMeta.Warnings { keyCount := historyJob.ReorgMeta.WarningsCount[key] @@ -1286,12 +1284,12 @@ func (d *ddl) DoDDLJob(ctx sessionctx.Context, job *model.Job) error { } appendMultiChangeWarningsToOwnerCtx(ctx, historyJob) - logutil.BgLogger().Info("DDL job is finished", zap.String("category", "ddl"), zap.Int64("jobID", jobID)) + logutil.DDLLogger().Info("DDL job is finished", zap.Int64("jobID", jobID)) return nil } if historyJob.Error != nil { - logutil.BgLogger().Info("DDL job is failed", zap.String("category", "ddl"), zap.Int64("jobID", jobID)) + logutil.DDLLogger().Info("DDL job is failed", zap.Int64("jobID", jobID)) return errors.Trace(historyJob.Error) } panic("When the state is JobStateRollbackDone or JobStateCancelled, historyJob.Error should never be nil") @@ -1348,13 +1346,13 @@ func (d *ddl) startCleanDeadTableLock() { } deadLockTables, err := d.tableLockCkr.GetDeadLockedTables(d.ctx, d.infoCache.GetLatest().AllSchemas()) if err != nil { - logutil.BgLogger().Info("get dead table lock failed.", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Info("get dead table lock failed.", zap.Error(err)) continue } for se, tables := range deadLockTables { err := d.CleanDeadTableLock(tables, se) if err != nil { - logutil.BgLogger().Info("clean dead table lock failed.", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Info("clean dead table lock failed.", zap.Error(err)) } } case <-d.ctx.Done(): @@ -1402,10 +1400,10 @@ func (d *ddl) SwitchMDL(enable bool) error { return err }) if err != nil { - logutil.BgLogger().Warn("switch metadata lock feature", zap.String("category", "ddl"), zap.Bool("enable", enable), zap.Error(err)) + logutil.DDLLogger().Warn("switch metadata lock feature", zap.Bool("enable", enable), zap.Error(err)) return err } - logutil.BgLogger().Info("switch metadata lock feature", zap.String("category", "ddl"), zap.Bool("enable", enable)) + logutil.DDLLogger().Info("switch metadata lock feature", zap.Bool("enable", enable)) return nil } @@ -1439,7 +1437,7 @@ func (d *ddl) SwitchFastCreateTable(val bool) error { } variable.EnableFastCreateTable.Store(val) - logutil.BgLogger().Info("switch fast create table", zap.String("category", "ddl"), zap.Bool("val", val)) + logutil.DDLLogger().Info("switch fast create table", zap.Bool("val", val)) return nil } @@ -1538,7 +1536,7 @@ func delayForAsyncCommit() { } cfg := config.GetGlobalConfig().TiKVClient.AsyncCommit duration := cfg.SafeWindow + cfg.AllowedClockDrift - logutil.BgLogger().Info("sleep before DDL finishes to make async commit and 1PC safe", + logutil.DDLLogger().Info("sleep before DDL finishes to make async commit and 1PC safe", zap.Duration("duration", duration)) time.Sleep(duration) } @@ -1767,7 +1765,7 @@ func processJobs(process func(*sess.Session, *model.Job, model.AdminCommandOpera for _, job := range jobs { i, ok := jobMap[job.ID] if !ok { - logutil.BgLogger().Debug("Job ID from meta is not consistent with requested job id,", + logutil.DDLLogger().Debug("Job ID from meta is not consistent with requested job id,", zap.Int64("fetched job ID", job.ID)) jobErrs[i] = dbterror.ErrInvalidDDLJob.GenWithStackByArgs(job.ID) continue diff --git a/pkg/ddl/ddl_api.go b/pkg/ddl/ddl_api.go index 63a8051ba04c6..866c0f8087d03 100644 --- a/pkg/ddl/ddl_api.go +++ b/pkg/ddl/ddl_api.go @@ -34,6 +34,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/ddl/label" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/ddl/resourcegroup" ddlutil "github.com/pingcap/tidb/pkg/ddl/util" rg "github.com/pingcap/tidb/pkg/domain/resourcegroup" @@ -67,7 +68,6 @@ import ( "github.com/pingcap/tidb/pkg/util/dbterror/exeerrors" "github.com/pingcap/tidb/pkg/util/domainutil" "github.com/pingcap/tidb/pkg/util/hack" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/mathutil" "github.com/pingcap/tidb/pkg/util/mock" "github.com/pingcap/tidb/pkg/util/set" @@ -343,7 +343,7 @@ func (d *ddl) waitPendingTableThreshold(sctx sessionctx.Context, schemaID int64, for retry := 0; retry < configRetry; retry++ { done, killed := isSessionDone(sctx) if done { - logutil.BgLogger().Info("abort batch add TiFlash replica", zap.Int64("schemaID", schemaID), zap.Uint32("isKilled", killed)) + logutil.DDLLogger().Info("abort batch add TiFlash replica", zap.Int64("schemaID", schemaID), zap.Uint32("isKilled", killed)) return true, originVersion, pendingCount, false } originVersion, pendingCount = d.getPendingTiFlashTableCount(sctx, originVersion, pendingCount) @@ -352,11 +352,16 @@ func (d *ddl) waitPendingTableThreshold(sctx sessionctx.Context, schemaID int64, // If there are not many unavailable tables, we don't need a force check. return false, originVersion, pendingCount, false } - logutil.BgLogger().Info("too many unavailable tables, wait", zap.Uint32("threshold", threshold), zap.Uint32("currentPendingCount", pendingCount), zap.Int64("schemaID", schemaID), zap.Int64("tableID", tableID), zap.Duration("time", configWaitTime)) + logutil.DDLLogger().Info("too many unavailable tables, wait", + zap.Uint32("threshold", threshold), + zap.Uint32("currentPendingCount", pendingCount), + zap.Int64("schemaID", schemaID), + zap.Int64("tableID", tableID), + zap.Duration("time", configWaitTime)) delay = configWaitTime time.Sleep(delay) } - logutil.BgLogger().Info("too many unavailable tables, timeout", zap.Int64("schemaID", schemaID), zap.Int64("tableID", tableID)) + logutil.DDLLogger().Info("too many unavailable tables, timeout", zap.Int64("schemaID", schemaID), zap.Int64("tableID", tableID)) // If timeout here, we will trigger a ddl job, to force sync schema. However, it doesn't mean we remove limiter, // so there is a force check immediately after that. return false, originVersion, pendingCount, true @@ -393,20 +398,20 @@ func (d *ddl) ModifySchemaSetTiFlashReplica(sctx sessionctx.Context, stmt *ast.A var pendingCount uint32 forceCheck := false - logutil.BgLogger().Info("start batch add TiFlash replicas", zap.Int("total", total), zap.Int64("schemaID", dbInfo.ID)) + logutil.DDLLogger().Info("start batch add TiFlash replicas", zap.Int("total", total), zap.Int64("schemaID", dbInfo.ID)) threshold := uint32(sctx.GetSessionVars().BatchPendingTiFlashCount) for _, tbl := range tbls { tbl := tbl.Meta() done, killed := isSessionDone(sctx) if done { - logutil.BgLogger().Info("abort batch add TiFlash replica", zap.Int64("schemaID", dbInfo.ID), zap.Uint32("isKilled", killed)) + logutil.DDLLogger().Info("abort batch add TiFlash replica", zap.Int64("schemaID", dbInfo.ID), zap.Uint32("isKilled", killed)) return nil } tbReplicaInfo := tbl.TiFlashReplica if !shouldModifyTiFlashReplica(tbReplicaInfo, tiflashReplica) { - logutil.BgLogger().Info("skip repeated processing table", + logutil.DDLLogger().Info("skip repeated processing table", zap.Int64("tableID", tbl.ID), zap.Int64("schemaID", dbInfo.ID), zap.String("tableName", tbl.Name.String()), @@ -418,7 +423,7 @@ func (d *ddl) ModifySchemaSetTiFlashReplica(sctx sessionctx.Context, stmt *ast.A // If table is not supported, add err to warnings. err = isTableTiFlashSupported(dbName, tbl) if err != nil { - logutil.BgLogger().Info("skip processing table", zap.Int64("tableID", tbl.ID), + logutil.DDLLogger().Info("skip processing table", zap.Int64("tableID", tbl.ID), zap.Int64("schemaID", dbInfo.ID), zap.String("tableName", tbl.Name.String()), zap.String("schemaName", dbInfo.Name.String()), @@ -435,7 +440,7 @@ func (d *ddl) ModifySchemaSetTiFlashReplica(sctx sessionctx.Context, stmt *ast.A finished := false finished, originVersion, pendingCount, forceCheck = d.waitPendingTableThreshold(sctx, dbInfo.ID, tbl.ID, originVersion, pendingCount, threshold) if finished { - logutil.BgLogger().Info("abort batch add TiFlash replica", zap.Int64("schemaID", dbInfo.ID)) + logutil.DDLLogger().Info("abort batch add TiFlash replica", zap.Int64("schemaID", dbInfo.ID)) return nil } } @@ -455,7 +460,12 @@ func (d *ddl) ModifySchemaSetTiFlashReplica(sctx sessionctx.Context, stmt *ast.A if err != nil { oneFail = tbl.ID fail++ - logutil.BgLogger().Info("processing schema table error", zap.Int64("tableID", tbl.ID), zap.Int64("schemaID", dbInfo.ID), zap.String("tableName", tbl.Name.String()), zap.String("schemaName", dbInfo.Name.String()), zap.Error(err)) + logutil.DDLLogger().Info("processing schema table error", + zap.Int64("tableID", tbl.ID), + zap.Int64("schemaID", dbInfo.ID), + zap.Stringer("tableName", tbl.Name), + zap.Stringer("schemaName", dbInfo.Name), + zap.Error(err)) } else { succ++ } @@ -466,7 +476,7 @@ func (d *ddl) ModifySchemaSetTiFlashReplica(sctx sessionctx.Context, stmt *ast.A } msg := fmt.Sprintf("In total %v tables: %v succeed, %v failed%v, %v skipped", total, succ, fail, failStmt, skip) sctx.GetSessionVars().StmtCtx.SetMessage(msg) - logutil.BgLogger().Info("finish batch add TiFlash replica", zap.Int64("schemaID", dbInfo.ID)) + logutil.DDLLogger().Info("finish batch add TiFlash replica", zap.Int64("schemaID", dbInfo.ID)) return nil } @@ -960,7 +970,7 @@ func decodeEnumSetBinaryLiteralToUTF8(tp *types.FieldType, chs string) { } s, err := enc.Transform(nil, hack.Slice(elem), charset.OpDecodeReplace) if err != nil { - logutil.BgLogger().Warn("decode enum binary literal to utf-8 failed", zap.Error(err)) + logutil.DDLLogger().Warn("decode enum binary literal to utf-8 failed", zap.Error(err)) } tp.SetElem(i, string(hack.String(s))) } @@ -1120,17 +1130,17 @@ func adjustBlobTypesFlen(tp *types.FieldType, colCharset string) error { l := tp.GetFlen() * cs.Maxlen if tp.GetType() == mysql.TypeBlob { if l <= tinyBlobMaxLength { - logutil.BgLogger().Info(fmt.Sprintf("Automatically convert BLOB(%d) to TINYBLOB", tp.GetFlen())) + logutil.DDLLogger().Info(fmt.Sprintf("Automatically convert BLOB(%d) to TINYBLOB", tp.GetFlen())) tp.SetFlen(tinyBlobMaxLength) tp.SetType(mysql.TypeTinyBlob) } else if l <= blobMaxLength { tp.SetFlen(blobMaxLength) } else if l <= mediumBlobMaxLength { - logutil.BgLogger().Info(fmt.Sprintf("Automatically convert BLOB(%d) to MEDIUMBLOB", tp.GetFlen())) + logutil.DDLLogger().Info(fmt.Sprintf("Automatically convert BLOB(%d) to MEDIUMBLOB", tp.GetFlen())) tp.SetFlen(mediumBlobMaxLength) tp.SetType(mysql.TypeMediumBlob) } else if l <= longBlobMaxLength { - logutil.BgLogger().Info(fmt.Sprintf("Automatically convert BLOB(%d) to LONGBLOB", tp.GetFlen())) + logutil.DDLLogger().Info(fmt.Sprintf("Automatically convert BLOB(%d) to LONGBLOB", tp.GetFlen())) tp.SetFlen(longBlobMaxLength) tp.SetType(mysql.TypeLongBlob) } @@ -1614,7 +1624,7 @@ func setYearDefaultValue(c *table.Column, hasDefaultValue bool) { if c.GetType() == mysql.TypeYear && mysql.HasNotNullFlag(c.GetFlag()) { if err := c.SetDefaultValue("0000"); err != nil { - logutil.BgLogger().Error("set default value failed", zap.Error(err)) + logutil.DDLLogger().Error("set default value failed", zap.Error(err)) } } } @@ -1628,11 +1638,11 @@ func setTimestampDefaultValue(c *table.Column, hasDefaultValue bool, setOnUpdate if mysql.HasTimestampFlag(c.GetFlag()) && mysql.HasNotNullFlag(c.GetFlag()) { if setOnUpdateNow { if err := c.SetDefaultValue(types.ZeroDatetimeStr); err != nil { - logutil.BgLogger().Error("set default value failed", zap.Error(err)) + logutil.DDLLogger().Error("set default value failed", zap.Error(err)) } } else { if err := c.SetDefaultValue(strings.ToUpper(ast.CurrentTimestamp)); err != nil { - logutil.BgLogger().Error("set default value failed", zap.Error(err)) + logutil.DDLLogger().Error("set default value failed", zap.Error(err)) } } } @@ -3110,7 +3120,7 @@ func preSplitAndScatter(ctx sessionctx.Context, store kv.Storage, tbInfo *model. ) val, err := ctx.GetSessionVars().GetGlobalSystemVar(context.Background(), variable.TiDBScatterRegion) if err != nil { - logutil.BgLogger().Warn("won't scatter region", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Warn("won't scatter region", zap.Error(err)) } else { scatterRegion = variable.TiDBOptOn(val) } @@ -3127,7 +3137,7 @@ func preSplitAndScatter(ctx sessionctx.Context, store kv.Storage, tbInfo *model. } func (d *ddl) FlashbackCluster(ctx sessionctx.Context, flashbackTS uint64) error { - logutil.BgLogger().Info("get flashback cluster job", zap.String("category", "ddl"), zap.String("flashbackTS", oracle.GetTimeFromTS(flashbackTS).String())) + logutil.DDLLogger().Info("get flashback cluster job", zap.Stringer("flashbackTS", oracle.GetTimeFromTS(flashbackTS))) nowTS, err := ctx.GetStore().GetOracle().GetTimestamp(d.ctx, &oracle.Option{}) if err != nil { return errors.Trace(err) @@ -6979,7 +6989,7 @@ func (d *ddl) dropTableObject( tempTableType := tableInfo.Meta().TempTableType if config.CheckTableBeforeDrop && tempTableType == model.TempTableNone { - logutil.BgLogger().Warn("admin check table before drop", + logutil.DDLLogger().Warn("admin check table before drop", zap.String("database", fullti.Schema.O), zap.String("table", fullti.Name.O), ) @@ -7727,8 +7737,8 @@ func newReorgMetaFromVariables(job *model.Job, sctx sessionctx.Context) (*model. } if hasSysDB(job) { if reorgMeta.IsDistReorg { - logutil.BgLogger().Info("cannot use distributed task execution on system DB", - zap.String("category", "ddl"), zap.Stringer("job", job)) + logutil.DDLLogger().Info("cannot use distributed task execution on system DB", + zap.Stringer("job", job)) } reorgMeta.IsDistReorg = false reorgMeta.IsFastReorg = false @@ -8484,8 +8494,7 @@ func (d *ddl) RepairTable(ctx sessionctx.Context, createStmt *ast.CreateTableStm return dbterror.ErrRepairTableFail.GenWithStackByArgs("Column " + newOne.Name.L + " type should be the same") } if newOne.GetFlen() != old.GetFlen() { - logutil.BgLogger().Warn("admin repair table : Column "+newOne.Name.L+" flen is not equal to the old one", - zap.String("category", "ddl")) + logutil.DDLLogger().Warn("admin repair table : Column " + newOne.Name.L + " flen is not equal to the old one") } newTableInfo.Columns[i].ID = old.ID } @@ -8934,7 +8943,7 @@ func (d *ddl) AddResourceGroup(ctx sessionctx.Context, stmt *ast.CreateResourceG return err } - logutil.BgLogger().Debug("create resource group", zap.String("name", groupName.O), zap.Stringer("resource group settings", groupInfo.ResourceGroupSettings)) + logutil.DDLLogger().Debug("create resource group", zap.String("name", groupName.O), zap.Stringer("resource group settings", groupInfo.ResourceGroupSettings)) groupIDs, err := d.genGlobalIDs(1) if err != nil { return err @@ -9048,7 +9057,7 @@ func (d *ddl) AlterResourceGroup(ctx sessionctx.Context, stmt *ast.AlterResource return err } - logutil.BgLogger().Debug("alter resource group", zap.String("name", groupName.L), zap.Stringer("new resource group settings", newGroupInfo.ResourceGroupSettings)) + logutil.DDLLogger().Debug("alter resource group", zap.String("name", groupName.L), zap.Stringer("new resource group settings", newGroupInfo.ResourceGroupSettings)) job := &model.Job{ SchemaID: newGroupInfo.ID, diff --git a/pkg/ddl/ddl_history.go b/pkg/ddl/ddl_history.go index f0c35a256288a..9e07f019d0c77 100644 --- a/pkg/ddl/ddl_history.go +++ b/pkg/ddl/ddl_history.go @@ -23,13 +23,13 @@ import ( "github.com/pingcap/errors" sess "github.com/pingcap/tidb/pkg/ddl/internal/session" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/ddl/util" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/meta" "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessiontxn" - "github.com/pingcap/tidb/pkg/util/logutil" "go.uber.org/zap" ) @@ -43,7 +43,7 @@ const ( func AddHistoryDDLJob(sess *sess.Session, t *meta.Meta, job *model.Job, updateRawArgs bool) error { err := addHistoryDDLJob2Table(sess, job, updateRawArgs) if err != nil { - logutil.BgLogger().Info("failed to add DDL job to history table", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Info("failed to add DDL job to history table", zap.Error(err)) } // we always add history DDL job to job list at this moment. return t.AddHistoryDDLJob(job, updateRawArgs) diff --git a/pkg/ddl/ddl_tiflash_api.go b/pkg/ddl/ddl_tiflash_api.go index 719187dd83a9f..3281beef38de6 100644 --- a/pkg/ddl/ddl_tiflash_api.go +++ b/pkg/ddl/ddl_tiflash_api.go @@ -30,6 +30,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/pkg/ddl/logutil" ddlutil "github.com/pingcap/tidb/pkg/ddl/util" "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/infoschema" @@ -39,7 +40,6 @@ import ( "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/engine" "github.com/pingcap/tidb/pkg/util/intest" - "github.com/pingcap/tidb/pkg/util/logutil" pd "github.com/tikv/pd/client/http" atomicutil "go.uber.org/atomic" "go.uber.org/zap" @@ -278,17 +278,17 @@ func LoadTiFlashReplicaInfo(tblInfo *model.TableInfo, tableList *[]TiFlashReplic } if pi := tblInfo.GetPartitionInfo(); pi != nil { for _, p := range pi.Definitions { - logutil.BgLogger().Debug(fmt.Sprintf("Table %v has partition %v\n", tblInfo.ID, p.ID)) + logutil.DDLLogger().Debug(fmt.Sprintf("Table %v has partition %v\n", tblInfo.ID, p.ID)) *tableList = append(*tableList, TiFlashReplicaStatus{p.ID, tblInfo.TiFlashReplica.Count, tblInfo.TiFlashReplica.LocationLabels, tblInfo.TiFlashReplica.IsPartitionAvailable(p.ID), tblInfo.TiFlashReplica.Available, false, true}) } // partitions that in adding mid-state for _, p := range pi.AddingDefinitions { - logutil.BgLogger().Debug(fmt.Sprintf("Table %v has partition adding %v\n", tblInfo.ID, p.ID)) + logutil.DDLLogger().Debug(fmt.Sprintf("Table %v has partition adding %v\n", tblInfo.ID, p.ID)) *tableList = append(*tableList, TiFlashReplicaStatus{p.ID, tblInfo.TiFlashReplica.Count, tblInfo.TiFlashReplica.LocationLabels, tblInfo.TiFlashReplica.IsPartitionAvailable(p.ID), tblInfo.TiFlashReplica.Available, true, true}) } } else { - logutil.BgLogger().Debug(fmt.Sprintf("Table %v has no partition\n", tblInfo.ID)) + logutil.DDLLogger().Debug(fmt.Sprintf("Table %v has no partition\n", tblInfo.ID)) *tableList = append(*tableList, TiFlashReplicaStatus{tblInfo.ID, tblInfo.TiFlashReplica.Count, tblInfo.TiFlashReplica.LocationLabels, tblInfo.TiFlashReplica.Available, tblInfo.TiFlashReplica.Available, false, false}) } } @@ -321,7 +321,7 @@ func (d *ddl) UpdateTiFlashHTTPAddress(store *pd.StoreInfo) error { } } if origin != httpAddr { - logutil.BgLogger().Warn(fmt.Sprintf("Update status addr of %v from %v to %v", key, origin, httpAddr)) + logutil.DDLLogger().Warn(fmt.Sprintf("Update status addr of %v from %v to %v", key, origin, httpAddr)) err := ddlutil.PutKVToEtcd(d.ctx, d.etcdCli, 1, key, httpAddr) if err != nil { return errors.Trace(err) @@ -345,7 +345,7 @@ func updateTiFlashStores(pollTiFlashContext *TiFlashManagementContext) error { pollTiFlashContext.TiFlashStores[store.Store.ID] = store } } - logutil.BgLogger().Debug("updateTiFlashStores finished", zap.Int("TiFlash store count", len(pollTiFlashContext.TiFlashStores))) + logutil.DDLLogger().Debug("updateTiFlashStores finished", zap.Int("TiFlash store count", len(pollTiFlashContext.TiFlashStores))) return nil } @@ -361,7 +361,7 @@ func PollAvailableTableProgress(schemas infoschema.InfoSchema, _ sessionctx.Cont if availableTableID.IsPartition { table, _, _ = schemas.FindTableByPartitionID(availableTableID.ID) if table == nil { - logutil.BgLogger().Info("get table by partition failed, may be dropped or truncated", + logutil.DDLLogger().Info("get table by partition failed, may be dropped or truncated", zap.Int64("partitionID", availableTableID.ID), ) pollTiFlashContext.UpdatingProgressTables.Remove(element) @@ -372,7 +372,7 @@ func PollAvailableTableProgress(schemas infoschema.InfoSchema, _ sessionctx.Cont var ok bool table, ok = schemas.TableByID(availableTableID.ID) if !ok { - logutil.BgLogger().Info("get table id failed, may be dropped or truncated", + logutil.DDLLogger().Info("get table id failed, may be dropped or truncated", zap.Int64("tableID", availableTableID.ID), ) pollTiFlashContext.UpdatingProgressTables.Remove(element) @@ -382,7 +382,7 @@ func PollAvailableTableProgress(schemas infoschema.InfoSchema, _ sessionctx.Cont } tableInfo := table.Meta() if tableInfo.TiFlashReplica == nil { - logutil.BgLogger().Info("table has no TiFlash replica", + logutil.DDLLogger().Info("table has no TiFlash replica", zap.Int64("tableID or partitionID", availableTableID.ID), zap.Bool("IsPartition", availableTableID.IsPartition), ) @@ -406,7 +406,7 @@ func PollAvailableTableProgress(schemas infoschema.InfoSchema, _ sessionctx.Cont } err = infosync.UpdateTiFlashProgressCache(availableTableID.ID, progress) if err != nil { - logutil.BgLogger().Error("update tiflash sync progress cache failed", + logutil.DDLLogger().Error("update tiflash sync progress cache failed", zap.Error(err), zap.Int64("tableID", availableTableID.ID), zap.Bool("IsPartition", availableTableID.IsPartition), @@ -466,7 +466,7 @@ func (d *ddl) refreshTiFlashTicker(ctx sessionctx.Context, pollTiFlashContext *T if !ok { tb, _, _ := is.FindTableByPartitionID(phyTable.ID) if tb == nil { - logutil.BgLogger().Info("waitForAddPartition", zap.String("category", "ddl")) + logutil.DDLLogger().Info("waitForAddPartition") sleepSecond := val.(int) time.Sleep(time.Duration(sleepSecond) * time.Second) } @@ -490,13 +490,13 @@ func (d *ddl) refreshTiFlashTicker(ctx sessionctx.Context, pollTiFlashContext *T if !available && !tb.LogicalTableAvailable { enabled, inqueue, _ := pollTiFlashContext.Backoff.Tick(tb.ID) if inqueue && !enabled { - logutil.BgLogger().Info("Escape checking available status due to backoff", zap.Int64("tableId", tb.ID)) + logutil.DDLLogger().Info("Escape checking available status due to backoff", zap.Int64("tableId", tb.ID)) continue } progress, err := infosync.CalculateTiFlashProgress(tb.ID, tb.Count, pollTiFlashContext.TiFlashStores) if err != nil { - logutil.BgLogger().Error("get tiflash sync progress failed", + logutil.DDLLogger().Error("get tiflash sync progress failed", zap.Error(err), zap.Int64("tableID", tb.ID), ) @@ -505,7 +505,7 @@ func (d *ddl) refreshTiFlashTicker(ctx sessionctx.Context, pollTiFlashContext *T err = infosync.UpdateTiFlashProgressCache(tb.ID, progress) if err != nil { - logutil.BgLogger().Error("get tiflash sync progress from cache failed", + logutil.DDLLogger().Error("get tiflash sync progress from cache failed", zap.Error(err), zap.Int64("tableID", tb.ID), zap.Bool("IsPartition", tb.IsPartition), @@ -520,10 +520,10 @@ func (d *ddl) refreshTiFlashTicker(ctx sessionctx.Context, pollTiFlashContext *T }) if !avail { - logutil.BgLogger().Info("Tiflash replica is not available", zap.Int64("tableID", tb.ID), zap.Float64("progress", progress)) + logutil.DDLLogger().Info("Tiflash replica is not available", zap.Int64("tableID", tb.ID), zap.Float64("progress", progress)) pollTiFlashContext.Backoff.Put(tb.ID) } else { - logutil.BgLogger().Info("Tiflash replica is available", zap.Int64("tableID", tb.ID), zap.Float64("progress", progress)) + logutil.DDLLogger().Info("Tiflash replica is available", zap.Int64("tableID", tb.ID), zap.Float64("progress", progress)) pollTiFlashContext.Backoff.Remove(tb.ID) } failpoint.Inject("skipUpdateTableReplicaInfoInLoop", func() { @@ -533,9 +533,9 @@ func (d *ddl) refreshTiFlashTicker(ctx sessionctx.Context, pollTiFlashContext *T if err := d.UpdateTableReplicaInfo(ctx, tb.ID, avail); err != nil { if infoschema.ErrTableNotExists.Equal(err) && tb.IsPartition { // May be due to blocking add partition - logutil.BgLogger().Info("updating TiFlash replica status err, maybe false alarm by blocking add", zap.Error(err), zap.Int64("tableID", tb.ID), zap.Bool("isPartition", tb.IsPartition)) + logutil.DDLLogger().Info("updating TiFlash replica status err, maybe false alarm by blocking add", zap.Error(err), zap.Int64("tableID", tb.ID), zap.Bool("isPartition", tb.IsPartition)) } else { - logutil.BgLogger().Error("updating TiFlash replica status err", zap.Error(err), zap.Int64("tableID", tb.ID), zap.Bool("isPartition", tb.IsPartition)) + logutil.DDLLogger().Error("updating TiFlash replica status err", zap.Error(err), zap.Int64("tableID", tb.ID), zap.Bool("isPartition", tb.IsPartition)) } } } else { @@ -551,7 +551,7 @@ func (d *ddl) refreshTiFlashTicker(ctx sessionctx.Context, pollTiFlashContext *T func (d *ddl) PollTiFlashRoutine() { pollTiflashContext, err := NewTiFlashManagementContext() if err != nil { - logutil.BgLogger().Fatal("TiFlashManagement init failed", zap.Error(err)) + logutil.DDLLogger().Fatal("TiFlashManagement init failed", zap.Error(err)) } hasSetTiFlashGroup := false @@ -564,7 +564,7 @@ func (d *ddl) PollTiFlashRoutine() { } if d.IsTiFlashPollEnabled() { if d.sessPool == nil { - logutil.BgLogger().Error("failed to get sessionPool for refreshTiFlashTicker") + logutil.DDLLogger().Error("failed to get sessionPool for refreshTiFlashTicker") return } failpoint.Inject("BeforeRefreshTiFlashTickeLoop", func() { @@ -575,7 +575,7 @@ func (d *ddl) PollTiFlashRoutine() { // We should set tiflash rule group a higher index than other placement groups to forbid override by them. // Once `SetTiFlashGroupConfig` succeed, we do not need to invoke it again. If failed, we should retry it util success. if err = infosync.SetTiFlashGroupConfig(d.ctx); err != nil { - logutil.BgLogger().Warn("SetTiFlashGroupConfig failed", zap.Error(err)) + logutil.DDLLogger().Warn("SetTiFlashGroupConfig failed", zap.Error(err)) nextSetTiFlashGroupTime = time.Now().Add(time.Minute) } else { hasSetTiFlashGroup = true @@ -591,7 +591,7 @@ func (d *ddl) PollTiFlashRoutine() { case *infosync.MockTiFlashError: // If we have not set up MockTiFlash instance, for those tests without TiFlash, just suppress. default: - logutil.BgLogger().Warn("refreshTiFlashTicker returns error", zap.Error(err)) + logutil.DDLLogger().Warn("refreshTiFlashTicker returns error", zap.Error(err)) } } } else { @@ -602,7 +602,7 @@ func (d *ddl) PollTiFlashRoutine() { if sctx != nil { d.sessPool.Put(sctx) } - logutil.BgLogger().Error("failed to get session for pollTiFlashReplicaStatus", zap.Error(err)) + logutil.DDLLogger().Error("failed to get session for pollTiFlashReplicaStatus", zap.Error(err)) } } } diff --git a/pkg/ddl/ddl_worker.go b/pkg/ddl/ddl_worker.go index 898da94134877..4fb7d5a716a96 100644 --- a/pkg/ddl/ddl_worker.go +++ b/pkg/ddl/ddl_worker.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/kvrpcpb" sess "github.com/pingcap/tidb/pkg/ddl/internal/session" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/ddl/util" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/meta" @@ -42,7 +43,7 @@ import ( pumpcli "github.com/pingcap/tidb/pkg/tidb-binlog/pump_client" tidbutil "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/dbterror" - "github.com/pingcap/tidb/pkg/util/logutil" + tidblogutil "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/resourcegrouptag" "github.com/pingcap/tidb/pkg/util/topsql" topsqlstate "github.com/pingcap/tidb/pkg/util/topsql/state" @@ -142,14 +143,14 @@ func newWorker(ctx context.Context, tp workerType, sessPool *sess.Pool, delRange delRangeManager: delRangeMgr, } worker.addingDDLJobKey = addingDDLJobPrefix + worker.typeStr() - worker.logCtx = logutil.WithFields(context.Background(), zap.String("worker", worker.String()), zap.String("category", "ddl")) + worker.logCtx = tidblogutil.WithFields(context.Background(), zap.String("worker", worker.String()), zap.String("category", "ddl")) return worker } func (w *worker) jobLogger(job *model.Job) *zap.Logger { - logger := logutil.Logger(w.logCtx) + logger := tidblogutil.Logger(w.logCtx) if job != nil { - logger = logutil.LoggerWithTraceInfo( + logger = tidblogutil.LoggerWithTraceInfo( logger.With(zap.Int64("jobID", job.ID)), job.TraceInfo, ) @@ -182,7 +183,7 @@ func (w *worker) Close() { w.sessPool.Put(w.sess.Session()) } w.wg.Wait() - logutil.Logger(w.logCtx).Info("DDL worker closed", zap.Duration("take time", time.Since(startTime))) + tidblogutil.Logger(w.logCtx).Info("DDL worker closed", zap.Duration("take time", time.Since(startTime))) } func (dc *ddlCtx) asyncNotifyByEtcd(etcdPath string, jobID int64, jobType string) { @@ -194,8 +195,11 @@ func (dc *ddlCtx) asyncNotifyByEtcd(etcdPath string, jobID int64, jobType string timeStart := time.Now() err := util.PutKVToEtcd(dc.ctx, dc.etcdCli, 1, etcdPath, jobIDStr) if err != nil { - logutil.BgLogger().Info("notify handling DDL job failed", zap.String("category", "ddl"), - zap.String("etcdPath", etcdPath), zap.Int64("jobID", jobID), zap.String("type", jobType), zap.Error(err)) + logutil.DDLLogger().Info("notify handling DDL job failed", + zap.String("etcdPath", etcdPath), + zap.Int64("jobID", jobID), + zap.String("type", jobType), + zap.Error(err)) } metrics.DDLWorkerHistogram.WithLabelValues(metrics.WorkerNotifyDDLJob, jobType, metrics.RetLabel(err)).Observe(time.Since(timeStart).Seconds()) } @@ -250,9 +254,12 @@ func (d *ddl) addBatchDDLJobsV1(tasks []*limitJobTask) { metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds()) } if err != nil { - logutil.BgLogger().Warn("add DDL jobs failed", zap.String("category", "ddl"), zap.String("jobs", jobs), zap.Error(err)) + logutil.DDLLogger().Warn("add DDL jobs failed", zap.String("jobs", jobs), zap.Error(err)) } else { - logutil.BgLogger().Info("add DDL jobs", zap.String("category", "ddl"), zap.Int("batch count", len(tasks)), zap.String("jobs", jobs), zap.Bool("table", toTable)) + logutil.DDLLogger().Info("add DDL jobs", + zap.Int("batch count", len(tasks)), + zap.String("jobs", jobs), + zap.Bool("table", toTable)) } } @@ -263,9 +270,11 @@ func (d *ddl) addBatchLocalDDLJobs(tasks []*limitJobTask) { for _, task := range tasks { task.NotifyError(err) } - logutil.BgLogger().Error("add DDL jobs failed", zap.String("category", "ddl"), zap.Bool("local_mode", true), zap.Error(err)) + logutil.DDLLogger().Error("add DDL jobs failed", zap.Bool("local_mode", true), zap.Error(err)) } else { - logutil.BgLogger().Info("add DDL jobs", zap.String("category", "ddl"), zap.Bool("local_mode", true), zap.Int("batch count", len(tasks))) + logutil.DDLLogger().Info("add DDL jobs", + zap.Bool("local_mode", true), + zap.Int("batch count", len(tasks))) } } @@ -292,7 +301,9 @@ func buildJobDependence(t *meta.Meta, curJob *model.Job) error { return errors.Trace(err) } if isDependent { - logutil.BgLogger().Info("current DDL job depends on other job", zap.String("category", "ddl"), zap.String("currentJob", curJob.String()), zap.String("dependentJob", job.String())) + logutil.DDLLogger().Info("current DDL job depends on other job", + zap.Stringer("currentJob", curJob), + zap.Stringer("dependentJob", job)) curJob.DependencyID = job.ID break } @@ -468,11 +479,11 @@ func (d *ddl) addBatchDDLJobs(tasks []*limitJobTask) error { // currently doesn't support pause job in local mode. if d.stateSyncer.IsUpgradingState() && !hasSysDB(job) && !job.LocalMode { if err = pauseRunningJob(sess.NewSession(se), job, model.AdminCommandBySystem); err != nil { - logutil.BgLogger().Warn("pause user DDL by system failed", zap.String("category", "ddl-upgrading"), zap.Stringer("job", job), zap.Error(err)) + logutil.DDLUpgradingLogger().Warn("pause user DDL by system failed", zap.Stringer("job", job), zap.Error(err)) task.cacheErr = err continue } - logutil.BgLogger().Info("pause user DDL by system successful", zap.String("category", "ddl-upgrading"), zap.Stringer("job", job)) + logutil.DDLUpgradingLogger().Info("pause user DDL by system successful", zap.Stringer("job", job)) } if _, err := job.Encode(true); err != nil { @@ -518,7 +529,7 @@ func (d *ddl) combineBatchCreateTableJobs(tasks []*limitJobTask) ([]*limitJobTas if err != nil { return tasks, err } - logutil.BgLogger().Info("combine jobs to batch create table job", zap.String("category", "ddl"), zap.Int("len", len(tasks))) + logutil.DDLLogger().Info("combine jobs to batch create table job", zap.Int("len", len(tasks))) jobTask := &limitJobTask{job, []chan error{}, nil} // combine the error chans. @@ -612,14 +623,14 @@ func cleanMDLInfo(pool *sess.Pool, jobID int64, ec *clientv3.Client, cleanETCD b se.GetSessionVars().SetDiskFullOpt(kvrpcpb.DiskFullOpt_AllowedOnAlmostFull) _, err := se.Execute(context.Background(), sql, "delete-mdl-info") if err != nil { - logutil.BgLogger().Warn("unexpected error when clean mdl info", zap.Int64("job ID", jobID), zap.Error(err)) + logutil.DDLLogger().Warn("unexpected error when clean mdl info", zap.Int64("job ID", jobID), zap.Error(err)) return } if cleanETCD && ec != nil { path := fmt.Sprintf("%s/%d/", util.DDLAllSchemaVersionsByJob, jobID) _, err = ec.Delete(context.Background(), path, clientv3.WithPrefix()) if err != nil { - logutil.BgLogger().Warn("delete versions failed", zap.String("category", "ddl"), zap.Int64("job ID", jobID), zap.Error(err)) + logutil.DDLLogger().Warn("delete versions failed", zap.Int64("job ID", jobID), zap.Error(err)) } } } @@ -1382,13 +1393,13 @@ func waitSchemaChanged(d *ddlCtx, waitTime time.Duration, latestSchemaVersion in }() if latestSchemaVersion == 0 { - logutil.Logger(d.ctx).Info("schema version doesn't change", zap.String("category", "ddl"), zap.Int64("jobID", job.ID)) + tidblogutil.Logger(d.ctx).Info("schema version doesn't change", zap.String("category", "ddl"), zap.Int64("jobID", job.ID)) return nil } err = d.schemaSyncer.OwnerUpdateGlobalVersion(d.ctx, latestSchemaVersion) if err != nil { - logutil.Logger(d.ctx).Info("update latest schema version failed", zap.String("category", "ddl"), zap.Int64("ver", latestSchemaVersion), zap.Error(err)) + tidblogutil.Logger(d.ctx).Info("update latest schema version failed", zap.String("category", "ddl"), zap.Int64("ver", latestSchemaVersion), zap.Error(err)) if variable.EnableMDL.Load() { return err } diff --git a/pkg/ddl/ddl_workerpool.go b/pkg/ddl/ddl_workerpool.go index c02383c9ae0ca..bee4b6d64235e 100644 --- a/pkg/ddl/ddl_workerpool.go +++ b/pkg/ddl/ddl_workerpool.go @@ -17,9 +17,8 @@ package ddl import ( "github.com/ngaut/pools" "github.com/pingcap/errors" - "github.com/pingcap/tidb/pkg/util/logutil" + "github.com/pingcap/tidb/pkg/ddl/logutil" "go.uber.org/atomic" - "go.uber.org/zap" ) // workerPool is used to new worker. @@ -80,7 +79,7 @@ func (wp *workerPool) close() { return } wp.exit.Store(true) - logutil.BgLogger().Info("closing workerPool", zap.String("category", "ddl")) + logutil.DDLLogger().Info("closing workerPool") wp.resPool.Close() } diff --git a/pkg/ddl/delete_range.go b/pkg/ddl/delete_range.go index 131efb10f9c1e..c7ea1781d2de4 100644 --- a/pkg/ddl/delete_range.go +++ b/pkg/ddl/delete_range.go @@ -25,13 +25,13 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/kvrpcpb" sess "github.com/pingcap/tidb/pkg/ddl/internal/session" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/ddl/util" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/parser/terror" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/tablecodec" - "github.com/pingcap/tidb/pkg/util/logutil" topsqlstate "github.com/pingcap/tidb/pkg/util/topsql/state" "go.uber.org/zap" ) @@ -103,13 +103,13 @@ func (dr *delRange) addDelRangeJob(ctx context.Context, job *model.Job) error { wrapper := newDelRangeExecWrapper(sctx) err = AddDelRangeJobInternal(ctx, wrapper, job) if err != nil { - logutil.BgLogger().Error("add job into delete-range table failed", zap.String("category", "ddl"), zap.Int64("jobID", job.ID), zap.String("jobType", job.Type.String()), zap.Error(err)) + logutil.DDLLogger().Error("add job into delete-range table failed", zap.Int64("jobID", job.ID), zap.String("jobType", job.Type.String()), zap.Error(err)) return errors.Trace(err) } if !dr.storeSupport { dr.emulatorCh <- struct{}{} } - logutil.BgLogger().Info("add job into delete-range table", zap.String("category", "ddl"), zap.Int64("jobID", job.ID), zap.String("jobType", job.Type.String())) + logutil.DDLLogger().Info("add job into delete-range table", zap.Int64("jobID", job.ID), zap.String("jobType", job.Type.String())) return nil } @@ -159,7 +159,7 @@ func (dr *delRange) start() { // clear implements delRangeManager interface. func (dr *delRange) clear() { - logutil.BgLogger().Info("closing delRange", zap.String("category", "ddl")) + logutil.DDLLogger().Info("closing delRange") close(dr.quitCh) dr.wait.Wait() } @@ -169,7 +169,7 @@ func (dr *delRange) clear() { // deletes all keys in each DelRangeTask. func (dr *delRange) startEmulator() { defer dr.wait.Done() - logutil.BgLogger().Info("start delRange emulator", zap.String("category", "ddl")) + logutil.DDLLogger().Info("start delRange emulator") for { select { case <-dr.emulatorCh: @@ -186,7 +186,7 @@ func (dr *delRange) startEmulator() { func (dr *delRange) doDelRangeWork() error { sctx, err := dr.sessPool.Get() if err != nil { - logutil.BgLogger().Error("delRange emulator get session failed", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Error("delRange emulator get session failed", zap.Error(err)) return errors.Trace(err) } defer dr.sessPool.Put(sctx) @@ -194,13 +194,13 @@ func (dr *delRange) doDelRangeWork() error { ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL) ranges, err := util.LoadDeleteRanges(ctx, sctx, math.MaxInt64) if err != nil { - logutil.BgLogger().Error("delRange emulator load tasks failed", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Error("delRange emulator load tasks failed", zap.Error(err)) return errors.Trace(err) } for _, r := range ranges { if err := dr.doTask(sctx, r); err != nil { - logutil.BgLogger().Error("delRange emulator do task failed", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Error("delRange emulator do task failed", zap.Error(err)) return errors.Trace(err) } } @@ -252,11 +252,11 @@ func (dr *delRange) doTask(sctx sessionctx.Context, r util.DelRangeTask) error { } if finish { if err := util.CompleteDeleteRange(sctx, r, true); err != nil { - logutil.BgLogger().Error("delRange emulator complete task failed", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Error("delRange emulator complete task failed", zap.Error(err)) return errors.Trace(err) } startKey, endKey := r.Range() - logutil.BgLogger().Info("delRange emulator complete task", zap.String("category", "ddl"), + logutil.DDLLogger().Info("delRange emulator complete task", zap.String("category", "ddl"), zap.Int64("jobID", r.JobID), zap.Int64("elementID", r.ElementID), zap.Stringer("startKey", startKey), @@ -264,7 +264,7 @@ func (dr *delRange) doTask(sctx sessionctx.Context, r util.DelRangeTask) error { break } if err := util.UpdateDeleteRange(sctx, r, newStartKey, oldStartKey); err != nil { - logutil.BgLogger().Error("delRange emulator update task failed", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Error("delRange emulator update task failed", zap.Error(err)) } oldStartKey = newStartKey } @@ -419,7 +419,7 @@ func insertJobIntoDeleteRangeTable(ctx context.Context, wrapper DelRangeExecWrap } func doBatchDeleteIndiceRange(ctx context.Context, wrapper DelRangeExecWrapper, jobID, tableID int64, indexIDs []int64, ea *elementIDAlloc, comment string) error { - logutil.BgLogger().Info("insert into delete-range indices", zap.String("category", "ddl"), zap.Int64("jobID", jobID), zap.Int64("tableID", tableID), zap.Int64s("indexIDs", indexIDs), zap.String("comment", comment)) + logutil.DDLLogger().Info("insert into delete-range indices", zap.Int64("jobID", jobID), zap.Int64("tableID", tableID), zap.Int64s("indexIDs", indexIDs), zap.String("comment", comment)) var buf strings.Builder buf.WriteString(insertDeleteRangeSQLPrefix) wrapper.PrepareParamsList(len(indexIDs) * 5) @@ -444,7 +444,7 @@ func doBatchDeleteIndiceRange(ctx context.Context, wrapper DelRangeExecWrapper, } func doBatchDeleteTablesRange(ctx context.Context, wrapper DelRangeExecWrapper, jobID int64, tableIDs []int64, ea *elementIDAlloc, comment string) error { - logutil.BgLogger().Info("insert into delete-range table", zap.String("category", "ddl"), zap.Int64("jobID", jobID), zap.Int64s("tableIDs", tableIDs), zap.String("comment", comment)) + logutil.DDLLogger().Info("insert into delete-range table", zap.Int64("jobID", jobID), zap.Int64s("tableIDs", tableIDs), zap.String("comment", comment)) var buf strings.Builder buf.WriteString(insertDeleteRangeSQLPrefix) wrapper.PrepareParamsList(len(tableIDs) * 5) diff --git a/pkg/ddl/index.go b/pkg/ddl/index.go index a48b01ab6646a..f0468554be25b 100644 --- a/pkg/ddl/index.go +++ b/pkg/ddl/index.go @@ -34,6 +34,7 @@ import ( "github.com/pingcap/tidb/pkg/ddl/copr" "github.com/pingcap/tidb/pkg/ddl/ingest" sess "github.com/pingcap/tidb/pkg/ddl/internal/session" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/disttask/framework/handle" "github.com/pingcap/tidb/pkg/disttask/framework/proto" "github.com/pingcap/tidb/pkg/disttask/framework/scheduler" @@ -61,7 +62,7 @@ import ( "github.com/pingcap/tidb/pkg/util/chunk" "github.com/pingcap/tidb/pkg/util/codec" "github.com/pingcap/tidb/pkg/util/dbterror" - "github.com/pingcap/tidb/pkg/util/logutil" + tidblogutil "github.com/pingcap/tidb/pkg/util/logutil" decoder "github.com/pingcap/tidb/pkg/util/rowDecoder" "github.com/pingcap/tidb/pkg/util/size" "github.com/pingcap/tidb/pkg/util/sqlexec" @@ -650,7 +651,7 @@ func (w *worker) onCreateIndex(d *ddlCtx, t *meta.Meta, job *model.Job, isPK boo job.State = model.JobStateCancelled return ver, err } - logutil.BgLogger().Info("run add index job", zap.String("category", "ddl"), zap.String("job", job.String()), zap.Reflect("indexInfo", indexInfo)) + logutil.DDLLogger().Info("run add index job", zap.Stringer("job", job), zap.Reflect("indexInfo", indexInfo)) } allIndexInfos = append(allIndexInfos, indexInfo) } @@ -777,7 +778,7 @@ SwitchIndexState: if !job.ReorgMeta.IsDistReorg && job.ReorgMeta.ReorgTp == model.ReorgTypeLitMerge { ingest.LitBackCtxMgr.Unregister(job.ID) } - logutil.BgLogger().Info("run add index job done", zap.String("category", "ddl"), + logutil.DDLLogger().Info("run add index job done", zap.String("charset", job.Charset), zap.String("collation", job.Collate)) default: @@ -813,7 +814,7 @@ func pickBackfillType(ctx context.Context, job *model.Job) (model.ReorgType, err } } // The lightning environment is unavailable, but we can still use the txn-merge backfill. - logutil.BgLogger().Info("fallback to txn-merge backfill process", zap.String("category", "ddl"), + logutil.DDLLogger().Info("fallback to txn-merge backfill process", zap.Bool("lightning env initialized", ingest.LitInitialized)) job.ReorgMeta.ReorgTp = model.ReorgTypeTxnMerge return model.ReorgTypeTxnMerge, nil @@ -853,7 +854,7 @@ func doReorgWorkForCreateIndex(w *worker, d *ddlCtx, t *meta.Meta, job *model.Jo } switch allIndexInfos[0].BackfillState { case model.BackfillStateRunning: - logutil.BgLogger().Info("index backfill state running", zap.String("category", "ddl"), + logutil.DDLLogger().Info("index backfill state running", zap.Int64("job ID", job.ID), zap.String("table", tbl.Meta().Name.O), zap.Bool("ingest mode", reorgTp == model.ReorgTypeLitMerge), zap.String("index", allIndexInfos[0].Name.O)) @@ -876,8 +877,10 @@ func doReorgWorkForCreateIndex(w *worker, d *ddlCtx, t *meta.Meta, job *model.Jo ver, err = updateVersionAndTableInfo(d, t, job, tbl.Meta(), true) return false, ver, errors.Trace(err) case model.BackfillStateReadyToMerge: - logutil.BgLogger().Info("index backfill state ready to merge", zap.String("category", "ddl"), zap.Int64("job ID", job.ID), - zap.String("table", tbl.Meta().Name.O), zap.String("index", allIndexInfos[0].Name.O)) + logutil.DDLLogger().Info("index backfill state ready to merge", + zap.Int64("job ID", job.ID), + zap.String("table", tbl.Meta().Name.O), + zap.String("index", allIndexInfos[0].Name.O)) for _, indexInfo := range allIndexInfos { indexInfo.BackfillState = model.BackfillStateMerging } @@ -921,7 +924,7 @@ func runIngestReorgJob(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job, if ok && bc.Done() { return true, 0, nil } - ctx := logutil.WithCategory(w.ctx, "ddl-ingest") + ctx := tidblogutil.WithCategory(w.ctx, "ddl-ingest") var discovery pd.ServiceDiscovery if d != nil { //nolint:forcetypeassert @@ -935,7 +938,7 @@ func runIngestReorgJob(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job, done, ver, err = runReorgJobAndHandleErr(w, d, t, job, tbl, allIndexInfos, false) if err != nil { if !errorIsRetryable(err, job) { - logutil.BgLogger().Warn("run reorg job failed, convert job to rollback", zap.String("category", "ddl"), + logutil.DDLLogger().Warn("run reorg job failed, convert job to rollback", zap.String("job", job.String()), zap.Error(err)) ver, err = convertAddIdxJob2RollbackJob(d, t, job, tbl.Meta(), allIndexInfos, err) } @@ -951,10 +954,10 @@ func runIngestReorgJob(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job, err = convertToKeyExistsErr(err, indexInfo, tbl.Meta()) } if kv.ErrKeyExists.Equal(err) { - logutil.BgLogger().Warn("import index duplicate key, convert job to rollback", zap.String("category", "ddl"), zap.String("job", job.String()), zap.Error(err)) + logutil.DDLLogger().Warn("import index duplicate key, convert job to rollback", zap.Stringer("job", job), zap.Error(err)) ver, err = convertAddIdxJob2RollbackJob(d, t, job, tbl.Meta(), allIndexInfos, err) } else { - logutil.BgLogger().Warn("lightning import error", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Warn("lightning import error", zap.Error(err)) if !errorIsRetryable(err, job) { ver, err = convertAddIdxJob2RollbackJob(d, t, job, tbl.Meta(), allIndexInfos, err) } @@ -1052,10 +1055,10 @@ func runReorgJobAndHandleErr(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job, err = convertToKeyExistsErr(err, allIndexInfos[0], tbl.Meta()) } if !errorIsRetryable(err, job) { - logutil.BgLogger().Warn("run add index job failed, convert job to rollback", zap.String("category", "ddl"), zap.String("job", job.String()), zap.Error(err)) + logutil.DDLLogger().Warn("run add index job failed, convert job to rollback", zap.Stringer("job", job), zap.Error(err)) ver, err = convertAddIdxJob2RollbackJob(d, t, job, tbl.Meta(), allIndexInfos, err) if err1 := rh.RemoveDDLReorgHandle(job, reorgInfo.elements); err1 != nil { - logutil.BgLogger().Warn("run add index job failed, convert job to rollback, RemoveDDLReorgHandle failed", zap.String("category", "ddl"), zap.String("job", job.String()), zap.Error(err1)) + logutil.DDLLogger().Warn("run add index job failed, convert job to rollback, RemoveDDLReorgHandle failed", zap.Stringer("job", job), zap.Error(err1)) } } return false, ver, errors.Trace(err) @@ -1358,7 +1361,7 @@ func newAddIndexTxnWorker( eleTypeKey []byte, ) (*addIndexTxnWorker, error) { if !bytes.Equal(eleTypeKey, meta.IndexElementKey) { - logutil.BgLogger().Error("Element type for addIndexTxnWorker incorrect", + logutil.DDLLogger().Error("Element type for addIndexTxnWorker incorrect", zap.Int64("job ID", jobID), zap.ByteString("element type", eleTypeKey), zap.Int64("element ID", elements[0].ID)) return nil, errors.Errorf("element type is not index, typeKey: %v", eleTypeKey) } @@ -1520,7 +1523,7 @@ func (w *baseIndexWorker) fetchRowColVals(txn kv.Transaction, taskRange reorgBac taskDone = true } - logutil.BgLogger().Debug("txn fetches handle info", zap.String("category", "ddl"), zap.Stringer("worker", w), zap.Uint64("txnStartTS", txn.StartTS()), + logutil.DDLLogger().Debug("txn fetches handle info", zap.Stringer("worker", w), zap.Uint64("txnStartTS", txn.StartTS()), zap.String("taskRange", taskRange.String()), zap.Duration("takeTime", time.Since(startTime))) return w.idxRecords, w.getNextKey(taskRange, taskDone), taskDone, errors.Trace(err) } @@ -1554,7 +1557,7 @@ func genKeyExistsErr(key, value []byte, idxInfo *model.IndexInfo, tblInfo *model indexName := fmt.Sprintf("%s.%s", tblInfo.Name.String(), idxInfo.Name.String()) valueStr, err := tables.GenIndexValueFromIndex(key, value, tblInfo, idxInfo) if err != nil { - logutil.BgLogger().Warn("decode index key value / column value failed", zap.String("index", indexName), + logutil.DDLLogger().Warn("decode index key value / column value failed", zap.String("index", indexName), zap.String("key", hex.EncodeToString(key)), zap.String("value", hex.EncodeToString(value)), zap.Error(err)) return errors.Trace(kv.ErrKeyExists.FastGenByArgs(key, indexName)) } @@ -1922,10 +1925,10 @@ var MockDMLExecutionStateBeforeImport func() func (w *worker) addPhysicalTableIndex(t table.PhysicalTable, reorgInfo *reorgInfo) error { if reorgInfo.mergingTmpIdx { - logutil.BgLogger().Info("start to merge temp index", zap.String("category", "ddl"), zap.String("job", reorgInfo.Job.String()), zap.String("reorgInfo", reorgInfo.String())) + logutil.DDLLogger().Info("start to merge temp index", zap.Stringer("job", reorgInfo.Job), zap.Stringer("reorgInfo", reorgInfo)) return w.writePhysicalTableRecord(w.sessPool, t, typeAddIndexMergeTmpWorker, reorgInfo) } - logutil.BgLogger().Info("start to add table index", zap.String("category", "ddl"), zap.String("job", reorgInfo.Job.String()), zap.String("reorgInfo", reorgInfo.String())) + logutil.DDLLogger().Info("start to add table index", zap.Stringer("job", reorgInfo.Job), zap.Stringer("reorgInfo", reorgInfo)) return w.writePhysicalTableRecord(w.sessPool, t, typeAddIndexWorker, reorgInfo) } @@ -1991,7 +1994,7 @@ func checkDuplicateForUniqueIndex(ctx context.Context, t table.Table, reorgInfo return errors.New("unexpected error, can't find index info") } if indexInfo.Unique { - ctx := logutil.WithCategory(ctx, "ddl-ingest") + ctx := tidblogutil.WithCategory(ctx, "ddl-ingest") if bc == nil { bc, err = ingest.LitBackCtxMgr.Register(ctx, reorgInfo.ID, indexInfo.Unique, nil, discovery, reorgInfo.ReorgMeta.ResourceGroupName) if err != nil { @@ -2050,16 +2053,15 @@ func (w *worker) executeDistTask(t table.Table, reorgInfo *reorgInfo) error { // It's possible that the task state is succeed but the ddl job is paused. // When task in succeed state, we can skip the dist task execution/scheduing process. if task.State == proto.TaskStateSucceed { - logutil.BgLogger().Info( + logutil.DDLLogger().Info( "task succeed, start to resume the ddl job", - zap.String("category", "ddl"), zap.String("task-key", taskKey)) return nil } g.Go(func() error { defer close(done) backoffer := backoff.NewExponential(scheduler.RetrySQLInterval, 2, scheduler.RetrySQLMaxInterval) - err := handle.RunWithRetry(ctx, scheduler.RetrySQLTimes, backoffer, logutil.BgLogger(), + err := handle.RunWithRetry(ctx, scheduler.RetrySQLTimes, backoffer, logutil.DDLLogger(), func(context.Context) (bool, error) { return true, handle.ResumeTask(w.ctx, taskKey) }, @@ -2070,7 +2072,7 @@ func (w *worker) executeDistTask(t table.Table, reorgInfo *reorgInfo) error { err = handle.WaitTaskDoneOrPaused(ctx, task.ID) if err := w.isReorgRunnable(reorgInfo.Job.ID, true); err != nil { if dbterror.ErrPausedDDLJob.Equal(err) { - logutil.BgLogger().Warn("job paused by user", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Warn("job paused by user", zap.Error(err)) return dbterror.ErrPausedDDLJob.GenWithStackByArgs(reorgInfo.Job.ID) } } @@ -2084,7 +2086,7 @@ func (w *worker) executeDistTask(t table.Table, reorgInfo *reorgInfo) error { return err } concurrency := min(workerCntLimit, cpuCount) - logutil.BgLogger().Info("adjusted add-index task concurrency", + logutil.DDLLogger().Info("adjusted add-index task concurrency", zap.Int("worker-cnt", workerCntLimit), zap.Int("task-concurrency", concurrency), zap.String("task-key", taskKey)) rowSize := estimateTableRowSize(w.ctx, w.store, w.sess.GetRestrictedSQLExecutor(), t) @@ -2109,7 +2111,7 @@ func (w *worker) executeDistTask(t table.Table, reorgInfo *reorgInfo) error { }) if err := w.isReorgRunnable(reorgInfo.Job.ID, true); err != nil { if dbterror.ErrPausedDDLJob.Equal(err) { - logutil.BgLogger().Warn("job paused by user", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Warn("job paused by user", zap.Error(err)) return dbterror.ErrPausedDDLJob.GenWithStackByArgs(reorgInfo.Job.ID) } } @@ -2131,7 +2133,7 @@ func (w *worker) executeDistTask(t table.Table, reorgInfo *reorgInfo) error { if err = w.isReorgRunnable(reorgInfo.Job.ID, true); err != nil { if dbterror.ErrPausedDDLJob.Equal(err) { if err = handle.PauseTask(w.ctx, taskKey); err != nil { - logutil.BgLogger().Error("pause task error", zap.String("category", "ddl"), zap.String("task_key", taskKey), zap.Error(err)) + logutil.DDLLogger().Error("pause task error", zap.String("task_key", taskKey), zap.Error(err)) continue } failpoint.Inject("syncDDLTaskPause", func() { @@ -2143,7 +2145,7 @@ func (w *worker) executeDistTask(t table.Table, reorgInfo *reorgInfo) error { return errors.Trace(err) } if err = handle.CancelTask(w.ctx, taskKey); err != nil { - logutil.BgLogger().Error("cancel task error", zap.String("category", "ddl"), zap.String("task_key", taskKey), zap.Error(err)) + logutil.DDLLogger().Error("cancel task error", zap.String("task_key", taskKey), zap.Error(err)) // continue to cancel task. continue } @@ -2173,7 +2175,7 @@ func estimateTableRowSize( defer util.Recover(metrics.LabelDDL, "estimateTableRowSize", nil, false) var gErr error defer func() { - logutil.Logger(ctx).Info("estimate row size", + tidblogutil.Logger(ctx).Info("estimate row size", zap.Int64("tableID", tbl.Meta().ID), zap.Int("size", sizeInBytes), zap.Error(gErr)) }() rows, _, err := exec.ExecRestrictedSQL(ctx, nil, @@ -2248,17 +2250,17 @@ func estimateRowSizeFromRegion(ctx context.Context, store kv.Storage, tbl table. func (w *worker) updateJobRowCount(taskKey string, jobID int64) { taskMgr, err := storage.GetTaskManager() if err != nil { - logutil.BgLogger().Warn("cannot get task manager", zap.String("category", "ddl"), zap.String("task_key", taskKey), zap.Error(err)) + logutil.DDLLogger().Warn("cannot get task manager", zap.String("task_key", taskKey), zap.Error(err)) return } task, err := taskMgr.GetTaskByKey(w.ctx, taskKey) if err != nil { - logutil.BgLogger().Warn("cannot get task", zap.String("category", "ddl"), zap.String("task_key", taskKey), zap.Error(err)) + logutil.DDLLogger().Warn("cannot get task", zap.String("task_key", taskKey), zap.Error(err)) return } rowCount, err := taskMgr.GetSubtaskRowCount(w.ctx, task.ID, proto.BackfillStepReadIndex) if err != nil { - logutil.BgLogger().Warn("cannot get subtask row count", zap.String("category", "ddl"), zap.String("task_key", taskKey), zap.Error(err)) + logutil.DDLLogger().Warn("cannot get subtask row count", zap.String("task_key", taskKey), zap.Error(err)) return } w.getReorgCtx(jobID).setRowCount(rowCount) @@ -2291,7 +2293,7 @@ func getNextPartitionInfo(reorg *reorgInfo, t table.PartitionedTable, currPhysic pid, err := findNextPartitionID(currPhysicalTableID, nextPartitionDefs) if err != nil { // Fatal error, should not run here. - logutil.BgLogger().Error("find next partition ID failed", zap.String("category", "ddl"), zap.Reflect("table", t), zap.Error(err)) + logutil.DDLLogger().Error("find next partition ID failed", zap.Reflect("table", t), zap.Error(err)) return 0, nil, nil, errors.Trace(err) } if pid == 0 { @@ -2346,7 +2348,7 @@ func updateReorgInfo(sessPool *sess.Pool, t table.PartitionedTable, reorg *reorg // Write the reorg info to store so the whole reorganize process can recover from panic. err = reorg.UpdateReorgMeta(reorg.StartKey, sessPool) - logutil.BgLogger().Info("job update reorgInfo", zap.String("category", "ddl"), + logutil.DDLLogger().Info("job update reorgInfo", zap.Int64("jobID", reorg.Job.ID), zap.Stringer("element", reorg.currElement), zap.Int64("partitionTableID", pid), @@ -2389,7 +2391,7 @@ func indexColumnSliceEqual(a, b []*model.IndexColumn) bool { return false } if len(a) == 0 { - logutil.BgLogger().Warn("admin repair table : index's columns length equal to 0", zap.String("category", "ddl")) + logutil.DDLLogger().Warn("admin repair table : index's columns length equal to 0") return true } // Accelerate the compare by eliminate index bound check. @@ -2475,7 +2477,7 @@ func (w *cleanUpIndexWorker) BackfillData(handleRange reorgBackfillTask) (taskCt // cleanupPhysicalTableIndex handles the drop partition reorganization state for a non-partitioned table or a partition. func (w *worker) cleanupPhysicalTableIndex(t table.PhysicalTable, reorgInfo *reorgInfo) error { - logutil.BgLogger().Info("start to clean up index", zap.String("category", "ddl"), zap.String("job", reorgInfo.Job.String()), zap.String("reorgInfo", reorgInfo.String())) + logutil.DDLLogger().Info("start to clean up index", zap.Stringer("job", reorgInfo.Job), zap.Stringer("reorgInfo", reorgInfo)) return w.writePhysicalTableRecord(w.sessPool, t, typeCleanUpIndexWorker, reorgInfo) } @@ -2533,7 +2535,7 @@ func (w *worker) updateReorgInfoForPartitions(t table.PartitionedTable, reorg *r // Write the reorg info to store so the whole reorganize process can recover from panic. err = reorg.UpdateReorgMeta(reorg.StartKey, w.sessPool) - logutil.BgLogger().Info("job update reorg info", zap.String("category", "ddl"), zap.Int64("jobID", reorg.Job.ID), + logutil.DDLLogger().Info("job update reorg info", zap.Int64("jobID", reorg.Job.ID), zap.Stringer("element", reorg.currElement), zap.Int64("partition table ID", pid), zap.String("start key", hex.EncodeToString(start)), zap.String("end key", hex.EncodeToString(end)), zap.Error(err)) diff --git a/pkg/ddl/index_merge_tmp.go b/pkg/ddl/index_merge_tmp.go index 8b39fdf0e2dc4..ba4619f3c8d33 100644 --- a/pkg/ddl/index_merge_tmp.go +++ b/pkg/ddl/index_merge_tmp.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/meta" "github.com/pingcap/tidb/pkg/parser/model" @@ -28,7 +29,6 @@ import ( "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/table/tables" "github.com/pingcap/tidb/pkg/tablecodec" - "github.com/pingcap/tidb/pkg/util/logutil" kvutil "github.com/tikv/client-go/v2/util" "go.uber.org/zap" ) @@ -379,7 +379,7 @@ func (w *mergeIndexWorker) fetchTempIndexVals( nextKey = lastKey } - logutil.BgLogger().Debug("merge temp index txn fetches handle info", zap.String("category", "ddl"), zap.Uint64("txnStartTS", txn.StartTS()), + logutil.DDLLogger().Debug("merge temp index txn fetches handle info", zap.Uint64("txnStartTS", txn.StartTS()), zap.String("taskRange", taskRange.String()), zap.Duration("takeTime", time.Since(startTime))) return w.tmpIdxRecords, nextKey.Next(), taskDone, errors.Trace(err) } diff --git a/pkg/ddl/ingest/BUILD.bazel b/pkg/ddl/ingest/BUILD.bazel index cf54254a7db40..570fafbecac3b 100644 --- a/pkg/ddl/ingest/BUILD.bazel +++ b/pkg/ddl/ingest/BUILD.bazel @@ -21,6 +21,7 @@ go_library( deps = [ "//pkg/config", "//pkg/ddl/internal/session", + "//pkg/ddl/logutil", "//pkg/ddl/util", "//pkg/kv", "//pkg/lightning/backend", diff --git a/pkg/ddl/ingest/backend_mgr.go b/pkg/ddl/ingest/backend_mgr.go index 537752479291f..5276184a4913e 100644 --- a/pkg/ddl/ingest/backend_mgr.go +++ b/pkg/ddl/ingest/backend_mgr.go @@ -25,6 +25,7 @@ import ( "time" "github.com/pingcap/failpoint" + ddllogutil "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/config" "github.com/pingcap/tidb/pkg/util/generic" @@ -96,7 +97,7 @@ func NewLitBackendCtxMgr(path string, memQuota uint64, getProcessingJobIDs Filte litDiskRoot.UpdateUsage() err := litDiskRoot.StartupCheck() if err != nil { - litLogger.Warn("ingest backfill may not be available", zap.Error(err)) + ddllogutil.DDLIngestLogger().Warn("ingest backfill may not be available", zap.Error(err)) } return mgr } @@ -105,7 +106,7 @@ func NewLitBackendCtxMgr(path string, memQuota uint64, getProcessingJobIDs Filte func (m *litBackendCtxMgr) CheckMoreTasksAvailable(ctx context.Context) (bool, error) { m.cleanupSortPath(ctx) if err := m.diskRoot.PreCheckUsage(); err != nil { - litLogger.Info("ingest backfill is not available", zap.Error(err)) + ddllogutil.DDLIngestLogger().Info("ingest backfill is not available", zap.Error(err)) return false, err } return true, nil @@ -244,7 +245,7 @@ func createLocalBackend( return nil, err } - litLogger.Info("create local backend for adding index", + ddllogutil.DDLIngestLogger().Info("create local backend for adding index", zap.String("sortDir", cfg.lightning.TikvImporter.SortedKVDir), zap.String("keyspaceName", cfg.keyspaceName)) // We disable the switch TiKV mode feature for now, diff --git a/pkg/ddl/ingest/checkpoint.go b/pkg/ddl/ingest/checkpoint.go index 17644c5334a95..cde9bcbbc20ea 100644 --- a/pkg/ddl/ingest/checkpoint.go +++ b/pkg/ddl/ingest/checkpoint.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/config" sess "github.com/pingcap/tidb/pkg/ddl/internal/session" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/ddl/util" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/meta" @@ -112,7 +113,7 @@ func NewCheckpointManager(ctx context.Context, flushCtrl FlushController, cm.updateCheckpointLoop() cm.updaterWg.Done() }() - litLogger.Info("create checkpoint manager", + logutil.DDLIngestLogger().Info("create checkpoint manager", zap.Int64("jobID", jobID), zap.Int64s("indexIDs", indexIDs)) return cm, nil } @@ -211,7 +212,7 @@ func (s *CheckpointManager) progressLocalSyncMinKey() { func (s *CheckpointManager) Close() { s.updaterExitCh <- struct{}{} s.updaterWg.Wait() - litLogger.Info("close checkpoint manager", + logutil.DDLIngestLogger().Info("close checkpoint manager", zap.Int64("jobID", s.jobID), zap.Int64s("indexIDs", s.indexIDs)) } @@ -219,7 +220,7 @@ func (s *CheckpointManager) Close() { func (s *CheckpointManager) Sync() { _, _, _, err := TryFlushAllIndexes(s.flushCtrl, FlushModeForceLocal, s.indexIDs) if err != nil { - litLogger.Warn("flush local engine failed", zap.Error(err)) + logutil.DDLIngestLogger().Warn("flush local engine failed", zap.Error(err)) } s.mu.Lock() s.progressLocalSyncMinKey() @@ -234,7 +235,7 @@ func (s *CheckpointManager) Sync() { func (s *CheckpointManager) Reset(newPhysicalID int64, start, end kv.Key) { s.mu.Lock() defer s.mu.Unlock() - litLogger.Info("reset checkpoint manager", + logutil.DDLIngestLogger().Info("reset checkpoint manager", zap.Int64("newPhysicalID", newPhysicalID), zap.Int64("oldPhysicalID", s.pidLocal), zap.Int64s("indexIDs", s.indexIDs), zap.Int64("jobID", s.jobID), zap.Int("localCnt", s.localCnt)) if s.pidLocal != newPhysicalID { @@ -308,7 +309,7 @@ func (s *CheckpointManager) resumeCheckpoint() error { s.minKeySyncLocal = cp.LocalSyncKey s.localCnt = cp.LocalKeyCount } - litLogger.Info("resume checkpoint", + logutil.DDLIngestLogger().Info("resume checkpoint", zap.Int64("job ID", s.jobID), zap.Int64s("index IDs", s.indexIDs), zap.String("local checkpoint", hex.EncodeToString(s.minKeySyncLocal)), zap.String("global checkpoint", hex.EncodeToString(s.minKeySyncGlobal)), @@ -317,7 +318,7 @@ func (s *CheckpointManager) resumeCheckpoint() error { zap.String("current instance", s.instanceAddr)) return nil } - litLogger.Info("checkpoint is empty", + logutil.DDLIngestLogger().Info("checkpoint is empty", zap.Int64("job ID", s.jobID), zap.Int64s("index IDs", s.indexIDs)) return nil }) @@ -374,7 +375,7 @@ func (s *CheckpointManager) updateCheckpoint() error { s.mu.Unlock() return nil }) - litLogger.Info("update checkpoint", + logutil.DDLIngestLogger().Info("update checkpoint", zap.Int64("job ID", s.jobID), zap.Int64s("index IDs", s.indexIDs), zap.String("local checkpoint", hex.EncodeToString(currentLocalKey)), zap.String("global checkpoint", hex.EncodeToString(currentGlobalKey)), @@ -391,7 +392,7 @@ func (s *CheckpointManager) updateCheckpointLoop() { case wg := <-s.updaterCh: err := s.updateCheckpoint() if err != nil { - litLogger.Error("update checkpoint failed", zap.Error(err)) + logutil.DDLIngestLogger().Error("update checkpoint failed", zap.Error(err)) } wg.Done() case <-ticker.C: @@ -403,7 +404,7 @@ func (s *CheckpointManager) updateCheckpointLoop() { s.mu.Unlock() err := s.updateCheckpoint() if err != nil { - litLogger.Error("update checkpoint failed", zap.Error(err)) + logutil.DDLIngestLogger().Error("update checkpoint failed", zap.Error(err)) } case <-s.updaterExitCh: return diff --git a/pkg/ddl/ingest/disk_root.go b/pkg/ddl/ingest/disk_root.go index fa450305f1c3f..90e3fa4f62922 100644 --- a/pkg/ddl/ingest/disk_root.go +++ b/pkg/ddl/ingest/disk_root.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/pkg/ddl/logutil" lcom "github.com/pingcap/tidb/pkg/lightning/common" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/util/dbterror" @@ -67,7 +68,7 @@ func (d *diskRootImpl) UpdateUsage() { var capacity, used uint64 sz, err := lcom.GetStorageSize(d.path) if err != nil { - litLogger.Error(LitErrGetStorageQuota, zap.Error(err)) + logutil.DDLIngestLogger().Error(LitErrGetStorageQuota, zap.Error(err)) } else { capacity, used = sz.Capacity, sz.Capacity-sz.Available } @@ -84,7 +85,7 @@ func (d *diskRootImpl) ShouldImport() bool { d.mu.RLock() defer d.mu.RUnlock() if d.bcUsed > variable.DDLDiskQuota.Load() { - litLogger.Info("disk usage is over quota", + logutil.DDLIngestLogger().Info("disk usage is over quota", zap.Uint64("quota", variable.DDLDiskQuota.Load()), zap.String("usage", d.usageInfo())) return true @@ -93,7 +94,7 @@ func (d *diskRootImpl) ShouldImport() bool { return false } if float64(d.used) >= float64(d.capacity)*capacityThreshold { - litLogger.Warn("available disk space is less than 10%, "+ + logutil.DDLIngestLogger().Warn("available disk space is less than 10%, "+ "this may degrade the performance, "+ "please make sure the disk available space is larger than @@tidb_ddl_disk_quota before adding index", zap.String("usage", d.usageInfo())) @@ -127,7 +128,7 @@ func (d *diskRootImpl) PreCheckUsage() error { return dbterror.ErrIngestCheckEnvFailed.FastGenByArgs(err.Error()) } if RiskOfDiskFull(sz.Available, sz.Capacity) { - litLogger.Warn("available disk space is less than 10%, cannot use ingest mode", + logutil.DDLIngestLogger().Warn("available disk space is less than 10%, cannot use ingest mode", zap.String("sort path", d.path), zap.String("usage", d.usageInfo())) msg := fmt.Sprintf("no enough space in %s", d.path) diff --git a/pkg/ddl/ingest/env.go b/pkg/ddl/ingest/env.go index 46457e45be3b2..c74d7f69faadd 100644 --- a/pkg/ddl/ingest/env.go +++ b/pkg/ddl/ingest/env.go @@ -20,17 +20,15 @@ import ( "strconv" "github.com/pingcap/tidb/pkg/config" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/lightning/log" "github.com/pingcap/tidb/pkg/util" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/memory" "github.com/pingcap/tidb/pkg/util/size" "go.uber.org/zap" ) var ( - litLogger = logutil.BgLogger().With(zap.String(logutil.LogFieldCategory, "ddl-ingest")) - // LitBackCtxMgr is the entry for the lightning backfill process. LitBackCtxMgr BackendCtxMgr // LitMemRoot is used to track the memory usage of the lightning backfill process. @@ -47,10 +45,10 @@ const defaultMemoryQuota = 2 * size.GB // InitGlobalLightningEnv initialize Lightning backfill environment. func InitGlobalLightningEnv(filterProcessingJobIDs FilterProcessingJobIDsFunc) { - log.SetAppLogger(litLogger) + log.SetAppLogger(logutil.DDLIngestLogger()) globalCfg := config.GetGlobalConfig() if globalCfg.Store != "tikv" { - litLogger.Warn(LitWarnEnvInitFail, + logutil.DDLIngestLogger().Warn(LitWarnEnvInitFail, zap.String("storage limitation", "only support TiKV storage"), zap.String("current storage", globalCfg.Store), zap.Bool("lightning is initialized", LitInitialized)) @@ -58,14 +56,14 @@ func InitGlobalLightningEnv(filterProcessingJobIDs FilterProcessingJobIDsFunc) { } sortPath, err := genLightningDataDir() if err != nil { - litLogger.Warn(LitWarnEnvInitFail, + logutil.DDLIngestLogger().Warn(LitWarnEnvInitFail, zap.Error(err), zap.Bool("lightning is initialized", LitInitialized)) return } memTotal, err := memory.MemTotal() if err != nil { - litLogger.Warn("get total memory fail", zap.Error(err)) + logutil.DDLIngestLogger().Warn("get total memory fail", zap.Error(err)) memTotal = defaultMemoryQuota } else { memTotal = memTotal / 2 @@ -73,7 +71,7 @@ func InitGlobalLightningEnv(filterProcessingJobIDs FilterProcessingJobIDsFunc) { LitBackCtxMgr = NewLitBackendCtxMgr(sortPath, memTotal, filterProcessingJobIDs) litRLimit = util.GenRLimit("ddl-ingest") LitInitialized = true - litLogger.Info(LitInfoEnvInitSucc, + logutil.DDLIngestLogger().Info(LitInfoEnvInitSucc, zap.Uint64("memory limitation", memTotal), zap.String("disk usage info", litDiskRoot.UsageInfo()), zap.Uint64("max open file number", litRLimit), @@ -89,18 +87,18 @@ func genLightningDataDir() (string, error) { if _, err := os.Stat(sortPath); err != nil { if !os.IsNotExist(err) { - litLogger.Error(LitErrStatDirFail, + logutil.DDLIngestLogger().Error(LitErrStatDirFail, zap.String("sort path", sortPath), zap.Error(err)) return "", err } } err := os.MkdirAll(sortPath, 0o700) if err != nil { - litLogger.Error(LitErrCreateDirFail, + logutil.DDLIngestLogger().Error(LitErrCreateDirFail, zap.String("sort path", sortPath), zap.Error(err)) return "", err } - litLogger.Info(LitInfoSortDir, zap.String("data path", sortPath)) + logutil.DDLIngestLogger().Info(LitInfoSortDir, zap.String("data path", sortPath)) return sortPath, nil } diff --git a/pkg/ddl/ingest/mock.go b/pkg/ddl/ingest/mock.go index acc43bf06d7a8..8829a6ca7c2fa 100644 --- a/pkg/ddl/ingest/mock.go +++ b/pkg/ddl/ingest/mock.go @@ -19,11 +19,11 @@ import ( "encoding/hex" "sync" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/table" - "github.com/pingcap/tidb/pkg/util/logutil" pd "github.com/tikv/pd/client" clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" @@ -50,7 +50,7 @@ func (m *MockBackendCtxMgr) CheckMoreTasksAvailable(context.Context) (bool, erro // Register implements BackendCtxMgr.Register interface. func (m *MockBackendCtxMgr) Register(ctx context.Context, jobID int64, unique bool, etcdClient *clientv3.Client, pdSvcDiscovery pd.ServiceDiscovery, resourceGroupName string) (BackendCtx, error) { - logutil.BgLogger().Info("mock backend mgr register", zap.Int64("jobID", jobID)) + logutil.DDLIngestLogger().Info("mock backend mgr register", zap.Int64("jobID", jobID)) if mockCtx, ok := m.runningJobs[jobID]; ok { return mockCtx, nil } @@ -68,7 +68,7 @@ func (m *MockBackendCtxMgr) Unregister(jobID int64) { if mCtx, ok := m.runningJobs[jobID]; ok { mCtx.sessCtx.StmtCommit(context.Background()) err := mCtx.sessCtx.CommitTxn(context.Background()) - logutil.BgLogger().Info("mock backend mgr unregister", zap.Int64("jobID", jobID), zap.Error(err)) + logutil.DDLIngestLogger().Info("mock backend mgr unregister", zap.Int64("jobID", jobID), zap.Error(err)) delete(m.runningJobs, jobID) if mCtx.checkpointMgr != nil { mCtx.checkpointMgr.Close() @@ -78,7 +78,7 @@ func (m *MockBackendCtxMgr) Unregister(jobID int64) { // Load implements BackendCtxMgr.Load interface. func (m *MockBackendCtxMgr) Load(jobID int64) (BackendCtx, bool) { - logutil.BgLogger().Info("mock backend mgr load", zap.Int64("jobID", jobID)) + logutil.DDLIngestLogger().Info("mock backend mgr load", zap.Int64("jobID", jobID)) if mockCtx, ok := m.runningJobs[jobID]; ok { return mockCtx, true } @@ -101,24 +101,24 @@ type MockBackendCtx struct { // Register implements BackendCtx.Register interface. func (m *MockBackendCtx) Register(jobID, indexID int64, _, _ string) (Engine, error) { - logutil.BgLogger().Info("mock backend ctx register", zap.Int64("jobID", jobID), zap.Int64("indexID", indexID)) + logutil.DDLIngestLogger().Info("mock backend ctx register", zap.Int64("jobID", jobID), zap.Int64("indexID", indexID)) return &MockEngineInfo{sessCtx: m.sessCtx, mu: &m.mu}, nil } // Unregister implements BackendCtx.Unregister interface. func (*MockBackendCtx) Unregister(jobID, indexID int64) { - logutil.BgLogger().Info("mock backend ctx unregister", zap.Int64("jobID", jobID), zap.Int64("indexID", indexID)) + logutil.DDLIngestLogger().Info("mock backend ctx unregister", zap.Int64("jobID", jobID), zap.Int64("indexID", indexID)) } // CollectRemoteDuplicateRows implements BackendCtx.CollectRemoteDuplicateRows interface. func (*MockBackendCtx) CollectRemoteDuplicateRows(indexID int64, _ table.Table) error { - logutil.BgLogger().Info("mock backend ctx collect remote duplicate rows", zap.Int64("indexID", indexID)) + logutil.DDLIngestLogger().Info("mock backend ctx collect remote duplicate rows", zap.Int64("indexID", indexID)) return nil } // FinishImport implements BackendCtx.FinishImport interface. func (*MockBackendCtx) FinishImport(indexID int64, _ bool, _ table.Table) error { - logutil.BgLogger().Info("mock backend ctx finish import", zap.Int64("indexID", indexID)) + logutil.DDLIngestLogger().Info("mock backend ctx finish import", zap.Int64("indexID", indexID)) return nil } @@ -195,7 +195,7 @@ func (m *MockEngineInfo) SetHook(onWrite func(key, val []byte)) { // CreateWriter implements Engine.CreateWriter interface. func (m *MockEngineInfo) CreateWriter(id int) (Writer, error) { - logutil.BgLogger().Info("mock engine info create writer", zap.Int("id", id)) + logutil.DDLIngestLogger().Info("mock engine info create writer", zap.Int("id", id)) return &MockWriter{sessCtx: m.sessCtx, mu: m.mu, onWrite: m.onWrite}, nil } @@ -208,7 +208,7 @@ type MockWriter struct { // WriteRow implements Writer.WriteRow interface. func (m *MockWriter) WriteRow(_ context.Context, key, idxVal []byte, _ kv.Handle) error { - logutil.BgLogger().Info("mock writer write row", + logutil.DDLIngestLogger().Info("mock writer write row", zap.String("key", hex.EncodeToString(key)), zap.String("idxVal", hex.EncodeToString(idxVal))) m.mu.Lock() diff --git a/pkg/ddl/internal/session/BUILD.bazel b/pkg/ddl/internal/session/BUILD.bazel index b929e0f3cfdb7..62c006a7b1e4c 100644 --- a/pkg/ddl/internal/session/BUILD.bazel +++ b/pkg/ddl/internal/session/BUILD.bazel @@ -9,6 +9,7 @@ go_library( importpath = "github.com/pingcap/tidb/pkg/ddl/internal/session", visibility = ["//pkg/ddl:__subpackages__"], deps = [ + "//pkg/ddl/logutil", "//pkg/domain/infosync", "//pkg/kv", "//pkg/metrics", @@ -17,13 +18,11 @@ go_library( "//pkg/sessionctx", "//pkg/sessiontxn", "//pkg/util/chunk", - "//pkg/util/logutil", "//pkg/util/mock", "//pkg/util/sqlexec", "@com_github_ngaut_pools//:pools", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", - "@org_uber_go_zap//:zap", ], ) diff --git a/pkg/ddl/internal/session/session_pool.go b/pkg/ddl/internal/session/session_pool.go index 4767ed3c11cd2..66b8cef0372b8 100644 --- a/pkg/ddl/internal/session/session_pool.go +++ b/pkg/ddl/internal/session/session_pool.go @@ -20,13 +20,12 @@ import ( "github.com/ngaut/pools" "github.com/pingcap/errors" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/sessionctx" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/mock" - "go.uber.org/zap" ) // Pool is used to new Session. @@ -97,7 +96,7 @@ func (sg *Pool) Close() { if sg.mu.closed || sg.resPool == nil { return } - logutil.BgLogger().Info("closing session pool", zap.String("category", "ddl")) + logutil.DDLLogger().Info("closing session pool") sg.resPool.Close() sg.mu.closed = true } diff --git a/pkg/ddl/job_table.go b/pkg/ddl/job_table.go index e7ebe5b5936b6..64b74ed9b0aef 100644 --- a/pkg/ddl/job_table.go +++ b/pkg/ddl/job_table.go @@ -30,6 +30,7 @@ import ( "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/tidb/pkg/ddl/ingest" sess "github.com/pingcap/tidb/pkg/ddl/internal/session" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/ddl/syncer" "github.com/pingcap/tidb/pkg/ddl/util" "github.com/pingcap/tidb/pkg/kv" @@ -43,7 +44,7 @@ import ( tidb_util "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/dbterror" "github.com/pingcap/tidb/pkg/util/intest" - "github.com/pingcap/tidb/pkg/util/logutil" + tidblogutil "github.com/pingcap/tidb/pkg/util/logutil" clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) @@ -129,10 +130,10 @@ func (d *ddl) getJob(se *sess.Session, tp jobType, filter func(*model.Job) (bool } if b { if err = d.markJobProcessing(se, &job); err != nil { - logutil.BgLogger().Warn( + logutil.DDLLogger().Warn( "[ddl] handle ddl job failed: mark job is processing meet error", zap.Error(err), - zap.String("job", job.String())) + zap.Stringer("job", &job)) return nil, errors.Trace(err) } return &job, nil @@ -169,13 +170,13 @@ func (d *ddl) processJobDuringUpgrade(sess *sess.Session, job *model.Job) (isRun if err != nil { isCannotPauseDDLJobErr := dbterror.ErrCannotPauseDDLJob.Equal(err) - logutil.BgLogger().Warn("pause the job failed", zap.String("category", "ddl-upgrading"), zap.Stringer("job", job), + logutil.DDLUpgradingLogger().Warn("pause the job failed", zap.Stringer("job", job), zap.Bool("isRunnable", isCannotPauseDDLJobErr), zap.Error(err)) if isCannotPauseDDLJobErr { return true, nil } } else { - logutil.BgLogger().Warn("pause the job successfully", zap.String("category", "ddl-upgrading"), zap.Stringer("job", job)) + logutil.DDLUpgradingLogger().Warn("pause the job successfully", zap.Stringer("job", job)) } return false, nil @@ -185,14 +186,14 @@ func (d *ddl) processJobDuringUpgrade(sess *sess.Session, job *model.Job) (isRun var errs []error errs, err = ResumeJobsBySystem(sess.Session(), []int64{job.ID}) if len(errs) > 0 && errs[0] != nil { - logutil.BgLogger().Warn("normal cluster state, resume the job failed", zap.String("category", "ddl-upgrading"), zap.Stringer("job", job), zap.Error(errs[0])) + logutil.DDLUpgradingLogger().Warn("normal cluster state, resume the job failed", zap.Stringer("job", job), zap.Error(errs[0])) return false, errs[0] } if err != nil { - logutil.BgLogger().Warn("normal cluster state, resume the job failed", zap.String("category", "ddl-upgrading"), zap.Stringer("job", job), zap.Error(err)) + logutil.DDLUpgradingLogger().Warn("normal cluster state, resume the job failed", zap.Stringer("job", job), zap.Error(err)) return false, err } - logutil.BgLogger().Warn("normal cluster state, resume the job successfully", zap.String("category", "ddl-upgrading"), zap.Stringer("job", job)) + logutil.DDLUpgradingLogger().Warn("normal cluster state, resume the job successfully", zap.Stringer("job", job)) return false, errors.Errorf("system paused job:%d need to be resumed", job.ID) } @@ -257,7 +258,7 @@ func (d *ddl) startLocalWorkerLoop() { func (d *ddl) startDispatchLoop() { sessCtx, err := d.sessPool.Get() if err != nil { - logutil.BgLogger().Fatal("dispatch loop get session failed, it should not happen, please try restart TiDB", zap.Error(err)) + logutil.DDLLogger().Fatal("dispatch loop get session failed, it should not happen, please try restart TiDB", zap.Error(err)) } defer d.sessPool.Put(sessCtx) se := sess.NewSession(sessCtx) @@ -266,7 +267,7 @@ func (d *ddl) startDispatchLoop() { notifyDDLJobByEtcdCh = d.etcdCli.Watch(d.ctx, addingDDLJobConcurrent) } if err := d.checkAndUpdateClusterState(true); err != nil { - logutil.BgLogger().Fatal("dispatch loop get cluster state failed, it should not happen, please try restart TiDB", zap.Error(err)) + logutil.DDLLogger().Fatal("dispatch loop get cluster state failed, it should not happen, please try restart TiDB", zap.Error(err)) } ticker := time.NewTicker(dispatchLoopWaitingDuration) defer ticker.Stop() @@ -285,7 +286,7 @@ func (d *ddl) startDispatchLoop() { if ingest.ResignOwnerForTest.Load() { err2 := d.ownerManager.ResignOwner(context.Background()) if err2 != nil { - logutil.BgLogger().Info("resign meet error", zap.Error(err2)) + logutil.DDLLogger().Info("resign meet error", zap.Error(err2)) } ingest.ResignOwnerForTest.Store(false) } @@ -295,7 +296,7 @@ func (d *ddl) startDispatchLoop() { case <-ticker.C: case _, ok := <-notifyDDLJobByEtcdCh: if !ok { - logutil.BgLogger().Warn("start worker watch channel closed", zap.String("category", "ddl"), zap.String("watch key", addingDDLJobConcurrent)) + logutil.DDLLogger().Warn("start worker watch channel closed", zap.String("watch key", addingDDLJobConcurrent)) notifyDDLJobByEtcdCh = d.etcdCli.Watch(d.ctx, addingDDLJobConcurrent) time.Sleep(time.Second) continue @@ -327,10 +328,10 @@ func (d *ddl) checkAndUpdateClusterState(needUpdate bool) error { oldState := d.stateSyncer.IsUpgradingState() stateInfo, err := d.stateSyncer.GetGlobalState(d.ctx) if err != nil { - logutil.BgLogger().Warn("get global state failed", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Warn("get global state failed", zap.Error(err)) return errors.Trace(err) } - logutil.BgLogger().Info("get global state and global state change", zap.String("category", "ddl"), + logutil.DDLLogger().Info("get global state and global state change", zap.Bool("oldState", oldState), zap.Bool("currState", d.stateSyncer.IsUpgradingState())) if !d.isOwner() { return nil @@ -342,17 +343,17 @@ func (d *ddl) checkAndUpdateClusterState(needUpdate bool) error { } err = d.ownerManager.SetOwnerOpValue(d.ctx, ownerOp) if err != nil { - logutil.BgLogger().Warn("the owner sets global state to owner operator value failed", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Warn("the owner sets global state to owner operator value failed", zap.Error(err)) return errors.Trace(err) } - logutil.BgLogger().Info("the owner sets owner operator value", zap.String("category", "ddl"), zap.Stringer("ownerOp", ownerOp)) + logutil.DDLLogger().Info("the owner sets owner operator value", zap.Stringer("ownerOp", ownerOp)) return nil } func (d *ddl) loadDDLJobAndRun(se *sess.Session, pool *workerPool, getJob func(*sess.Session) (*model.Job, error)) { wk, err := pool.get() if err != nil || wk == nil { - logutil.BgLogger().Debug(fmt.Sprintf("[ddl] no %v worker available now", pool.tp()), zap.Error(err)) + logutil.DDLLogger().Debug(fmt.Sprintf("[ddl] no %v worker available now", pool.tp()), zap.Error(err)) return } @@ -407,7 +408,7 @@ func (d *ddl) delivery2LocalWorker(pool *workerPool, task *limitJobTask) { err := wk.HandleLocalDDLJob(d.ddlCtx, job) pool.put(wk) if err != nil { - logutil.BgLogger().Info("handle ddl job failed", zap.String("category", "ddl"), zap.Error(err), zap.String("job", job.String())) + logutil.DDLLogger().Info("handle ddl job failed", zap.Error(err), zap.Stringer("job", job)) } task.NotifyError(err) }) @@ -461,7 +462,7 @@ func (d *ddl) delivery2Worker(wk *worker, pool *workerPool, job *model.Job) { logCtx := wk.logCtx pool.put(wk) if err != nil { - logutil.Logger(logCtx).Info("handle ddl job failed", zap.String("category", "ddl"), zap.Error(err), zap.String("job", job.String())) + tidblogutil.Logger(logCtx).Info("handle ddl job failed", zap.Error(err), zap.Stringer("job", job)) } else { failpoint.Inject("mockDownBeforeUpdateGlobalVersion", func(val failpoint.Value) { if val.(bool) { @@ -553,7 +554,7 @@ func insertDDLJobs2Table(se *sess.Session, updateRawArgs bool, jobs ...*model.Jo se.GetSessionVars().SetDiskFullOpt(kvrpcpb.DiskFullOpt_AllowedOnAlmostFull) ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnDDL) _, err := se.Execute(ctx, sql.String(), "insert_job") - logutil.BgLogger().Debug("add job to mysql.tidb_ddl_job table", zap.String("category", "ddl"), zap.String("sql", sql.String())) + logutil.DDLLogger().Debug("add job to mysql.tidb_ddl_job table", zap.String("sql", sql.String())) return errors.Trace(err) } @@ -656,7 +657,7 @@ func getCheckpointReorgHandle(se *sess.Session, job *model.Job) (startKey, endKe return nil, nil, 0, errors.Trace(err) } if cp := reorgMeta.Checkpoint; cp != nil { - logutil.BgLogger().Info("resume physical table ID from checkpoint", zap.String("category", "ddl-ingest"), + logutil.DDLIngestLogger().Info("resume physical table ID from checkpoint", zap.Int64("jobID", job.ID), zap.String("start", hex.EncodeToString(cp.StartKey)), zap.String("end", hex.EncodeToString(cp.EndKey)), @@ -701,7 +702,7 @@ func initDDLReorgHandle(s *sess.Session, jobID int64, startKey kv.Key, endKey kv return s.RunInTxn(func(se *sess.Session) error { _, err := se.Execute(context.Background(), del, "init_handle") if err != nil { - logutil.BgLogger().Info("initDDLReorgHandle failed to delete", zap.Int64("jobID", jobID), zap.Error(err)) + logutil.DDLLogger().Info("initDDLReorgHandle failed to delete", zap.Int64("jobID", jobID), zap.Error(err)) } _, err = se.Execute(context.Background(), ins, "init_handle") return err diff --git a/pkg/ddl/logutil/BUILD.bazel b/pkg/ddl/logutil/BUILD.bazel new file mode 100644 index 0000000000000..6226c1609fd4f --- /dev/null +++ b/pkg/ddl/logutil/BUILD.bazel @@ -0,0 +1,12 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "logutil", + srcs = ["logutil.go"], + importpath = "github.com/pingcap/tidb/pkg/ddl/logutil", + visibility = ["//visibility:public"], + deps = [ + "//pkg/util/logutil", + "@org_uber_go_zap//:zap", + ], +) diff --git a/pkg/ddl/logutil/logutil.go b/pkg/ddl/logutil/logutil.go new file mode 100644 index 0000000000000..3d7c4ff5675f2 --- /dev/null +++ b/pkg/ddl/logutil/logutil.go @@ -0,0 +1,40 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logutil + +import ( + "github.com/pingcap/tidb/pkg/util/logutil" + "go.uber.org/zap" +) + +// DDLLogger with category "ddl" is used to log DDL related messages. Do not use +// it to log the message that is not related to DDL. +func DDLLogger() *zap.Logger { + return logutil.BgLogger().With(zap.String(logutil.LogFieldCategory, "ddl")) +} + +// DDLUpgradingLogger with category "ddl-upgrading" is used to log DDL related +// messages during the upgrading process. Do not use it to log the message that +// is not related to DDL. +func DDLUpgradingLogger() *zap.Logger { + return logutil.BgLogger().With(zap.String(logutil.LogFieldCategory, "ddl-upgrading")) +} + +// DDLIngestLogger with category "ddl-ingest" is used to log DDL related messages +// during the fast DDL process. Do not use it to log the message that is not +// related to DDL. +func DDLIngestLogger() *zap.Logger { + return logutil.BgLogger().With(zap.String(logutil.LogFieldCategory, "ddl-ingest")) +} diff --git a/pkg/ddl/partition.go b/pkg/ddl/partition.go index de9fa7ebbe962..15ea2271e73e6 100644 --- a/pkg/ddl/partition.go +++ b/pkg/ddl/partition.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" sess "github.com/pingcap/tidb/pkg/ddl/internal/session" "github.com/pingcap/tidb/pkg/ddl/label" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/ddl/placement" "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/expression" @@ -57,7 +58,6 @@ import ( "github.com/pingcap/tidb/pkg/util/collate" "github.com/pingcap/tidb/pkg/util/dbterror" "github.com/pingcap/tidb/pkg/util/hack" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/mathutil" "github.com/pingcap/tidb/pkg/util/mock" decoder "github.com/pingcap/tidb/pkg/util/rowDecoder" @@ -148,7 +148,7 @@ func (w *worker) onAddTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (v if tblInfo.TiFlashReplica != nil { // Must set placement rule, and make sure it succeeds. if err := infosync.ConfigureTiFlashPDForPartitions(true, &tblInfo.Partition.AddingDefinitions, tblInfo.TiFlashReplica.Count, &tblInfo.TiFlashReplica.LocationLabels, tblInfo.ID); err != nil { - logutil.BgLogger().Error("ConfigureTiFlashPDForPartitions fails", zap.Error(err)) + logutil.DDLLogger().Error("ConfigureTiFlashPDForPartitions fails", zap.Error(err)) return ver, errors.Trace(err) } } @@ -206,7 +206,7 @@ func (w *worker) onAddTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (v err = infosync.UpdateTiFlashProgressCache(d.ID, 1) if err != nil { // just print log, progress will be updated in `refreshTiFlashTicker` - logutil.BgLogger().Error("update tiflash sync progress cache failed", + logutil.DDLLogger().Error("update tiflash sync progress cache failed", zap.Error(err), zap.Int64("tableID", tblInfo.ID), zap.Int64("partitionID", d.ID), @@ -458,11 +458,11 @@ func checkPartitionReplica(replicaCount uint64, addingDefinitions []model.Partit continue } needWait = true - logutil.BgLogger().Info("partition replicas check failed in replica-only DDL state", zap.String("category", "ddl"), zap.Int64("pID", pd.ID), zap.Uint64("wait region ID", region.Meta.Id), zap.Bool("tiflash peer at least one", tiflashPeerAtLeastOne), zap.Time("check time", time.Now())) + logutil.DDLLogger().Info("partition replicas check failed in replica-only DDL state", zap.Int64("pID", pd.ID), zap.Uint64("wait region ID", region.Meta.Id), zap.Bool("tiflash peer at least one", tiflashPeerAtLeastOne), zap.Time("check time", time.Now())) return needWait, nil } } - logutil.BgLogger().Info("partition replicas check ok in replica-only DDL state", zap.String("category", "ddl")) + logutil.DDLLogger().Info("partition replicas check ok in replica-only DDL state") return needWait, nil } @@ -2411,7 +2411,7 @@ func clearTruncatePartitionTiflashStatus(tblInfo *model.TableInfo, newPartitions e = errors.New("enforced error") }) if e != nil { - logutil.BgLogger().Error("ConfigureTiFlashPDForPartitions fails", zap.Error(e)) + logutil.DDLLogger().Error("ConfigureTiFlashPDForPartitions fails", zap.Error(e)) return e } tblInfo.TiFlashReplica.Available = false @@ -2556,8 +2556,8 @@ func (w *worker) onExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo } if defID != partDef.ID { - logutil.BgLogger().Info("Exchange partition id changed, updating to actual id", zap.String("category", "ddl"), - zap.String("job", job.String()), zap.Int64("defID", defID), zap.Int64("partDef.ID", partDef.ID)) + logutil.DDLLogger().Info("Exchange partition id changed, updating to actual id", + zap.Stringer("job", job), zap.Int64("defID", defID), zap.Int64("partDef.ID", partDef.ID)) job.Args[0] = partDef.ID defID = partDef.ID err = updateDDLJob2Table(w.sess, job, true) @@ -2599,8 +2599,8 @@ func (w *worker) onExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo if defID != partDef.ID { // Should never happen, should have been updated above, in previous state! - logutil.BgLogger().Error("Exchange partition id changed, updating to actual id", zap.String("category", "ddl"), - zap.String("job", job.String()), zap.Int64("defID", defID), zap.Int64("partDef.ID", partDef.ID)) + logutil.DDLLogger().Error("Exchange partition id changed, updating to actual id", + zap.Stringer("job", job), zap.Int64("defID", defID), zap.Int64("partDef.ID", partDef.ID)) job.Args[0] = partDef.ID defID = partDef.ID err = updateDDLJob2Table(w.sess, job, true) @@ -2877,7 +2877,7 @@ func (w *worker) onReorganizePartition(d *ddlCtx, t *meta.Meta, job *model.Job) if tblInfo.TiFlashReplica != nil { // Must set placement rule, and make sure it succeeds. if err := infosync.ConfigureTiFlashPDForPartitions(true, &tblInfo.Partition.AddingDefinitions, tblInfo.TiFlashReplica.Count, &tblInfo.TiFlashReplica.LocationLabels, tblInfo.ID); err != nil { - logutil.BgLogger().Error("ConfigureTiFlashPDForPartitions fails", zap.Error(err)) + logutil.DDLLogger().Error("ConfigureTiFlashPDForPartitions fails", zap.Error(err)) job.State = model.JobStateCancelled return ver, errors.Trace(err) } @@ -3217,10 +3217,10 @@ func doPartitionReorgWork(w *worker, d *ddlCtx, t *meta.Meta, job *model.Job, tb return false, ver, errors.Trace(err) } if err1 := rh.RemoveDDLReorgHandle(job, reorgInfo.elements); err1 != nil { - logutil.BgLogger().Warn("reorg partition job failed, RemoveDDLReorgHandle failed, can't convert job to rollback", zap.String("category", "ddl"), - zap.String("job", job.String()), zap.Error(err1)) + logutil.DDLLogger().Warn("reorg partition job failed, RemoveDDLReorgHandle failed, can't convert job to rollback", + zap.Stringer("job", job), zap.Error(err1)) } - logutil.BgLogger().Warn("reorg partition job failed, convert job to rollback", zap.String("category", "ddl"), zap.String("job", job.String()), zap.Error(err)) + logutil.DDLLogger().Warn("reorg partition job failed, convert job to rollback", zap.Stringer("job", job), zap.Error(err)) ver, err = convertAddTablePartitionJob2RollbackJob(d, t, job, err, tbl.Meta()) return false, ver, errors.Trace(err) } @@ -3390,7 +3390,10 @@ func (w *reorgPartitionWorker) fetchRowColVals(txn kv.Transaction, taskRange reo taskDone = true } - logutil.BgLogger().Debug("txn fetches handle info", zap.String("category", "ddl"), zap.Uint64("txnStartTS", txn.StartTS()), zap.String("taskRange", taskRange.String()), zap.Duration("takeTime", time.Since(startTime))) + logutil.DDLLogger().Debug("txn fetches handle info", + zap.Uint64("txnStartTS", txn.StartTS()), + zap.Stringer("taskRange", &taskRange), + zap.Duration("takeTime", time.Since(startTime))) return w.rowRecords, getNextHandleKey(taskRange, taskDone, lastAccessedHandle), taskDone, errors.Trace(err) } @@ -3487,7 +3490,7 @@ func (w *worker) reorgPartitionDataAndIndex(t table.Table, reorgInfo *reorgInfo) reorgInfo.currElement = reorgInfo.elements[i+1] // Write the reorg info to store so the whole reorganize process can recover from panic. err = reorgInfo.UpdateReorgMeta(reorgInfo.StartKey, w.sessPool) - logutil.BgLogger().Info("update column and indexes", zap.String("category", "ddl"), + logutil.DDLLogger().Info("update column and indexes", zap.Int64("jobID", reorgInfo.Job.ID), zap.ByteString("elementType", reorgInfo.currElement.TypeKey), zap.Int64("elementID", reorgInfo.currElement.ID), @@ -4050,7 +4053,7 @@ func isPartExprUnsigned(tbInfo *model.TableInfo) bool { ctx := mock.NewContext() expr, err := expression.ParseSimpleExpr(ctx, tbInfo.Partition.Expr, expression.WithTableInfo("", tbInfo)) if err != nil { - logutil.BgLogger().Error("isPartExpr failed parsing expression!", zap.Error(err)) + logutil.DDLLogger().Error("isPartExpr failed parsing expression!", zap.Error(err)) return false } if mysql.HasUnsignedFlag(expr.GetType().GetFlag()) { diff --git a/pkg/ddl/reorg.go b/pkg/ddl/reorg.go index 4f99100c8c1c6..51e7b3893e3b0 100644 --- a/pkg/ddl/reorg.go +++ b/pkg/ddl/reorg.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/ddl/ingest" sess "github.com/pingcap/tidb/pkg/ddl/internal/session" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/distsql" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/meta" @@ -44,7 +45,6 @@ import ( "github.com/pingcap/tidb/pkg/util/chunk" "github.com/pingcap/tidb/pkg/util/codec" "github.com/pingcap/tidb/pkg/util/dbterror" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/mock" "github.com/pingcap/tidb/pkg/util/ranger" "github.com/pingcap/tipb/go-tipb" @@ -227,9 +227,9 @@ func (w *worker) runReorgJob(reorgInfo *reorgInfo, tblInfo *model.TableInfo, rowCount := rc.getRowCount() job.SetRowCount(rowCount) if err != nil { - logutil.BgLogger().Warn("run reorg job done", zap.String("category", "ddl"), zap.Int64("handled rows", rowCount), zap.Error(err)) + logutil.DDLLogger().Warn("run reorg job done", zap.Int64("handled rows", rowCount), zap.Error(err)) } else { - logutil.BgLogger().Info("run reorg job done", zap.String("category", "ddl"), zap.Int64("handled rows", rowCount)) + logutil.DDLLogger().Info("run reorg job done", zap.Int64("handled rows", rowCount)) } // Update a job's warnings. @@ -245,7 +245,7 @@ func (w *worker) runReorgJob(reorgInfo *reorgInfo, tblInfo *model.TableInfo, return errors.Trace(err) } case <-w.ctx.Done(): - logutil.BgLogger().Info("run reorg job quit", zap.String("category", "ddl")) + logutil.DDLLogger().Info("run reorg job quit") d.removeReorgCtx(job.ID) // We return dbterror.ErrWaitReorgTimeout here too, so that outer loop will break. return dbterror.ErrWaitReorgTimeout @@ -259,7 +259,7 @@ func (w *worker) runReorgJob(reorgInfo *reorgInfo, tblInfo *model.TableInfo, rc.resetWarnings() - logutil.BgLogger().Info("run reorg job wait timeout", zap.String("category", "ddl"), + logutil.DDLLogger().Info("run reorg job wait timeout", zap.Duration("wait time", waitTimeout), zap.Int64("total added row count", rowCount)) // If timeout, we will return, check the owner and retry to wait job done again. @@ -291,7 +291,7 @@ func overwriteReorgInfoFromGlobalCheckpoint(w *worker, sess *sess.Session, job * if bc.GetCheckpointManager() == nil { mgr, err := ingest.NewCheckpointManager(w.ctx, bc, w.sessPool, job.ID, extractElemIDs(reorgInfo)) if err != nil { - logutil.BgLogger().Warn("create checkpoint manager failed", zap.String("category", "ddl-ingest"), zap.Error(err)) + logutil.DDLIngestLogger().Warn("create checkpoint manager failed", zap.Error(err)) } bc.AttachCheckpointManager(mgr) } @@ -343,7 +343,7 @@ func updateBackfillProgress(w *worker, reorgInfo *reorgInfo, tblInfo *model.Tabl if progress > 1 { progress = 1 } - logutil.BgLogger().Debug("update progress", zap.String("category", "ddl"), + logutil.DDLLogger().Debug("update progress", zap.Float64("progress", progress), zap.Int64("addedRowCount", addedRowCount), zap.Int64("totalCount", totalCount)) @@ -421,7 +421,7 @@ func (dc *ddlCtx) isReorgRunnable(jobID int64, isDistReorg bool) error { } if dc.isReorgPaused(jobID) { - logutil.BgLogger().Warn("job paused by user", zap.String("category", "ddl"), zap.String("ID", dc.uuid)) + logutil.DDLLogger().Warn("job paused by user", zap.String("ID", dc.uuid)) return dbterror.ErrPausedDDLJob.GenWithStackByArgs(jobID) } @@ -431,7 +431,7 @@ func (dc *ddlCtx) isReorgRunnable(jobID int64, isDistReorg bool) error { } if !dc.isOwner() { // If it's not the owner, we will try later, so here just returns an error. - logutil.BgLogger().Info("DDL is not the DDL owner", zap.String("category", "ddl"), zap.String("ID", dc.uuid)) + logutil.DDLLogger().Info("DDL is not the DDL owner", zap.String("ID", dc.uuid)) return errors.Trace(dbterror.ErrNotOwner) } return nil @@ -639,7 +639,7 @@ func getTableRange(ctx *JobContext, d *ddlCtx, tbl table.PhysicalTable, snapshot endHandleKey = tablecodec.EncodeRecordKey(tbl.RecordPrefix(), maxHandle).Next() } if isEmptyTable || endHandleKey.Cmp(startHandleKey) <= 0 { - logutil.BgLogger().Info("get noop table range", zap.String("category", "ddl"), + logutil.DDLLogger().Info("get noop table range", zap.String("table", fmt.Sprintf("%v", tbl.Meta())), zap.Int64("table/partition ID", tbl.GetPhysicalID()), zap.String("start key", hex.EncodeToString(startHandleKey)), @@ -715,7 +715,7 @@ func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job, return nil, errors.Trace(err) } } - logutil.BgLogger().Info("job get table range", zap.String("category", "ddl"), + logutil.DDLLogger().Info("job get table range", zap.Int64("jobID", job.ID), zap.Int64("physicalTableID", pid), zap.String("startKey", hex.EncodeToString(start)), zap.String("endKey", hex.EncodeToString(end))) @@ -750,7 +750,7 @@ func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job, // We'll try to remove it in the next major TiDB version. if meta.ErrDDLReorgElementNotExist.Equal(err) { job.SnapshotVer = 0 - logutil.BgLogger().Warn("get reorg info, the element does not exist", zap.String("category", "ddl"), zap.String("job", job.String())) + logutil.DDLLogger().Warn("get reorg info, the element does not exist", zap.Stringer("job", job)) if job.IsCancelling() { return nil, nil } @@ -795,7 +795,7 @@ func getReorgInfoFromPartitions(ctx *JobContext, d *ddlCtx, rh *reorgHandler, jo if err != nil { return nil, errors.Trace(err) } - logutil.BgLogger().Info("job get table range", zap.String("category", "ddl"), + logutil.DDLLogger().Info("job get table range", zap.Int64("job ID", job.ID), zap.Int64("physical table ID", pid), zap.String("start key", hex.EncodeToString(start)), zap.String("end key", hex.EncodeToString(end))) @@ -816,7 +816,7 @@ func getReorgInfoFromPartitions(ctx *JobContext, d *ddlCtx, rh *reorgHandler, jo // We'll try to remove it in the next major TiDB version. if meta.ErrDDLReorgElementNotExist.Equal(err) { job.SnapshotVer = 0 - logutil.BgLogger().Warn("get reorg info, the element does not exist", zap.String("category", "ddl"), zap.String("job", job.String())) + logutil.DDLLogger().Warn("get reorg info, the element does not exist", zap.Stringer("job", job)) } return &info, errors.Trace(err) } @@ -898,7 +898,7 @@ func CleanupDDLReorgHandles(job *model.Job, s *sess.Session) { err := cleanDDLReorgHandles(s, job) if err != nil { // ignore error, cleanup is not that critical - logutil.BgLogger().Warn("Failed removing the DDL reorg entry in tidb_ddl_reorg", zap.String("job", job.String()), zap.Error(err)) + logutil.DDLLogger().Warn("Failed removing the DDL reorg entry in tidb_ddl_reorg", zap.Stringer("job", job), zap.Error(err)) } } @@ -916,8 +916,7 @@ func (r *reorgHandler) GetDDLReorgHandle(job *model.Job) (element *meta.Element, // For old version TiDB, the semantic is still [start_key, end_key], we need to adjust it in new version TiDB. func adjustEndKeyAcrossVersion(job *model.Job, endKey kv.Key) kv.Key { if job.ReorgMeta != nil && job.ReorgMeta.Version == model.ReorgMetaVersion0 { - logutil.BgLogger().Info("adjust range end key for old version ReorgMetas", - zap.String("category", "ddl"), + logutil.DDLLogger().Info("adjust range end key for old version ReorgMetas", zap.Int64("jobID", job.ID), zap.Int64("reorgMetaVersion", job.ReorgMeta.Version), zap.String("endKey", hex.EncodeToString(endKey))) diff --git a/pkg/ddl/reorg_partition_test.go b/pkg/ddl/reorg_partition_test.go index bab9b060637db..480eea96a4e5b 100644 --- a/pkg/ddl/reorg_partition_test.go +++ b/pkg/ddl/reorg_partition_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/ddl/util/callback" "github.com/pingcap/tidb/pkg/domain" "github.com/pingcap/tidb/pkg/errno" @@ -31,7 +32,6 @@ import ( "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/tablecodec" "github.com/pingcap/tidb/pkg/testkit" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/mathutil" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -125,7 +125,7 @@ func getAllDataForPhysicalTable(t *testing.T, ctx sessionctx.Context, physTable tblID, kv, _ := tablecodec.DecodeRecordKey(it.Key()) require.Equal(t, pid, tblID) vals, _ := tablecodec.DecodeValuesBytesToStrings(it.Value()) - logutil.BgLogger().Info("Record", + logutil.DDLLogger().Info("Record", zap.Int64("pid", tblID), zap.Stringer("key", kv), zap.Strings("values", vals)) diff --git a/pkg/ddl/resource_group.go b/pkg/ddl/resource_group.go index c3419bcd60531..caca77ec438e4 100644 --- a/pkg/ddl/resource_group.go +++ b/pkg/ddl/resource_group.go @@ -20,6 +20,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/ddl/resourcegroup" "github.com/pingcap/tidb/pkg/domain/infosync" rg "github.com/pingcap/tidb/pkg/domain/resourcegroup" @@ -27,7 +28,6 @@ import ( "github.com/pingcap/tidb/pkg/meta" "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/util/dbterror" - "github.com/pingcap/tidb/pkg/util/logutil" "go.uber.org/zap" ) @@ -49,7 +49,7 @@ func onCreateResourceGroup(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, // check if resource group value is valid and convert to proto format. protoGroup, err := resourcegroup.NewGroupFromOptions(groupInfo.Name.L, groupInfo.ResourceGroupSettings) if err != nil { - logutil.BgLogger().Warn("convert to resource group failed", zap.Error(err)) + logutil.DDLLogger().Warn("convert to resource group failed", zap.Error(err)) job.State = model.JobStateCancelled return ver, errors.Trace(err) } @@ -67,7 +67,7 @@ func onCreateResourceGroup(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, defer cancel() err = infosync.AddResourceGroup(ctx, protoGroup) if err != nil { - logutil.BgLogger().Warn("create resource group failed", zap.String("group-name", groupInfo.Name.L), zap.Error(err)) + logutil.DDLLogger().Warn("create resource group failed", zap.String("group-name", groupInfo.Name.L), zap.Error(err)) // TiDB will add the group to the resource manager when it bootstraps. // here order to compatible with keyspace mode TiDB to skip the exist error with default group. if !strings.Contains(err.Error(), alreadyExists) || groupInfo.Name.L != rg.DefaultResourceGroupName { @@ -96,7 +96,7 @@ func onAlterResourceGroup(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ // check if resource group value is valid and convert to proto format. protoGroup, err := resourcegroup.NewGroupFromOptions(alterGroupInfo.Name.L, alterGroupInfo.ResourceGroupSettings) if err != nil { - logutil.BgLogger().Warn("convert to resource group failed", zap.Error(err)) + logutil.DDLLogger().Warn("convert to resource group failed", zap.Error(err)) job.State = model.JobStateCancelled return ver, errors.Trace(err) } @@ -117,7 +117,7 @@ func onAlterResourceGroup(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ err = infosync.ModifyResourceGroup(context.TODO(), protoGroup) if err != nil { - logutil.BgLogger().Warn("update resource group failed", zap.Error(err)) + logutil.DDLLogger().Warn("update resource group failed", zap.Error(err)) job.State = model.JobStateCancelled return ver, errors.Trace(err) } diff --git a/pkg/ddl/sanity_check.go b/pkg/ddl/sanity_check.go index 3f20d31b6e479..8dec33dd42f8f 100644 --- a/pkg/ddl/sanity_check.go +++ b/pkg/ddl/sanity_check.go @@ -21,13 +21,13 @@ import ( "github.com/pingcap/errors" sess "github.com/pingcap/tidb/pkg/ddl/internal/session" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/util/intest" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/mathutil" "go.uber.org/zap" ) @@ -38,12 +38,12 @@ func (d *ddl) checkDeleteRangeCnt(job *model.Job) { if strings.Contains(err.Error(), "Not Supported") { return // For mock session, we don't support executing SQLs. } - logutil.BgLogger().Error("query delete range count failed", zap.Error(err)) + logutil.DDLLogger().Error("query delete range count failed", zap.Error(err)) panic(err) } expectedCnt, err := expectedDeleteRangeCnt(delRangeCntCtx{idxIDs: map[int64]struct{}{}}, job) if err != nil { - logutil.BgLogger().Error("decode job's delete range count failed", zap.Error(err)) + logutil.DDLLogger().Error("decode job's delete range count failed", zap.Error(err)) panic(err) } if actualCnt != expectedCnt { diff --git a/pkg/ddl/schema_test.go b/pkg/ddl/schema_test.go index ee461daabd7f6..07ece0969383e 100644 --- a/pkg/ddl/schema_test.go +++ b/pkg/ddl/schema_test.go @@ -23,6 +23,7 @@ import ( "github.com/ngaut/pools" "github.com/pingcap/tidb/pkg/ddl" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/infoschema" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/meta" @@ -33,7 +34,6 @@ import ( "github.com/pingcap/tidb/pkg/sessiontxn" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/types" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/stretchr/testify/require" "go.uber.org/zap" ) @@ -373,14 +373,14 @@ func TestRenameTableAutoIDs(t *testing.T) { if len(res) == 1 && res[0][col] == s { break } - logutil.BgLogger().Info("Could not find match", zap.String("tableName", tableName), zap.String("s", s), zap.Int("colNum", col)) + logutil.DDLLogger().Info("Could not find match", zap.String("tableName", tableName), zap.String("s", s), zap.Int("colNum", col)) for i := range res { strs := make([]string, 0, len(res[i])) for j := range res[i] { strs = append(strs, res[i][j].(string)) } - logutil.BgLogger().Info("ddl jobs", zap.Strings("jobs", strs)) + logutil.DDLLogger().Info("ddl jobs", zap.Strings("jobs", strs)) } time.Sleep(10 * time.Millisecond) } diff --git a/pkg/ddl/split_region.go b/pkg/ddl/split_region.go index bbd97b89e4571..e5eb0134bb2bf 100644 --- a/pkg/ddl/split_region.go +++ b/pkg/ddl/split_region.go @@ -18,6 +18,7 @@ import ( "context" "github.com/pingcap/errors" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/meta/autoid" "github.com/pingcap/tidb/pkg/parser/model" @@ -25,7 +26,6 @@ import ( "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/tablecodec" "github.com/pingcap/tidb/pkg/types" - "github.com/pingcap/tidb/pkg/util/logutil" tikverr "github.com/tikv/client-go/v2/error" "go.uber.org/zap" ) @@ -110,7 +110,7 @@ func preSplitPhysicalTableByShardRowID(ctx context.Context, store kv.SplittableS var err error regionIDs, err := store.SplitRegions(ctx, splitTableKeys, scatter, &tbInfo.ID) if err != nil { - logutil.BgLogger().Warn("pre split some table regions failed", zap.String("category", "ddl"), + logutil.DDLLogger().Warn("pre split some table regions failed", zap.Stringer("table", tbInfo.Name), zap.Int("successful region count", len(regionIDs)), zap.Error(err)) } regionIDs = append(regionIDs, splitIndexRegion(store, tbInfo, scatter)...) @@ -123,7 +123,7 @@ func SplitRecordRegion(ctx context.Context, store kv.SplittableStore, physicalTa regionIDs, err := store.SplitRegions(ctx, [][]byte{tableStartKey}, scatter, &tableID) if err != nil { // It will be automatically split by TiKV later. - logutil.BgLogger().Warn("split table region failed", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Warn("split table region failed", zap.Error(err)) } if len(regionIDs) == 1 { return regionIDs[0] @@ -139,7 +139,7 @@ func splitIndexRegion(store kv.SplittableStore, tblInfo *model.TableInfo, scatte } regionIDs, err := store.SplitRegions(context.Background(), splitKeys, scatter, &tblInfo.ID) if err != nil { - logutil.BgLogger().Warn("pre split some table index regions failed", zap.String("category", "ddl"), + logutil.DDLLogger().Warn("pre split some table index regions failed", zap.Stringer("table", tblInfo.Name), zap.Int("successful region count", len(regionIDs)), zap.Error(err)) } return regionIDs @@ -150,7 +150,7 @@ func WaitScatterRegionFinish(ctx context.Context, store kv.SplittableStore, regi for _, regionID := range regionIDs { err := store.WaitScatterRegionFinish(ctx, regionID, 0) if err != nil { - logutil.BgLogger().Warn("wait scatter region failed", zap.String("category", "ddl"), zap.Uint64("regionID", regionID), zap.Error(err)) + logutil.DDLLogger().Warn("wait scatter region failed", zap.Uint64("regionID", regionID), zap.Error(err)) // We don't break for PDError because it may caused by ScatterRegion request failed. if _, ok := errors.Cause(err).(*tikverr.PDError); !ok { break diff --git a/pkg/ddl/syncer/BUILD.bazel b/pkg/ddl/syncer/BUILD.bazel index a1cc2e9b691dd..a18b5f63e9a4b 100644 --- a/pkg/ddl/syncer/BUILD.bazel +++ b/pkg/ddl/syncer/BUILD.bazel @@ -9,12 +9,12 @@ go_library( importpath = "github.com/pingcap/tidb/pkg/ddl/syncer", visibility = ["//visibility:public"], deps = [ + "//pkg/ddl/logutil", "//pkg/ddl/util", "//pkg/domain/infosync", "//pkg/metrics", "//pkg/sessionctx/variable", "//pkg/util", - "//pkg/util/logutil", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", "@io_etcd_go_etcd_api_v3//mvccpb", diff --git a/pkg/ddl/syncer/state_syncer.go b/pkg/ddl/syncer/state_syncer.go index da5785a2e5566..d65a7b6585911 100644 --- a/pkg/ddl/syncer/state_syncer.go +++ b/pkg/ddl/syncer/state_syncer.go @@ -21,10 +21,10 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/ddl/util" "github.com/pingcap/tidb/pkg/metrics" tidbutil "github.com/pingcap/tidb/pkg/util" - "github.com/pingcap/tidb/pkg/util/logutil" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" @@ -155,7 +155,7 @@ func (*serverStateSyncer) getKeyValue(ctx context.Context, etcdCli *clientv3.Cli resp, err = etcdCli.Get(childCtx, key, opts...) cancel() if err != nil { - logutil.BgLogger().Info("get key failed", zap.String("key", key), zap.Error(err)) + logutil.DDLLogger().Info("get key failed", zap.String("key", key), zap.Error(err)) time.Sleep(200 * time.Millisecond) continue } @@ -184,7 +184,7 @@ func (s *serverStateSyncer) GetGlobalState(ctx context.Context) (*StateInfo, err } err = state.Unmarshal(kvs[0].Value) if err != nil { - logutil.BgLogger().Warn("get global state failed", zap.String("key", s.etcdPath), zap.ByteString("value", kvs[0].Value), zap.Error(err)) + logutil.DDLLogger().Warn("get global state failed", zap.String("key", s.etcdPath), zap.ByteString("value", kvs[0].Value), zap.Error(err)) return nil, errors.Trace(err) } s.clusterState.Store(state) diff --git a/pkg/ddl/syncer/syncer.go b/pkg/ddl/syncer/syncer.go index 4ef458d6b4fcf..8401fdd9666f9 100644 --- a/pkg/ddl/syncer/syncer.go +++ b/pkg/ddl/syncer/syncer.go @@ -27,12 +27,12 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/ddl/util" "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/metrics" "github.com/pingcap/tidb/pkg/sessionctx/variable" tidbutil "github.com/pingcap/tidb/pkg/util" - "github.com/pingcap/tidb/pkg/util/logutil" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" "go.uber.org/zap" @@ -101,7 +101,7 @@ func (w *watcher) Rewatch(ctx context.Context, etcdCli *clientv3.Client, path st w.Lock() w.wCh = wCh w.Unlock() - logutil.BgLogger().Info("syncer rewatch global info finished", zap.String("category", "ddl")) + logutil.DDLLogger().Info("syncer rewatch global info finished") }() } @@ -189,7 +189,7 @@ func (s *schemaVersionSyncer) Done() <-chan struct{} { failpoint.Inject("ErrorMockSessionDone", func(val failpoint.Value) { if val.(bool) { err := s.loadSession().Close() - logutil.BgLogger().Error("close session failed", zap.Error(err)) + logutil.DDLLogger().Error("close session failed", zap.Error(err)) } }) @@ -327,7 +327,7 @@ func (s *schemaVersionSyncer) OwnerCheckAllVersions(ctx context.Context, jobID i // Get all the schema versions from ETCD. resp, err := s.etcdCli.Get(ctx, path, clientv3.WithPrefix()) if err != nil { - logutil.BgLogger().Info("syncer check all versions failed, continue checking.", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Info("syncer check all versions failed, continue checking.", zap.Error(err)) continue } @@ -349,7 +349,10 @@ func (s *schemaVersionSyncer) OwnerCheckAllVersions(ctx context.Context, jobID i succ = false if notMatchVerCnt%intervalCnt == 0 { for _, info := range updatedMap { - logutil.BgLogger().Info("syncer check all versions, someone is not synced", zap.String("category", "ddl"), zap.String("info", info), zap.Int64("ddl job id", jobID), zap.Int64("ver", latestVer)) + logutil.DDLLogger().Info("syncer check all versions, someone is not synced", + zap.String("info", info), + zap.Int64("ddl job id", jobID), + zap.Int64("ver", latestVer)) } } } @@ -378,13 +381,13 @@ func (s *schemaVersionSyncer) OwnerCheckAllVersions(ctx context.Context, jobID i func isUpdatedLatestVersion(key, val string, latestVer int64, notMatchVerCnt, intervalCnt int, nodeAlive bool) bool { ver, err := strconv.Atoi(val) if err != nil { - logutil.BgLogger().Info("syncer check all versions, convert value to int failed, continue checking.", zap.String("category", "ddl"), + logutil.DDLLogger().Info("syncer check all versions, convert value to int failed, continue checking.", zap.String("ddl", key), zap.String("value", val), zap.Error(err)) return false } if int64(ver) < latestVer && nodeAlive { if notMatchVerCnt%intervalCnt == 0 { - logutil.BgLogger().Info("syncer check all versions, someone is not synced, continue checking", zap.String("category", "ddl"), + logutil.DDLLogger().Info("syncer check all versions, someone is not synced, continue checking", zap.String("ddl", key), zap.Int("currentVer", ver), zap.Int64("latestVer", latestVer)) } return false @@ -395,6 +398,6 @@ func isUpdatedLatestVersion(key, val string, latestVer int64, notMatchVerCnt, in func (s *schemaVersionSyncer) Close() { err := s.removeSelfVersionPath() if err != nil { - logutil.BgLogger().Error("remove self version path failed", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Error("remove self version path failed", zap.Error(err)) } } diff --git a/pkg/ddl/table.go b/pkg/ddl/table.go index 2705f18974c0c..63523481509dd 100644 --- a/pkg/ddl/table.go +++ b/pkg/ddl/table.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/failpoint" sess "github.com/pingcap/tidb/pkg/ddl/internal/session" "github.com/pingcap/tidb/pkg/ddl/label" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/ddl/placement" "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/infoschema" @@ -44,7 +45,6 @@ import ( tidb_util "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/dbterror" "github.com/pingcap/tidb/pkg/util/gcutil" - "github.com/pingcap/tidb/pkg/util/logutil" "go.uber.org/zap" ) @@ -115,7 +115,7 @@ func createTable(d *ddlCtx, t *meta.Meta, job *model.Job, fkCheck bool) (*model. if tbInfo.TiFlashReplica != nil { replicaInfo := tbInfo.TiFlashReplica if pi := tbInfo.GetPartitionInfo(); pi != nil { - logutil.BgLogger().Info("Set TiFlash replica pd rule for partitioned table when creating", zap.Int64("tableID", tbInfo.ID)) + logutil.DDLLogger().Info("Set TiFlash replica pd rule for partitioned table when creating", zap.Int64("tableID", tbInfo.ID)) if e := infosync.ConfigureTiFlashPDForPartitions(false, &pi.Definitions, replicaInfo.Count, &replicaInfo.LocationLabels, tbInfo.ID); e != nil { job.State = model.JobStateCancelled return tbInfo, errors.Trace(e) @@ -126,7 +126,7 @@ func createTable(d *ddlCtx, t *meta.Meta, job *model.Job, fkCheck bool) (*model. return tbInfo, errors.Trace(e) } } else { - logutil.BgLogger().Info("Set TiFlash replica pd rule when creating", zap.Int64("tableID", tbInfo.ID)) + logutil.DDLLogger().Info("Set TiFlash replica pd rule when creating", zap.Int64("tableID", tbInfo.ID)) if e := infosync.ConfigureTiFlashPDForTable(tbInfo.ID, replicaInfo.Count, &replicaInfo.LocationLabels); e != nil { job.State = model.JobStateCancelled return tbInfo, errors.Trace(e) @@ -402,7 +402,7 @@ func onDropTableOrView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ er if tblInfo.TiFlashReplica != nil { e := infosync.DeleteTiFlashTableSyncProgress(tblInfo) if e != nil { - logutil.BgLogger().Error("DeleteTiFlashTableSyncProgress fails", zap.Error(e)) + logutil.DDLLogger().Error("DeleteTiFlashTableSyncProgress fails", zap.Error(e)) } } // Placement rules cannot be removed immediately after drop table / truncate table, because the @@ -759,7 +759,7 @@ func (w *worker) onTruncateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver i if tblInfo.TiFlashReplica != nil { e := infosync.DeleteTiFlashTableSyncProgress(tblInfo) if e != nil { - logutil.BgLogger().Error("DeleteTiFlashTableSyncProgress fails", zap.Error(e)) + logutil.DDLLogger().Error("DeleteTiFlashTableSyncProgress fails", zap.Error(e)) } } @@ -805,13 +805,13 @@ func (w *worker) onTruncateTable(d *ddlCtx, t *meta.Meta, job *model.Job) (ver i // Set PD rules for TiFlash if pi := tblInfo.GetPartitionInfo(); pi != nil { if e := infosync.ConfigureTiFlashPDForPartitions(true, &pi.Definitions, tblInfo.TiFlashReplica.Count, &tblInfo.TiFlashReplica.LocationLabels, tblInfo.ID); e != nil { - logutil.BgLogger().Error("ConfigureTiFlashPDForPartitions fails", zap.Error(err)) + logutil.DDLLogger().Error("ConfigureTiFlashPDForPartitions fails", zap.Error(err)) job.State = model.JobStateCancelled return ver, errors.Trace(e) } } else { if e := infosync.ConfigureTiFlashPDForTable(newTableID, tblInfo.TiFlashReplica.Count, &tblInfo.TiFlashReplica.LocationLabels); e != nil { - logutil.BgLogger().Error("ConfigureTiFlashPDForTable fails", zap.Error(err)) + logutil.DDLLogger().Error("ConfigureTiFlashPDForTable fails", zap.Error(err)) job.State = model.JobStateCancelled return ver, errors.Trace(e) } @@ -1377,7 +1377,7 @@ func (w *worker) onSetTableFlashReplica(d *ddlCtx, t *meta.Meta, job *model.Job) } // We should check this first, in order to avoid creating redundant DDL jobs. if pi := tblInfo.GetPartitionInfo(); pi != nil { - logutil.BgLogger().Info("Set TiFlash replica pd rule for partitioned table", zap.Int64("tableID", tblInfo.ID)) + logutil.DDLLogger().Info("Set TiFlash replica pd rule for partitioned table", zap.Int64("tableID", tblInfo.ID)) if e := infosync.ConfigureTiFlashPDForPartitions(false, &pi.Definitions, replicaInfo.Count, &replicaInfo.Labels, tblInfo.ID); e != nil { job.State = model.JobStateCancelled return ver, errors.Trace(e) @@ -1388,7 +1388,7 @@ func (w *worker) onSetTableFlashReplica(d *ddlCtx, t *meta.Meta, job *model.Job) return ver, errors.Trace(e) } } else { - logutil.BgLogger().Info("Set TiFlash replica pd rule", zap.Int64("tableID", tblInfo.ID)) + logutil.DDLLogger().Info("Set TiFlash replica pd rule", zap.Int64("tableID", tblInfo.ID)) if e := infosync.ConfigureTiFlashPDForTable(tblInfo.ID, replicaInfo.Count, &replicaInfo.Labels); e != nil { job.State = model.JobStateCancelled return ver, errors.Trace(e) @@ -1409,7 +1409,7 @@ func (w *worker) onSetTableFlashReplica(d *ddlCtx, t *meta.Meta, job *model.Job) if tblInfo.TiFlashReplica != nil { err = infosync.DeleteTiFlashTableSyncProgress(tblInfo) if err != nil { - logutil.BgLogger().Error("DeleteTiFlashTableSyncProgress fails", zap.Error(err)) + logutil.DDLLogger().Error("DeleteTiFlashTableSyncProgress fails", zap.Error(err)) } } tblInfo.TiFlashReplica = nil @@ -1472,7 +1472,7 @@ func onUpdateFlashReplicaStatus(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in newIDs = append(newIDs, tblInfo.TiFlashReplica.AvailablePartitionIDs[i+1:]...) tblInfo.TiFlashReplica.AvailablePartitionIDs = newIDs tblInfo.TiFlashReplica.Available = false - logutil.BgLogger().Info("TiFlash replica become unavailable", zap.Int64("tableID", tblInfo.ID), zap.Int64("partitionID", id)) + logutil.DDLLogger().Info("TiFlash replica become unavailable", zap.Int64("tableID", tblInfo.ID), zap.Int64("partitionID", id)) break } } @@ -1483,7 +1483,7 @@ func onUpdateFlashReplicaStatus(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in } if tblInfo.TiFlashReplica.Available { - logutil.BgLogger().Info("TiFlash replica available", zap.Int64("tableID", tblInfo.ID)) + logutil.DDLLogger().Info("TiFlash replica available", zap.Int64("tableID", tblInfo.ID)) } ver, err = updateVersionAndTableInfo(d, t, job, tblInfo, true) if err != nil { diff --git a/pkg/ddl/tests/adminpause/BUILD.bazel b/pkg/ddl/tests/adminpause/BUILD.bazel index f5917cb6181fb..30c658808ec7f 100644 --- a/pkg/ddl/tests/adminpause/BUILD.bazel +++ b/pkg/ddl/tests/adminpause/BUILD.bazel @@ -11,10 +11,10 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/ddl", + "//pkg/ddl/logutil", "//pkg/domain", "//pkg/parser/model", "//pkg/testkit", - "//pkg/util/logutil", ], ) diff --git a/pkg/ddl/tests/adminpause/global.go b/pkg/ddl/tests/adminpause/global.go index 3c7ddb2a5eb25..c75a2ee4d1afc 100644 --- a/pkg/ddl/tests/adminpause/global.go +++ b/pkg/ddl/tests/adminpause/global.go @@ -19,15 +19,15 @@ import ( "time" ddlctrl "github.com/pingcap/tidb/pkg/ddl" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/domain" "github.com/pingcap/tidb/pkg/testkit" - "github.com/pingcap/tidb/pkg/util/logutil" ) const dbTestLease = 600 * time.Millisecond // Logger is the global logger in this package -var Logger = logutil.BgLogger() +var Logger = logutil.DDLLogger() func prepareDomain(t *testing.T) (*domain.Domain, *testkit.TestKit, *testkit.TestKit) { store, dom := testkit.CreateMockStoreAndDomainWithSchemaLease(t, dbTestLease) diff --git a/pkg/ddl/tests/partition/BUILD.bazel b/pkg/ddl/tests/partition/BUILD.bazel index 855385296d697..786e251b233fc 100644 --- a/pkg/ddl/tests/partition/BUILD.bazel +++ b/pkg/ddl/tests/partition/BUILD.bazel @@ -12,6 +12,7 @@ go_test( deps = [ "//pkg/config", "//pkg/ddl", + "//pkg/ddl/logutil", "//pkg/ddl/testutil", "//pkg/ddl/util/callback", "//pkg/domain", @@ -35,7 +36,6 @@ go_test( "//pkg/types", "//pkg/util/codec", "//pkg/util/dbterror", - "//pkg/util/logutil", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", "@com_github_stretchr_testify//assert", diff --git a/pkg/ddl/tests/partition/db_partition_test.go b/pkg/ddl/tests/partition/db_partition_test.go index 51b685dade9d8..d9b6e2c61fd23 100644 --- a/pkg/ddl/tests/partition/db_partition_test.go +++ b/pkg/ddl/tests/partition/db_partition_test.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/ddl" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/ddl/testutil" "github.com/pingcap/tidb/pkg/ddl/util/callback" "github.com/pingcap/tidb/pkg/domain" @@ -50,7 +51,6 @@ import ( "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/codec" "github.com/pingcap/tidb/pkg/util/dbterror" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -2187,7 +2187,7 @@ func checkPartitionDelRangeDone(t *testing.T, tk *testkit.TestKit, store kv.Stor done := waitGCDeleteRangeDone(t, tk, oldPID) if !done { // Takes too long, give up the check. - logutil.BgLogger().Info("truncate partition table", + logutil.DDLLogger().Info("truncate partition table", zap.Int64("id", oldPID), zap.Stringer("duration", time.Since(startTime)), ) @@ -2571,7 +2571,7 @@ func TestDropSchemaWithPartitionTable(t *testing.T) { done := waitGCDeleteRangeDone(t, tk, tableIDs[2]) if !done { // Takes too long, give up the check. - logutil.BgLogger().Info("drop schema", + logutil.DDLLogger().Info("drop schema", zap.Int64("id", tableIDs[0]), zap.Stringer("duration", time.Since(startTime)), ) @@ -3206,7 +3206,7 @@ func TestRemovePartitioningAutoIDs(t *testing.T) { for j := range res[i] { strs = append(strs, res[i][j].(string)) } - logutil.BgLogger().Info("ddl jobs", zap.Strings("jobs", strs)) + logutil.DDLLogger().Info("ddl jobs", zap.Strings("jobs", strs)) } time.Sleep(10 * time.Millisecond) } diff --git a/pkg/ddl/tests/tiflash/BUILD.bazel b/pkg/ddl/tests/tiflash/BUILD.bazel index 1668f154eaf30..cce04874bc8a0 100644 --- a/pkg/ddl/tests/tiflash/BUILD.bazel +++ b/pkg/ddl/tests/tiflash/BUILD.bazel @@ -12,6 +12,7 @@ go_test( deps = [ "//pkg/config", "//pkg/ddl", + "//pkg/ddl/logutil", "//pkg/ddl/placement", "//pkg/ddl/util", "//pkg/domain", @@ -30,7 +31,6 @@ go_test( "//pkg/testkit/testsetup", "//pkg/types", "//pkg/util", - "//pkg/util/logutil", "//pkg/util/sqlkiller", "@com_github_pingcap_failpoint//:failpoint", "@com_github_pingcap_kvproto//pkg/metapb", diff --git a/pkg/ddl/tests/tiflash/ddl_tiflash_test.go b/pkg/ddl/tests/tiflash/ddl_tiflash_test.go index f81357007bd6f..9c02497bbe0df 100644 --- a/pkg/ddl/tests/tiflash/ddl_tiflash_test.go +++ b/pkg/ddl/tests/tiflash/ddl_tiflash_test.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/tidb/pkg/ddl" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/ddl/placement" ddlutil "github.com/pingcap/tidb/pkg/ddl/util" "github.com/pingcap/tidb/pkg/domain" @@ -46,7 +47,6 @@ import ( "github.com/pingcap/tidb/pkg/testkit/external" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/sqlkiller" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" @@ -833,7 +833,7 @@ func execWithTimeout(t *testing.T, tk *testkit.TestKit, to time.Duration, sql st return false, e case <-ctx.Done(): // Exceed given timeout - logutil.BgLogger().Info("execWithTimeout meet timeout", zap.String("sql", sql)) + logutil.DDLLogger().Info("execWithTimeout meet timeout", zap.String("sql", sql)) require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/BatchAddTiFlashSendDone", "return(true)")) } @@ -911,7 +911,7 @@ func TestTiFlashBatchRateLimiter(t *testing.T) { mu.Lock() defer mu.Unlock() tk.Session().Close() - logutil.BgLogger().Info("session closed") + logutil.DDLLogger().Info("session closed") }) mu.Lock() timeOut, err = execWithTimeout(t, tk, time.Second*2, "alter database tiflash_ddl_limit set tiflash replica 1") @@ -1398,7 +1398,7 @@ type TestDDLCallback struct { // OnJobRunBefore is used to run the user customized logic of `onJobRunBefore` first. func (tc *TestDDLCallback) OnJobRunBefore(job *model.Job) { - logutil.BgLogger().Info("on job run before", zap.String("job", job.String())) + logutil.DDLLogger().Info("on job run before", zap.String("job", job.String())) if tc.OnJobRunBeforeExported != nil { tc.OnJobRunBeforeExported(job) return diff --git a/pkg/ddl/testutil/BUILD.bazel b/pkg/ddl/testutil/BUILD.bazel index 54d11c7cc4084..4cfa751c86c17 100644 --- a/pkg/ddl/testutil/BUILD.bazel +++ b/pkg/ddl/testutil/BUILD.bazel @@ -6,6 +6,7 @@ go_library( importpath = "github.com/pingcap/tidb/pkg/ddl/testutil", visibility = ["//visibility:public"], deps = [ + "//pkg/ddl/logutil", "//pkg/domain", "//pkg/kv", "//pkg/parser/model", @@ -15,7 +16,6 @@ go_library( "//pkg/table", "//pkg/table/tables", "//pkg/types", - "//pkg/util/logutil", "@com_github_pingcap_errors//:errors", "@com_github_stretchr_testify//require", "@org_uber_go_zap//:zap", diff --git a/pkg/ddl/testutil/testutil.go b/pkg/ddl/testutil/testutil.go index c12f4e5b01cf5..fb6e44df1c2fb 100644 --- a/pkg/ddl/testutil/testutil.go +++ b/pkg/ddl/testutil/testutil.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/pingcap/errors" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/domain" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/parser/model" @@ -29,7 +30,6 @@ import ( "github.com/pingcap/tidb/pkg/table" "github.com/pingcap/tidb/pkg/table/tables" "github.com/pingcap/tidb/pkg/types" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/stretchr/testify/require" "go.uber.org/zap" ) @@ -93,7 +93,7 @@ func ExtractAllTableHandles(se sessiontypes.Session, dbName, tbName string) ([]i func FindIdxInfo(dom *domain.Domain, dbName, tbName, idxName string) *model.IndexInfo { tbl, err := dom.InfoSchema().TableByName(model.NewCIStr(dbName), model.NewCIStr(tbName)) if err != nil { - logutil.BgLogger().Warn("cannot find table", zap.String("dbName", dbName), zap.String("tbName", tbName)) + logutil.DDLLogger().Warn("cannot find table", zap.String("dbName", dbName), zap.String("tbName", tbName)) return nil } return tbl.Meta().FindIndexByName(idxName) diff --git a/pkg/ddl/util/BUILD.bazel b/pkg/ddl/util/BUILD.bazel index 0c24368f9e97b..71bef12a86dcd 100644 --- a/pkg/ddl/util/BUILD.bazel +++ b/pkg/ddl/util/BUILD.bazel @@ -10,13 +10,13 @@ go_library( importpath = "github.com/pingcap/tidb/pkg/ddl/util", visibility = ["//visibility:public"], deps = [ + "//pkg/ddl/logutil", "//pkg/kv", "//pkg/parser/model", "//pkg/parser/terror", "//pkg/sessionctx", "//pkg/sessionctx/variable", "//pkg/util/chunk", - "//pkg/util/logutil", "//pkg/util/mock", "//pkg/util/sqlexec", "@com_github_pingcap_errors//:errors", diff --git a/pkg/ddl/util/callback/BUILD.bazel b/pkg/ddl/util/callback/BUILD.bazel index c656f29eb348c..a5a677b5f9bc2 100644 --- a/pkg/ddl/util/callback/BUILD.bazel +++ b/pkg/ddl/util/callback/BUILD.bazel @@ -7,10 +7,10 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/ddl", + "//pkg/ddl/logutil", "//pkg/infoschema", "//pkg/parser/model", "//pkg/sessionctx", - "//pkg/util/logutil", "@org_uber_go_zap//:zap", ], ) diff --git a/pkg/ddl/util/callback/callback.go b/pkg/ddl/util/callback/callback.go index 0303d4077de42..10fc9c7c022c3 100644 --- a/pkg/ddl/util/callback/callback.go +++ b/pkg/ddl/util/callback/callback.go @@ -19,10 +19,10 @@ import ( "sync/atomic" "github.com/pingcap/tidb/pkg/ddl" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/infoschema" "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/sessionctx" - "github.com/pingcap/tidb/pkg/util/logutil" "go.uber.org/zap" ) @@ -67,11 +67,11 @@ func (tc *TestDDLCallback) OnChanged(err error) error { if err != nil { return err } - logutil.BgLogger().Info("performing DDL change, must reload") + logutil.DDLLogger().Info("performing DDL change, must reload") if tc.Do != nil { err = tc.Do.Reload() if err != nil { - logutil.BgLogger().Error("performing DDL change failed", zap.Error(err)) + logutil.DDLLogger().Error("performing DDL change failed", zap.Error(err)) } } return nil @@ -81,7 +81,7 @@ func (tc *TestDDLCallback) OnChanged(err error) error { func (tc *TestDDLCallback) OnSchemaStateChanged(schemaVer int64) { if tc.Do != nil { if err := tc.Do.Reload(); err != nil { - logutil.BgLogger().Warn("reload failed on schema state changed", zap.Error(err)) + logutil.DDLLogger().Warn("reload failed on schema state changed", zap.Error(err)) } } @@ -93,7 +93,7 @@ func (tc *TestDDLCallback) OnSchemaStateChanged(schemaVer int64) { // OnJobRunBefore is used to run the user customized logic of `onJobRunBefore` first. func (tc *TestDDLCallback) OnJobRunBefore(job *model.Job) { - logutil.BgLogger().Info("on job run before", zap.String("job", job.String())) + logutil.DDLLogger().Info("on job run before", zap.String("job", job.String())) if tc.OnJobRunBeforeExported != nil { tc.OnJobRunBeforeExported(job) return @@ -108,7 +108,7 @@ func (tc *TestDDLCallback) OnJobRunBefore(job *model.Job) { // OnJobRunAfter is used to run the user customized logic of `OnJobRunAfter` first. func (tc *TestDDLCallback) OnJobRunAfter(job *model.Job) { - logutil.BgLogger().Info("on job run after", zap.String("job", job.String())) + logutil.DDLLogger().Info("on job run after", zap.String("job", job.String())) if tc.OnJobRunAfterExported != nil { tc.OnJobRunAfterExported(job) return @@ -119,7 +119,7 @@ func (tc *TestDDLCallback) OnJobRunAfter(job *model.Job) { // OnJobUpdated is used to run the user customized logic of `OnJobUpdated` first. func (tc *TestDDLCallback) OnJobUpdated(job *model.Job) { - logutil.BgLogger().Info("on job updated", zap.String("job", job.String())) + logutil.DDLLogger().Info("on job updated", zap.String("job", job.String())) if onJobUpdatedExportedFunc := tc.OnJobUpdatedExported.Load(); onJobUpdatedExportedFunc != nil { (*onJobUpdatedExportedFunc)(job) return diff --git a/pkg/ddl/util/dead_table_lock_checker.go b/pkg/ddl/util/dead_table_lock_checker.go index 0dd404dc7fb8e..f0236e164ca6c 100644 --- a/pkg/ddl/util/dead_table_lock_checker.go +++ b/pkg/ddl/util/dead_table_lock_checker.go @@ -20,8 +20,8 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/parser/model" - "github.com/pingcap/tidb/pkg/util/logutil" clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" ) @@ -59,7 +59,7 @@ func (d *DeadTableLockChecker) getAliveServers(ctx context.Context) (map[string] resp, err = d.etcdCli.Get(childCtx, DDLAllSchemaVersions, clientv3.WithPrefix()) cancel() if err != nil { - logutil.BgLogger().Info("clean dead table lock get alive servers failed.", zap.String("category", "ddl"), zap.Error(err)) + logutil.DDLLogger().Info("clean dead table lock get alive servers failed.", zap.Error(err)) time.Sleep(defaultRetryInterval) continue } diff --git a/pkg/ddl/util/util.go b/pkg/ddl/util/util.go index fa61d6f5f541e..a8c6bef2c86fc 100644 --- a/pkg/ddl/util/util.go +++ b/pkg/ddl/util/util.go @@ -24,12 +24,12 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" + "github.com/pingcap/tidb/pkg/ddl/logutil" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/parser/terror" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/util/chunk" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/mock" "github.com/pingcap/tidb/pkg/util/sqlexec" "github.com/tikv/client-go/v2/tikvrpc" @@ -283,7 +283,7 @@ func DeleteKeyFromEtcd(key string, etcdCli *clientv3.Client, retryCnt int, timeo if err == nil { return nil } - logutil.BgLogger().Warn("etcd-cli delete key failed", zap.String("category", "ddl"), zap.String("key", key), zap.Error(err), zap.Int("retryCnt", i)) + logutil.DDLLogger().Warn("etcd-cli delete key failed", zap.String("key", key), zap.Error(err), zap.Int("retryCnt", i)) } return errors.Trace(err) } @@ -305,7 +305,7 @@ func PutKVToEtcdMono(ctx context.Context, etcdCli *clientv3.Client, retryCnt int resp, err = etcdCli.Get(childCtx, key) if err != nil { cancel() - logutil.BgLogger().Warn("etcd-cli put kv failed", zap.String("category", "ddl"), zap.String("key", key), zap.String("value", val), zap.Error(err), zap.Int("retryCnt", i)) + logutil.DDLLogger().Warn("etcd-cli put kv failed", zap.String("key", key), zap.String("value", val), zap.Error(err), zap.Int("retryCnt", i)) time.Sleep(KeyOpRetryInterval) continue } @@ -330,7 +330,7 @@ func PutKVToEtcdMono(ctx context.Context, etcdCli *clientv3.Client, retryCnt int err = errors.New("performing compare-and-swap during PutKVToEtcd failed") } - logutil.BgLogger().Warn("etcd-cli put kv failed", zap.String("category", "ddl"), zap.String("key", key), zap.String("value", val), zap.Error(err), zap.Int("retryCnt", i)) + logutil.DDLLogger().Warn("etcd-cli put kv failed", zap.String("key", key), zap.String("value", val), zap.Error(err), zap.Int("retryCnt", i)) time.Sleep(KeyOpRetryInterval) } return errors.Trace(err) @@ -354,7 +354,7 @@ func PutKVToEtcd(ctx context.Context, etcdCli *clientv3.Client, retryCnt int, ke if err == nil { return nil } - logutil.BgLogger().Warn("etcd-cli put kv failed", zap.String("category", "ddl"), zap.String("key", key), zap.String("value", val), zap.Error(err), zap.Int("retryCnt", i)) + logutil.DDLLogger().Warn("etcd-cli put kv failed", zap.String("key", key), zap.String("value", val), zap.Error(err), zap.Int("retryCnt", i)) time.Sleep(KeyOpRetryInterval) } return errors.Trace(err) diff --git a/pkg/util/logutil/log.go b/pkg/util/logutil/log.go index 798b3a1c1f366..39ee17e75e564 100644 --- a/pkg/util/logutil/log.go +++ b/pkg/util/logutil/log.go @@ -223,7 +223,9 @@ func Logger(ctx context.Context) *zap.Logger { return log.L() } -// BgLogger is alias of `logutil.BgLogger()` +// BgLogger is alias of `logutil.BgLogger()`. It's initialized in tidb-server's +// main function. Don't use it in `init` or equivalent functions otherwise it +// will print to stdout. func BgLogger() *zap.Logger { return log.L() } From 944fff519c90039747affb94067439ff9541f2fb Mon Sep 17 00:00:00 2001 From: yibin Date: Fri, 26 Apr 2024 17:46:02 +0800 Subject: [PATCH 17/20] executor: Fix index join hash produces redundant rows for left outer anti semi join type (#52908) close pingcap/tidb#52902 --- pkg/executor/index_lookup_hash_join.go | 3 +-- .../test/jointest/hashjoin/BUILD.bazel | 2 +- .../test/jointest/hashjoin/hash_join_test.go | 25 +++++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/pkg/executor/index_lookup_hash_join.go b/pkg/executor/index_lookup_hash_join.go index d443845624913..8dba9d3673215 100644 --- a/pkg/executor/index_lookup_hash_join.go +++ b/pkg/executor/index_lookup_hash_join.go @@ -28,7 +28,6 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/executor/internal/exec" "github.com/pingcap/tidb/pkg/expression" - plannercore "github.com/pingcap/tidb/pkg/planner/core" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/channel" @@ -736,7 +735,7 @@ func (iw *indexHashJoinInnerWorker) getMatchedOuterRows(innerRow chunk.Row, task return nil, nil, nil } joinType := JoinerType(iw.joiner) - isSemiJoin := joinType == plannercore.SemiJoin || joinType == plannercore.LeftOuterSemiJoin + isSemiJoin := joinType.IsSemiJoin() for ; matchedOuterEntry != nil; matchedOuterEntry = matchedOuterEntry.next { ptr := matchedOuterEntry.ptr outerRow := task.outerResult.GetRow(ptr) diff --git a/pkg/executor/test/jointest/hashjoin/BUILD.bazel b/pkg/executor/test/jointest/hashjoin/BUILD.bazel index 944eeb60c8537..11f52cb3a8aae 100644 --- a/pkg/executor/test/jointest/hashjoin/BUILD.bazel +++ b/pkg/executor/test/jointest/hashjoin/BUILD.bazel @@ -9,7 +9,7 @@ go_test( ], flaky = True, race = "on", - shard_count = 10, + shard_count = 11, deps = [ "//pkg/config", "//pkg/meta/autoid", diff --git a/pkg/executor/test/jointest/hashjoin/hash_join_test.go b/pkg/executor/test/jointest/hashjoin/hash_join_test.go index c93b93748bb61..6de6676196eb4 100644 --- a/pkg/executor/test/jointest/hashjoin/hash_join_test.go +++ b/pkg/executor/test/jointest/hashjoin/hash_join_test.go @@ -169,6 +169,31 @@ func TestIndexNestedLoopHashJoin(t *testing.T) { tk.MustExec("drop table orders") } +func TestIssue52902(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + // index hash join with semi join + require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/planner/core/MockOnlyEnableIndexHashJoin", "return(true)")) + defer func() { + require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/planner/core/MockOnlyEnableIndexHashJoin")) + }() + tk.MustExec("use test") + tk.MustExec("drop table if exists t0") + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1 (x int, y int)") + tk.MustExec("create table t0 (a int, b int, key (`b`))") + tk.MustExec("insert into t1 values(103, 600)") + tk.MustExec("insert into t1 values(100, 200)") + tk.MustExec("insert into t0 values( 105, 400)") + tk.MustExec("insert into t0 values( 104, 300)") + tk.MustExec("insert into t0 values( 103, 300)") + tk.MustExec("insert into t0 values( 102, 200)") + tk.MustExec("insert into t0 values( 101, 200)") + tk.MustExec("insert into t0 values( 100, 200)") + tk.MustQuery("select * from t1 where 1 = 1 and case when t1.x < 1000 then 1 = 1 " + + "when t1.x < 2000 then not exists (select 1 from t0 where t0.b = t1.y) else 1 = 1 end").Check(testkit.Rows("100 200", "103 600")) +} + func TestHashJoin(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) From 938d056077f8ef4eb69c13f391af0d2bd21e77a9 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Fri, 26 Apr 2024 19:34:26 +0800 Subject: [PATCH 18/20] *: upgrade rules_go (#52894) --- WORKSPACE | 10 +++++----- build/nogo_config.json | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 0b35e7cf8972a..5c81b9ffbb158 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -16,12 +16,12 @@ versions.check(minimum_bazel_version = "6.0.0") http_archive( name = "io_bazel_rules_go", - sha256 = "80a98277ad1311dacd837f9b16db62887702e9f1d1c4c9f796d0121a46c8e184", + sha256 = "af47f30e9cbd70ae34e49866e201b3f77069abb111183f2c0297e7e74ba6bbc0", urls = [ - "http://bazel-cache.pingcap.net:8080/bazelbuild/rules_go/releases/download/v0.46.0/rules_go-v0.46.0.zip", - "http://ats.apps.svc/bazelbuild/rules_go/releases/download/v0.46.0/rules_go-v0.46.0.zip", - "https://github.com/bazelbuild/rules_go/releases/download/v0.46.0/rules_go-v0.46.0.zip", - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.46.0/rules_go-v0.46.0.zip", + "http://bazel-cache.pingcap.net:8080/bazelbuild/rules_go/releases/download/v0.47.0/rules_go-v0.47.0.zip", + "http://ats.apps.svc/bazelbuild/rules_go/releases/download/v0.47.0/rules_go-v0.47.0.zip", + "https://github.com/bazelbuild/rules_go/releases/download/v0.47.0/rules_go-v0.47.0.zip", + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.47.0/rules_go-v0.47.0.zip", ], ) diff --git a/build/nogo_config.json b/build/nogo_config.json index 39ae55d92fcf5..58d37523fe211 100644 --- a/build/nogo_config.json +++ b/build/nogo_config.json @@ -1083,7 +1083,7 @@ "SA1029": { "exclude_files": { "pkg/parser/parser.go": "parser/parser.go code", - "/external/": "no need to vet third party code", + "external/": "no need to vet third party code", ".*_generated\\.go$": "ignore generated code", ".*mock.go$": "ignore generated code", ".*_test\\.go$": "ignore test code" From e0b6b0ce5e61d25e57e9230394a6ee96f852d983 Mon Sep 17 00:00:00 2001 From: Zhou Kunqin <25057648+time-and-fate@users.noreply.github.com> Date: Fri, 26 Apr 2024 21:56:57 +0800 Subject: [PATCH 19/20] planner: add a fix control to allow generate index merge path when normal index lookup path exists (#52872) close pingcap/tidb#52869 --- pkg/planner/core/indexmerge_path.go | 16 +++- pkg/planner/util/fixcontrol/get.go | 3 + .../r/planner/core/indexmerge_path.result | 94 +++++++++++++++++++ .../t/planner/core/indexmerge_path.test | 18 ++++ 4 files changed, 127 insertions(+), 4 deletions(-) diff --git a/pkg/planner/core/indexmerge_path.go b/pkg/planner/core/indexmerge_path.go index 13c05e34760b6..2483e57f2c1ca 100644 --- a/pkg/planner/core/indexmerge_path.go +++ b/pkg/planner/core/indexmerge_path.go @@ -32,6 +32,7 @@ import ( "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/util" "github.com/pingcap/tidb/pkg/planner/util/debugtrace" + "github.com/pingcap/tidb/pkg/planner/util/fixcontrol" "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/chunk" @@ -743,10 +744,17 @@ func (ds *DataSource) generateIndexMerge4NormalIndex(regularPathCount int, index needConsiderIndexMerge := true // if current index merge hint is nil, once there is a no-access-cond in one of possible access path. if len(ds.indexMergeHints) == 0 { - for i := 1; i < len(ds.possibleAccessPaths); i++ { - if len(ds.possibleAccessPaths[i].AccessConds) != 0 { - needConsiderIndexMerge = false - break + skipRangeScanCheck := fixcontrol.GetBoolWithDefault( + ds.SCtx().GetSessionVars().GetOptimizerFixControlMap(), + fixcontrol.Fix52869, + false, + ) + if !skipRangeScanCheck { + for i := 1; i < len(ds.possibleAccessPaths); i++ { + if len(ds.possibleAccessPaths[i].AccessConds) != 0 { + needConsiderIndexMerge = false + break + } } } if needConsiderIndexMerge { diff --git a/pkg/planner/util/fixcontrol/get.go b/pkg/planner/util/fixcontrol/get.go index d7dcc38a295ca..6739b41f3d41b 100644 --- a/pkg/planner/util/fixcontrol/get.go +++ b/pkg/planner/util/fixcontrol/get.go @@ -47,6 +47,9 @@ const ( // Fix49736 controls whether to force the optimizer to use plan cache even if there is risky optimization. // This fix-control is test-only. Fix49736 uint64 = 49736 + // Fix52869 controls whether to disable the limitation that index merge path won't be generated automatically when + // there exist other single-index access paths that do range scan. + Fix52869 uint64 = 52869 ) // GetStr fetches the given key from the fix control map as a string type. diff --git a/tests/integrationtest/r/planner/core/indexmerge_path.result b/tests/integrationtest/r/planner/core/indexmerge_path.result index e3dd09f1a19eb..696560821af9f 100644 --- a/tests/integrationtest/r/planner/core/indexmerge_path.result +++ b/tests/integrationtest/r/planner/core/indexmerge_path.result @@ -985,3 +985,97 @@ Selection 31.95 root or(json_contains(planner__core__indexmerge_path.t1.col_45, ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:idx_17(cast(`col_45` as double array)) range:[0.5226516826882698,0.5226516826882698], keep order:false, stats:pseudo └─Selection(Probe) 26.59 cop[tikv] or(lt(planner__core__indexmerge_path.t1.col_44, 1980-03-18 00:00:00.000000), gt(planner__core__indexmerge_path.t1.col_44, 2011-10-24 00:00:00.000000)) └─TableRowIDScan 39.94 cop[tikv] table:t1 keep order:false, stats:pseudo +drop table if exists t; +create table t(pk varbinary(255) primary key, a int, b varchar(50), c int, d varchar(45), index ia(a), index ib(b), index ic(c), index id(d)); +EXPLAIN format = brief SELECT /*+ use_index_merge(t) */ * FROM t WHERE a = 1 AND (b = '2' OR c = 3 OR d = '4'); +id estRows task access object operator info +IndexMerge 0.03 root type: union +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ib(b) range:["2","2"], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ic(c) range:[3,3], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:id(d) range:["4","4"], keep order:false, stats:pseudo +└─Selection(Probe) 0.03 cop[tikv] eq(planner__core__indexmerge_path.t.a, 1) + └─TableRowIDScan 29.97 cop[tikv] table:t keep order:false, stats:pseudo +EXPLAIN format = brief SELECT /*+ set_var(tidb_opt_fix_control='52869:on') */ * FROM t WHERE a = 1 AND (b = '2' OR c = 3 OR d = '4'); +id estRows task access object operator info +IndexMerge 0.03 root type: union +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ib(b) range:["2","2"], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ic(c) range:[3,3], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:id(d) range:["4","4"], keep order:false, stats:pseudo +└─Selection(Probe) 0.03 cop[tikv] eq(planner__core__indexmerge_path.t.a, 1) + └─TableRowIDScan 29.97 cop[tikv] table:t keep order:false, stats:pseudo +EXPLAIN format = brief SELECT * FROM t WHERE a = 1 AND (b = '2' OR c = 3 OR d = '4'); +id estRows task access object operator info +IndexLookUp 0.03 root +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ia(a) range:[1,1], keep order:false, stats:pseudo +└─Selection(Probe) 0.03 cop[tikv] or(eq(planner__core__indexmerge_path.t.b, "2"), or(eq(planner__core__indexmerge_path.t.c, 3), eq(planner__core__indexmerge_path.t.d, "4"))) + └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo +EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR d = '4'); +id estRows task access object operator info +TableReader 9.99 root data:Selection +└─Selection 9.99 cop[tikv] gt(planner__core__indexmerge_path.t.a, 1), or(eq(planner__core__indexmerge_path.t.b, "2"), or(eq(planner__core__indexmerge_path.t.c, 3), eq(planner__core__indexmerge_path.t.d, "4"))) + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR b = '4' OR c = 5); +id estRows task access object operator info +TableReader 13.32 root data:Selection +└─Selection 13.32 cop[tikv] gt(planner__core__indexmerge_path.t.a, 1), or(or(eq(planner__core__indexmerge_path.t.b, "2"), eq(planner__core__indexmerge_path.t.c, 3)), or(eq(planner__core__indexmerge_path.t.b, "4"), eq(planner__core__indexmerge_path.t.c, 5))) + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR b = '4' OR c = 5 OR b = '12' OR c = 13); +id estRows task access object operator info +TableReader 19.97 root data:Selection +└─Selection 19.97 cop[tikv] gt(planner__core__indexmerge_path.t.a, 1), or(or(eq(planner__core__indexmerge_path.t.b, "2"), or(eq(planner__core__indexmerge_path.t.c, 3), eq(planner__core__indexmerge_path.t.b, "4"))), or(eq(planner__core__indexmerge_path.t.c, 5), or(eq(planner__core__indexmerge_path.t.b, "12"), eq(planner__core__indexmerge_path.t.c, 13)))) + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +EXPLAIN format = brief SELECT * FROM t WHERE a = 1 AND (c = 13 OR c = 15 OR c = 5 OR b = '12' OR c = 13 OR b = '11'); +id estRows task access object operator info +IndexLookUp 0.05 root +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ia(a) range:[1,1], keep order:false, stats:pseudo +└─Selection(Probe) 0.05 cop[tikv] or(or(eq(planner__core__indexmerge_path.t.c, 13), or(eq(planner__core__indexmerge_path.t.c, 15), eq(planner__core__indexmerge_path.t.c, 5))), or(eq(planner__core__indexmerge_path.t.b, "12"), or(eq(planner__core__indexmerge_path.t.c, 13), eq(planner__core__indexmerge_path.t.b, "11")))) + └─TableRowIDScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo +SET @@tidb_opt_fix_control = '52869:on'; +EXPLAIN format = brief SELECT * FROM t WHERE a = 1 AND (b = '2' OR c = 3 OR d = '4'); +id estRows task access object operator info +IndexMerge 0.03 root type: union +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ib(b) range:["2","2"], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ic(c) range:[3,3], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:id(d) range:["4","4"], keep order:false, stats:pseudo +└─Selection(Probe) 0.03 cop[tikv] eq(planner__core__indexmerge_path.t.a, 1) + └─TableRowIDScan 29.97 cop[tikv] table:t keep order:false, stats:pseudo +EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR d = '4'); +id estRows task access object operator info +IndexMerge 9.99 root type: union +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ib(b) range:["2","2"], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ic(c) range:[3,3], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:id(d) range:["4","4"], keep order:false, stats:pseudo +└─Selection(Probe) 9.99 cop[tikv] gt(planner__core__indexmerge_path.t.a, 1) + └─TableRowIDScan 29.97 cop[tikv] table:t keep order:false, stats:pseudo +EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR b = '4' OR c = 5); +id estRows task access object operator info +IndexMerge 13.32 root type: union +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ib(b) range:["2","2"], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ic(c) range:[3,3], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ib(b) range:["4","4"], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ic(c) range:[5,5], keep order:false, stats:pseudo +└─Selection(Probe) 13.32 cop[tikv] gt(planner__core__indexmerge_path.t.a, 1) + └─TableRowIDScan 39.96 cop[tikv] table:t keep order:false, stats:pseudo +EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR b = '4' OR c = 5 OR b = '12' OR c = 13); +id estRows task access object operator info +IndexMerge 19.97 root type: union +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ib(b) range:["2","2"], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ic(c) range:[3,3], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ib(b) range:["4","4"], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ic(c) range:[5,5], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ib(b) range:["12","12"], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ic(c) range:[13,13], keep order:false, stats:pseudo +└─Selection(Probe) 19.97 cop[tikv] gt(planner__core__indexmerge_path.t.a, 1) + └─TableRowIDScan 59.91 cop[tikv] table:t keep order:false, stats:pseudo +EXPLAIN format = brief SELECT * FROM t WHERE a = 1 AND (c = 13 OR c = 15 OR c = 5 OR b = '12' OR c = 13 OR b = '11'); +id estRows task access object operator info +IndexMerge 0.05 root type: union +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ic(c) range:[13,13], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ic(c) range:[15,15], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ic(c) range:[5,5], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ib(b) range:["12","12"], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ic(c) range:[13,13], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:ib(b) range:["11","11"], keep order:false, stats:pseudo +└─Selection(Probe) 0.05 cop[tikv] eq(planner__core__indexmerge_path.t.a, 1) + └─TableRowIDScan 49.94 cop[tikv] table:t keep order:false, stats:pseudo +SET @@tidb_opt_fix_control = default; diff --git a/tests/integrationtest/t/planner/core/indexmerge_path.test b/tests/integrationtest/t/planner/core/indexmerge_path.test index 52fa5cf5cb249..e398b39dba679 100644 --- a/tests/integrationtest/t/planner/core/indexmerge_path.test +++ b/tests/integrationtest/t/planner/core/indexmerge_path.test @@ -407,3 +407,21 @@ CREATE TABLE `t1` ( INSERT INTO t1 VALUES('1988-07-19','[0.9233398239291353, 0.9396459773262974, 0.540018481999012, 0.181978533893545]',x'652539286c5f7e6b482a7265575a'); SELECT col_44, col_45 FROM t1 WHERE NOT (col_44 BETWEEN '1980-03-18' AND '2011-10-24') GROUP BY col_46,col_45 HAVING JSON_CONTAINS(col_45, '0.540018481999012') OR JSON_OVERLAPS(col_45, '[0.5785147169732324,0.8314968898215304,0.5226516826882698]'); EXPLAIN format=brief SELECT /*+ use_index_merge(t1) */ col_44, col_45 FROM t1 WHERE NOT (col_44 BETWEEN '1980-03-18' AND '2011-10-24') GROUP BY col_46,col_45 HAVING JSON_CONTAINS(col_45, '0.540018481999012') OR JSON_OVERLAPS(col_45, '[0.5785147169732324,0.8314968898215304,0.5226516826882698]'); + +# TestIssue52869 +drop table if exists t; +create table t(pk varbinary(255) primary key, a int, b varchar(50), c int, d varchar(45), index ia(a), index ib(b), index ic(c), index id(d)); +EXPLAIN format = brief SELECT /*+ use_index_merge(t) */ * FROM t WHERE a = 1 AND (b = '2' OR c = 3 OR d = '4'); +EXPLAIN format = brief SELECT /*+ set_var(tidb_opt_fix_control='52869:on') */ * FROM t WHERE a = 1 AND (b = '2' OR c = 3 OR d = '4'); +EXPLAIN format = brief SELECT * FROM t WHERE a = 1 AND (b = '2' OR c = 3 OR d = '4'); +EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR d = '4'); +EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR b = '4' OR c = 5); +EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR b = '4' OR c = 5 OR b = '12' OR c = 13); +EXPLAIN format = brief SELECT * FROM t WHERE a = 1 AND (c = 13 OR c = 15 OR c = 5 OR b = '12' OR c = 13 OR b = '11'); +SET @@tidb_opt_fix_control = '52869:on'; +EXPLAIN format = brief SELECT * FROM t WHERE a = 1 AND (b = '2' OR c = 3 OR d = '4'); +EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR d = '4'); +EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR b = '4' OR c = 5); +EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR b = '4' OR c = 5 OR b = '12' OR c = 13); +EXPLAIN format = brief SELECT * FROM t WHERE a = 1 AND (c = 13 OR c = 15 OR c = 5 OR b = '12' OR c = 13 OR b = '11'); +SET @@tidb_opt_fix_control = default; From c73d6c5a98ad76df5b97b25dc8c90d1df2e8be35 Mon Sep 17 00:00:00 2001 From: yibin Date: Sat, 27 Apr 2024 00:08:56 +0800 Subject: [PATCH 20/20] planner: Fix projection expression and schema not match error (#52913) ref pingcap/tidb#52828 --- .../mpp/testdata/integration_suite_out.json | 18 +++++++++--------- .../testdata/plan_suite_out.json | 4 ++-- pkg/planner/core/task.go | 1 + 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/pkg/planner/core/casetest/mpp/testdata/integration_suite_out.json b/pkg/planner/core/casetest/mpp/testdata/integration_suite_out.json index 0c339b9b06b6c..a75f09037a2ba 100644 --- a/pkg/planner/core/casetest/mpp/testdata/integration_suite_out.json +++ b/pkg/planner/core/casetest/mpp/testdata/integration_suite_out.json @@ -714,7 +714,7 @@ " │ └─TableFullScan 4.00 mpp[tiflash] table:d3_t pushed down filter:empty, keep order:false", " └─ExchangeReceiver(Probe) 64.00 mpp[tiflash] ", " └─ExchangeSender 64.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.fact_t.d3_k, collate: binary]", - " └─Projection 64.00 mpp[tiflash] test.fact_t.d3_k", + " └─Projection 64.00 mpp[tiflash] test.fact_t.d3_k, test.fact_t.d2_k", " └─HashJoin 64.00 mpp[tiflash] inner join, equal:[eq(test.fact_t.d2_k, test.d2_t.d2_k)]", " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d2_t.d2_k, collate: binary]", @@ -722,7 +722,7 @@ " │ └─TableFullScan 4.00 mpp[tiflash] table:d2_t pushed down filter:empty, keep order:false", " └─ExchangeReceiver(Probe) 32.00 mpp[tiflash] ", " └─ExchangeSender 32.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.fact_t.d2_k, collate: binary]", - " └─Projection 32.00 mpp[tiflash] test.fact_t.d2_k, test.fact_t.d3_k", + " └─Projection 32.00 mpp[tiflash] test.fact_t.d2_k, test.fact_t.d3_k, test.fact_t.d1_k", " └─HashJoin 32.00 mpp[tiflash] inner join, equal:[eq(test.d1_t.d1_k, test.fact_t.d1_k)]", " ├─ExchangeReceiver(Build) 4.00 mpp[tiflash] ", " │ └─ExchangeSender 4.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.d1_t.d1_k, collate: binary]", @@ -1015,7 +1015,7 @@ "TableReader 1.00 root MppVersion: 2, data:ExchangeSender", "└─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", " └─Projection 1.00 mpp[tiflash] Column#13", - " └─Projection 1.00 mpp[tiflash] Column#13", + " └─Projection 1.00 mpp[tiflash] Column#13, Column#23, Column#24", " └─HashJoin 1.00 mpp[tiflash] left outer join, equal:[eq(test.t3.v1, test.t1.v1) eq(test.t3.v2, test.t1.v2)]", " ├─ExchangeReceiver(Build) 1.00 mpp[tiflash] ", " │ └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#23, collate: binary], [name: Column#24, collate: binary]", @@ -1045,7 +1045,7 @@ " └─ExchangeReceiver 2.00 mpp[tiflash] ", " └─ExchangeSender 2.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.t2.v1, collate: binary], [name: test.t2.v2, collate: binary]", " └─HashAgg 2.00 mpp[tiflash] group by:test.t2.v1, test.t2.v2, funcs:count(1)->Column#22", - " └─Projection 2.00 mpp[tiflash] test.t2.v1, test.t2.v2", + " └─Projection 2.00 mpp[tiflash] test.t2.v1, test.t2.v2, test.t1.v1, test.t1.v2", " └─HashJoin 2.00 mpp[tiflash] left outer join, equal:[eq(test.t1.v1, test.t2.v1) eq(test.t1.v2, test.t2.v2)]", " ├─ExchangeReceiver(Build) 2.00 mpp[tiflash] ", " │ └─ExchangeSender 2.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.t1.v1, collate: binary], [name: test.t1.v2, collate: binary]", @@ -1066,7 +1066,7 @@ " └─HashAgg 1.00 mpp[tiflash] group by:test.t2.v1, test.t2.v2, funcs:count(1)->Column#9, funcs:firstrow(test.t2.v1)->test.t2.v1, funcs:firstrow(test.t2.v2)->test.t2.v2", " └─ExchangeReceiver 1.00 mpp[tiflash] ", " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.t2.v1, collate: binary], [name: test.t2.v2, collate: binary]", - " └─Projection 1.00 mpp[tiflash] test.t2.v1, test.t2.v2", + " └─Projection 1.00 mpp[tiflash] test.t2.v1, test.t2.v2, test.t3.v1, test.t3.v2", " └─HashJoin 1.00 mpp[tiflash] left outer join, equal:[eq(test.t3.v1, test.t2.v1) eq(test.t3.v2, test.t2.v2)]", " ├─ExchangeReceiver(Build) 1.00 mpp[tiflash] ", " │ └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.t3.v1, collate: binary], [name: test.t3.v2, collate: binary]", @@ -1593,7 +1593,7 @@ "TableReader 12500.00 root MppVersion: 2, data:ExchangeSender", "└─ExchangeSender 12500.00 mpp[tiflash] ExchangeType: PassThrough", " └─Projection 12500.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c1, test.t.c2, test.t.c3", - " └─Projection 12500.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c1, test.t.c2, test.t.c3", + " └─Projection 12500.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c1, test.t.c2, test.t.c3, Column#14, Column#16", " └─HashJoin 12500.00 mpp[tiflash] inner join, equal:[eq(Column#13, Column#14) eq(Column#15, Column#16)]", " ├─ExchangeReceiver(Build) 10000.00 mpp[tiflash] ", " │ └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#23, collate: binary], [name: Column#24, collate: binary]", @@ -1612,7 +1612,7 @@ "TableReader 7976.02 root MppVersion: 2, data:ExchangeSender", "└─ExchangeSender 7976.02 mpp[tiflash] ExchangeType: PassThrough", " └─Projection 7976.02 mpp[tiflash] test.t.c1, test.t.c2, test.t.c5, Column#7, test.t.c1, test.t.c2, test.t.c3, Column#14", - " └─Projection 7976.02 mpp[tiflash] Column#7, test.t.c1, test.t.c2, test.t.c5, Column#14, test.t.c1, test.t.c2, test.t.c3", + " └─Projection 7976.02 mpp[tiflash] Column#7, test.t.c1, test.t.c2, test.t.c5, Column#14, test.t.c1, test.t.c2, test.t.c3, Column#59, Column#60", " └─HashJoin 7976.02 mpp[tiflash] inner join, equal:[eq(test.t.c1, test.t.c2) eq(test.t.c2, test.t.c3) eq(test.t.c5, test.t.c1)]", " ├─ExchangeReceiver(Build) 7976.02 mpp[tiflash] ", " │ └─ExchangeSender 7976.02 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.t.c1, collate: binary], [name: Column#58, collate: binary], [name: test.t.c5, collate: binary]", @@ -1658,7 +1658,7 @@ "TableReader 12462.54 root MppVersion: 2, data:ExchangeSender", "└─ExchangeSender 12462.54 mpp[tiflash] ExchangeType: PassThrough", " └─Projection 12462.54 mpp[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5", - " └─Projection 12462.54 mpp[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5", + " └─Projection 12462.54 mpp[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, Column#19, Column#21, Column#22", " └─HashJoin 12462.54 mpp[tiflash] inner join, equal:[eq(test.t.c2, test.t.c1) eq(test.t.c3, test.t.c2) eq(test.t.c1, test.t.c3) eq(test.t.c3, test.t.c4) eq(test.t.c5, test.t.c1)]", " ├─ExchangeReceiver(Build) 9970.03 mpp[tiflash] ", " │ └─ExchangeSender 9970.03 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.t.c2, collate: binary], [name: Column#18, collate: binary], [name: Column#20, collate: binary], [name: test.t.c3, collate: binary], [name: test.t.c5, collate: binary]", @@ -1678,7 +1678,7 @@ "TableReader 12500.00 root MppVersion: 2, data:ExchangeSender", "└─ExchangeSender 12500.00 mpp[tiflash] ExchangeType: PassThrough", " └─Projection 12500.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5", - " └─Projection 12500.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5", + " └─Projection 12500.00 mpp[tiflash] test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, test.t.c1, test.t.c2, test.t.c3, test.t.c4, test.t.c5, Column#14", " └─HashJoin 12500.00 mpp[tiflash] inner join, equal:[eq(Column#13, Column#14)]", " ├─ExchangeReceiver(Build) 10000.00 mpp[tiflash] ", " │ └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#18, collate: binary]", diff --git a/pkg/planner/core/casetest/physicalplantest/testdata/plan_suite_out.json b/pkg/planner/core/casetest/physicalplantest/testdata/plan_suite_out.json index 4b106ec137efe..6ffc8c1fde96b 100644 --- a/pkg/planner/core/casetest/physicalplantest/testdata/plan_suite_out.json +++ b/pkg/planner/core/casetest/physicalplantest/testdata/plan_suite_out.json @@ -3088,7 +3088,7 @@ "TableReader 8000.00 root MppVersion: 2, data:ExchangeSender", "└─ExchangeSender 8000.00 mpp[tiflash] ExchangeType: PassThrough", " └─Projection 8000.00 mpp[tiflash] Column#6, Column#5, test.employee.empid, test.employee.deptid, test.employee.salary", - " └─Projection 8000.00 mpp[tiflash] test.employee.empid, test.employee.deptid, test.employee.salary, Column#6, Column#5", + " └─Projection 8000.00 mpp[tiflash] test.employee.empid, test.employee.deptid, test.employee.salary, Column#6, Column#5, Column#26", " └─HashJoin 8000.00 mpp[tiflash] inner join, equal:[eq(test.employee.deptid, Column#6)]", " ├─ExchangeReceiver(Build) 6400.00 mpp[tiflash] ", " │ └─ExchangeSender 6400.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#6, collate: binary]", @@ -3114,7 +3114,7 @@ "TableReader 8000.00 root MppVersion: 2, data:ExchangeSender", "└─ExchangeSender 8000.00 mpp[tiflash] ExchangeType: PassThrough", " └─Projection 8000.00 mpp[tiflash] test.employee.empid, test.employee.deptid, test.employee.salary, Column#10, Column#9", - " └─Projection 8000.00 mpp[tiflash] test.employee.empid, test.employee.deptid, test.employee.salary, Column#10, Column#9", + " └─Projection 8000.00 mpp[tiflash] test.employee.empid, test.employee.deptid, test.employee.salary, Column#10, Column#9, Column#26", " └─HashJoin 8000.00 mpp[tiflash] inner join, equal:[eq(test.employee.deptid, Column#10)]", " ├─ExchangeReceiver(Build) 6400.00 mpp[tiflash] ", " │ └─ExchangeSender 6400.00 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: Column#10, collate: binary]", diff --git a/pkg/planner/core/task.go b/pkg/planner/core/task.go index 19bc02024b877..36fae9b96ceba 100644 --- a/pkg/planner/core/task.go +++ b/pkg/planner/core/task.go @@ -447,6 +447,7 @@ func (p *PhysicalHashJoin) attach2TaskForMpp(tasks ...base.Task) base.Task { proj.SetSchema(p.Schema().Clone()) for _, hashCol := range hashColArray { if !proj.Schema().Contains(hashCol) { + proj.Exprs = append(proj.Exprs, hashCol) proj.Schema().Append(hashCol) } }