From 6c5abe1ec044412cb4c0776e4a610ccf6e89a629 Mon Sep 17 00:00:00 2001 From: yisaer Date: Tue, 17 Jan 2023 14:25:40 +0800 Subject: [PATCH 1/6] add test --- executor/historical_stats_test.go | 45 +++++++++++++++++++++++++++++++ sessionctx/variable/sysvar.go | 2 +- statistics/handle/gc.go | 7 +++-- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/executor/historical_stats_test.go b/executor/historical_stats_test.go index 0b00d3182f019..4150961546e85 100644 --- a/executor/historical_stats_test.go +++ b/executor/historical_stats_test.go @@ -183,6 +183,51 @@ func TestGCHistoryStatsAfterDropTable(t *testing.T) { tableInfo.Meta().ID)).Check(testkit.Rows("0")) } +func TestAssertHistoricalStatsAfterAlterTable(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("set global tidb_enable_historical_stats = 1") + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int, b varchar(10),c int, KEY `idx` (`c`))") + tk.MustExec("analyze table test.t") + is := dom.InfoSchema() + tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + // dump historical stats + h := dom.StatsHandle() + hsWorker := dom.GetHistoricalStatsWorker() + tblID := hsWorker.GetOneHistoricalStatsTable() + err = hsWorker.DumpHistoricalStats(tblID, h) + require.Nil(t, err) + + time.Sleep(1 * time.Second) + snapshot := oracle.GoTimeToTS(time.Now()) + jsTable, err := h.DumpHistoricalStatsBySnapshot("test", tableInfo.Meta(), snapshot) + require.NoError(t, err) + require.NotNil(t, jsTable) + require.NotEqual(t, jsTable.Version, uint64(0)) + originVersion := jsTable.Version + + // assert historical stats non-change after drop column + tk.MustExec("alter table t drop column b") + h.GCStats(is, 0) + snapshot = oracle.GoTimeToTS(time.Now()) + jsTable, err = h.DumpHistoricalStatsBySnapshot("test", tableInfo.Meta(), snapshot) + require.NoError(t, err) + require.NotNil(t, jsTable) + require.Equal(t, jsTable.Version, originVersion) + + // assert historical stats non-change after drop index + tk.MustExec("alter table t drop index idx") + h.GCStats(is, 0) + snapshot = oracle.GoTimeToTS(time.Now()) + jsTable, err = h.DumpHistoricalStatsBySnapshot("test", tableInfo.Meta(), snapshot) + require.NoError(t, err) + require.NotNil(t, jsTable) + require.Equal(t, jsTable.Version, originVersion) +} + func TestGCOutdatedHistoryStats(t *testing.T) { store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 376b6d6d9f4fd..438996ed2a2c1 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -731,7 +731,7 @@ var defaultSysVars = []*SysVar{ return nil }}, {Scope: ScopeGlobal, Name: TiDBEnableTelemetry, Value: BoolToOnOff(DefTiDBEnableTelemetry), Type: TypeBool}, - {Scope: ScopeGlobal, Name: TiDBEnableHistoricalStats, Value: Off, Type: TypeBool}, + {Scope: ScopeGlobal, Name: TiDBEnableHistoricalStats, Value: On, Type: TypeBool}, /* tikv gc metrics */ {Scope: ScopeGlobal, Name: TiDBGCEnable, Value: On, Type: TypeBool, GetGlobal: func(_ context.Context, s *SessionVars) (string, error) { return getTiDBTableValue(s, "tikv_gc_enable", On) diff --git a/statistics/handle/gc.go b/statistics/handle/gc.go index f16e2c9719088..af20eeb35a363 100644 --- a/statistics/handle/gc.go +++ b/statistics/handle/gc.go @@ -53,8 +53,11 @@ func (h *Handle) GCStats(is infoschema.InfoSchema, ddlLease time.Duration) error if err := h.gcTableStats(is, row.GetInt64(0)); err != nil { return errors.Trace(err) } - if err := h.gcHistoryStatsFromKV(row.GetInt64(0)); err != nil { - return errors.Trace(err) + _, existed := is.TableByID(row.GetInt64(0)) + if !existed { + if err := h.gcHistoryStatsFromKV(row.GetInt64(0)); err != nil { + return errors.Trace(err) + } } } if err := h.ClearOutdatedHistoryStats(); err != nil { From 61419d3ccb1b05942a029d67152b3ce49586219b Mon Sep 17 00:00:00 2001 From: yisaer Date: Tue, 17 Jan 2023 19:30:02 +0800 Subject: [PATCH 2/6] fix --- executor/historical_stats_test.go | 1 + executor/set_test.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/executor/historical_stats_test.go b/executor/historical_stats_test.go index 4150961546e85..9c38f3b29aea3 100644 --- a/executor/historical_stats_test.go +++ b/executor/historical_stats_test.go @@ -174,6 +174,7 @@ func TestGCHistoryStatsAfterDropTable(t *testing.T) { tableInfo.Meta().ID)).Check(testkit.Rows("1")) // drop the table and gc stats tk.MustExec("drop table t") + is = dom.InfoSchema() h.GCStats(is, 0) // assert stats_history tables delete the record of dropped table diff --git a/executor/set_test.go b/executor/set_test.go index 1b2b4186bb4a3..01a2fc7979efc 100644 --- a/executor/set_test.go +++ b/executor/set_test.go @@ -645,7 +645,7 @@ func TestSetVar(t *testing.T) { tk.MustQuery("select @@tidb_enable_tso_follower_proxy").Check(testkit.Rows("0")) require.Error(t, tk.ExecToErr("set tidb_enable_tso_follower_proxy = 1")) - tk.MustQuery("select @@tidb_enable_historical_stats").Check(testkit.Rows("0")) + tk.MustQuery("select @@tidb_enable_historical_stats").Check(testkit.Rows("1")) tk.MustExec("set global tidb_enable_historical_stats = 1") tk.MustQuery("select @@tidb_enable_historical_stats").Check(testkit.Rows("1")) tk.MustExec("set global tidb_enable_historical_stats = 0") From 3da9bf2ebcba4f232602e59f758256f2f157a50d Mon Sep 17 00:00:00 2001 From: yisaer Date: Mon, 30 Jan 2023 17:43:55 +0800 Subject: [PATCH 3/6] fix 40843 --- domain/historical_stats.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/domain/historical_stats.go b/domain/historical_stats.go index 07e82bafeb58c..5b4074eecb27d 100644 --- a/domain/historical_stats.go +++ b/domain/historical_stats.go @@ -35,6 +35,9 @@ type HistoricalStatsWorker struct { // SendTblToDumpHistoricalStats send tableID to worker to dump historical stats func (w *HistoricalStatsWorker) SendTblToDumpHistoricalStats(tableID int64) { + if !enableDumpHistoricalStats.Load() { + return + } w.tblCH <- tableID } From 708ac3a0258428d5d97b3cca58b57f450c422822 Mon Sep 17 00:00:00 2001 From: yisaer Date: Mon, 30 Jan 2023 19:17:08 +0800 Subject: [PATCH 4/6] fix --- domain/historical_stats.go | 9 ++++++++- executor/historical_stats_test.go | 13 +++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/domain/historical_stats.go b/domain/historical_stats.go index 5b4074eecb27d..981876f54e15d 100644 --- a/domain/historical_stats.go +++ b/domain/historical_stats.go @@ -16,6 +16,7 @@ package domain import ( "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/metrics" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/sessionctx" @@ -35,7 +36,13 @@ type HistoricalStatsWorker struct { // SendTblToDumpHistoricalStats send tableID to worker to dump historical stats func (w *HistoricalStatsWorker) SendTblToDumpHistoricalStats(tableID int64) { - if !enableDumpHistoricalStats.Load() { + send := enableDumpHistoricalStats.Load() + failpoint.Inject("sendHistoricalStats", func(val failpoint.Value) { + if val.(bool) { + send = true + } + }) + if !send { return } w.tblCH <- tableID diff --git a/executor/historical_stats_test.go b/executor/historical_stats_test.go index 9c38f3b29aea3..f4d95ec5b76b5 100644 --- a/executor/historical_stats_test.go +++ b/executor/historical_stats_test.go @@ -17,6 +17,7 @@ package executor_test import ( "encoding/json" "fmt" + "github.com/pingcap/failpoint" "strconv" "testing" "time" @@ -30,6 +31,8 @@ import ( ) func TestRecordHistoryStatsAfterAnalyze(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/domain/sendHistoricalStats", "return(true)") + defer failpoint.Disable("github.com/pingcap/tidb/domain/sendHistoricalStats") store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) @@ -150,6 +153,8 @@ func TestRecordHistoryStatsMetaAfterAnalyze(t *testing.T) { } func TestGCHistoryStatsAfterDropTable(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/domain/sendHistoricalStats", "return(true)") + defer failpoint.Disable("github.com/pingcap/tidb/domain/sendHistoricalStats") store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) tk.MustExec("set global tidb_enable_historical_stats = 1") @@ -185,6 +190,8 @@ func TestGCHistoryStatsAfterDropTable(t *testing.T) { } func TestAssertHistoricalStatsAfterAlterTable(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/domain/sendHistoricalStats", "return(true)") + defer failpoint.Disable("github.com/pingcap/tidb/domain/sendHistoricalStats") store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) tk.MustExec("set global tidb_enable_historical_stats = 1") @@ -230,6 +237,8 @@ func TestAssertHistoricalStatsAfterAlterTable(t *testing.T) { } func TestGCOutdatedHistoryStats(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/domain/sendHistoricalStats", "return(true)") + defer failpoint.Disable("github.com/pingcap/tidb/domain/sendHistoricalStats") store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) tk.MustExec("set global tidb_enable_historical_stats = 1") @@ -265,6 +274,8 @@ func TestGCOutdatedHistoryStats(t *testing.T) { } func TestPartitionTableHistoricalStats(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/domain/sendHistoricalStats", "return(true)") + defer failpoint.Disable("github.com/pingcap/tidb/domain/sendHistoricalStats") store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) tk.MustExec("set global tidb_enable_historical_stats = 1") @@ -292,6 +303,8 @@ PARTITION p0 VALUES LESS THAN (6) } func TestDumpHistoricalStatsByTable(t *testing.T) { + failpoint.Enable("github.com/pingcap/tidb/domain/sendHistoricalStats", "return(true)") + defer failpoint.Disable("github.com/pingcap/tidb/domain/sendHistoricalStats") store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) tk.MustExec("set global tidb_enable_historical_stats = 1") From 3155f4f369db78e4c18813c5e6ddd10c3f3f3635 Mon Sep 17 00:00:00 2001 From: yisaer Date: Mon, 30 Jan 2023 19:37:22 +0800 Subject: [PATCH 5/6] fix --- executor/historical_stats_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/historical_stats_test.go b/executor/historical_stats_test.go index f4d95ec5b76b5..becb1e82212f8 100644 --- a/executor/historical_stats_test.go +++ b/executor/historical_stats_test.go @@ -17,11 +17,11 @@ package executor_test import ( "encoding/json" "fmt" - "github.com/pingcap/failpoint" "strconv" "testing" "time" + "github.com/pingcap/failpoint" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/statistics/handle" From bb181162773fc22b3e7c13690b71a702e2ba9855 Mon Sep 17 00:00:00 2001 From: yisaer Date: Mon, 30 Jan 2023 21:18:55 +0800 Subject: [PATCH 6/6] fix --- domain/historical_stats.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/domain/historical_stats.go b/domain/historical_stats.go index 981876f54e15d..6d4125b75f5d7 100644 --- a/domain/historical_stats.go +++ b/domain/historical_stats.go @@ -21,6 +21,8 @@ import ( "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/statistics/handle" + "github.com/pingcap/tidb/util/logutil" + "go.uber.org/zap" ) var ( @@ -45,7 +47,12 @@ func (w *HistoricalStatsWorker) SendTblToDumpHistoricalStats(tableID int64) { if !send { return } - w.tblCH <- tableID + select { + case w.tblCH <- tableID: + return + default: + logutil.BgLogger().Warn("discard dump historical stats task", zap.Int64("table-id", tableID)) + } } // DumpHistoricalStats dump stats by given tableID