From e954bcfc74f85f7592d68014dbcd807bd9fa2fba Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 22 May 2024 18:53:47 +0800 Subject: [PATCH] *: not load unnecessay data during the non-lite-init-stats (#53399) close pingcap/tidb#53425 --- pkg/planner/cardinality/selectivity_test.go | 5 +- pkg/statistics/handle/bootstrap.go | 5 +- .../handle/handletest/statstest/stats_test.go | 66 ++++++++++--------- 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/pkg/planner/cardinality/selectivity_test.go b/pkg/planner/cardinality/selectivity_test.go index f8f993f59682c..318cb3a454911 100644 --- a/pkg/planner/cardinality/selectivity_test.go +++ b/pkg/planner/cardinality/selectivity_test.go @@ -1338,12 +1338,11 @@ func TestBuiltinInEstWithoutStats(t *testing.T) { "└─Selection 0.08 cop[tikv] in(test.t.a, 1, 2, 3, 4, 5, 6, 7, 8)", " └─TableFullScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo", )) - h.Clear() require.NoError(t, h.InitStats(is)) tk.MustQuery("explain format='brief' select * from t where a in (1, 2, 3, 4, 5, 6, 7, 8)").Check(testkit.Rows( - "TableReader 0.08 root data:Selection", - "└─Selection 0.08 cop[tikv] in(test.t.a, 1, 2, 3, 4, 5, 6, 7, 8)", + "TableReader 8.00 root data:Selection", + "└─Selection 8.00 cop[tikv] in(test.t.a, 1, 2, 3, 4, 5, 6, 7, 8)", " └─TableFullScan 10.00 cop[tikv] table:t keep order:false, stats:pseudo", )) } diff --git a/pkg/statistics/handle/bootstrap.go b/pkg/statistics/handle/bootstrap.go index d238c620fec8d..6acd746bd26d0 100644 --- a/pkg/statistics/handle/bootstrap.go +++ b/pkg/statistics/handle/bootstrap.go @@ -322,7 +322,10 @@ func (h *Handle) initStatsHistogramsByPaging(is infoschema.InfoSchema, cache sta } }() sctx := se.(sessionctx.Context) - sql := "select HIGH_PRIORITY table_id, is_index, hist_id, distinct_count, version, null_count, cm_sketch, tot_col_size, stats_ver, correlation, flag, last_analyze_pos from mysql.stats_histograms where table_id >= %? and table_id < %?" + // Why do we need to add `is_index=1` in the SQL? + // because it is aligned to the `initStatsTopN` function, which only loads the topn of the index too. + // the other will be loaded by sync load. + sql := "select HIGH_PRIORITY table_id, is_index, hist_id, distinct_count, version, null_count, cm_sketch, tot_col_size, stats_ver, correlation, flag, last_analyze_pos from mysql.stats_histograms where table_id >= %? and table_id < %? and is_index=1" rc, err := util.Exec(sctx, sql, task.StartTid, task.EndTid) if err != nil { return errors.Trace(err) diff --git a/pkg/statistics/handle/handletest/statstest/stats_test.go b/pkg/statistics/handle/handletest/statstest/stats_test.go index 3d9a0c1066635..7d7e0bdd4d78d 100644 --- a/pkg/statistics/handle/handletest/statstest/stats_test.go +++ b/pkg/statistics/handle/handletest/statstest/stats_test.go @@ -262,23 +262,12 @@ func TestInitStats(t *testing.T) { h.Clear() require.NoError(t, h.InitStats(is)) table0 := h.GetTableStats(tbl.Meta()) - cols := table0.Columns - require.Equal(t, uint8(0x36), cols[1].LastAnalyzePos.GetBytes()[0]) - require.Equal(t, uint8(0x37), cols[2].LastAnalyzePos.GetBytes()[0]) - require.Equal(t, uint8(0x38), cols[3].LastAnalyzePos.GetBytes()[0]) + idx := table0.Indices + require.Equal(t, uint8(0x3), idx[1].LastAnalyzePos.GetBytes()[0]) h.Clear() require.NoError(t, h.Update(is)) // Index and pk are loaded. needed := fmt.Sprintf(`Table:%v RealtimeCount:6 -column:1 ndv:6 totColSize:0 -num: 1 lower_bound: 1 upper_bound: 1 repeats: 1 ndv: 0 -num: 1 lower_bound: 2 upper_bound: 2 repeats: 1 ndv: 0 -num: 1 lower_bound: 3 upper_bound: 3 repeats: 1 ndv: 0 -num: 1 lower_bound: 4 upper_bound: 4 repeats: 1 ndv: 0 -num: 1 lower_bound: 5 upper_bound: 5 repeats: 1 ndv: 0 -num: 1 lower_bound: 6 upper_bound: 6 repeats: 1 ndv: 0 -column:2 ndv:6 totColSize:6 -column:3 ndv:6 totColSize:6 index:1 ndv:6 num: 1 lower_bound: 1 upper_bound: 1 repeats: 1 ndv: 0 num: 1 lower_bound: 2 upper_bound: 2 repeats: 1 ndv: 0 @@ -328,26 +317,30 @@ func TestInitStats51358(t *testing.T) { } func TestInitStatsVer2(t *testing.T) { - restore := config.RestoreFunc() - defer restore() - config.UpdateGlobal(func(conf *config.Config) { - config.GetGlobalConfig().Performance.LiteInitStats = false - config.GetGlobalConfig().Performance.ConcurrentlyInitStats = false - }) - initStatsVer2(t) + originValue := config.GetGlobalConfig().Performance.LiteInitStats + concurrentlyInitStatsValue := config.GetGlobalConfig().Performance.ConcurrentlyInitStats + defer func() { + config.GetGlobalConfig().Performance.LiteInitStats = originValue + config.GetGlobalConfig().Performance.ConcurrentlyInitStats = concurrentlyInitStatsValue + }() + config.GetGlobalConfig().Performance.LiteInitStats = false + config.GetGlobalConfig().Performance.ConcurrentlyInitStats = false + initStatsVer2(t, false) } func TestInitStatsVer2Concurrency(t *testing.T) { - restore := config.RestoreFunc() - defer restore() - config.UpdateGlobal(func(conf *config.Config) { - config.GetGlobalConfig().Performance.LiteInitStats = false - config.GetGlobalConfig().Performance.ConcurrentlyInitStats = true - }) - initStatsVer2(t) + originValue := config.GetGlobalConfig().Performance.LiteInitStats + concurrentlyInitStatsValue := config.GetGlobalConfig().Performance.ConcurrentlyInitStats + defer func() { + config.GetGlobalConfig().Performance.LiteInitStats = originValue + config.GetGlobalConfig().Performance.ConcurrentlyInitStats = concurrentlyInitStatsValue + }() + config.GetGlobalConfig().Performance.LiteInitStats = false + config.GetGlobalConfig().Performance.ConcurrentlyInitStats = true + initStatsVer2(t, true) } -func initStatsVer2(t *testing.T) { +func initStatsVer2(t *testing.T, isConcurrency bool) { store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) tk.MustExec("use test") @@ -366,10 +359,19 @@ func initStatsVer2(t *testing.T) { h.Clear() require.NoError(t, h.InitStats(is)) table0 := h.GetTableStats(tbl.Meta()) - cols := table0.Columns - require.Equal(t, uint8(0x33), cols[1].LastAnalyzePos.GetBytes()[0]) - require.Equal(t, uint8(0x33), cols[2].LastAnalyzePos.GetBytes()[0]) - require.Equal(t, uint8(0x33), cols[3].LastAnalyzePos.GetBytes()[0]) + if isConcurrency { + idx := table0.Indices + require.Equal(t, uint8(0x3), idx[1].LastAnalyzePos.GetBytes()[0]) + require.Equal(t, uint8(0x3), idx[2].LastAnalyzePos.GetBytes()[0]) + } else { + cols := table0.Columns + require.Equal(t, uint8(0x33), cols[1].LastAnalyzePos.GetBytes()[0]) + require.Equal(t, uint8(0x33), cols[2].LastAnalyzePos.GetBytes()[0]) + require.Equal(t, uint8(0x33), cols[3].LastAnalyzePos.GetBytes()[0]) + idx := table0.Indices + require.Equal(t, uint8(0x3), idx[1].LastAnalyzePos.GetBytes()[0]) + require.Equal(t, uint8(0x3), idx[2].LastAnalyzePos.GetBytes()[0]) + } h.Clear() require.NoError(t, h.InitStats(is)) table1 := h.GetTableStats(tbl.Meta())