From fc74c657d18c477412a1743a75b9e5f2b2bd55ca Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 28 Mar 2023 11:37:19 +0800 Subject: [PATCH 1/4] oomtest: add message filer for oom test to avoid flaky test Signed-off-by: Weizhen Wang --- executor/oomtest/BUILD.bazel | 1 + executor/oomtest/oom_test.go | 39 ++++++++++++++++++++++++++++++------ util/set/BUILD.bazel | 1 + util/set/string_set.go | 7 +++++++ 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/executor/oomtest/BUILD.bazel b/executor/oomtest/BUILD.bazel index 367e215f04287..ea66329264da9 100644 --- a/executor/oomtest/BUILD.bazel +++ b/executor/oomtest/BUILD.bazel @@ -10,6 +10,7 @@ go_test( deps = [ "//testkit", "//testkit/testsetup", + "//util/set", "//util/syncutil", "@com_github_pingcap_failpoint//:failpoint", "@com_github_pingcap_log//:log", diff --git a/executor/oomtest/oom_test.go b/executor/oomtest/oom_test.go index fbb998526a26e..e91d23fd30630 100644 --- a/executor/oomtest/oom_test.go +++ b/executor/oomtest/oom_test.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/tidb/testkit" "github.com/pingcap/tidb/testkit/testsetup" + "github.com/pingcap/tidb/util/set" "github.com/pingcap/tidb/util/syncutil" "github.com/stretchr/testify/require" "go.uber.org/goleak" @@ -53,6 +54,9 @@ func TestMemTracker4UpdateExec(t *testing.T) { log.SetLevel(zap.InfoLevel) oom.SetTracker("") + oom.ClearMessageFilter() + oom.AddMessageFilter("expensive_query during bootstrap phase") + oom.AddMessageFilter("schemaLeaseChecker is not set for this transaction") tk.MustExec("insert into t_MemTracker4UpdateExec values (1,1,1), (2,2,2), (3,3,3)") require.Equal(t, "schemaLeaseChecker is not set for this transaction", oom.GetTracker()) @@ -75,6 +79,9 @@ func TestMemTracker4InsertAndReplaceExec(t *testing.T) { log.SetLevel(zap.InfoLevel) oom.SetTracker("") + oom.AddMessageFilter( + "schemaLeaseChecker is not set for this transaction", + "expensive_query during bootstrap phase") tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)") require.Equal(t, "schemaLeaseChecker is not set for this transaction", oom.GetTracker()) @@ -84,6 +91,8 @@ func TestMemTracker4InsertAndReplaceExec(t *testing.T) { tk.Session().GetSessionVars().MemQuotaQuery = -1 oom.SetTracker("") + oom.ClearMessageFilter() + oom.AddMessageFilter("expensive_query during bootstrap phase") tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)") require.Equal(t, "", oom.GetTracker()) @@ -93,6 +102,7 @@ func TestMemTracker4InsertAndReplaceExec(t *testing.T) { tk.Session().GetSessionVars().MemQuotaQuery = -1 oom.SetTracker("") + oom.ClearMessageFilter() tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec select * from t") require.Equal(t, "", oom.GetTracker()) @@ -102,6 +112,7 @@ func TestMemTracker4InsertAndReplaceExec(t *testing.T) { tk.Session().GetSessionVars().MemQuotaQuery = -1 oom.SetTracker("") + oom.ClearMessageFilter() tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec select * from t") require.Equal(t, "", oom.GetTracker()) @@ -114,6 +125,7 @@ func TestMemTracker4InsertAndReplaceExec(t *testing.T) { tk.Session().GetSessionVars().BatchInsert = true oom.SetTracker("") + oom.ClearMessageFilter() tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)") require.Equal(t, "", oom.GetTracker()) @@ -123,6 +135,8 @@ func TestMemTracker4InsertAndReplaceExec(t *testing.T) { tk.Session().GetSessionVars().MemQuotaQuery = -1 oom.SetTracker("") + oom.ClearMessageFilter() + oom.AddMessageFilter("expensive_query during bootstrap phase") tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)") require.Equal(t, "", oom.GetTracker()) @@ -146,6 +160,8 @@ func TestMemTracker4DeleteExec(t *testing.T) { tk.MustExec("insert into MemTracker4DeleteExec1 values(1,1,1), (2,2,2), (3,3,3), (4,4,4), (5,5,5)") oom.SetTracker("") + oom.ClearMessageFilter() + oom.AddMessageFilter("expensive_query during bootstrap phase") tk.MustExec("delete from MemTracker4DeleteExec1") require.Equal(t, "", oom.GetTracker()) @@ -166,8 +182,9 @@ func TestMemTracker4DeleteExec(t *testing.T) { require.Equal(t, "", oom.GetTracker()) tk.MustExec("insert into MemTracker4DeleteExec1 values(1,1,1)") tk.MustExec("insert into MemTracker4DeleteExec2 values(1,1,1)") - + oom.ClearMessageFilter() oom.SetTracker("") + oom.AddMessageFilter("memory exceeds quota, rateLimitAction delegate to fallback action") require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/store/copr/disableFixedRowCountHint", "return")) defer func() { @@ -184,15 +201,26 @@ var oom *oomCapture func registerHook() { conf := &log.Config{Level: os.Getenv("log_level"), File: log.FileLogConfig{}} _, r, _ := log.InitLogger(conf) - oom = &oomCapture{r.Core, "", syncutil.Mutex{}} + oom = &oomCapture{r.Core, "", syncutil.Mutex{}, set.NewStringSet()} lg := zap.New(oom) log.ReplaceGlobals(lg, r) } type oomCapture struct { zapcore.Core - tracker string - mu syncutil.Mutex + tracker string + mu syncutil.Mutex + messageFilter set.StringSet +} + +func (h *oomCapture) AddMessageFilter(vals ...string) { + for _, val := range vals { + h.messageFilter.Insert(val) + } +} + +func (h *oomCapture) ClearMessageFilter() { + h.messageFilter.Clear() } func (h *oomCapture) SetTracker(tracker string) { @@ -223,8 +251,7 @@ func (h *oomCapture) Write(entry zapcore.Entry, fields []zapcore.Field) error { return nil } // They are just common background task and not related to the oom. - if entry.Message == "SetTiFlashGroupConfig" || - entry.Message == "record table item load status failed due to not finding item" { + if !h.messageFilter.Exist(entry.Message) { return nil } diff --git a/util/set/BUILD.bazel b/util/set/BUILD.bazel index 69b7df38fc921..7c6201a6286fe 100644 --- a/util/set/BUILD.bazel +++ b/util/set/BUILD.bazel @@ -14,6 +14,7 @@ go_library( deps = [ "//util/hack", "//util/memory", + "@org_golang_x_exp//maps", ], ) diff --git a/util/set/string_set.go b/util/set/string_set.go index 9c2d6946f2683..dd73c652d429d 100644 --- a/util/set/string_set.go +++ b/util/set/string_set.go @@ -14,6 +14,8 @@ package set +import "golang.org/x/exp/maps" + // StringSet is a string set. type StringSet map[string]struct{} @@ -57,3 +59,8 @@ func (s StringSet) Count() int { func (s StringSet) Empty() bool { return len(s) == 0 } + +// Clear clears the set. +func (s StringSet) Clear() { + maps.Clear(s) +} From 87332930afaaa3c2b11f36aaa660198b0c1535ce Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 28 Mar 2023 13:41:55 +0800 Subject: [PATCH 2/4] update Signed-off-by: Weizhen Wang --- executor/oomtest/oom_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/oomtest/oom_test.go b/executor/oomtest/oom_test.go index e91d23fd30630..9562309d9335f 100644 --- a/executor/oomtest/oom_test.go +++ b/executor/oomtest/oom_test.go @@ -251,7 +251,7 @@ func (h *oomCapture) Write(entry zapcore.Entry, fields []zapcore.Field) error { return nil } // They are just common background task and not related to the oom. - if !h.messageFilter.Exist(entry.Message) { + if !h.messageFilter.Empty() && !h.messageFilter.Exist(entry.Message) { return nil } From d28f92589db9ed4bd818a1ed604f05982373fabe Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Tue, 28 Mar 2023 13:43:25 +0800 Subject: [PATCH 3/4] update Signed-off-by: Weizhen Wang --- executor/oomtest/oom_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/executor/oomtest/oom_test.go b/executor/oomtest/oom_test.go index 9562309d9335f..e99129d3d46be 100644 --- a/executor/oomtest/oom_test.go +++ b/executor/oomtest/oom_test.go @@ -86,6 +86,8 @@ func TestMemTracker4InsertAndReplaceExec(t *testing.T) { tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)") require.Equal(t, "schemaLeaseChecker is not set for this transaction", oom.GetTracker()) tk.Session().GetSessionVars().MemQuotaQuery = 1 + oom.ClearMessageFilter() + oom.AddMessageFilter("expensive_query during bootstrap phase") tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)") require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker()) tk.Session().GetSessionVars().MemQuotaQuery = -1 @@ -96,6 +98,7 @@ func TestMemTracker4InsertAndReplaceExec(t *testing.T) { tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)") require.Equal(t, "", oom.GetTracker()) + oom.AddMessageFilter("expensive_query during bootstrap phase") tk.Session().GetSessionVars().MemQuotaQuery = 1 tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)") require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker()) @@ -106,6 +109,7 @@ func TestMemTracker4InsertAndReplaceExec(t *testing.T) { tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec select * from t") require.Equal(t, "", oom.GetTracker()) + oom.AddMessageFilter("expensive_query during bootstrap phase") tk.Session().GetSessionVars().MemQuotaQuery = 1 tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec select * from t") require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker()) @@ -116,6 +120,7 @@ func TestMemTracker4InsertAndReplaceExec(t *testing.T) { tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec select * from t") require.Equal(t, "", oom.GetTracker()) + oom.AddMessageFilter("expensive_query during bootstrap phase") tk.Session().GetSessionVars().MemQuotaQuery = 1 tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec select * from t") require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker()) @@ -129,6 +134,7 @@ func TestMemTracker4InsertAndReplaceExec(t *testing.T) { tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)") require.Equal(t, "", oom.GetTracker()) + oom.AddMessageFilter("expensive_query during bootstrap phase") tk.Session().GetSessionVars().MemQuotaQuery = 1 tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)") require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker()) @@ -140,6 +146,7 @@ func TestMemTracker4InsertAndReplaceExec(t *testing.T) { tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)") require.Equal(t, "", oom.GetTracker()) + oom.AddMessageFilter("expensive_query during bootstrap phase") tk.Session().GetSessionVars().MemQuotaQuery = 1 tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)") require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker()) From c373e6c31f6e519ceed7064b93efa6ef07a334d9 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 12 Apr 2023 15:39:45 +0800 Subject: [PATCH 4/4] admintest: move admin test into new package Signed-off-by: Weizhen Wang --- executor/oomtest/oom_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/executor/oomtest/oom_test.go b/executor/oomtest/oom_test.go index e99129d3d46be..cd8490e189cad 100644 --- a/executor/oomtest/oom_test.go +++ b/executor/oomtest/oom_test.go @@ -55,8 +55,9 @@ func TestMemTracker4UpdateExec(t *testing.T) { oom.SetTracker("") oom.ClearMessageFilter() - oom.AddMessageFilter("expensive_query during bootstrap phase") - oom.AddMessageFilter("schemaLeaseChecker is not set for this transaction") + oom.AddMessageFilter( + "expensive_query during bootstrap phase", + "schemaLeaseChecker is not set for this transaction") tk.MustExec("insert into t_MemTracker4UpdateExec values (1,1,1), (2,2,2), (3,3,3)") require.Equal(t, "schemaLeaseChecker is not set for this transaction", oom.GetTracker())