Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

oomtest: add message filer for oom test to avoid flaky test #42628

Merged
merged 4 commits into from
Apr 12, 2023
Merged

oomtest: add message filer for oom test to avoid flaky test #42628

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fc74c65
oomtest: add message filer for oom test to avoid flaky test
hawkingrei Mar 28, 2023
8733293
update
hawkingrei Mar 28, 2023
d28f925
update
hawkingrei Mar 28, 2023
c373e6c
admintest: move admin test into new package
hawkingrei Apr 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions executor/oomtest/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ go_test(
deps = [
"//testkit",
"//testkit/testsetup",
"//util/set",
"//util/syncutil",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_log//:log",
Expand Down
47 changes: 41 additions & 6 deletions executor/oomtest/oom_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/pingcap/log"
"github.com/pingcap/tidb/testkit"
"github.com/pingcap/tidb/testkit/testsetup"
"github.com/pingcap/tidb/util/set"
"github.com/pingcap/tidb/util/syncutil"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
Expand Down Expand Up @@ -53,6 +54,10 @@ func TestMemTracker4UpdateExec(t *testing.T) {
log.SetLevel(zap.InfoLevel)

oom.SetTracker("")
oom.ClearMessageFilter()
oom.AddMessageFilter(
"expensive_query during bootstrap phase",
"schemaLeaseChecker is not set for this transaction")

tk.MustExec("insert into t_MemTracker4UpdateExec values (1,1,1), (2,2,2), (3,3,3)")
require.Equal(t, "schemaLeaseChecker is not set for this transaction", oom.GetTracker())
Expand All @@ -75,36 +80,48 @@ func TestMemTracker4InsertAndReplaceExec(t *testing.T) {
log.SetLevel(zap.InfoLevel)

oom.SetTracker("")
oom.AddMessageFilter(
"schemaLeaseChecker is not set for this transaction",
"expensive_query during bootstrap phase")

tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
require.Equal(t, "schemaLeaseChecker is not set for this transaction", oom.GetTracker())
tk.Session().GetSessionVars().MemQuotaQuery = 1
oom.ClearMessageFilter()
oom.AddMessageFilter("expensive_query during bootstrap phase")
tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker())
tk.Session().GetSessionVars().MemQuotaQuery = -1

oom.SetTracker("")
oom.ClearMessageFilter()
oom.AddMessageFilter("expensive_query during bootstrap phase")

tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
require.Equal(t, "", oom.GetTracker())
oom.AddMessageFilter("expensive_query during bootstrap phase")
tk.Session().GetSessionVars().MemQuotaQuery = 1
tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker())
tk.Session().GetSessionVars().MemQuotaQuery = -1

oom.SetTracker("")
oom.ClearMessageFilter()

tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec select * from t")
require.Equal(t, "", oom.GetTracker())
oom.AddMessageFilter("expensive_query during bootstrap phase")
tk.Session().GetSessionVars().MemQuotaQuery = 1
tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec select * from t")
require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker())
tk.Session().GetSessionVars().MemQuotaQuery = -1

oom.SetTracker("")
oom.ClearMessageFilter()

tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec select * from t")
require.Equal(t, "", oom.GetTracker())
oom.AddMessageFilter("expensive_query during bootstrap phase")
tk.Session().GetSessionVars().MemQuotaQuery = 1
tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec select * from t")
require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker())
Expand All @@ -114,18 +131,23 @@ func TestMemTracker4InsertAndReplaceExec(t *testing.T) {
tk.Session().GetSessionVars().BatchInsert = true

oom.SetTracker("")
oom.ClearMessageFilter()

tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
require.Equal(t, "", oom.GetTracker())
oom.AddMessageFilter("expensive_query during bootstrap phase")
tk.Session().GetSessionVars().MemQuotaQuery = 1
tk.MustExec("insert into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker())
tk.Session().GetSessionVars().MemQuotaQuery = -1

oom.SetTracker("")
oom.ClearMessageFilter()
oom.AddMessageFilter("expensive_query during bootstrap phase")

tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
require.Equal(t, "", oom.GetTracker())
oom.AddMessageFilter("expensive_query during bootstrap phase")
tk.Session().GetSessionVars().MemQuotaQuery = 1
tk.MustExec("replace into t_MemTracker4InsertAndReplaceExec values (1,1,1), (2,2,2), (3,3,3)")
require.Equal(t, "expensive_query during bootstrap phase", oom.GetTracker())
Expand All @@ -146,6 +168,8 @@ func TestMemTracker4DeleteExec(t *testing.T) {
tk.MustExec("insert into MemTracker4DeleteExec1 values(1,1,1), (2,2,2), (3,3,3), (4,4,4), (5,5,5)")

oom.SetTracker("")
oom.ClearMessageFilter()
oom.AddMessageFilter("expensive_query during bootstrap phase")

tk.MustExec("delete from MemTracker4DeleteExec1")
require.Equal(t, "", oom.GetTracker())
Expand All @@ -166,8 +190,9 @@ func TestMemTracker4DeleteExec(t *testing.T) {
require.Equal(t, "", oom.GetTracker())
tk.MustExec("insert into MemTracker4DeleteExec1 values(1,1,1)")
tk.MustExec("insert into MemTracker4DeleteExec2 values(1,1,1)")

oom.ClearMessageFilter()
oom.SetTracker("")
oom.AddMessageFilter("memory exceeds quota, rateLimitAction delegate to fallback action")

require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/store/copr/disableFixedRowCountHint", "return"))
defer func() {
Expand All @@ -184,15 +209,26 @@ var oom *oomCapture
func registerHook() {
conf := &log.Config{Level: os.Getenv("log_level"), File: log.FileLogConfig{}}
_, r, _ := log.InitLogger(conf)
oom = &oomCapture{r.Core, "", syncutil.Mutex{}}
oom = &oomCapture{r.Core, "", syncutil.Mutex{}, set.NewStringSet()}
lg := zap.New(oom)
log.ReplaceGlobals(lg, r)
}

type oomCapture struct {
zapcore.Core
tracker string
mu syncutil.Mutex
tracker string
mu syncutil.Mutex
messageFilter set.StringSet
}

func (h *oomCapture) AddMessageFilter(vals ...string) {
for _, val := range vals {
h.messageFilter.Insert(val)
}
}

func (h *oomCapture) ClearMessageFilter() {
h.messageFilter.Clear()
}

func (h *oomCapture) SetTracker(tracker string) {
Expand Down Expand Up @@ -223,8 +259,7 @@ func (h *oomCapture) Write(entry zapcore.Entry, fields []zapcore.Field) error {
return nil
}
// They are just common background task and not related to the oom.
if entry.Message == "SetTiFlashGroupConfig" ||
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where is the original filter?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The filter is a whitelist to filter available logs for the test. But I found that more and more logs need to be set in, and there will be conflicts between them. so I have to set the specifier filter at the test checker.

entry.Message == "record table item load status failed due to not finding item" {
if !h.messageFilter.Empty() && !h.messageFilter.Exist(entry.Message) {
return nil
}

Expand Down
1 change: 1 addition & 0 deletions util/set/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ go_library(
deps = [
"//util/hack",
"//util/memory",
"@org_golang_x_exp//maps",
],
)

Expand Down
7 changes: 7 additions & 0 deletions util/set/string_set.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

package set

import "golang.org/x/exp/maps"

// StringSet is a string set.
type StringSet map[string]struct{}

Expand Down Expand Up @@ -57,3 +59,8 @@ func (s StringSet) Count() int {
func (s StringSet) Empty() bool {
return len(s) == 0
}

// Clear clears the set.
func (s StringSet) Clear() {
maps.Clear(s)
}