Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

*: add global variable tidb_slow_log_masking to control masking slow log query #17637

Merged
merged 8 commits into from
Jun 5, 2020
Merged

*: add global variable tidb_slow_log_masking to control masking slow log query #17637

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
595e287
*: add global variable tidb_slow_log_masking to control masking slow …
crazycs520 Jun 3, 2020
05c73e8
add test
crazycs520 Jun 3, 2020
af0b987
Merge branch 'master' into slow-log-masking
crazycs520 Jun 3, 2020
aaec4d9
Merge branch 'master' into slow-log-masking
crazycs520 Jun 3, 2020
76f31ae
Merge branch 'master' into slow-log-masking
crazycs520 Jun 4, 2020
ec4dac6
Merge branch 'master' into slow-log-masking
bb7133 Jun 4, 2020
9bdd384
Merge branch 'master' into slow-log-masking
crazycs520 Jun 4, 2020
857bd48
Merge branch 'master' into slow-log-masking
crazycs520 Jun 5, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions executor/adapter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -817,7 +817,13 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool, hasMoreResults bool) {
if (!enable || costTime < threshold) && level > zapcore.DebugLevel {
return
}
sql := FormatSQL(a.Text, sessVars.PreparedParams)
var sql stringutil.StringerFunc
normalizedSQL, digest := sessVars.StmtCtx.SQLDigest()
if sessVars.EnableSlowLogMasking {
sql = FormatSQL(normalizedSQL, nil)
} else {
sql = FormatSQL(a.Text, sessVars.PreparedParams)
}

var tableIDs, indexNames string
if len(sessVars.StmtCtx.TableIDs) > 0 {
Expand All @@ -831,7 +837,6 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool, hasMoreResults bool) {
statsInfos := plannercore.GetStatsInfo(a.Plan)
memMax := sessVars.StmtCtx.MemTracker.MaxConsumed()
diskMax := sessVars.StmtCtx.DiskTracker.MaxConsumed()
_, digest := sessVars.StmtCtx.SQLDigest()
_, planDigest := getPlanDigest(a.Ctx, a.Plan)
slowItems := &variable.SlowQueryLogItems{
TxnTS: txnTS,
Expand Down
11 changes: 11 additions & 0 deletions executor/set_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,17 @@ func (s *testSuite5) TestSetVar(c *C) {
c.Assert(err, NotNil)
c.Assert(err.Error(), Equals, "tidb_metric_query_range_duration(9) cannot be smaller than 10 or larger than 216000")
tk.MustQuery("select @@session.tidb_metric_query_range_duration;").Check(testkit.Rows("120"))

// test for tidb_slow_log_masking
tk.MustQuery(`select @@global.tidb_slow_log_masking;`).Check(testkit.Rows("0"))
tk.MustExec("set global tidb_slow_log_masking = 1")
tk.MustQuery(`select @@global.tidb_slow_log_masking;`).Check(testkit.Rows("1"))
tk.MustExec("set global tidb_slow_log_masking = 0")
tk.MustQuery(`select @@global.tidb_slow_log_masking;`).Check(testkit.Rows("0"))
_, err = tk.Exec("set session tidb_slow_log_masking = 0")
c.Assert(err, NotNil)
_, err = tk.Exec(`select @@session.tidb_slow_log_masking;`)
c.Assert(err, NotNil)
}

func (s *testSuite5) TestSetCharset(c *C) {
Expand Down
1 change: 1 addition & 0 deletions session/session.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2056,6 +2056,7 @@ var builtinGlobalVariable = []string{
variable.TiDBStoreLimit,
variable.TiDBAllowAutoRandExplicitInsert,
variable.TiDBEnableClusteredIndex,
variable.TiDBSlowLogMasking,
}

var (
Expand Down
6 changes: 6 additions & 0 deletions sessionctx/variable/session.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -612,6 +612,9 @@ type SessionVars struct {

// EnableClusteredIndex indicates whether to enable clustered index when creating a new table.
EnableClusteredIndex bool

// EnableSlowLogMasking indicates that whether masking the query data when log slow query.
EnableSlowLogMasking bool
}

// PreparedParams contains the parameters of the current prepared statement when executing it.
Expand Down Expand Up @@ -702,6 +705,7 @@ func NewSessionVars() *SessionVars {
SelectLimit: math.MaxUint64,
AllowAutoRandExplicitInsert: DefTiDBAllowAutoRandExplicitInsert,
EnableClusteredIndex: DefTiDBEnableClusteredIndex,
EnableSlowLogMasking: DefTiDBSlowLogMasking,
}
vars.KVVars = kv.NewVariables(&vars.Killed)
vars.Concurrency = Concurrency{
Expand Down Expand Up @@ -1312,6 +1316,8 @@ func (s *SessionVars) SetSystemVar(name string, val string) error {
s.AllowAutoRandExplicitInsert = TiDBOptOn(val)
case TiDBEnableClusteredIndex:
s.EnableClusteredIndex = TiDBOptOn(val)
case TiDBSlowLogMasking:
s.EnableSlowLogMasking = TiDBOptOn(val)
}
s.systems[name] = val
return nil
Expand Down
1 change: 1 addition & 0 deletions sessionctx/variable/sysvar.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -723,6 +723,7 @@ var defaultSysVars = []*SysVar{
{ScopeSession, TiDBEnableCollectExecutionInfo, BoolToIntStr(logutil.DefaultTiDBEnableSlowLog)},
{ScopeSession, TiDBAllowAutoRandExplicitInsert, boolToOnOff(DefTiDBAllowAutoRandExplicitInsert)},
{ScopeGlobal | ScopeSession, TiDBEnableClusteredIndex, BoolToIntStr(DefTiDBEnableClusteredIndex)},
{ScopeGlobal, TiDBSlowLogMasking, BoolToIntStr(DefTiDBSlowLogMasking)},
}

// SynonymsSysVariables is synonyms of system variables.
Expand Down
4 changes: 4 additions & 0 deletions sessionctx/variable/tidb_vars.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,9 @@ const (

// TiDBEnableClusteredIndex indicates if clustered index feature is enabled.
TiDBEnableClusteredIndex = "tidb_enable_clustered_index"

// TiDBSlowLogMasking indicates that whether masking the query data when log slow query.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does it only effective for parameterized queries?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's your mean of parameterized queries?

Actually the masking SQL is also used to generate the SQL digest.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, I mean prepared statements. For non prepared statements, will they have effect?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, It also works on non-prepared statement.

TiDBSlowLogMasking = "tidb_slow_log_masking"
)

// Default TiDB system variable values.
Expand Down Expand Up @@ -495,6 +498,7 @@ const (
DefTidbEnableCollectExecutionInfo = false
DefTiDBAllowAutoRandExplicitInsert = false
DefTiDBEnableClusteredIndex = false
DefTiDBSlowLogMasking = false
)

// Process global variables.
Expand Down