Skip to content

Commit

Permalink
This is an automated cherry-pick of pingcap#56356
Browse files Browse the repository at this point in the history 
Signed-off-by: ti-chi-bot <[email protected]>
  • Loading branch information
@crazycs520 @ti-chi-bot
crazycs520 authored and ti-chi-bot committed Jan 6, 2025
1 parent 1bdef59 commit c95ff2c
Show file tree
Hide file tree
Showing 5 changed files with 2,554 additions and 28 deletions.
78 changes: 58 additions & 20 deletions executor/slow_query.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -944,18 +944,6 @@ func (e *slowQueryRetriever) getAllFiles(ctx context.Context, sctx sessionctx.Co
e.stats.totalFileNum = totalFileNum
}()
}
if e.extractor == nil || !e.extractor.Enable {
totalFileNum = 1
//nolint: gosec
file, err := os.Open(logFilePath)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
return []logFile{{file: file}}, nil
}
var logFiles []logFile
logDir := filepath.Dir(logFilePath)
ext := filepath.Ext(logFilePath)
Expand Down Expand Up @@ -999,17 +987,20 @@ func (e *slowQueryRetriever) getAllFiles(ctx context.Context, sctx sessionctx.Co
return handleErr(err)
}
start := types.NewTime(types.FromGoTime(fileStartTime), mysql.TypeDatetime, types.MaxFsp)
notInAllTimeRanges := true
for _, tr := range e.checker.timeRanges {
if start.Compare(tr.endTime) <= 0 {
notInAllTimeRanges = false
break
if e.checker.enableTimeCheck {
notInAllTimeRanges := true
for _, tr := range e.checker.timeRanges {
if start.Compare(tr.endTime) <= 0 {
notInAllTimeRanges = false
break
}
}
if notInAllTimeRanges {
return nil
}
}
if notInAllTimeRanges {
return nil
}

<<<<<<< HEAD:executor/slow_query.go
// Get the file end time.
fileEndTime, err := e.getFileEndTime(ctx, file)
if err != nil {
Expand All @@ -1021,6 +1012,28 @@ func (e *slowQueryRetriever) getAllFiles(ctx context.Context, sctx sessionctx.Co
if !(start.Compare(tr.endTime) > 0 || end.Compare(tr.startTime) < 0) {
inTimeRanges = true
break
=======
// If we want to get the end time from a compressed file,
// we need uncompress the whole file which is very slow and consume a lot of memory.
if !compressed {
// Get the file end time.
fileEndTime, err := e.getFileEndTime(ctx, file)
if err != nil {
return handleErr(err)
}
if e.checker.enableTimeCheck {
end := types.NewTime(types.FromGoTime(fileEndTime), mysql.TypeDatetime, types.MaxFsp)
inTimeRanges := false
for _, tr := range e.checker.timeRanges {
if !(start.Compare(tr.endTime) > 0 || end.Compare(tr.startTime) < 0) {
inTimeRanges = true
break
}
}
if !inTimeRanges {
return nil
}
>>>>>>> 484c1ae88ee (executor: fix issue that query slow_query table return wrong result (#56356)):pkg/executor/slow_query.go
}
}
if !inTimeRanges {
Expand Down Expand Up @@ -1048,7 +1061,32 @@ func (e *slowQueryRetriever) getAllFiles(ctx context.Context, sctx sessionctx.Co
slices.SortFunc(logFiles, func(i, j logFile) int {
return cmp.Compare(i.start.UnixNano(), j.start.UnixNano())
})
<<<<<<< HEAD:executor/slow_query.go
return logFiles, err
=======
// Assume no time range overlap in log files and remove unnecessary log files for compressed files.
var ret []logFile
for i, file := range logFiles {
if i == len(logFiles)-1 || !file.compressed || !e.checker.enableTimeCheck {
ret = append(ret, file)
continue
}
start := types.NewTime(types.FromGoTime(logFiles[i].start), mysql.TypeDatetime, types.MaxFsp)
// use next file.start as endTime
end := types.NewTime(types.FromGoTime(logFiles[i+1].start), mysql.TypeDatetime, types.MaxFsp)
inTimeRanges := false
for _, tr := range e.checker.timeRanges {
if !(start.Compare(tr.endTime) > 0 || end.Compare(tr.startTime) < 0) {
inTimeRanges = true
break
}
}
if inTimeRanges {
ret = append(ret, file)
}
}
return ret, err
>>>>>>> 484c1ae88ee (executor: fix issue that query slow_query table return wrong result (#56356)):pkg/executor/slow_query.go
}

func (e *slowQueryRetriever) getFileStartTime(ctx context.Context, file *os.File) (time.Time, error) {
Expand Down
161 changes: 161 additions & 0 deletions executor/slow_query_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -351,13 +351,174 @@ select 6;
select 7;`
logData := []string{logData0, logData1, logData2, logData3}

<<<<<<< HEAD:executor/slow_query_test.go
fileName0 := "tidb-slow-2020-02-14T19-04-05.01.log"
fileName1 := "tidb-slow-2020-02-15T19-04-05.01.log"
fileName2 := "tidb-slow-2020-02-16T19-04-05.01.log"
fileName3 := "tidb-slow.log"
fileNames := []string{fileName0, fileName1, fileName2, fileName3}
prepareLogs(t, logData, fileNames)
defer func() {
=======
fileName0 := "tidb-slow-retriever-2020-02-14T19-04-05.01.log"
fileName1 := "tidb-slow-retriever-2020-02-15T19-04-05.01.log"
fileName2 := "tidb-slow-retriever-2020-02-16T19-04-05.01.log"
fileName3 := "tidb-slow-retriever.log"
defer config.RestoreFunc()()
config.UpdateGlobal(func(conf *config.Config) {
conf.Log.SlowQueryFile = fileName3
})
for k := 0; k < 2; k++ {
// k = 0 for normal files
// k = 1 for compressed files
var fileNames []string
if k == 0 {
fileNames = []string{fileName0, fileName1, fileName2, fileName3}
} else {
fileNames = []string{fileName0 + ".gz", fileName1 + ".gz", fileName2 + ".gz", fileName3}
}
prepareLogs(t, logData, fileNames)

cases := []struct {
startTime string
endTime string
files []string
querys []string
}{
{
startTime: "2020-02-15T18:00:00.000000+08:00",
endTime: "2020-02-17T20:00:00.000000+08:00",
files: []string{fileName1, fileName2, fileName3},
querys: []string{
"select 1;",
"select 2;",
"select 3;",
"select 4;",
"select 5;",
"select 6;",
},
},
{
startTime: "2020-02-15T18:00:02.000000+08:00",
endTime: "2020-02-16T20:00:00.000000+08:00",
files: []string{fileName1, fileName2, fileName3},
querys: []string{
"select 2;",
"select 3;",
"select 4;",
"select 5;",
},
},
{
startTime: "2020-02-16T18:00:03.000000+08:00",
endTime: "2020-02-16T18:59:00.000000+08:00",
files: []string{fileName2},
querys: []string{
"select 4;",
},
},
{
startTime: "2020-02-16T18:00:03.000000+08:00",
endTime: "2020-02-16T20:00:00.000000+08:00",
files: []string{fileName2, fileName3},
querys: []string{
"select 4;",
"select 5;",
},
},
{
startTime: "2020-02-16T19:00:00.000000+08:00",
endTime: "2020-02-17T17:00:00.000000+08:00",
files: []string{fileName3},
querys: []string{
"select 5;",
},
},
{
startTime: "2010-01-01T00:00:00.000000+08:00",
endTime: "2010-01-01T01:00:00.000000+08:00",
files: []string{},
},
{
startTime: "2020-03-01T00:00:00.000000+08:00",
endTime: "2010-03-01T01:00:00.000000+08:00",
files: []string{},
},
{
startTime: "",
endTime: "",
files: []string{fileName1, fileName2, fileName3},
querys: []string{
"select 1;",
"select 2;",
"select 3;",
"select 4;",
"select 5;",
"select 6;",
"select 7;",
},
},
{
startTime: "2020-04-15T18:00:05.299063744+08:00",
endTime: "2020-04-15T18:00:05.299063744+08:00",
files: []string{fileName3},
querys: []string{
"select 7;",
},
},
}

loc, err := time.LoadLocation("Asia/Shanghai")
require.NoError(t, err)
sctx := mock.NewContext()
sctx.ResetSessionAndStmtTimeZone(loc)
sctx.GetSessionVars().SlowQueryFile = fileName3
for i, cas := range cases {
extractor := &plannercore.SlowQueryExtractor{Enable: len(cas.startTime) > 0 && len(cas.endTime) > 0}
if extractor.Enable {
startTime, err := ParseTime(cas.startTime)
require.NoError(t, err)
endTime, err := ParseTime(cas.endTime)
require.NoError(t, err)
extractor.TimeRanges = []*plannercore.TimeRange{{StartTime: startTime, EndTime: endTime}}
}
retriever, err := newSlowQueryRetriever()
require.NoError(t, err)
retriever.extractor = extractor
err = retriever.initialize(context.Background(), sctx)
require.NoError(t, err)
comment := fmt.Sprintf("compressed: %v, case id: %v", k, i)
if len(retriever.files) > 0 {
var reader *bufio.Reader
reader, err := retriever.getNextReader()
require.NoError(t, err, comment)
rows, err := parseLog(retriever, sctx, reader)
require.NoError(t, err, comment)
require.Equal(t, len(rows), len(cas.querys), comment)
for i, row := range rows {
require.Equal(t, row[len(row)-1].GetString(), cas.querys[i], comment)
}
}

if k == 0 {
require.Equal(t, len(retriever.files), len(cas.files), comment)
for i, file := range retriever.files {
require.Equal(t, file.file.Name(), cas.files[i], comment)
}
} else {
// for compressed file, sometimes it will contains one more file.
require.True(t, (len(retriever.files) == len(cas.files)) || (len(retriever.files) == len(cas.files)+1), comment)
var fileNames []string
for _, file := range retriever.files {
fileNames = append(fileNames, strings.TrimSuffix(file.file.Name(), ".gz"))
}
for _, file := range cas.files {
require.Contains(t, fileNames, file, comment)
}
}
require.NoError(t, retriever.close())
}
>>>>>>> 484c1ae88ee (executor: fix issue that query slow_query table return wrong result (#56356)):pkg/executor/slow_query_test.go
removeFiles(fileNames)
}()

Expand Down
Loading

0 comments on commit c95ff2c

Please sign in to comment.