Skip to content

Commit

Permalink
Merge branch 'resource_manager/time_function' into resource_manager/s…
Browse files Browse the repository at this point in the history 
…hort_calibrate
  • Loading branch information
@CabinfeverB
CabinfeverB committed Jun 27, 2023
2 parents 767267a + 7cd36ef commit 42e40ad
Show file tree
Hide file tree
Showing 6 changed files with 6,297 additions and 6,107 deletions.
42 changes: 41 additions & 1 deletion executor/calibrate_resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (

"github.com/docker/go-units"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/infoschema"
"github.com/pingcap/tidb/kv"
Expand Down Expand Up @@ -130,12 +131,26 @@ func (e *calibrateResourceExec) parseCalibrateDuration(ctx context.Context) (sta
return
}
startTime = oracle.GetTimeFromTS(ts)
if len(op.StrValue) > 0 {
dur, err = duration.ParseDuration(op.StrValue)
if err != nil {
return
}
startTime = startTime.Add(-dur)
}
case ast.CalibrateEndTime:
ts, err = staleread.CalculateAsOfTsExpr(ctx, e.ctx, op.Ts)
if err != nil {
return
}
endTime = oracle.GetTimeFromTS(ts)
if len(op.StrValue) > 0 {
dur, err = duration.ParseDuration(op.StrValue)
if err != nil {
return
}
endTime = endTime.Add(-dur)
}
case ast.CalibrateDuration:
dur, err = duration.ParseDuration(op.StrValue)
if err != nil {
Expand Down Expand Up @@ -164,7 +179,6 @@ func (e *calibrateResourceExec) parseCalibrateDuration(ctx context.Context) (sta
if dur < minDuration {
err = errors.Errorf("the duration of calibration is too short, which could lead to inaccurate output. Please make the duration between %s and %s", minDuration.String(), maxDuration.String())
}

return
}

Expand Down Expand Up @@ -216,6 +230,32 @@ func (e *calibrateResourceExec) dynamicCalibrate(ctx context.Context, req *chunk
if err != nil {
return err
}
failpoint.Inject("mockMetricsDataFilter", func() {
ret := make([]*timePointValue, 0)
for _, point := range tikvCPUs.vals {
if point.tp.After(endTs) || point.tp.Before(startTs) {
continue
}
ret = append(ret, point)
}
tikvCPUs.vals = ret
ret = make([]*timePointValue, 0)
for _, point := range tidbCPUs.vals {
if point.tp.After(endTs) || point.tp.Before(startTs) {
continue
}
ret = append(ret, point)
}
tidbCPUs.vals = ret
ret = make([]*timePointValue, 0)
for _, point := range rus.vals {
if point.tp.After(endTs) || point.tp.Before(startTs) {
continue
}
ret = append(ret, point)
}
rus.vals = ret
})
quotas := make([]float64, 0)
lowCount := 0
for {
Expand Down
87 changes: 83 additions & 4 deletions executor/calibrate_resource_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"context"
"encoding/json"
"testing"
"time"

"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
Expand Down Expand Up @@ -79,15 +80,22 @@ func TestCalibrateResource(t *testing.T) {
// Mock for metric table data.
fpName := "github.com/pingcap/tidb/executor/mockMetricsTableData"
require.NoError(t, failpoint.Enable(fpName, "return"))
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/executor/mockMetricsDataFilter", "return(true)"))
defer func() {
require.NoError(t, failpoint.Disable(fpName))
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/executor/mockMetricsDataFilter"))
}()

datetime := func(s string) types.Time {
time, err := types.ParseTime(tk.Session().GetSessionVars().StmtCtx, s, mysql.TypeDatetime, types.MaxFsp, nil)
require.NoError(t, err)
return time
}
now := time.Now()
datetimeBeforeNow := func(dur time.Duration) types.Time {
time := types.NewTime(types.FromGoTime(now.Add(-dur)), mysql.TypeDatetime, types.MaxFsp)
return time
}

mockData := make(map[string][][]types.Datum)
ctx := context.WithValue(context.Background(), "__mockMetricsTableData", mockData)
Expand Down Expand Up @@ -125,8 +133,79 @@ func TestCalibrateResource(t *testing.T) {
}
tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE").Check(testkit.Rows("38760"))

// construct data for dynamic calibrate
ru1 := [][]types.Datum{
types.MakeDatums(datetimeBeforeNow(20*time.Minute), 2200.0),
types.MakeDatums(datetimeBeforeNow(10*time.Minute), 2200.0),
types.MakeDatums(datetimeBeforeNow(9*time.Minute), 2100.0),
types.MakeDatums(datetimeBeforeNow(8*time.Minute), 2250.0),
types.MakeDatums(datetimeBeforeNow(7*time.Minute), 2300.0),
types.MakeDatums(datetimeBeforeNow(6*time.Minute), 2230.0),
types.MakeDatums(datetimeBeforeNow(5*time.Minute), 2210.0),
types.MakeDatums(datetimeBeforeNow(4*time.Minute), 2250.0),
types.MakeDatums(datetimeBeforeNow(3*time.Minute), 2330.0),
types.MakeDatums(datetimeBeforeNow(2*time.Minute), 2330.0),
types.MakeDatums(datetimeBeforeNow(1*time.Minute), 2300.0),
types.MakeDatums(datetimeBeforeNow(2*time.Second), 2280.0),
}
mockData["resource_manager_resource_unit"] = ru1
cpu1 := [][]types.Datum{
types.MakeDatums(datetimeBeforeNow(20*time.Minute), "tidb-0", "tidb", 1.212),
types.MakeDatums(datetimeBeforeNow(10*time.Minute), "tidb-0", "tidb", 1.212),
types.MakeDatums(datetimeBeforeNow(9*time.Minute), "tidb-0", "tidb", 1.233),
types.MakeDatums(datetimeBeforeNow(8*time.Minute), "tidb-0", "tidb", 1.234),
types.MakeDatums(datetimeBeforeNow(7*time.Minute), "tidb-0", "tidb", 1.213),
types.MakeDatums(datetimeBeforeNow(6*time.Minute), "tidb-0", "tidb", 1.209),
types.MakeDatums(datetimeBeforeNow(5*time.Minute), "tidb-0", "tidb", 1.213),
types.MakeDatums(datetimeBeforeNow(4*time.Minute), "tidb-0", "tidb", 1.236),
types.MakeDatums(datetimeBeforeNow(3*time.Minute), "tidb-0", "tidb", 1.228),
types.MakeDatums(datetimeBeforeNow(2*time.Minute), "tidb-0", "tidb", 1.219),
types.MakeDatums(datetimeBeforeNow(1*time.Minute), "tidb-0", "tidb", 1.220),
types.MakeDatums(datetimeBeforeNow(2*time.Second), "tidb-0", "tidb", 1.221),
types.MakeDatums(datetimeBeforeNow(20*time.Minute), "tikv-1", "tikv", 2.212),
types.MakeDatums(datetimeBeforeNow(10*time.Minute), "tikv-1", "tikv", 2.212),
types.MakeDatums(datetimeBeforeNow(9*time.Minute), "tikv-1", "tikv", 2.233),
types.MakeDatums(datetimeBeforeNow(8*time.Minute), "tikv-1", "tikv", 2.234),
types.MakeDatums(datetimeBeforeNow(7*time.Minute), "tikv-1", "tikv", 2.213),
types.MakeDatums(datetimeBeforeNow(6*time.Minute), "tikv-1", "tikv", 2.209),
types.MakeDatums(datetimeBeforeNow(5*time.Minute), "tikv-1", "tikv", 2.213),
types.MakeDatums(datetimeBeforeNow(4*time.Minute), "tikv-1", "tikv", 2.236),
types.MakeDatums(datetimeBeforeNow(3*time.Minute), "tikv-1", "tikv", 2.228),
types.MakeDatums(datetimeBeforeNow(2*time.Minute), "tikv-1", "tikv", 2.219),
types.MakeDatums(datetimeBeforeNow(1*time.Minute), "tikv-1", "tikv", 2.220),
types.MakeDatums(datetimeBeforeNow(2*time.Second), "tikv-1", "tikv", 2.281),
types.MakeDatums(datetimeBeforeNow(20*time.Minute), "tikv-0", "tikv", 2.282),
types.MakeDatums(datetimeBeforeNow(10*time.Minute), "tikv-0", "tikv", 2.282),
types.MakeDatums(datetimeBeforeNow(9*time.Minute), "tikv-0", "tikv", 2.283),
types.MakeDatums(datetimeBeforeNow(8*time.Minute), "tikv-0", "tikv", 2.284),
types.MakeDatums(datetimeBeforeNow(7*time.Minute), "tikv-0", "tikv", 2.283),
types.MakeDatums(datetimeBeforeNow(6*time.Minute), "tikv-0", "tikv", 2.289),
types.MakeDatums(datetimeBeforeNow(5*time.Minute), "tikv-0", "tikv", 2.283),
types.MakeDatums(datetimeBeforeNow(4*time.Minute), "tikv-0", "tikv", 2.286),
types.MakeDatums(datetimeBeforeNow(3*time.Minute), "tikv-0", "tikv", 2.288),
types.MakeDatums(datetimeBeforeNow(2*time.Minute), "tikv-0", "tikv", 2.289),
types.MakeDatums(datetimeBeforeNow(1*time.Minute), "tikv-0", "tikv", 2.280),
types.MakeDatums(datetimeBeforeNow(2*time.Second), "tikv-0", "tikv", 2.281),
types.MakeDatums(datetimeBeforeNow(20*time.Minute), "tikv-2", "tikv", 2.112),
types.MakeDatums(datetimeBeforeNow(10*time.Minute), "tikv-2", "tikv", 2.112),
types.MakeDatums(datetimeBeforeNow(9*time.Minute), "tikv-2", "tikv", 2.133),
types.MakeDatums(datetimeBeforeNow(8*time.Minute), "tikv-2", "tikv", 2.134),
types.MakeDatums(datetimeBeforeNow(7*time.Minute), "tikv-2", "tikv", 2.113),
types.MakeDatums(datetimeBeforeNow(6*time.Minute), "tikv-2", "tikv", 2.109),
types.MakeDatums(datetimeBeforeNow(5*time.Minute), "tikv-2", "tikv", 2.113),
types.MakeDatums(datetimeBeforeNow(4*time.Minute), "tikv-2", "tikv", 2.136),
types.MakeDatums(datetimeBeforeNow(3*time.Minute), "tikv-2", "tikv", 2.128),
types.MakeDatums(datetimeBeforeNow(2*time.Minute), "tikv-2", "tikv", 2.119),
types.MakeDatums(datetimeBeforeNow(1*time.Minute), "tikv-2", "tikv", 2.120),
types.MakeDatums(datetimeBeforeNow(2*time.Second), "tikv-2", "tikv", 2.281),
}
mockData["process_cpu_usage"] = cpu1

tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE START_TIME now() - '11m'").Check(testkit.Rows("8161"))
tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE START_TIME now() - '11m' DURATION '11m'").Check(testkit.Rows("8161"))
tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE START_TIME now() - '11m' END_TIME now()").Check(testkit.Rows("8161"))

// construct data for dynamic calibrate
ru1 = [][]types.Datum{
types.MakeDatums(datetime("2020-02-12 10:35:00"), 2200.0),
types.MakeDatums(datetime("2020-02-12 10:36:00"), 2100.0),
types.MakeDatums(datetime("2020-02-12 10:37:00"), 2250.0),
Expand All @@ -141,7 +220,7 @@ func TestCalibrateResource(t *testing.T) {
}
mockData["resource_manager_resource_unit"] = ru1

cpu1 := [][]types.Datum{
cpu1 = [][]types.Datum{
types.MakeDatums(datetime("2020-02-12 10:35:00"), "tidb-0", "tidb", 1.212),
types.MakeDatums(datetime("2020-02-12 10:36:00"), "tidb-0", "tidb", 1.233),
types.MakeDatums(datetime("2020-02-12 10:37:00"), "tidb-0", "tidb", 1.234),
Expand Down Expand Up @@ -353,7 +432,7 @@ func TestCalibrateResource(t *testing.T) {
types.MakeDatums(datetime("2020-02-12 10:48:00"), "tikv-2", "tikv", 3.220),
}
mockData["process_cpu_usage"] = cpu2Mofidy
tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE START_TIME '2020-02-12 10:25:00' DURATION '20m'").Check(testkit.Rows("5616"))
tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE START_TIME '2020-02-12 10:25:00' DURATION '20m'").Check(testkit.Rows("5631"))

ruModify3 := [][]types.Datum{
types.MakeDatums(datetime("2020-02-12 10:25:00"), 5.0),
Expand Down Expand Up @@ -382,7 +461,7 @@ func TestCalibrateResource(t *testing.T) {
}
mockData["resource_manager_resource_unit"] = ruModify3
// because there are 20s difference in two time points, the result is changed.
tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE START_TIME '2020-02-12 10:25:00' DURATION '20m'").Check(testkit.Rows("5613"))
tk.MustQueryWithContext(ctx, "CALIBRATE RESOURCE START_TIME '2020-02-12 10:25:00' DURATION '20m'").Check(testkit.Rows("5633"))

ru2 := [][]types.Datum{
types.MakeDatums(datetime("2020-02-12 10:25:00"), 2200.0),
Expand Down
8 changes: 8 additions & 0 deletions parser/ast/misc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3859,11 +3859,19 @@ func (n *DynamicCalibrateResourceOption) Restore(ctx *format.RestoreCtx) error {
if err := n.Ts.Restore(ctx); err != nil {
return errors.Annotate(err, "An error occurred while splicing DynamicCalibrateResourceOption StartTime")
}
if len(n.StrValue) > 0 {
ctx.WritePlain(" - ")
ctx.WriteString(n.StrValue)
}
case CalibrateEndTime:
ctx.WriteKeyWord("END_TIME ")
if err := n.Ts.Restore(ctx); err != nil {
return errors.Annotate(err, "An error occurred while splicing DynamicCalibrateResourceOption EndTime")
}
if len(n.StrValue) > 0 {
ctx.WritePlain(" - ")
ctx.WriteString(n.StrValue)
}
case CalibrateDuration:
ctx.WriteKeyWord("DURATION ")
ctx.WriteString(n.StrValue)
Expand Down
Loading

0 comments on commit 42e40ad

Please sign in to comment.