diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index be3a8410a97c..faa4a6411840 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -1,23 +1,19 @@ --- name: "🐛 Bug Report" about: Something isn't working as expected -title: '' -labels: 'kind/bug' +labels: 'type/bug' --- -Please answer these questions before submitting your issue. Thanks! +## Bug Report -1. What did you do? -If possible, provide a recipe for reproducing the error. + +### What did you do? -2. What did you expect to see? + +### What did you expect to see? +### What did you see instead? -3. What did you see instead? - - - -4. What version of PD are you using (`pd-server -V`)? - +### What version of PD are you using (`pd-server -V`)? diff --git a/.github/ISSUE_TEMPLATE/development-task.md b/.github/ISSUE_TEMPLATE/development-task.md new file mode 100644 index 000000000000..7a6ea64f9c30 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/development-task.md @@ -0,0 +1,7 @@ +--- +name: "🗒️ Development Task" +about: As a developer, I want to record a development task. +labels: type/enhancement +--- + +## Development Task diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md index 9231fde9bf4d..c83a51155803 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -6,14 +6,18 @@ labels: type/enhancement ## Feature Request -### Describe your feature request related problem: +### Describe your feature request related problem + -### Describe the feature you'd like: +### Describe the feature you'd like + -### Describe alternatives you've considered: +### Describe alternatives you've considered + -### Teachability, Documentation, Adoption, Migration Strategy: +### Teachability, Documentation, Adoption, Migration Strategy + \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/performance-challenge-program.md b/.github/ISSUE_TEMPLATE/performance-challenge-program.md index b493d6f564bb..ba4762444529 100644 --- a/.github/ISSUE_TEMPLATE/performance-challenge-program.md +++ b/.github/ISSUE_TEMPLATE/performance-challenge-program.md @@ -2,9 +2,7 @@ name: "✈️ Performance Challenge Program" about: PCP Season1 Program title: 'PCP: ' -labels: '' -assignees: '' - +labels: 'PCP-S1' --- ## PCP Issue @@ -16,6 +14,6 @@ assignees: '' - Recommended Skills: - Mentor(s): -## Description +### Description - + diff --git a/.github/ISSUE_TEMPLATE/usability-challenge-program.md b/.github/ISSUE_TEMPLATE/usability-challenge-program.md index bcac0d8288a2..6c093d7e04ea 100644 --- a/.github/ISSUE_TEMPLATE/usability-challenge-program.md +++ b/.github/ISSUE_TEMPLATE/usability-challenge-program.md @@ -2,27 +2,27 @@ name: "🏪 Usability Challenge Program" about: UCP Season1 Program title: 'UCP: ' -labels: '' -assignees: '' - +labels: 'challenge-program-2' --- -## Description +## UCP Issue + +### Description - + -## Score +### Score - + -## Mentor +### Mentor - + -## Recommended Skills +### Recommended Skills -## Learning Materials +### Learning Materials diff --git a/.github/auto_assign.yml b/.github/auto_assign.yml index add04ad4f0f7..eda32d5c550f 100644 --- a/.github/auto_assign.yml +++ b/.github/auto_assign.yml @@ -1,9 +1,10 @@ -addReviewers: true -addAssignees: author +addReviewers: false +addAssignees: true numberOfReviewers: 0 numberOfAssignees: 1 -useReviewGroups: true -reviewGroups: +useReviewGroups: false +useAssigneeGroups: true +assigneeGroups: scheduling: - nolouch - disksing @@ -11,4 +12,7 @@ reviewGroups: - shafreeck - HunDunDM - rleungx -useAssigneeGroups: false +filterLabels: + # Run + include: + - contribution diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index a906c8728f3d..498312c1be74 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -5,37 +5,45 @@ PR Title Format: 2. *: what's changed --> -### What problem does this PR solve? +### What problem does this PR solve? + ### What is changed and how it works? +### Check List -### Check List + -Tests +Tests - - Unit test - - Integration test - - Manual test (add detailed scripts or steps below) - - No code + + +- Unit test +- Integration test +- Manual test (add detailed scripts or steps below) +- No code Code changes - - Has configuration change - - Has HTTP API interfaces change (Don't forget to [add the declarative for API](https://github.com/pingcap/pd/blob/master/docs/development.md#updating-api-documentation)) - - Has persistent data change +- Has configuration change +- Has HTTP API interfaces change (Don't forget to [add the declarative for API](https://github.com/pingcap/pd/blob/master/docs/development.md#updating-api-documentation)) +- Has persistent data change Side effects - - Possible performance regression - - Increased code complexity - - Breaking backward compatibility +- Possible performance regression +- Increased code complexity +- Breaking backward compatibility Related changes -- PR to update `pingcap/docs`/`pingcap/docs-cn`: -- PR to update `pingcap/tidb-ansible`: +- PR to update [`pingcap/docs`](https://github.com/pingcap/docs)/[`pingcap/docs-cn`](https://github.com/pingcap/docs-cn): +- PR to update [`pingcap/tidb-ansible`](https://github.com/pingcap/tidb-ansible): - Need to cherry-pick to the release branch -### Release note +### Release note + + + + diff --git a/client/base_client.go b/client/base_client.go index 9916f459acb8..5e38370cbb86 100644 --- a/client/base_client.go +++ b/client/base_client.go @@ -47,6 +47,8 @@ type baseClient struct { security SecurityOption gRPCDialOptions []grpc.DialOption + + timeout time.Duration } // SecurityOption records options about tls @@ -66,6 +68,13 @@ func WithGRPCDialOptions(opts ...grpc.DialOption) ClientOption { } } +// WithCustomTimeoutOption configures the client with timeout option. +func WithCustomTimeoutOption(timeout time.Duration) ClientOption { + return func(c *baseClient) { + c.timeout = timeout + } +} + // newBaseClient returns a new baseClient. func newBaseClient(ctx context.Context, urls []string, security SecurityOption, opts ...ClientOption) (*baseClient, error) { ctx1, cancel := context.WithCancel(ctx) @@ -75,6 +84,7 @@ func newBaseClient(ctx context.Context, urls []string, security SecurityOption, ctx: ctx1, cancel: cancel, security: security, + timeout: defaultPDTimeout, } c.connMu.clientConns = make(map[string]*grpc.ClientConn) for _, opt := range opts { @@ -163,7 +173,7 @@ func (c *baseClient) initClusterID() error { ctx, cancel := context.WithCancel(c.ctx) defer cancel() for _, u := range c.urls { - timeoutCtx, timeoutCancel := context.WithTimeout(ctx, pdTimeout) + timeoutCtx, timeoutCancel := context.WithTimeout(ctx, c.timeout) members, err := c.getMembers(timeoutCtx, u) timeoutCancel() if err != nil || members.GetHeader() == nil { diff --git a/client/client.go b/client/client.go index 0ecce28eaaf2..cdabd30dd8f0 100644 --- a/client/client.go +++ b/client/client.go @@ -111,7 +111,7 @@ type tsoRequest struct { } const ( - pdTimeout = 3 * time.Second + defaultPDTimeout = 3 * time.Second dialTimeout = 3 * time.Second updateLeaderTimeout = time.Second // Use a shorter timeout to recover faster from network isolation. maxMergeTSORequests = 10000 @@ -238,7 +238,7 @@ func (c *client) tsLoop() { } done := make(chan struct{}) dl := deadline{ - timer: time.After(pdTimeout), + timer: time.After(c.timeout), done: done, cancel: cancel, } @@ -455,7 +455,7 @@ func (c *client) GetRegion(ctx context.Context, key []byte) (*Region, error) { start := time.Now() defer func() { cmdDurationGetRegion.Observe(time.Since(start).Seconds()) }() - ctx, cancel := context.WithTimeout(ctx, pdTimeout) + ctx, cancel := context.WithTimeout(ctx, c.timeout) resp, err := c.leaderClient().GetRegion(ctx, &pdpb.GetRegionRequest{ Header: c.requestHeader(), RegionKey: key, @@ -478,7 +478,7 @@ func (c *client) GetPrevRegion(ctx context.Context, key []byte) (*Region, error) start := time.Now() defer func() { cmdDurationGetPrevRegion.Observe(time.Since(start).Seconds()) }() - ctx, cancel := context.WithTimeout(ctx, pdTimeout) + ctx, cancel := context.WithTimeout(ctx, c.timeout) resp, err := c.leaderClient().GetPrevRegion(ctx, &pdpb.GetRegionRequest{ Header: c.requestHeader(), RegionKey: key, @@ -501,7 +501,7 @@ func (c *client) GetRegionByID(ctx context.Context, regionID uint64) (*Region, e start := time.Now() defer func() { cmdDurationGetRegionByID.Observe(time.Since(start).Seconds()) }() - ctx, cancel := context.WithTimeout(ctx, pdTimeout) + ctx, cancel := context.WithTimeout(ctx, c.timeout) resp, err := c.leaderClient().GetRegionByID(ctx, &pdpb.GetRegionByIDRequest{ Header: c.requestHeader(), RegionId: regionID, @@ -527,7 +527,7 @@ func (c *client) ScanRegions(ctx context.Context, key, endKey []byte, limit int) var cancel context.CancelFunc scanCtx := ctx if _, ok := ctx.Deadline(); !ok { - scanCtx, cancel = context.WithTimeout(ctx, pdTimeout) + scanCtx, cancel = context.WithTimeout(ctx, c.timeout) defer cancel() } @@ -553,7 +553,7 @@ func (c *client) GetStore(ctx context.Context, storeID uint64) (*metapb.Store, e start := time.Now() defer func() { cmdDurationGetStore.Observe(time.Since(start).Seconds()) }() - ctx, cancel := context.WithTimeout(ctx, pdTimeout) + ctx, cancel := context.WithTimeout(ctx, c.timeout) resp, err := c.leaderClient().GetStore(ctx, &pdpb.GetStoreRequest{ Header: c.requestHeader(), StoreId: storeID, @@ -589,7 +589,7 @@ func (c *client) GetAllStores(ctx context.Context, opts ...GetStoreOption) ([]*m start := time.Now() defer func() { cmdDurationGetAllStores.Observe(time.Since(start).Seconds()) }() - ctx, cancel := context.WithTimeout(ctx, pdTimeout) + ctx, cancel := context.WithTimeout(ctx, c.timeout) resp, err := c.leaderClient().GetAllStores(ctx, &pdpb.GetAllStoresRequest{ Header: c.requestHeader(), ExcludeTombstoneStores: options.excludeTombstone, @@ -613,7 +613,7 @@ func (c *client) UpdateGCSafePoint(ctx context.Context, safePoint uint64) (uint6 start := time.Now() defer func() { cmdDurationUpdateGCSafePoint.Observe(time.Since(start).Seconds()) }() - ctx, cancel := context.WithTimeout(ctx, pdTimeout) + ctx, cancel := context.WithTimeout(ctx, c.timeout) resp, err := c.leaderClient().UpdateGCSafePoint(ctx, &pdpb.UpdateGCSafePointRequest{ Header: c.requestHeader(), SafePoint: safePoint, @@ -641,7 +641,7 @@ func (c *client) UpdateServiceGCSafePoint(ctx context.Context, serviceID string, start := time.Now() defer func() { cmdDurationUpdateServiceGCSafePoint.Observe(time.Since(start).Seconds()) }() - ctx, cancel := context.WithTimeout(ctx, pdTimeout) + ctx, cancel := context.WithTimeout(ctx, c.timeout) resp, err := c.leaderClient().UpdateServiceGCSafePoint(ctx, &pdpb.UpdateServiceGCSafePointRequest{ Header: c.requestHeader(), ServiceId: []byte(serviceID), @@ -666,7 +666,7 @@ func (c *client) ScatterRegion(ctx context.Context, regionID uint64) error { start := time.Now() defer func() { cmdDurationScatterRegion.Observe(time.Since(start).Seconds()) }() - ctx, cancel := context.WithTimeout(ctx, pdTimeout) + ctx, cancel := context.WithTimeout(ctx, c.timeout) resp, err := c.leaderClient().ScatterRegion(ctx, &pdpb.ScatterRegionRequest{ Header: c.requestHeader(), RegionId: regionID, @@ -689,7 +689,7 @@ func (c *client) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOpe start := time.Now() defer func() { cmdDurationGetOperator.Observe(time.Since(start).Seconds()) }() - ctx, cancel := context.WithTimeout(ctx, pdTimeout) + ctx, cancel := context.WithTimeout(ctx, c.timeout) defer cancel() return c.leaderClient().GetOperator(ctx, &pdpb.GetOperatorRequest{ Header: c.requestHeader(), diff --git a/conf/config.toml b/conf/config.toml index ea8637466425..9b7d45c6d0a0 100644 --- a/conf/config.toml +++ b/conf/config.toml @@ -123,3 +123,7 @@ location-labels = [] ## is running behind a reverse proxy. Do not configure it if you access ## Dashboard directly. # public-path-prefix = "/dashboard" + +## When enabled, request will be proxied to the instance running Dashboard +## internally instead of result in a 307 redirection. +# internal-proxy = false diff --git a/go.mod b/go.mod index fcc743b1c818..9aac2a31d7b8 100644 --- a/go.mod +++ b/go.mod @@ -8,13 +8,13 @@ require ( github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e github.com/coreos/go-semver v0.2.0 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/docker/go-units v0.4.0 github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e // indirect github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 // indirect - github.com/elazarl/go-bindata-assetfs v1.0.0 github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect - github.com/go-openapi/spec v0.19.7 // indirect - github.com/go-openapi/swag v0.19.8 // indirect + github.com/go-openapi/spec v0.19.8 // indirect + github.com/go-openapi/swag v0.19.9 // indirect github.com/go-playground/overalls v0.0.0-20180201144345-22ec1a223b7c github.com/go-playground/universal-translator v0.17.0 // indirect github.com/gogo/protobuf v1.3.1 @@ -28,7 +28,6 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.14.3 // indirect github.com/json-iterator/go v1.1.9 // indirect github.com/juju/ratelimit v1.0.1 - github.com/kevinburke/go-bindata v3.18.0+incompatible github.com/leodido/go-urn v1.2.0 // indirect github.com/mailru/easyjson v0.7.1 // indirect github.com/mattn/go-shellwords v1.0.3 @@ -38,7 +37,7 @@ require ( github.com/opentracing/opentracing-go v1.0.2 github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d - github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200526165651-52e570bf6f0a + github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200604095604-967424d77384 github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12 github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9 github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d @@ -52,19 +51,19 @@ require ( github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.1 github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba - github.com/swaggo/swag v1.6.6-0.20200323071853-8e21f4cefeea + github.com/swaggo/swag v1.6.6-0.20200529100950-7c765ddd0476 github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3 + github.com/urfave/cli/v2 v2.2.0 // indirect github.com/urfave/negroni v0.3.0 go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 go.uber.org/goleak v0.10.0 go.uber.org/zap v1.13.0 - golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect - golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224 + golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d google.golang.org/grpc v1.25.1 gopkg.in/go-playground/validator.v9 v9.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 - gopkg.in/yaml.v2 v2.2.8 // indirect + gopkg.in/yaml.v2 v2.3.0 // indirect honnef.co/go/tools v0.0.1-2020.1.3 ) diff --git a/go.sum b/go.sum index 086d7fa33414..821c8a8b8ae4 100644 --- a/go.sum +++ b/go.sum @@ -46,6 +46,8 @@ github.com/corona10/goimagehash v1.0.2 h1:pUfB0LnsJASMPGEZLj7tGY251vF+qLGqOgEP4r github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -61,8 +63,6 @@ github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e h1:Fw7ZmgiklsLh github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= -github.com/elazarl/go-bindata-assetfs v1.0.0 h1:G/bYguwHIzWq9ZoyUQqrjTmJbbYn3j3CKKpKinvZLFk= -github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= @@ -90,8 +90,6 @@ github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6 github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/gin-gonic/gin v1.5.0 h1:fi+bqFAx/oLK54somfCtEZs9HeH1LHVoEPUgARpTqyc= github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= -github.com/go-bindata/go-bindata/v3 v3.1.3 h1:F0nVttLC3ws0ojc7p60veTurcOm//D4QBODNM7EGrCI= -github.com/go-bindata/go-bindata/v3 v3.1.3/go.mod h1:1/zrpXsLD8YDIbhZRqXzm1Ghc7NhEvIN9+Z6R5/xH4I= github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs= github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -111,14 +109,14 @@ github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL9 github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.19.4 h1:ixzUSnHTd6hCemgtAJgluaTSGYpLNpJY4mA2DIkdOAo= github.com/go-openapi/spec v0.19.4/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/spec v0.19.7 h1:0xWSeMd35y5avQAThZR2PkEuqSosoS5t6gDH4L8n11M= -github.com/go-openapi/spec v0.19.7/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/spec v0.19.8 h1:qAdZLh1r6QF/hI/gTq+TJTvsQUodZsM7KLqkAJdiJNg= +github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.8 h1:vfK6jLhs7OI4tAXkvkooviaE1JEPcw3mutyegLHHjmk= -github.com/go-openapi/swag v0.19.8/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= +github.com/go-openapi/swag v0.19.9 h1:1IxuqvBUU3S2Bi4YC7tlP9SJF1gVpCvqN0T2Qof4azE= +github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= @@ -215,8 +213,6 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kevinburke/go-bindata v3.18.0+incompatible h1:NfOP49jFW7KyBl7UwTg0xkhSfHjESEwe2VMrcnSHG20= -github.com/kevinburke/go-bindata v3.18.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0 h1:reN85Pxc5larApoH1keMBiu2GWtPqXQ1nc9gx+jOU+E= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= @@ -296,8 +292,8 @@ github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8= -github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200526165651-52e570bf6f0a h1:FE7NgHM9ubPb7elnJUot9XBRLWDqhyXWIby5r1q1JXA= -github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200526165651-52e570bf6f0a/go.mod h1:c+hdTnQglVGcBvQwLZvCUYzvi1+1SYHfnWH/XlKI/BI= +github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200604095604-967424d77384 h1:cr6P7P9Pcu4YWlOSj7klgn4aXdEpAiW+RVHKvzCBIws= +github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200604095604-967424d77384/go.mod h1:9yaAM77sPfa5/f6sdxr3jSkKfIz463KRHyiFHiGjdes= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4 h1:iRtOAQ6FXkY/BGvst3CDfTva4nTqh6CL8WXvanLdbu0= @@ -359,8 +355,12 @@ github.com/shirou/gopsutil v2.19.10+incompatible h1:lA4Pi29JEVIQIgATSeftHSY0rMGI github.com/shirou/gopsutil v2.19.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 h1:mj/nMDAwTBiaCqMEs4cYCqF7pO6Np7vhy1D1wcQGz+E= +github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= @@ -381,16 +381,12 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 h1:PyYN9JH5jY9j6av01SpfRMb+1DWg/i3MbGOKPxJ2wjM= github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14/go.mod h1:gxQT6pBGRuIGunNf/+tSOB5OHvguWi8Tbt82WOkf35E= github.com/swaggo/gin-swagger v1.2.0/go.mod h1:qlH2+W7zXGZkczuL+r2nEBR2JTT+/lX05Nn6vPhc7OI= -github.com/swaggo/http-swagger v0.0.0-20200103000832-0e9263c4b516 h1:MauWjOaWBMls1+ZnLrxQs6qJG7VUDq4KU7JlHZjSdbo= -github.com/swaggo/http-swagger v0.0.0-20200103000832-0e9263c4b516/go.mod h1:O1lAbCgAAX/KZ80LM/OXwtWFI/5TvZlwxSg8Cq08PV0= github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba h1:lUPlXKqgbqT2SVg2Y+eT9mu5wbqMnG+i/+Q9nK7C0Rs= github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba/go.mod h1:O1lAbCgAAX/KZ80LM/OXwtWFI/5TvZlwxSg8Cq08PV0= github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+tv8Y= github.com/swaggo/swag v1.6.3/go.mod h1:wcc83tB4Mb2aNiL/HP4MFeQdpHUrca+Rp/DRNgWAUio= -github.com/swaggo/swag v1.6.5 h1:2C+t+xyK6p1sujqncYO/VnMvPZcBJjNdKKyxbOdAW8o= -github.com/swaggo/swag v1.6.5/go.mod h1:Y7ZLSS0d0DdxhWGVhQdu+Bu1QhaF5k0RD7FKdiAykeY= -github.com/swaggo/swag v1.6.6-0.20200323071853-8e21f4cefeea h1:e4navjNdMYbOJeqjea2DcUC00Djp1ewI7sJqmp1xd+I= -github.com/swaggo/swag v1.6.6-0.20200323071853-8e21f4cefeea/go.mod h1:xDhTyuFIujYiN3DKWC/H/83xcfHp+UE/IzWWampG7Zc= +github.com/swaggo/swag v1.6.6-0.20200529100950-7c765ddd0476 h1:UjnSXdNPIG+5FJ6xLQODEdk7gSnJlMldu3sPAxxCO+4= +github.com/swaggo/swag v1.6.6-0.20200529100950-7c765ddd0476/go.mod h1:xDhTyuFIujYiN3DKWC/H/83xcfHp+UE/IzWWampG7Zc= github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d h1:4J9HCZVpvDmj2tiKGSTUnb3Ok/9CEQb9oqu9LHKQQpc= github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/tidwall/gjson v1.3.5 h1:2oW9FBNu8qt9jy5URgrzsVx/T/KSn3qn/smJQ0crlDQ= @@ -412,18 +408,19 @@ github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3 h1:ZsIlNwu/G0zbChIZaWOeZ2TPGNmKMt46jZLXi3e8LFc= github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= +github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.1.1 h1:Qt8FeAtxE/vfdrLmR3rxR6JRE0RoVmbXu8+6kZtYU4k= github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= +github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/urfave/negroni v0.3.0 h1:PaXOb61mWeZJxc1Ji2xJjpVg9QfPo0rrB+lHyBxGNSU= github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yookoala/realpath v1.0.0 h1:7OA9pj4FZd+oZDsyvXWQvjn5oBdcHRTV44PpdMSuImQ= github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= @@ -501,9 +498,10 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 h1:eDrdRpKgkcCqKZQwyZRyeFZgfqt37SL7Kv3tok06cKE= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -527,6 +525,7 @@ golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= @@ -558,8 +557,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200225230052-807dcd883420/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224 h1:azwY/v0y0K4mFHVsg5+UrTgchqALYWpqVo6vL5OmkmI= -golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d h1:SR+e35rACZFBohNb4Om1ibX6N3iO0FtdbwqGSuD9dBU= +golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -613,8 +612,8 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json index b6f84a1bd1df..2bc93f41d1f1 100644 --- a/metrics/grafana/pd.json +++ b/metrics/grafana/pd.json @@ -3921,7 +3921,7 @@ "steppedLine": false, "targets": [ { - "expr": "pd_scheduler_store_status{store=~\"$store\", type=\"store_write_rate_bytes\"}", + "expr": "pd_scheduler_store_status{store=~\"$store\", type=\"store_write_rate_keys\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{address}}-store-{{store}}", @@ -3933,7 +3933,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Store write rate bytes", + "title": "Store write rate keys", "tooltip": { "shared": true, "sort": 0, diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go index 0ba7e22ffd86..d454cb87465f 100644 --- a/pkg/cache/cache_test.go +++ b/pkg/cache/cache_test.go @@ -15,6 +15,7 @@ package cache import ( "context" + "sort" "testing" "time" @@ -52,6 +53,8 @@ func (s *testRegionCacheSuite) TestExpireRegionCache(c *C) { c.Assert(cache.Len(), Equals, 3) + c.Assert(sortIDs(cache.GetKeys()), DeepEquals, []uint64{1, 2, 3}) + time.Sleep(2 * time.Second) value, ok = cache.Get(1) @@ -67,6 +70,7 @@ func (s *testRegionCacheSuite) TestExpireRegionCache(c *C) { c.Assert(value, Equals, 3.0) c.Assert(cache.Len(), Equals, 2) + c.Assert(sortIDs(cache.GetKeys()), DeepEquals, []uint64{2, 3}) cache.Remove(2) @@ -79,6 +83,13 @@ func (s *testRegionCacheSuite) TestExpireRegionCache(c *C) { c.Assert(value, Equals, 3.0) c.Assert(cache.Len(), Equals, 1) + c.Assert(sortIDs(cache.GetKeys()), DeepEquals, []uint64{3}) +} + +func sortIDs(ids []uint64) []uint64 { + ids = append(ids[:0:0], ids...) + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + return ids } func (s *testRegionCacheSuite) TestLRUCache(c *C) { diff --git a/pkg/cache/ttl.go b/pkg/cache/ttl.go index 60f6782dc948..19f8caf2de5e 100644 --- a/pkg/cache/ttl.go +++ b/pkg/cache/ttl.go @@ -83,6 +83,22 @@ func (c *TTL) Get(key uint64) (interface{}, bool) { return item.value, true } +// GetKeys returns all keys that are not expired. +func (c *TTL) GetKeys() []uint64 { + c.RLock() + defer c.RUnlock() + + var keys []uint64 + + now := time.Now() + for key, item := range c.items { + if item.expire.After(now) { + keys = append(keys, key) + } + } + return keys +} + // Remove eliminates an item from cache. func (c *TTL) Remove(key uint64) { c.Lock() @@ -152,6 +168,11 @@ func (c *TTLUint64) Put(id uint64) { c.TTL.Put(id, nil) } +// GetAll returns all ids. +func (c *TTLUint64) GetAll() []uint64 { + return c.TTL.GetKeys() +} + // Exists checks if an ID exists in cache. func (c *TTLUint64) Exists(id uint64) bool { _, ok := c.TTL.Get(id) diff --git a/pkg/dashboard/adapter/apiserver.go b/pkg/dashboard/adapter/adapter.go similarity index 50% rename from pkg/dashboard/adapter/apiserver.go rename to pkg/dashboard/adapter/adapter.go index 3747e956a727..41e6b6312e3f 100644 --- a/pkg/dashboard/adapter/apiserver.go +++ b/pkg/dashboard/adapter/adapter.go @@ -22,12 +22,11 @@ import ( "go.etcd.io/etcd/clientv3" "github.com/pingcap/pd/v4/pkg/dashboard/keyvisual/input" - "github.com/pingcap/pd/v4/pkg/dashboard/uiserver" "github.com/pingcap/pd/v4/server" ) -// NewAPIService uses the PD information to create a new apiserver.Service. -func NewAPIService(srv *server.Server, redirector http.Handler) (*apiserver.Service, error) { +// GenDashboardConfig generates a configuration for Dashboard Server. +func GenDashboardConfig(srv *server.Server) (*config.Config, error) { cfg := srv.GetConfig() etcdCfg, err := cfg.GenEmbedEtcdConfig() @@ -36,31 +35,30 @@ func NewAPIService(srv *server.Server, redirector http.Handler) (*apiserver.Serv } dashboardCfg := &config.Config{ - DataDir: cfg.DataDir, - PDEndPoint: etcdCfg.ACUrls[0].String(), - PathPrefix: cfg.Dashboard.PublicPathPrefix, + DataDir: cfg.DataDir, + PDEndPoint: etcdCfg.ACUrls[0].String(), + PublicPathPrefix: cfg.Dashboard.PublicPathPrefix, } - uiserver.InitAssetFS(cfg.Dashboard.PublicPathPrefix) - dashboardCfg.ClusterTLSConfig, err = cfg.Security.ToTLSConfig() - if err != nil { + + if dashboardCfg.ClusterTLSConfig, err = cfg.Security.ToTLSConfig(); err != nil { return nil, err } - dashboardCfg.TiDBTLSConfig, err = cfg.Dashboard.ToTiDBTLSConfig() - if err != nil { + if dashboardCfg.TiDBTLSConfig, err = cfg.Dashboard.ToTiDBTLSConfig(); err != nil { return nil, err } - s := apiserver.NewService( - dashboardCfg, - redirector, - uiserver.AssetFS(), - func(c *config.Config, httpClient *http.Client, etcdClient *clientv3.Client) *region.PDDataProvider { - return ®ion.PDDataProvider{ - EtcdClient: etcdClient, - PeriodicGetter: input.NewCorePeriodicGetter(srv), - } - }, - ) + dashboardCfg.NormalizePublicPathPrefix() - return s, nil + return dashboardCfg, nil +} + +// GenPDDataProviderConstructor generates a PDDataProviderConstructor for Dashboard API Service. +func GenPDDataProviderConstructor(srv *server.Server) apiserver.PDDataProviderConstructor { + // Get RegionInfos directly from Server, so dashboard Config and httpClient are not needed. + return func(c *config.Config, httpClient *http.Client, etcdClient *clientv3.Client) *region.PDDataProvider { + return ®ion.PDDataProvider{ + EtcdClient: etcdClient, + PeriodicGetter: input.NewCorePeriodicGetter(srv), + } + } } diff --git a/pkg/dashboard/adapter/manager.go b/pkg/dashboard/adapter/manager.go index 3d3e1b5b51d3..4ee2f281fd68 100644 --- a/pkg/dashboard/adapter/manager.go +++ b/pkg/dashboard/adapter/manager.go @@ -51,10 +51,7 @@ type Manager struct { // NewManager creates a new Manager. func NewManager(srv *server.Server, s *apiserver.Service, redirector *Redirector) *Manager { - ctx, cancel := context.WithCancel(srv.Context()) return &Manager{ - ctx: ctx, - cancel: cancel, srv: srv, service: s, redirector: redirector, @@ -63,7 +60,10 @@ func NewManager(srv *server.Server, s *apiserver.Service, redirector *Redirector // Start monitoring the dynamic config and control the dashboard. func (m *Manager) Start() { + m.ctx, m.cancel = context.WithCancel(m.srv.Context()) m.wg.Add(1) + m.isLeader = false + m.members = nil go m.serviceLoop() } diff --git a/pkg/dashboard/adapter/redirector.go b/pkg/dashboard/adapter/redirector.go index 9c710d3654a6..20d0350e21de 100644 --- a/pkg/dashboard/adapter/redirector.go +++ b/pkg/dashboard/adapter/redirector.go @@ -21,6 +21,7 @@ import ( "sync" "github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver" + "github.com/pingcap-incubator/tidb-dashboard/pkg/utils" ) const ( @@ -36,6 +37,9 @@ type Redirector struct { address string proxy *httputil.ReverseProxy + // The status of the dashboard in the cluster. + // It is not equal to `apiserver.Service.status`. + status *utils.ServiceStatus } // NewRedirector creates a new Redirector. @@ -43,6 +47,7 @@ func NewRedirector(name string, tlsConfig *tls.Config) *Redirector { return &Redirector{ name: name, tlsConfig: tlsConfig, + status: utils.NewServiceStatus(), } } @@ -56,11 +61,13 @@ func (h *Redirector) SetAddress(addr string) { } if addr == "" { + h.status.Stop() h.address = "" h.proxy = nil return } + h.status.Start() h.address = addr target, _ := url.Parse(addr) // error has been handled in checkAddress h.proxy = httputil.NewSingleHostReverseProxy(target) @@ -117,3 +124,8 @@ func (h *Redirector) ReverseProxy(w http.ResponseWriter, r *http.Request) { proxy.ServeHTTP(w, r) } + +// NewStatusAwareHandler returns a Handler that switches between different Handlers based on status. +func (h *Redirector) NewStatusAwareHandler(handler http.Handler) http.Handler { + return h.status.NewStatusAwareHandler(handler, apiserver.StoppedHandler) +} diff --git a/pkg/dashboard/dashboard.go b/pkg/dashboard/dashboard.go index 9044b2778595..9082e6b2ee76 100644 --- a/pkg/dashboard/dashboard.go +++ b/pkg/dashboard/dashboard.go @@ -21,9 +21,11 @@ import ( "time" "github.com/pingcap-incubator/tidb-dashboard/pkg/apiserver" + "github.com/pingcap-incubator/tidb-dashboard/pkg/config" + "github.com/pingcap-incubator/tidb-dashboard/pkg/uiserver" "github.com/pingcap/pd/v4/pkg/dashboard/adapter" - "github.com/pingcap/pd/v4/pkg/dashboard/uiserver" + ui "github.com/pingcap/pd/v4/pkg/dashboard/uiserver" "github.com/pingcap/pd/v4/server" ) @@ -32,14 +34,14 @@ var ( Name: "dashboard-api", Version: "v1", IsCore: false, - PathPrefix: "/dashboard/api/", + PathPrefix: config.APIPathPrefix, } uiServiceGroup = server.ServiceGroup{ Name: "dashboard-ui", Version: "v1", IsCore: false, - PathPrefix: "/dashboard/", + PathPrefix: config.UIPathPrefix, } ) @@ -50,22 +52,38 @@ func SetCheckInterval(d time.Duration) { // GetServiceBuilders returns all ServiceBuilders required by Dashboard func GetServiceBuilders() []server.HandlerBuilder { - var s *apiserver.Service - var redirector *adapter.Redirector + var ( + err error + cfg *config.Config + internalProxy bool + redirector *adapter.Redirector + assets http.FileSystem + s *apiserver.Service + ) + // The order of execution must be sequential. return []server.HandlerBuilder{ // Dashboard API Service func(ctx context.Context, srv *server.Server) (http.Handler, server.ServiceGroup, error) { - tlsConfig, err := srv.GetSecurityConfig().ToTLSConfig() - if err != nil { + if cfg, err = adapter.GenDashboardConfig(srv); err != nil { return nil, apiServiceGroup, err } + internalProxy = srv.GetConfig().Dashboard.InternalProxy + redirector = adapter.NewRedirector(srv.Name(), cfg.ClusterTLSConfig) + assets = ui.Assets(cfg) - redirector = adapter.NewRedirector(srv.Name(), tlsConfig) - - if s, err = adapter.NewAPIService(srv, http.HandlerFunc(redirector.ReverseProxy)); err != nil { - return nil, apiServiceGroup, err + var stoppedHandler http.Handler + if internalProxy { + stoppedHandler = http.HandlerFunc(redirector.ReverseProxy) + } else { + stoppedHandler = http.HandlerFunc(redirector.TemporaryRedirect) } + s = apiserver.NewService( + cfg, + stoppedHandler, + assets, + adapter.GenPDDataProviderConstructor(srv), + ) m := adapter.NewManager(srv, s, redirector) srv.AddStartCallback(m.Start) @@ -75,10 +93,18 @@ func GetServiceBuilders() []server.HandlerBuilder { }, // Dashboard UI func(context.Context, *server.Server) (http.Handler, server.ServiceGroup, error) { - handler := s.NewStatusAwareHandler( - http.StripPrefix(uiServiceGroup.PathPrefix, uiserver.Handler()), - http.HandlerFunc(redirector.TemporaryRedirect), - ) + if err != nil { + return nil, uiServiceGroup, err + } + + var handler http.Handler + uiHandler := http.StripPrefix(uiServiceGroup.PathPrefix, uiserver.Handler(assets)) + if internalProxy { + handler = redirector.NewStatusAwareHandler(uiHandler) + } else { + handler = s.NewStatusAwareHandler(uiHandler, http.HandlerFunc(redirector.TemporaryRedirect)) + } + return handler, uiServiceGroup, nil }, } diff --git a/pkg/dashboard/uiserver/embedded_assets_rewriter.go b/pkg/dashboard/uiserver/embedded_assets_rewriter.go index fe4d440deaee..b7a77a98242a 100644 --- a/pkg/dashboard/uiserver/embedded_assets_rewriter.go +++ b/pkg/dashboard/uiserver/embedded_assets_rewriter.go @@ -14,44 +14,30 @@ package uiserver import ( - "html" + "net/http" "os" - "strings" "sync" + "time" + + "github.com/pingcap-incubator/tidb-dashboard/pkg/config" + "github.com/pingcap-incubator/tidb-dashboard/pkg/uiserver" ) var once sync.Once -type modifiedFileInfo struct { - os.FileInfo - size int64 -} - -func (f modifiedFileInfo) Size() int64 { - return f.size -} - -func (f modifiedFileInfo) Sys() interface{} { - return nil -} - -// InitAssetFS init the static resources with given public path prefix. -func InitAssetFS(prefix string) { +// Assets returns the Assets FileSystem of the dashboard UI +func Assets(cfg *config.Config) http.FileSystem { once.Do(func() { - rewrite := func(assetPath string) { - a, err := _bindata[assetPath]() - if err != nil { - panic("Asset " + assetPath + " not found.") - } - tmplText := string(a.bytes) - updated := strings.ReplaceAll(tmplText, "__PUBLIC_PATH_PREFIX__", html.EscapeString(prefix)) - a.bytes = []byte(updated) - a.info = modifiedFileInfo{a.info, int64(len(a.bytes))} - _bindata[assetPath] = func() (*asset, error) { - return a, nil + uiserver.RewriteAssets(assets, cfg, func(fs http.FileSystem, f http.File, path, newContent string, bs []byte) { + m := fs.(vfsgen۰FS) + fi := f.(os.FileInfo) + m[path] = &vfsgen۰CompressedFileInfo{ + name: fi.Name(), + modTime: time.Now(), + uncompressedSize: int64(len(newContent)), + compressedContent: bs, } - } - rewrite("build/index.html") - rewrite("build/diagnoseReport.html") + }) }) + return assets } diff --git a/pkg/dashboard/uiserver/uiserver.go b/pkg/dashboard/uiserver/uiserver.go deleted file mode 100644 index a3d7551372ba..000000000000 --- a/pkg/dashboard/uiserver/uiserver.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package uiserver - -import ( - "io" - "net/http" - - assetfs "github.com/elazarl/go-bindata-assetfs" -) - -var ( - fs = assetFS() -) - -// Handler returns an http.Handler that serves the dashboard UI -func Handler() http.Handler { - if fs == nil { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = io.WriteString(w, "Dashboard UI is not built.\n") - }) - } - return http.FileServer(fs) -} - -// AssetFS returns the AssetFS of the dashboard UI -func AssetFS() *assetfs.AssetFS { - return fs -} diff --git a/pkg/dashboard/without_dashboard.go b/pkg/dashboard/without_dashboard.go index 81f9891643d3..9333de5d479e 100644 --- a/pkg/dashboard/without_dashboard.go +++ b/pkg/dashboard/without_dashboard.go @@ -21,6 +21,8 @@ import ( "net/http" "time" + "github.com/pingcap-incubator/tidb-dashboard/pkg/config" + "github.com/pingcap/pd/v4/server" ) @@ -29,7 +31,7 @@ var ( Name: "dashboard", Version: "v1", IsCore: false, - PathPrefix: "/dashboard/", + PathPrefix: config.UIPathPrefix, } ) diff --git a/pkg/testutil/leak.go b/pkg/testutil/leak.go index aa895810a341..6a4fc0b29f0e 100644 --- a/pkg/testutil/leak.go +++ b/pkg/testutil/leak.go @@ -26,6 +26,7 @@ var LeakOptions = []goleak.Option{ goleak.IgnoreTopFunction("google.golang.org/grpc.(*ccResolverWrapper).watcher"), goleak.IgnoreTopFunction("google.golang.org/grpc.(*addrConn).createTransport"), goleak.IgnoreTopFunction("google.golang.org/grpc.(*addrConn).resetTransport"), + goleak.IgnoreTopFunction("google.golang.org/grpc.(*Server).handleRawConn"), goleak.IgnoreTopFunction("go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop"), // TODO: remove the below options once we fixed the http connection leak problems goleak.IgnoreTopFunction("internal/poll.runtime_pollWait"), diff --git a/server/api/rule.go b/server/api/rule.go index b5491d4295e0..cea37148e11e 100644 --- a/server/api/rule.go +++ b/server/api/rule.go @@ -195,7 +195,7 @@ func (h *ruleHandler) checkRule(r *placement.Rule) error { if err != nil { return errors.Wrap(err, "end key is not hex format") } - if len(start) > 0 && bytes.Compare(end, start) <= 0 { + if len(end) > 0 && bytes.Compare(end, start) <= 0 { return errors.New("endKey should be greater than startKey") } diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 8df117597ed9..5da1ab1438eb 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/kvproto/pkg/replication_modepb" "github.com/pingcap/log" + "github.com/pingcap/pd/v4/pkg/cache" "github.com/pingcap/pd/v4/pkg/component" "github.com/pingcap/pd/v4/pkg/etcdutil" "github.com/pingcap/pd/v4/pkg/logutil" @@ -99,7 +100,8 @@ type RaftCluster struct { storesStats *statistics.StoresStats hotSpotCache *statistics.HotCache - coordinator *coordinator + coordinator *coordinator + suspectRegions *cache.TTLUint64 // suspectRegions are regions that may need fix wg sync.WaitGroup quit chan struct{} @@ -193,6 +195,7 @@ func (c *RaftCluster) InitCluster(id id.Allocator, opt *config.PersistOptions, s c.prepareChecker = newPrepareChecker() c.changedRegions = make(chan *core.RegionInfo, defaultChangedRegionsLimit) c.hotSpotCache = statistics.NewHotCache() + c.suspectRegions = cache.NewIDTTL(c.ctx, time.Minute, 3*time.Minute) } // Start starts a cluster. @@ -412,6 +415,29 @@ func (c *RaftCluster) SetStorage(s *core.Storage) { c.storage = s } +// AddSuspectRegions adds regions to suspect list. +func (c *RaftCluster) AddSuspectRegions(ids ...uint64) { + c.Lock() + defer c.Unlock() + for _, id := range ids { + c.suspectRegions.Put(id) + } +} + +// GetSuspectRegions gets all suspect regions. +func (c *RaftCluster) GetSuspectRegions() []uint64 { + c.RLock() + defer c.RUnlock() + return c.suspectRegions.GetAll() +} + +// RemoveSuspectRegion removes region from suspect list. +func (c *RaftCluster) RemoveSuspectRegion(id uint64) { + c.Lock() + defer c.Unlock() + c.suspectRegions.Remove(id) +} + // HandleStoreHeartbeat updates the store status. func (c *RaftCluster) HandleStoreHeartbeat(stats *pdpb.StoreStats) error { c.Lock() diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index a9ae9afb21c4..af1392a34097 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -14,6 +14,7 @@ package cluster import ( + "context" "fmt" "math/rand" "sync" @@ -332,13 +333,13 @@ func (s *testClusterInfoSuite) TestConcurrentRegionHeartbeat(c *C) { var wg sync.WaitGroup wg.Add(1) - c.Assert(failpoint.Enable("github.com/pingcap/pd/server/cluster/concurrentRegionHeartbeat", "return(true)"), IsNil) + c.Assert(failpoint.Enable("github.com/pingcap/pd/v4/server/cluster/concurrentRegionHeartbeat", "return(true)"), IsNil) go func() { defer wg.Done() cluster.processRegionHeartbeat(source) }() time.Sleep(100 * time.Millisecond) - c.Assert(failpoint.Disable("github.com/pingcap/pd/server/cluster/concurrentRegionHeartbeat"), IsNil) + c.Assert(failpoint.Disable("github.com/pingcap/pd/v4/server/cluster/concurrentRegionHeartbeat"), IsNil) c.Assert(cluster.processRegionHeartbeat(target), IsNil) wg.Wait() checkRegion(c, cluster.GetRegionByKey([]byte{}), target) @@ -670,7 +671,7 @@ func newTestCluster(opt *config.PersistOptions) *testCluster { } func newTestRaftCluster(id id.Allocator, opt *config.PersistOptions, storage *core.Storage, basicCluster *core.BasicCluster) *RaftCluster { - rc := &RaftCluster{} + rc := &RaftCluster{ctx: context.TODO()} rc.InitCluster(id, opt, storage, basicCluster) return rc } diff --git a/server/cluster/cluster_worker.go b/server/cluster/cluster_worker.go index 464c42453bfb..3e916921c04b 100644 --- a/server/cluster/cluster_worker.go +++ b/server/cluster/cluster_worker.go @@ -137,6 +137,11 @@ func (c *RaftCluster) HandleAskBatchSplit(request *pdpb.AskBatchSplitRequest) (* c.GetMergeChecker().RecordRegionSplit(recordRegions) } + // If region splits during the scheduling process, regions with abnormal + // status may be left, and these regions need to be checked with higher + // priority. + c.AddSuspectRegions(recordRegions...) + resp := &pdpb.AskBatchSplitResponse{Ids: splitIDs} return resp, nil diff --git a/server/cluster/cluster_worker_test.go b/server/cluster/cluster_worker_test.go index 1f3d4b1cc659..d41d1efba95b 100644 --- a/server/cluster/cluster_worker_test.go +++ b/server/cluster/cluster_worker_test.go @@ -18,7 +18,9 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" - // Register schedulers + "github.com/pingcap/pd/v4/pkg/mock/mockid" + "github.com/pingcap/pd/v4/server/core" + "github.com/pingcap/pd/v4/server/kv" _ "github.com/pingcap/pd/v4/server/schedulers" ) @@ -27,23 +29,27 @@ var _ = Suite(&testClusterWorkerSuite{}) type testClusterWorkerSuite struct{} func (s *testClusterWorkerSuite) TestReportSplit(c *C) { - var cluster RaftCluster + _, opt, err := newTestScheduleConfig() + c.Assert(err, IsNil) + cluster := newTestRaftCluster(mockid.NewIDAllocator(), opt, core.NewStorage(kv.NewMemoryKV()), core.NewBasicCluster()) left := &metapb.Region{Id: 1, StartKey: []byte("a"), EndKey: []byte("b")} right := &metapb.Region{Id: 2, StartKey: []byte("b"), EndKey: []byte("c")} - _, err := cluster.HandleReportSplit(&pdpb.ReportSplitRequest{Left: left, Right: right}) + _, err = cluster.HandleReportSplit(&pdpb.ReportSplitRequest{Left: left, Right: right}) c.Assert(err, IsNil) _, err = cluster.HandleReportSplit(&pdpb.ReportSplitRequest{Left: right, Right: left}) c.Assert(err, NotNil) } func (s *testClusterWorkerSuite) TestReportBatchSplit(c *C) { - var cluster RaftCluster + _, opt, err := newTestScheduleConfig() + c.Assert(err, IsNil) + cluster := newTestRaftCluster(mockid.NewIDAllocator(), opt, core.NewStorage(kv.NewMemoryKV()), core.NewBasicCluster()) regions := []*metapb.Region{ {Id: 1, StartKey: []byte(""), EndKey: []byte("a")}, {Id: 2, StartKey: []byte("a"), EndKey: []byte("b")}, {Id: 3, StartKey: []byte("b"), EndKey: []byte("c")}, {Id: 3, StartKey: []byte("c"), EndKey: []byte("")}, } - _, err := cluster.HandleBatchReportSplit(&pdpb.ReportBatchSplitRequest{Regions: regions}) + _, err = cluster.HandleBatchReportSplit(&pdpb.ReportBatchSplitRequest{Regions: regions}) c.Assert(err, IsNil) } diff --git a/server/cluster/coordinator.go b/server/cluster/coordinator.go index f05aba936153..55acc5ba14cc 100644 --- a/server/cluster/coordinator.go +++ b/server/cluster/coordinator.go @@ -103,6 +103,27 @@ func (c *coordinator) patrolRegions() { return } + // Check suspect regions first. + for _, id := range c.cluster.GetSuspectRegions() { + region := c.cluster.GetRegion(id) + if region == nil { + // the region could be recent split, continue to wait. + continue + } + if c.opController.GetOperator(id) != nil { + c.cluster.RemoveSuspectRegion(id) + continue + } + checkerIsBusy, ops := c.checkers.CheckRegion(region) + if checkerIsBusy { + continue + } + if len(ops) > 0 { + c.opController.AddWaitingOperator(ops...) + } + c.cluster.RemoveSuspectRegion(id) + } + regions := c.cluster.ScanRegions(key, nil, patrolScanRegionLimit) if len(regions) == 0 { // Resets the scan key. @@ -122,7 +143,7 @@ func (c *coordinator) patrolRegions() { } key = region.GetEndKey() - if ops != nil { + if len(ops) > 0 { c.opController.AddWaitingOperator(ops...) } } diff --git a/server/config/config.go b/server/config/config.go index e67db9276041..6bc8f8a3b733 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -29,15 +29,16 @@ import ( "github.com/coreos/go-semver/semver" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" - "github.com/pingcap/pd/v4/pkg/grpcutil" - "github.com/pingcap/pd/v4/pkg/metricutil" - "github.com/pingcap/pd/v4/pkg/typeutil" - "github.com/pingcap/pd/v4/server/schedule" "github.com/pkg/errors" "go.etcd.io/etcd/embed" "go.etcd.io/etcd/pkg/transport" "go.uber.org/zap" "go.uber.org/zap/zapcore" + + "github.com/pingcap/pd/v4/pkg/grpcutil" + "github.com/pingcap/pd/v4/pkg/metricutil" + "github.com/pingcap/pd/v4/pkg/typeutil" + "github.com/pingcap/pd/v4/server/schedule" ) // Config is the pd server configuration. @@ -210,8 +211,6 @@ const ( defaultDRWaitStoreTimeout = time.Minute defaultDRWaitSyncTimeout = time.Minute - - defaultPublicPathPrefix = "/dashboard" ) var ( @@ -466,11 +465,6 @@ func (c *Config) Adjust(meta *toml.MetaData) error { c.ReplicationMode.adjust(configMetaData.Child("replication-mode")) - if c.Dashboard.PublicPathPrefix == "" { - c.Dashboard.PublicPathPrefix = defaultPublicPathPrefix - } - c.Dashboard.PublicPathPrefix = strings.TrimRight(c.Dashboard.PublicPathPrefix, "/") - return nil } @@ -1128,6 +1122,7 @@ type DashboardConfig struct { TiDBCertPath string `toml:"tidb-cert-path" json:"tidb_cert_path"` TiDBKeyPath string `toml:"tidb-key-path" json:"tidb_key_path"` PublicPathPrefix string `toml:"public-path-prefix" json:"public_path_prefix"` + InternalProxy bool `toml:"internal-proxy" json:"internal_proxy"` } // ToTiDBTLSConfig generates tls config for connecting to TiDB, used by tidb-dashboard. diff --git a/server/core/store.go b/server/core/store.go index ef54bbd8c857..48b986ef9f8e 100644 --- a/server/core/store.go +++ b/server/core/store.go @@ -304,7 +304,7 @@ func (s *StoreInfo) RegionScore(highSpaceRatio, lowSpaceRatio float64, delta int available := float64(s.GetAvailable()) / mb used := float64(s.GetUsedSize()) / mb - if s.GetRegionSize() == 0 { + if s.GetRegionSize() == 0 || used == 0 { amplification = 1 } else { // because of rocksdb compression, region size is larger than actual used size diff --git a/server/core/store_test.go b/server/core/store_test.go index 928466eb6f41..4f790aea1ae2 100644 --- a/server/core/store_test.go +++ b/server/core/store_test.go @@ -15,6 +15,7 @@ package core import ( "fmt" + "math" "sync" "time" @@ -123,3 +124,19 @@ func (s *testStoreSuite) TestLowSpaceThreshold(c *C) { c.Assert(fmt.Sprintf("%.2f", threshold), Equals, fmt.Sprintf("%.2f", 100*0.2)) c.Assert(store.IsLowSpace(0.8), Equals, true) } + +func (s *testStoreSuite) TestRegionScore(c *C) { + stats := &pdpb.StoreStats{} + stats.Capacity = 512 * (1 << 20) // 512 MB + stats.Available = 100 * (1 << 20) // 100 MB + stats.UsedSize = 0 + + store := NewStoreInfo( + &metapb.Store{Id: 1}, + SetStoreStats(stats), + SetRegionSize(1), + ) + score := store.RegionScore(0.7, 0.9, 0) + // Region score should never be NaN, or /store API would fail. + c.Assert(math.IsNaN(score), Equals, false) +} diff --git a/server/grpc_service.go b/server/grpc_service.go index 4256493e3da5..0a12b30d1316 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -21,6 +21,7 @@ import ( "sync/atomic" "time" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" @@ -243,6 +244,9 @@ func (s *Server) PutStore(ctx context.Context, request *pdpb.PutStoreRequest) (* // GetAllStores implements gRPC PDServer. func (s *Server) GetAllStores(ctx context.Context, request *pdpb.GetAllStoresRequest) (*pdpb.GetAllStoresResponse, error) { + failpoint.Inject("customTimeout", func() { + time.Sleep(5 * time.Second) + }) if err := s.validateRequest(request.GetHeader()); err != nil { return nil, err } diff --git a/server/replication/replication_mode.go b/server/replication/replication_mode.go index 68cbe5037235..d97ec6ce90d8 100644 --- a/server/replication/replication_mode.go +++ b/server/replication/replication_mode.go @@ -287,7 +287,13 @@ func (m *ModeManager) drPersistStatus(status drAutoSyncStatus) error { data, _ := json.Marshal(status) if err := m.fileReplicater.ReplicateFileToAllMembers(ctx, drStatusFile, data); err != nil { log.Warn("failed to switch state", zap.String("replicate-mode", modeDRAutoSync), zap.String("new-state", status.State), zap.Error(err)) - return err + // Throw away the error to make it possible to switch to async when + // primary and dr DC are disconnected. This will result in the + // inability to accurately determine whether data is fully + // synchronized when using dr DC to disaster recovery. + // TODO: introduce PD's leader-follower connection timeout to solve + // this issue. More details: https://github.com/pingcap/pd/issues/2490 + return nil } } return nil @@ -329,9 +335,24 @@ func (m *ModeManager) tickDR() { drTickCounter.Inc() - canSync := m.checkCanSync() + totalPrimary, totalDr := m.config.DRAutoSync.PrimaryReplicas, m.config.DRAutoSync.DRReplicas + downPrimary, downDr := m.checkStoreStatus() - if !canSync && m.drGetState() != drStateAsync { + // canSync is true when every region has at least 1 replica in each DC. + canSync := downPrimary < totalPrimary && downDr < totalDr + + // hasMajority is true when every region has majority peer online. + var upPeers int + if downPrimary < totalPrimary { + upPeers += totalPrimary - downPrimary + } + if downDr < totalDr { + upPeers += totalDr - downDr + } + hasMajority := upPeers*2 > totalPrimary+totalDr + + // If hasMajority is false, the cluster is always unavailable. Switch to async won't help. + if !canSync && hasMajority && m.drGetState() != drStateAsync { m.drSwitchToAsync() } @@ -352,22 +373,21 @@ func (m *ModeManager) tickDR() { } } -func (m *ModeManager) checkCanSync() bool { +func (m *ModeManager) checkStoreStatus() (primaryFailCount, drFailCount int) { m.RLock() defer m.RUnlock() - var countPrimary, countDR int for _, s := range m.cluster.GetStores() { if !s.IsTombstone() && s.DownTime() >= m.config.DRAutoSync.WaitStoreTimeout.Duration { labelValue := s.GetLabelValue(m.config.DRAutoSync.LabelKey) if labelValue == m.config.DRAutoSync.Primary { - countPrimary++ + primaryFailCount++ } if labelValue == m.config.DRAutoSync.DR { - countDR++ + drFailCount++ } } } - return countPrimary < m.config.DRAutoSync.PrimaryReplicas && countDR < m.config.DRAutoSync.DRReplicas + return } var ( diff --git a/server/replication/replication_mode_test.go b/server/replication/replication_mode_test.go index 0f0852d98fc3..0ffafb77281c 100644 --- a/server/replication/replication_mode_test.go +++ b/server/replication/replication_mode_test.go @@ -14,6 +14,8 @@ package replication import ( + "context" + "errors" "testing" "time" @@ -127,6 +129,14 @@ func (s *testReplicationMode) TestStatus(c *C) { }) } +type mockFileReplicator struct { + err error +} + +func (rep *mockFileReplicator) ReplicateFileToAllMembers(context.Context, string, []byte) error { + return rep.err +} + func (s *testReplicationMode) TestStateSwitch(c *C) { store := core.NewStorage(kv.NewMemoryKV()) conf := config.ReplicationModeConfig{ReplicationMode: modeDRAutoSync, DRAutoSync: config.DRAutoSyncReplicationConfig{ @@ -139,7 +149,8 @@ func (s *testReplicationMode) TestStateSwitch(c *C) { WaitSyncTimeout: typeutil.Duration{Duration: time.Minute}, }} cluster := mockcluster.NewCluster(mockoption.NewScheduleOptions()) - rep, err := NewReplicationModeManager(conf, store, cluster, nil) + var replicator mockFileReplicator + rep, err := NewReplicationModeManager(conf, store, cluster, &replicator) c.Assert(err, IsNil) cluster.AddLabelsStore(1, 1, map[string]string{"zone": "zone1"}) @@ -165,15 +176,18 @@ func (s *testReplicationMode) TestStateSwitch(c *C) { c.Assert(rep.drGetState(), Equals, drStateSync) s.setStoreState(cluster, 2, "down") rep.tickDR() - c.Assert(rep.drGetState(), Equals, drStateAsync) - assertStateIDUpdate() - rep.drSwitchToSync() + c.Assert(rep.drGetState(), Equals, drStateSync) // cannot guarantee majority, keep sync. s.setStoreState(cluster, 1, "up") s.setStoreState(cluster, 2, "up") s.setStoreState(cluster, 5, "down") rep.tickDR() c.Assert(rep.drGetState(), Equals, drStateAsync) assertStateIDUpdate() + rep.drSwitchToSync() + replicator.err = errors.New("fail to replicate") + rep.tickDR() + c.Assert(rep.drGetState(), Equals, drStateAsync) + assertStateIDUpdate() // async -> sync_recover s.setStoreState(cluster, 5, "up") @@ -189,6 +203,7 @@ func (s *testReplicationMode) TestStateSwitch(c *C) { // sync_recover -> async rep.tickDR() c.Assert(rep.drGetState(), Equals, drStateSyncRecover) + s.setStoreState(cluster, 1, "up") s.setStoreState(cluster, 4, "down") rep.tickDR() c.Assert(rep.drGetState(), Equals, drStateAsync) diff --git a/server/schedule/placement/fit.go b/server/schedule/placement/fit.go index 5556ec24fe6a..8f35d30862c8 100644 --- a/server/schedule/placement/fit.go +++ b/server/schedule/placement/fit.go @@ -59,6 +59,9 @@ func (f *RegionFit) GetRuleFit(peerID uint64) *RuleFit { // It returns 1 when the first fit result is better. func CompareRegionFit(a, b *RegionFit) int { for i := range a.RuleFits { + if i >= len(b.RuleFits) { + break + } if cmp := compareRuleFit(a.RuleFits[i], b.RuleFits[i]); cmp != 0 { return cmp } diff --git a/server/schedule/placement/rule_list.go b/server/schedule/placement/rule_list.go index 8a9d27dce08f..a7fcff0aed14 100644 --- a/server/schedule/placement/rule_list.go +++ b/server/schedule/placement/rule_list.go @@ -15,7 +15,11 @@ package placement import ( "bytes" + "encoding/hex" "sort" + "strings" + + "github.com/pkg/errors" ) type splitPointType int @@ -68,9 +72,9 @@ type ruleList struct { ranges []rangeRules // ranges[i] contains rules apply to (ranges[i].startKey, ranges[i+1].startKey). } -func buildRuleList(rules map[[2]string]*Rule) ruleList { +func buildRuleList(rules map[[2]string]*Rule) (ruleList, error) { if len(rules) == 0 { - return ruleList{} + return ruleList{}, errors.New("no rule left") } // collect and sort split points. var points []splitPoint @@ -105,6 +109,15 @@ func buildRuleList(rules map[[2]string]*Rule) ruleList { if i == len(points)-1 || !bytes.Equal(p.key, points[i+1].key) { // next key is different, push sr to rl. rr := sr.rules + if len(rr) == 0 { + var endKey []byte + if i != len(points)-1 { + endKey = points[i+1].key + } + return ruleList{}, errors.Errorf("no rule for range {%s, %s}", + strings.ToUpper(hex.EncodeToString(p.key)), + strings.ToUpper(hex.EncodeToString(endKey))) + } if i != len(points)-1 { rr = append(rr[:0:0], rr...) // clone } @@ -115,7 +128,7 @@ func buildRuleList(rules map[[2]string]*Rule) ruleList { }) } } - return rl + return rl, nil } func (rl ruleList) getSplitKeys(start, end []byte) [][]byte { diff --git a/server/schedule/placement/rule_manager.go b/server/schedule/placement/rule_manager.go index fd398fa4e327..b749ece6c6af 100644 --- a/server/schedule/placement/rule_manager.go +++ b/server/schedule/placement/rule_manager.go @@ -69,7 +69,11 @@ func (m *RuleManager) Initialize(maxReplica int, locationLabels []string) error } m.rules[defaultRule.Key()] = defaultRule } - m.ruleList = buildRuleList(m.rules) + ruleList, err := buildRuleList(m.rules) + if err != nil { + return err + } + m.ruleList = ruleList m.initialized = true return nil } @@ -169,8 +173,13 @@ func (m *RuleManager) SetRule(rule *Rule) error { old := m.rules[rule.Key()] m.rules[rule.Key()] = rule - if err = m.store.SaveRule(rule.StoreKey(), rule); err != nil { - // restore + ruleList, err := buildRuleList(m.rules) + if err == nil { + err = m.store.SaveRule(rule.StoreKey(), rule) + } + + // restore + if err != nil { if old == nil { delete(m.rules, rule.Key()) } else { @@ -179,8 +188,8 @@ func (m *RuleManager) SetRule(rule *Rule) error { return err } + m.ruleList = ruleList log.Info("placement rule updated", zap.Stringer("rule", rule)) - m.ruleList = buildRuleList(m.rules) return nil } @@ -194,13 +203,21 @@ func (m *RuleManager) DeleteRule(group, id string) error { return nil } delete(m.rules, [2]string{group, id}) + + ruleList, err := buildRuleList(m.rules) + if err != nil { + // restore + m.rules[key] = old + return err + } + if err := m.store.DeleteRule(old.StoreKey()); err != nil { // restore m.rules[key] = old return err } + m.ruleList = ruleList log.Info("placement rule removed", zap.Stringer("rule", old)) - m.ruleList = buildRuleList(m.rules) return nil } diff --git a/server/schedule/placement/rule_manager_test.go b/server/schedule/placement/rule_manager_test.go index 011d3141064b..e65aad4f2a3c 100644 --- a/server/schedule/placement/rule_manager_test.go +++ b/server/schedule/placement/rule_manager_test.go @@ -89,7 +89,6 @@ func (s *testManagerSuite) TestSaveLoad(c *C) { } func (s *testManagerSuite) TestKeys(c *C) { - s.manager.DeleteRule("pd", "default") rules := []*Rule{ {GroupID: "1", ID: "1", Role: "voter", Count: 1, StartKeyHex: "", EndKeyHex: ""}, {GroupID: "2", ID: "2", Role: "voter", Count: 1, StartKeyHex: "11", EndKeyHex: "ff"}, @@ -100,6 +99,7 @@ func (s *testManagerSuite) TestKeys(c *C) { for _, r := range rules { s.manager.SetRule(r) } + s.manager.DeleteRule("pd", "default") splitKeys := [][]string{ {"", "", "11", "22", "44", "dd", "ee", "ff"}, @@ -161,6 +161,29 @@ func (s *testManagerSuite) TestKeys(c *C) { } } +func (s *testManagerSuite) TestRangeGap(c *C) { + // |-- default --| + // cannot delete the last rule + err := s.manager.DeleteRule("pd", "default") + c.Assert(err, NotNil) + + err = s.manager.SetRule(&Rule{GroupID: "pd", ID: "foo", StartKeyHex: "", EndKeyHex: "abcd", Role: "voter", Count: 1}) + c.Assert(err, IsNil) + // |-- default --| + // |-- foo --| + // still cannot delete default since it will cause ("abcd", "") has no rules inside. + err = s.manager.DeleteRule("pd", "default") + c.Assert(err, NotNil) + err = s.manager.SetRule(&Rule{GroupID: "pd", ID: "bar", StartKeyHex: "abcd", EndKeyHex: "", Role: "voter", Count: 1}) + c.Assert(err, IsNil) + // now default can be deleted. + err = s.manager.DeleteRule("pd", "default") + c.Assert(err, IsNil) + // cannot change range since it will cause ("abaa", "abcd") has no rules inside. + err = s.manager.SetRule(&Rule{GroupID: "pd", ID: "foo", StartKeyHex: "", EndKeyHex: "abaa", Role: "voter", Count: 1}) + c.Assert(err, NotNil) +} + func (s *testManagerSuite) dhex(hk string) []byte { k, err := hex.DecodeString(hk) if err != nil { diff --git a/tests/client/client_test.go b/tests/client/client_test.go index 70e8c75e6444..e9c249093d92 100644 --- a/tests/client/client_test.go +++ b/tests/client/client_test.go @@ -25,6 +25,7 @@ import ( "github.com/gogo/protobuf/proto" . "github.com/pingcap/check" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" pd "github.com/pingcap/pd/v4/client" @@ -187,6 +188,31 @@ func (s *clientTestSuite) TestLeaderTransfer(c *C) { wg.Wait() } +func (s *clientTestSuite) TestCustomTimeout(c *C) { + cluster, err := tests.NewTestCluster(s.ctx, 1) + c.Assert(err, IsNil) + defer cluster.Destroy() + + err = cluster.RunInitialServers() + c.Assert(err, IsNil) + cluster.WaitLeader() + + var endpoints []string + for _, s := range cluster.GetServers() { + endpoints = append(endpoints, s.GetConfig().AdvertiseClientUrls) + } + cli, err := pd.NewClientWithContext(s.ctx, endpoints, pd.SecurityOption{}, pd.WithCustomTimeoutOption(1*time.Second)) + c.Assert(err, IsNil) + + start := time.Now() + c.Assert(failpoint.Enable("github.com/pingcap/pd/v4/server/customTimeout", "return(true)"), IsNil) + _, err = cli.GetAllStores(context.TODO()) + c.Assert(failpoint.Disable("github.com/pingcap/pd/v4/server/customTimeout"), IsNil) + c.Assert(err, NotNil) + c.Assert(time.Since(start), GreaterEqual, 1*time.Second) + c.Assert(time.Since(start), Less, 2*time.Second) +} + func (s *clientTestSuite) waitLeader(c *C, cli client, leader string) { testutil.WaitUntil(c, func(c *C) bool { cli.ScheduleCheckLeader() diff --git a/tests/cluster.go b/tests/cluster.go index 273711d3e136..80e2da00200d 100644 --- a/tests/cluster.go +++ b/tests/cluster.go @@ -46,6 +46,13 @@ const ( Destroy ) +var ( + // WaitLeaderReturnDelay represents the time interval of WaitLeader sleep before returning. + WaitLeaderReturnDelay = 20 * time.Millisecond + // WaitLeaderCheckInterval represents the time interval of WaitLeader running check. + WaitLeaderCheckInterval = 500 * time.Millisecond +) + // TestServer is only for test. type TestServer struct { sync.RWMutex @@ -438,11 +445,11 @@ func (c *TestCluster) WaitLeader() string { } for name, num := range counter { if num == running && c.GetServer(name).IsLeader() { - time.Sleep(20 * time.Millisecond) + time.Sleep(WaitLeaderReturnDelay) return name } } - time.Sleep(500 * time.Millisecond) + time.Sleep(WaitLeaderCheckInterval) } return "" } diff --git a/tests/dashboard/race_test.go b/tests/dashboard/race_test.go new file mode 100644 index 000000000000..d45858123b1a --- /dev/null +++ b/tests/dashboard/race_test.go @@ -0,0 +1,61 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package dashboard_test + +import ( + "context" + "time" + + . "github.com/pingcap/check" + + "github.com/pingcap/pd/v4/pkg/dashboard" + "github.com/pingcap/pd/v4/server" + "github.com/pingcap/pd/v4/tests" + + // Register schedulers. + _ "github.com/pingcap/pd/v4/server/schedulers" +) + +var _ = Suite(&raceTestSuite{}) + +type raceTestSuite struct{} + +func (s *raceTestSuite) SetUpSuite(c *C) { + server.EnableZap = true + dashboard.SetCheckInterval(50 * time.Millisecond) + tests.WaitLeaderReturnDelay = 0 + tests.WaitLeaderCheckInterval = 20 * time.Millisecond +} + +func (s *raceTestSuite) TearDownSuite(c *C) { + server.EnableZap = false + dashboard.SetCheckInterval(time.Second) + tests.WaitLeaderReturnDelay = 20 * time.Millisecond + tests.WaitLeaderCheckInterval = 500 * time.Millisecond +} + +func (s *raceTestSuite) TestCancelDuringStarting(c *C) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + cluster, err := tests.NewTestCluster(ctx, 1) + c.Assert(err, IsNil) + defer cluster.Destroy() + err = cluster.RunInitialServers() + c.Assert(err, IsNil) + cluster.WaitLeader() + + time.Sleep(60 * time.Millisecond) + cancel() +} diff --git a/tests/dashboard/service_test.go b/tests/dashboard/service_test.go index 9eab1c02539e..b9e7320a0150 100644 --- a/tests/dashboard/service_test.go +++ b/tests/dashboard/service_test.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/pd/v4/pkg/dashboard" "github.com/pingcap/pd/v4/pkg/testutil" "github.com/pingcap/pd/v4/server" + "github.com/pingcap/pd/v4/server/config" "github.com/pingcap/pd/v4/tests" "github.com/pingcap/pd/v4/tests/pdctl" @@ -68,6 +69,16 @@ func (s *serverTestSuite) SetUpSuite(c *C) { func (s *serverTestSuite) TearDownSuite(c *C) { s.cancel() s.httpClient.CloseIdleConnections() + server.EnableZap = false + dashboard.SetCheckInterval(time.Second) +} + +func (s *serverTestSuite) TestDashboardRedirect(c *C) { + s.testDashboard(c, false) +} + +func (s *serverTestSuite) TestDashboardProxy(c *C) { + s.testDashboard(c, true) } func (s *serverTestSuite) checkRespCode(c *C, url string, code int) { @@ -83,20 +94,22 @@ func (s *serverTestSuite) waitForConfigSync() { time.Sleep(time.Second) } -func (s *serverTestSuite) checkServiceIsStarted(c *C, servers map[string]*tests.TestServer, leader *tests.TestServer) string { +func (s *serverTestSuite) checkServiceIsStarted(c *C, internalProxy bool, servers map[string]*tests.TestServer, leader *tests.TestServer) string { s.waitForConfigSync() dashboardAddress := leader.GetServer().GetPersistOptions().GetDashboardAddress() hasServiceNode := false for _, srv := range servers { c.Assert(srv.GetPersistOptions().GetDashboardAddress(), Equals, dashboardAddress) addr := srv.GetAddr() - if addr == dashboardAddress { + if addr == dashboardAddress || internalProxy { s.checkRespCode(c, fmt.Sprintf("%s/dashboard/", addr), http.StatusOK) s.checkRespCode(c, fmt.Sprintf("%s/dashboard/api/keyvisual/heatmaps", addr), http.StatusUnauthorized) - hasServiceNode = true + if addr == dashboardAddress { + hasServiceNode = true + } } else { s.checkRespCode(c, fmt.Sprintf("%s/dashboard/", addr), http.StatusTemporaryRedirect) - s.checkRespCode(c, fmt.Sprintf("%s/dashboard/api/keyvisual/heatmaps", addr), http.StatusUnauthorized) + s.checkRespCode(c, fmt.Sprintf("%s/dashboard/api/keyvisual/heatmaps", addr), http.StatusTemporaryRedirect) } } c.Assert(hasServiceNode, IsTrue) @@ -113,8 +126,10 @@ func (s *serverTestSuite) checkServiceIsStopped(c *C, servers map[string]*tests. } } -func (s *serverTestSuite) TestDashboard(c *C) { - cluster, err := tests.NewTestCluster(s.ctx, 3) +func (s *serverTestSuite) testDashboard(c *C, internalProxy bool) { + cluster, err := tests.NewTestCluster(s.ctx, 3, func(conf *config.Config) { + conf.Dashboard.InternalProxy = internalProxy + }) c.Assert(err, IsNil) defer cluster.Destroy() err = cluster.RunInitialServers() @@ -128,7 +143,7 @@ func (s *serverTestSuite) TestDashboard(c *C) { leaderAddr := leader.GetAddr() // auto select node - dashboardAddress1 := s.checkServiceIsStarted(c, servers, leader) + dashboardAddress1 := s.checkServiceIsStarted(c, internalProxy, servers, leader) // pd-ctl set another addr var dashboardAddress2 string @@ -141,7 +156,7 @@ func (s *serverTestSuite) TestDashboard(c *C) { args := []string{"-u", leaderAddr, "config", "set", "dashboard-address", dashboardAddress2} _, _, err = pdctl.ExecuteCommandC(cmd, args...) c.Assert(err, IsNil) - s.checkServiceIsStarted(c, servers, leader) + s.checkServiceIsStarted(c, internalProxy, servers, leader) c.Assert(leader.GetServer().GetPersistOptions().GetDashboardAddress(), Equals, dashboardAddress2) // pd-ctl set stop diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index 7eddbe857832..6172440f278a 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -669,7 +669,7 @@ func (s *clusterTestSuite) TestLoadClusterInfo(c *C) { } c.Assert(storage.Flush(), IsNil) - raftCluster = &cluster.RaftCluster{} + raftCluster = cluster.NewRaftCluster(s.ctx, svr.GetClusterRootPath(), svr.ClusterID(), syncer.NewRegionSyncer(svr), svr.GetClient(), svr.GetHTTPClient()) raftCluster.InitCluster(mockid.NewIDAllocator(), opt, storage, basicCluster) raftCluster, err = raftCluster.LoadClusterInfo() c.Assert(err, IsNil) diff --git a/tests/server/cluster/cluster_work_test.go b/tests/server/cluster/cluster_work_test.go index 3097781e7700..119fc17ef12e 100644 --- a/tests/server/cluster/cluster_work_test.go +++ b/tests/server/cluster/cluster_work_test.go @@ -15,6 +15,7 @@ package cluster_test import ( "context" + "sort" "time" . "github.com/pingcap/check" @@ -127,3 +128,37 @@ func (s *clusterWorkerTestSuite) TestAskSplit(c *C) { mergeChecker.Check(regions[0]) c.Assert(err, IsNil) } + +func (s *clusterWorkerTestSuite) TestSuspectRegions(c *C) { + cluster, err := tests.NewTestCluster(s.ctx, 1) + defer cluster.Destroy() + c.Assert(err, IsNil) + + err = cluster.RunInitialServers() + c.Assert(err, IsNil) + + cluster.WaitLeader() + leaderServer := cluster.GetServer(cluster.GetLeader()) + grpcPDClient := testutil.MustNewGrpcClient(c, leaderServer.GetAddr()) + clusterID := leaderServer.GetClusterID() + bootstrapCluster(c, clusterID, grpcPDClient, "127.0.0.1:0") + rc := leaderServer.GetRaftCluster() + opt := rc.GetOpt() + opt.SetSplitMergeInterval(time.Hour) + regions := rc.GetRegions() + + req := &pdpb.AskBatchSplitRequest{ + Header: &pdpb.RequestHeader{ + ClusterId: clusterID, + }, + Region: regions[0].GetMeta(), + SplitCount: 2, + } + res, err := rc.HandleAskBatchSplit(req) + c.Assert(err, IsNil) + ids := []uint64{regions[0].GetMeta().GetId(), res.Ids[0].NewRegionId, res.Ids[1].NewRegionId} + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + suspects := rc.GetSuspectRegions() + sort.Slice(suspects, func(i, j int) bool { return suspects[i] < suspects[j] }) + c.Assert(suspects, DeepEquals, ids) +} diff --git a/tools.go b/tools.go index b8fe4e2b0f94..f31ae8c48d20 100644 --- a/tools.go +++ b/tools.go @@ -17,7 +17,6 @@ package tools import ( _ "github.com/go-playground/overalls" - _ "github.com/kevinburke/go-bindata/go-bindata" _ "github.com/mgechev/revive" _ "github.com/pingcap/failpoint/failpoint-ctl" _ "github.com/sasha-s/go-deadlock" diff --git a/tools.json b/tools.json index 4136f361a24f..79f004caa47b 100644 --- a/tools.json +++ b/tools.json @@ -16,10 +16,6 @@ "Repository": "golang.org/x/tools/cmd/goimports", "Commit": "04b5d21e00f1f47bd824a6ade581e7189bacde87" }, - { - "Repository": "github.com/kevinburke/go-bindata/go-bindata", - "Commit": "c7c189834d8e8384b8aa7428c02e4deee6df6b17" - }, { "Repository": "gopkg.in/alecthomas/gometalinter.v2", "Commit": "46cc1ea3778b247666c2949669a3333c532fa9c6" diff --git a/tools/pd-ctl/README.md b/tools/pd-ctl/README.md index c6950021c493..ad1f6488a5ee 100644 --- a/tools/pd-ctl/README.md +++ b/tools/pd-ctl/README.md @@ -113,6 +113,7 @@ Usage: "high-space-ratio": 0.6, "hot-region-cache-hits-threshold": 3, "hot-region-schedule-limit": 4, + "key-type": "table", "leader-schedule-limit": 4, "leader-schedule-policy": "count", "low-space-ratio": 0.8, @@ -189,6 +190,12 @@ This option only works when key type is "table". >> config set enable-cross-table-merge true // Enable cross table merge. ``` +- `key-type` specifies the key encoding type used by the cluster. There are some strategics supported: ["table", "raw", "txn"], default: "table". When key type is "raw" or "txn", PD will be allowed to merge region cross table. + + ```bash + >> config set key-type raw // Enable cross table merge. + ``` + - `patrol-region-interval` controls the execution frequency that `replicaChecker` checks the health status of Regions. A shorter interval indicates a higher execution frequency. Generally, you do not need to adjust it. ```bash diff --git a/tools/pd-ctl/pdctl/command/config_command.go b/tools/pd-ctl/pdctl/command/config_command.go index 7c846a818bb6..6514d344284e 100644 --- a/tools/pd-ctl/pdctl/command/config_command.go +++ b/tools/pd-ctl/pdctl/command/config_command.go @@ -520,7 +520,10 @@ func putPlacementRulesFunc(cmd *cobra.Command, args []string) { return } fmt.Printf("saved rule %s/%s\n", r.GroupID, r.ID) - } else { + } + } + for _, r := range rules { + if r.Count == 0 { _, err = doRequest(cmd, path.Join(rulePrefix, r.GroupID, r.ID), http.MethodDelete) if err != nil { fmt.Printf("failed to delete rule %s/%s: %v\n", r.GroupID, r.ID, err)