Skip to content
This repository has been archived by the owner on Jul 24, 2024. It is now read-only.

Commit

Permalink
restore: remove tiflash replica before restore (#194)
Browse files Browse the repository at this point in the history 
* restore: remove tiflash replica before restore

Signed-off-by: 5kbpers <[email protected]>

* rename errSplit variable

Signed-off-by: 5kbpers <[email protected]>

* address comments

Signed-off-by: 5kbpers <[email protected]>

* check replica count by region info

Signed-off-by: 5kbpers <[email protected]>

* cleanup

Signed-off-by: 5kbpers <[email protected]>

* save tiflash replica count to backupmeta

Signed-off-by: 5kbpers <[email protected]>

* fix save crcxor

Signed-off-by: 5kbpers <[email protected]>

* fix decode the key of placement rule

Signed-off-by: 5kbpers <[email protected]>

* address lint

Signed-off-by: 5kbpers <[email protected]>

* address comments

Signed-off-by: 5kbpers <[email protected]>

* close domain after restoring tiflash-replica

Signed-off-by: 5kbpers <[email protected]>

* address comments

Signed-off-by: 5kbpers <[email protected]>

* address comments

Signed-off-by: 5kbpers <[email protected]>

* Update pkg/task/restore.go

Co-Authored-By: 3pointer <[email protected]>

Co-authored-by: 3pointer <[email protected]>
  • Loading branch information
@5kbpers @3pointer
5kbpers and 3pointer authored Mar 18, 2020
1 parent c3d26d9 commit 6268cde
Show file tree
Hide file tree
Showing 18 changed files with 526 additions and 129 deletions.
21 changes: 21 additions & 0 deletions cmd/restore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,15 @@ func runRestoreRawCommand(command *cobra.Command, cmdName string) error {
return task.RunRestoreRaw(GetDefaultContext(), gluetikv.Glue{}, cmdName, &cfg)
}

func runRestoreTiflashReplicaCommand(command *cobra.Command, cmdName string) error {
cfg := task.RestoreConfig{Config: task.Config{LogProgress: HasLogFile()}}
if err := cfg.ParseFromFlags(command.Flags()); err != nil {
return err
}

return task.RunRestoreTiflashReplica(GetDefaultContext(), tidbGlue, cmdName, &cfg)
}

// NewRestoreCommand returns a restore subcommand
func NewRestoreCommand() *cobra.Command {
command := &cobra.Command{
Expand All @@ -55,6 +64,7 @@ func NewRestoreCommand() *cobra.Command {
newDbRestoreCommand(),
newTableRestoreCommand(),
newRawRestoreCommand(),
newTiflashReplicaRestoreCommand(),
)
task.DefineRestoreFlags(command.PersistentFlags())

Expand Down Expand Up @@ -96,6 +106,17 @@ func newTableRestoreCommand() *cobra.Command {
return command
}

func newTiflashReplicaRestoreCommand() *cobra.Command {
command := &cobra.Command{
Use: "tiflash-replica",
Short: "restore the tiflash replica before the last restore, it must only be used after the last restore failed",
RunE: func(cmd *cobra.Command, _ []string) error {
return runRestoreTiflashReplicaCommand(cmd, "Restore TiFlash Replica")
},
}
return command
}

func newRawRestoreCommand() *cobra.Command {
command := &cobra.Command{
Use: "raw",
Expand Down
6 changes: 3 additions & 3 deletions cmd/validate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ func newCheckSumCommand() *cobra.Command {
return err
}

_, s, backupMeta, err := task.ReadBackupMeta(ctx, &cfg)
_, s, backupMeta, err := task.ReadBackupMeta(ctx, utils.MetaFile, &cfg)
if err != nil {
return err
}
Expand Down Expand Up @@ -151,7 +151,7 @@ func newBackupMetaCommand() *cobra.Command {
if err = cfg.ParseFromFlags(cmd.Flags()); err != nil {
return err
}
_, _, backupMeta, err := task.ReadBackupMeta(ctx, &cfg)
_, _, backupMeta, err := task.ReadBackupMeta(ctx, utils.MetaFile, &cfg)
if err != nil {
log.Error("read backupmeta failed", zap.Error(err))
return err
Expand Down Expand Up @@ -242,7 +242,7 @@ func decodeBackupMetaCommand() *cobra.Command {
if err := cfg.ParseFromFlags(cmd.Flags()); err != nil {
return err
}
_, s, backupMeta, err := task.ReadBackupMeta(ctx, &cfg)
_, s, backupMeta, err := task.ReadBackupMeta(ctx, utils.MetaFile, &cfg)
if err != nil {
return err
}
Expand Down
3 changes: 2 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,12 @@ require (
github.com/google/uuid v1.1.1
github.com/klauspost/cpuid v1.2.0 // indirect
github.com/mattn/go-runewidth v0.0.7 // indirect
github.com/montanaflynn/stats v0.5.0 // indirect
github.com/onsi/ginkgo v1.11.0 // indirect
github.com/onsi/gomega v1.8.1 // indirect
github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011
github.com/pingcap/kvproto v0.0.0-20200228095611-2cf9a243b8d5
github.com/pingcap/kvproto v0.0.0-20200317112120-78042b285b75
github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd
github.com/pingcap/parser v0.0.0-20200305120128-bde9faa0df84
github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3
Expand Down
5 changes: 5 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808 h1:pmpDGKLw4n82EtrNiLqB+xSz/JQwFOaZuMALYUHwX5s=
github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk=
github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdcNTgsos+vFzULLwyElndwn+5c=
github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI=
Expand Down Expand Up @@ -363,6 +365,8 @@ github.com/pingcap/kvproto v0.0.0-20200214064158-62d31900d88e/go.mod h1:IOdRDPLy
github.com/pingcap/kvproto v0.0.0-20200221034943-a2aa1d1e20a8/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
github.com/pingcap/kvproto v0.0.0-20200228095611-2cf9a243b8d5 h1:knEvP4R5v5b2T107/Q6VzB0C8/6T7NXB/V7Vl1FtQsg=
github.com/pingcap/kvproto v0.0.0-20200228095611-2cf9a243b8d5/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
github.com/pingcap/kvproto v0.0.0-20200317112120-78042b285b75 h1:DB3NTM0ilba/6sW+vccdEnP10bVvrVunDwWvRa0hSKc=
github.com/pingcap/kvproto v0.0.0-20200317112120-78042b285b75/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA=
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM=
Expand Down Expand Up @@ -486,6 +490,7 @@ github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljT
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3 h1:ZsIlNwu/G0zbChIZaWOeZ2TPGNmKMt46jZLXi3e8LFc=
github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
Expand Down
31 changes: 20 additions & 11 deletions pkg/conn/conn.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,25 +92,28 @@ func pdRequest(
return r, nil
}

// UnexpectedStoreBehavior is the action to do in GetAllTiKVStores when a
// non-TiKV store (e.g. TiFlash store) is found.
type UnexpectedStoreBehavior uint8
// StoreBehavior is the action to do in GetAllTiKVStores when a non-TiKV
// store (e.g. TiFlash store) is found.
type StoreBehavior uint8

const (
// ErrorOnTiFlash causes GetAllTiKVStores to return error when the store is
// found to be a TiFlash node.
ErrorOnTiFlash UnexpectedStoreBehavior = 0
ErrorOnTiFlash StoreBehavior = 0
// SkipTiFlash causes GetAllTiKVStores to skip the store when it is found to
// be a TiFlash node.
SkipTiFlash UnexpectedStoreBehavior = 1
SkipTiFlash StoreBehavior = 1
// TiFlashOnly caused GetAllTiKVStores to skip the store which is not a
// TiFlash node.
TiFlashOnly StoreBehavior = 2
)

// GetAllTiKVStores returns all TiKV stores registered to the PD client. The
// stores must not be a tombstone and must never contain a label `engine=tiflash`.
func GetAllTiKVStores(
ctx context.Context,
pdClient pd.Client,
unexpectedStoreBehavior UnexpectedStoreBehavior,
storeBehavior StoreBehavior,
) ([]*metapb.Store, error) {
// get all live stores.
stores, err := pdClient.GetAllStores(ctx, pd.WithExcludeTombstone())
Expand All @@ -122,15 +125,21 @@ func GetAllTiKVStores(
j := 0
skipStore:
for _, store := range stores {
var isTiFlash bool
for _, label := range store.Labels {
if label.Key == "engine" && label.Value == "tiflash" {
if unexpectedStoreBehavior == SkipTiFlash {
if storeBehavior == SkipTiFlash {
continue skipStore
} else if storeBehavior == ErrorOnTiFlash {
return nil, errors.Errorf(
"cannot restore to a cluster with active TiFlash stores (store %d at %s)", store.Id, store.Address)
}
return nil, errors.Errorf(
"cannot restore to a cluster with active TiFlash stores (store %d at %s)", store.Id, store.Address)
isTiFlash = true
}
}
if !isTiFlash && storeBehavior == TiFlashOnly {
continue skipStore
}
stores[j] = store
j++
}
Expand All @@ -145,7 +154,7 @@ func NewMgr(
storage tikv.Storage,
tlsConf *tls.Config,
securityOption pd.SecurityOption,
unexpectedStoreBehavior UnexpectedStoreBehavior,
storeBehavior StoreBehavior,
) (*Mgr, error) {
addrs := strings.Split(pdAddrs, ",")

Expand Down Expand Up @@ -190,7 +199,7 @@ func NewMgr(
log.Info("new mgr", zap.String("pdAddrs", pdAddrs))

// Check live tikv.
stores, err := GetAllTiKVStores(ctx, pdClient, unexpectedStoreBehavior)
stores, err := GetAllTiKVStores(ctx, pdClient, storeBehavior)
if err != nil {
log.Error("fail to get store", zap.Error(err))
return nil, err
Expand Down
46 changes: 29 additions & 17 deletions pkg/conn/conn_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,40 +162,40 @@ func (fpdc fakePDClient) GetAllStores(context.Context, ...pd.GetStoreOption) ([]

func (s *testClientSuite) TestGetAllTiKVStores(c *C) {
testCases := []struct {
stores []*metapb.Store
unexpectedStoreBehavior UnexpectedStoreBehavior
expectedStores map[uint64]int
expectedError string
stores []*metapb.Store
storeBehavior StoreBehavior
expectedStores map[uint64]int
expectedError string
}{
{
stores: []*metapb.Store{
{Id: 1},
},
unexpectedStoreBehavior: SkipTiFlash,
expectedStores: map[uint64]int{1: 1},
storeBehavior: SkipTiFlash,
expectedStores: map[uint64]int{1: 1},
},
{
stores: []*metapb.Store{
{Id: 1},
},
unexpectedStoreBehavior: ErrorOnTiFlash,
expectedStores: map[uint64]int{1: 1},
storeBehavior: ErrorOnTiFlash,
expectedStores: map[uint64]int{1: 1},
},
{
stores: []*metapb.Store{
{Id: 1},
{Id: 2, Labels: []*metapb.StoreLabel{{Key: "engine", Value: "tiflash"}}},
},
unexpectedStoreBehavior: SkipTiFlash,
expectedStores: map[uint64]int{1: 1},
storeBehavior: SkipTiFlash,
expectedStores: map[uint64]int{1: 1},
},
{
stores: []*metapb.Store{
{Id: 1},
{Id: 2, Labels: []*metapb.StoreLabel{{Key: "engine", Value: "tiflash"}}},
},
unexpectedStoreBehavior: ErrorOnTiFlash,
expectedError: "cannot restore to a cluster with active TiFlash stores.*",
storeBehavior: ErrorOnTiFlash,
expectedError: "cannot restore to a cluster with active TiFlash stores.*",
},
{
stores: []*metapb.Store{
Expand All @@ -206,8 +206,8 @@ func (s *testClientSuite) TestGetAllTiKVStores(c *C) {
{Id: 5, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tikv"}, {Key: "engine", Value: "tiflash"}}},
{Id: 6, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tiflash"}, {Key: "engine", Value: "tikv"}}},
},
unexpectedStoreBehavior: SkipTiFlash,
expectedStores: map[uint64]int{1: 1, 3: 1, 4: 1, 6: 1},
storeBehavior: SkipTiFlash,
expectedStores: map[uint64]int{1: 1, 3: 1, 4: 1, 6: 1},
},
{
stores: []*metapb.Store{
Expand All @@ -218,14 +218,26 @@ func (s *testClientSuite) TestGetAllTiKVStores(c *C) {
{Id: 5, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tikv"}, {Key: "engine", Value: "tiflash"}}},
{Id: 6, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tiflash"}, {Key: "engine", Value: "tikv"}}},
},
unexpectedStoreBehavior: ErrorOnTiFlash,
expectedError: "cannot restore to a cluster with active TiFlash stores.*",
storeBehavior: ErrorOnTiFlash,
expectedError: "cannot restore to a cluster with active TiFlash stores.*",
},
{
stores: []*metapb.Store{
{Id: 1},
{Id: 2, Labels: []*metapb.StoreLabel{{Key: "engine", Value: "tiflash"}}},
{Id: 3},
{Id: 4, Labels: []*metapb.StoreLabel{{Key: "engine", Value: "tikv"}}},
{Id: 5, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tikv"}, {Key: "engine", Value: "tiflash"}}},
{Id: 6, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tiflash"}, {Key: "engine", Value: "tikv"}}},
},
storeBehavior: TiFlashOnly,
expectedStores: map[uint64]int{2: 1, 5: 1},
},
}

for _, testCase := range testCases {
pdClient := fakePDClient{stores: testCase.stores}
stores, err := GetAllTiKVStores(context.Background(), pdClient, testCase.unexpectedStoreBehavior)
stores, err := GetAllTiKVStores(context.Background(), pdClient, testCase.storeBehavior)
if len(testCase.expectedError) != 0 {
c.Assert(err, ErrorMatches, testCase.expectedError)
continue
Expand Down
10 changes: 5 additions & 5 deletions pkg/restore/backoff.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,21 +90,21 @@ func (bo *importerBackoffer) Attempt() int {
return bo.attempt
}

type resetTSBackoffer struct {
type pdReqBackoffer struct {
attempt int
delayTime time.Duration
maxDelayTime time.Duration
}

func newResetTSBackoffer() utils.Backoffer {
return &resetTSBackoffer{
func newPDReqBackoffer() utils.Backoffer {
return &pdReqBackoffer{
attempt: resetTsRetryTime,
delayTime: resetTSWaitInterval,
maxDelayTime: resetTSMaxWaitInterval,
}
}

func (bo *resetTSBackoffer) NextBackoff(err error) time.Duration {
func (bo *pdReqBackoffer) NextBackoff(err error) time.Duration {
bo.delayTime = 2 * bo.delayTime
bo.attempt--
if bo.delayTime > bo.maxDelayTime {
Expand All @@ -113,6 +113,6 @@ func (bo *resetTSBackoffer) NextBackoff(err error) time.Duration {
return bo.delayTime
}

func (bo *resetTSBackoffer) Attempt() int {
func (bo *pdReqBackoffer) Attempt() int {
return bo.attempt
}
Loading

0 comments on commit 6268cde

Please sign in to comment.