restore: remove tiflash replica before restore

cmd/restore.go

@@ -30,6 +30,15 @@ func runRestoreRawCommand(command *cobra.Command, cmdName string) error {
 	return task.RunRestoreRaw(GetDefaultContext(), gluetikv.Glue{}, cmdName, &cfg)
 }
 
+func runRestoreTiflashReplicaCommand(command *cobra.Command, cmdName string) error {
+	cfg := task.RestoreConfig{Config: task.Config{LogProgress: HasLogFile()}}
+	if err := cfg.ParseFromFlags(command.Flags()); err != nil {
+		return err
+	}
+
+	return task.RunRestoreTiflashReplica(GetDefaultContext(), tidbGlue, cmdName, &cfg)
+}
+
 // NewRestoreCommand returns a restore subcommand
 func NewRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
@@ -55,6 +64,7 @@ func NewRestoreCommand() *cobra.Command {
 		newDbRestoreCommand(),
 		newTableRestoreCommand(),
 		newRawRestoreCommand(),
+		newTiflashReplicaRestoreCommand(),
 	)
 	task.DefineRestoreFlags(command.PersistentFlags())
 
@@ -96,6 +106,17 @@ func newTableRestoreCommand() *cobra.Command {
 	return command
 }
 
+func newTiflashReplicaRestoreCommand() *cobra.Command {
+	command := &cobra.Command{
+		Use:   "tiflash-replica",
+		Short: "restore the tiflash replica before the last restore, it must only be used after the last restore failed",
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			return runRestoreTiflashReplicaCommand(cmd, "Restore TiFlash Replica")
+		},
+	}
+	return command
+}
+
 func newRawRestoreCommand() *cobra.Command {
 	command := &cobra.Command{
 		Use:   "raw",

cmd/validate.go

@@ -63,7 +63,7 @@ func newCheckSumCommand() *cobra.Command {
 				return err
 			}
 
-			_, s, backupMeta, err := task.ReadBackupMeta(ctx, &cfg)
+			_, s, backupMeta, err := task.ReadBackupMeta(ctx, utils.MetaFile, &cfg)
 			if err != nil {
 				return err
 			}
@@ -151,7 +151,7 @@ func newBackupMetaCommand() *cobra.Command {
 			if err = cfg.ParseFromFlags(cmd.Flags()); err != nil {
 				return err
 			}
-			_, _, backupMeta, err := task.ReadBackupMeta(ctx, &cfg)
+			_, _, backupMeta, err := task.ReadBackupMeta(ctx, utils.MetaFile, &cfg)
 			if err != nil {
 				log.Error("read backupmeta failed", zap.Error(err))
 				return err
@@ -242,7 +242,7 @@ func decodeBackupMetaCommand() *cobra.Command {
 			if err := cfg.ParseFromFlags(cmd.Flags()); err != nil {
 				return err
 			}
-			_, s, backupMeta, err := task.ReadBackupMeta(ctx, &cfg)
+			_, s, backupMeta, err := task.ReadBackupMeta(ctx, utils.MetaFile, &cfg)
 			if err != nil {
 				return err
 			}

go.mod

@@ -16,11 +16,12 @@ require (
 	github.com/google/uuid v1.1.1
 	github.com/klauspost/cpuid v1.2.0 // indirect
 	github.com/mattn/go-runewidth v0.0.7 // indirect
+	github.com/montanaflynn/stats v0.5.0 // indirect
 	github.com/onsi/ginkgo v1.11.0 // indirect
 	github.com/onsi/gomega v1.8.1 // indirect
 	github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
 	github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011
-	github.com/pingcap/kvproto v0.0.0-20200228095611-2cf9a243b8d5
+	github.com/pingcap/kvproto v0.0.0-20200317112120-78042b285b75
 	github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd
 	github.com/pingcap/parser v0.0.0-20200305120128-bde9faa0df84
 	github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3

go.sum

@@ -309,6 +309,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808 h1:pmpDGKLw4n82EtrNiLqB+xSz/JQwFOaZuMALYUHwX5s=
 github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk=
+github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdcNTgsos+vFzULLwyElndwn+5c=
 github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI=
@@ -363,6 +365,8 @@ github.com/pingcap/kvproto v0.0.0-20200214064158-62d31900d88e/go.mod h1:IOdRDPLy
 github.com/pingcap/kvproto v0.0.0-20200221034943-a2aa1d1e20a8/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
 github.com/pingcap/kvproto v0.0.0-20200228095611-2cf9a243b8d5 h1:knEvP4R5v5b2T107/Q6VzB0C8/6T7NXB/V7Vl1FtQsg=
 github.com/pingcap/kvproto v0.0.0-20200228095611-2cf9a243b8d5/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
+github.com/pingcap/kvproto v0.0.0-20200317112120-78042b285b75 h1:DB3NTM0ilba/6sW+vccdEnP10bVvrVunDwWvRa0hSKc=
+github.com/pingcap/kvproto v0.0.0-20200317112120-78042b285b75/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
 github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA=
 github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM=
@@ -486,6 +490,7 @@ github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljT
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3 h1:ZsIlNwu/G0zbChIZaWOeZ2TPGNmKMt46jZLXi3e8LFc=
 github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=

pkg/conn/conn.go

@@ -92,25 +92,28 @@ func pdRequest(
 	return r, nil
 }
 
-// UnexpectedStoreBehavior is the action to do in GetAllTiKVStores when a
-// non-TiKV store (e.g. TiFlash store) is found.
-type UnexpectedStoreBehavior uint8
+// StoreBehavior is the action to do in GetAllTiKVStores when a non-TiKV
+// store (e.g. TiFlash store) is found.
+type StoreBehavior uint8
 
 const (
 	// ErrorOnTiFlash causes GetAllTiKVStores to return error when the store is
 	// found to be a TiFlash node.
-	ErrorOnTiFlash UnexpectedStoreBehavior = 0
+	ErrorOnTiFlash StoreBehavior = 0
 	// SkipTiFlash causes GetAllTiKVStores to skip the store when it is found to
 	// be a TiFlash node.
-	SkipTiFlash UnexpectedStoreBehavior = 1
+	SkipTiFlash StoreBehavior = 1
+	// TiFlashOnly caused GetAllTiKVStores to skip the store which is not a
+	// TiFlash node.
+	TiFlashOnly StoreBehavior = 2
 )
 
 // GetAllTiKVStores returns all TiKV stores registered to the PD client. The
 // stores must not be a tombstone and must never contain a label `engine=tiflash`.
 func GetAllTiKVStores(
 	ctx context.Context,
 	pdClient pd.Client,
-	unexpectedStoreBehavior UnexpectedStoreBehavior,
+	storeBehavior StoreBehavior,
 ) ([]*metapb.Store, error) {
 	// get all live stores.
 	stores, err := pdClient.GetAllStores(ctx, pd.WithExcludeTombstone())
@@ -122,15 +125,21 @@ func GetAllTiKVStores(
 	j := 0
 skipStore:
 	for _, store := range stores {
+		var isTiFlash bool
 		for _, label := range store.Labels {
 			if label.Key == "engine" && label.Value == "tiflash" {
-				if unexpectedStoreBehavior == SkipTiFlash {
+				if storeBehavior == SkipTiFlash {
 					continue skipStore
+				} else if storeBehavior == ErrorOnTiFlash {
+					return nil, errors.Errorf(
+						"cannot restore to a cluster with active TiFlash stores (store %d at %s)", store.Id, store.Address)
 				}
-				return nil, errors.Errorf(
-					"cannot restore to a cluster with active TiFlash stores (store %d at %s)", store.Id, store.Address)
+				isTiFlash = true
 			}
 		}
+		if !isTiFlash && storeBehavior == TiFlashOnly {
+			continue skipStore
+		}
 		stores[j] = store
 		j++
 	}
@@ -145,7 +154,7 @@ func NewMgr(
 	storage tikv.Storage,
 	tlsConf *tls.Config,
 	securityOption pd.SecurityOption,
-	unexpectedStoreBehavior UnexpectedStoreBehavior,
+	storeBehavior StoreBehavior,
 ) (*Mgr, error) {
 	addrs := strings.Split(pdAddrs, ",")
 
@@ -190,7 +199,7 @@ func NewMgr(
 	log.Info("new mgr", zap.String("pdAddrs", pdAddrs))
 
 	// Check live tikv.
-	stores, err := GetAllTiKVStores(ctx, pdClient, unexpectedStoreBehavior)
+	stores, err := GetAllTiKVStores(ctx, pdClient, storeBehavior)
 	if err != nil {
 		log.Error("fail to get store", zap.Error(err))
 		return nil, err

pkg/conn/conn_test.go

@@ -162,40 +162,40 @@ func (fpdc fakePDClient) GetAllStores(context.Context, ...pd.GetStoreOption) ([]
 
 func (s *testClientSuite) TestGetAllTiKVStores(c *C) {
 	testCases := []struct {
-		stores                  []*metapb.Store
-		unexpectedStoreBehavior UnexpectedStoreBehavior
-		expectedStores          map[uint64]int
-		expectedError           string
+		stores         []*metapb.Store
+		storeBehavior  StoreBehavior
+		expectedStores map[uint64]int
+		expectedError  string
 	}{
 		{
 			stores: []*metapb.Store{
 				{Id: 1},
 			},
-			unexpectedStoreBehavior: SkipTiFlash,
-			expectedStores:          map[uint64]int{1: 1},
+			storeBehavior:  SkipTiFlash,
+			expectedStores: map[uint64]int{1: 1},
 		},
 		{
 			stores: []*metapb.Store{
 				{Id: 1},
 			},
-			unexpectedStoreBehavior: ErrorOnTiFlash,
-			expectedStores:          map[uint64]int{1: 1},
+			storeBehavior:  ErrorOnTiFlash,
+			expectedStores: map[uint64]int{1: 1},
 		},
 		{
 			stores: []*metapb.Store{
 				{Id: 1},
 				{Id: 2, Labels: []*metapb.StoreLabel{{Key: "engine", Value: "tiflash"}}},
 			},
-			unexpectedStoreBehavior: SkipTiFlash,
-			expectedStores:          map[uint64]int{1: 1},
+			storeBehavior:  SkipTiFlash,
+			expectedStores: map[uint64]int{1: 1},
 		},
 		{
 			stores: []*metapb.Store{
 				{Id: 1},
 				{Id: 2, Labels: []*metapb.StoreLabel{{Key: "engine", Value: "tiflash"}}},
 			},
-			unexpectedStoreBehavior: ErrorOnTiFlash,
-			expectedError:           "cannot restore to a cluster with active TiFlash stores.*",
+			storeBehavior: ErrorOnTiFlash,
+			expectedError: "cannot restore to a cluster with active TiFlash stores.*",
 		},
 		{
 			stores: []*metapb.Store{
@@ -206,8 +206,8 @@ func (s *testClientSuite) TestGetAllTiKVStores(c *C) {
 				{Id: 5, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tikv"}, {Key: "engine", Value: "tiflash"}}},
 				{Id: 6, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tiflash"}, {Key: "engine", Value: "tikv"}}},
 			},
-			unexpectedStoreBehavior: SkipTiFlash,
-			expectedStores:          map[uint64]int{1: 1, 3: 1, 4: 1, 6: 1},
+			storeBehavior:  SkipTiFlash,
+			expectedStores: map[uint64]int{1: 1, 3: 1, 4: 1, 6: 1},
 		},
 		{
 			stores: []*metapb.Store{
@@ -218,14 +218,26 @@ func (s *testClientSuite) TestGetAllTiKVStores(c *C) {
 				{Id: 5, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tikv"}, {Key: "engine", Value: "tiflash"}}},
 				{Id: 6, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tiflash"}, {Key: "engine", Value: "tikv"}}},
 			},
-			unexpectedStoreBehavior: ErrorOnTiFlash,
-			expectedError:           "cannot restore to a cluster with active TiFlash stores.*",
+			storeBehavior: ErrorOnTiFlash,
+			expectedError: "cannot restore to a cluster with active TiFlash stores.*",
+		},
+		{
+			stores: []*metapb.Store{
+				{Id: 1},
+				{Id: 2, Labels: []*metapb.StoreLabel{{Key: "engine", Value: "tiflash"}}},
+				{Id: 3},
+				{Id: 4, Labels: []*metapb.StoreLabel{{Key: "engine", Value: "tikv"}}},
+				{Id: 5, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tikv"}, {Key: "engine", Value: "tiflash"}}},
+				{Id: 6, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tiflash"}, {Key: "engine", Value: "tikv"}}},
+			},
+			storeBehavior:  TiFlashOnly,
+			expectedStores: map[uint64]int{2: 1, 5: 1},
 		},
 	}
 
 	for _, testCase := range testCases {
 		pdClient := fakePDClient{stores: testCase.stores}
-		stores, err := GetAllTiKVStores(context.Background(), pdClient, testCase.unexpectedStoreBehavior)
+		stores, err := GetAllTiKVStores(context.Background(), pdClient, testCase.storeBehavior)
 		if len(testCase.expectedError) != 0 {
 			c.Assert(err, ErrorMatches, testCase.expectedError)
 			continue

pkg/restore/backoff.go

@@ -90,21 +90,21 @@ func (bo *importerBackoffer) Attempt() int {
 	return bo.attempt
 }
 
-type resetTSBackoffer struct {
+type pdReqBackoffer struct {
 	attempt      int
 	delayTime    time.Duration
 	maxDelayTime time.Duration
 }
 
-func newResetTSBackoffer() utils.Backoffer {
-	return &resetTSBackoffer{
+func newPDReqBackoffer() utils.Backoffer {
+	return &pdReqBackoffer{
 		attempt:      resetTsRetryTime,
 		delayTime:    resetTSWaitInterval,
 		maxDelayTime: resetTSMaxWaitInterval,
 	}
 }
 
-func (bo *resetTSBackoffer) NextBackoff(err error) time.Duration {
+func (bo *pdReqBackoffer) NextBackoff(err error) time.Duration {
 	bo.delayTime = 2 * bo.delayTime
 	bo.attempt--
 	if bo.delayTime > bo.maxDelayTime {
@@ -113,6 +113,6 @@ func (bo *resetTSBackoffer) NextBackoff(err error) time.Duration {
 	return bo.delayTime
 }
 
-func (bo *resetTSBackoffer) Attempt() int {
+func (bo *pdReqBackoffer) Attempt() int {
 	return bo.attempt
 }