From 5327efca4d2170c1eaf898b3bde5aad8d4ca9ce8 Mon Sep 17 00:00:00 2001 From: 5kbpers Date: Tue, 17 Mar 2020 13:40:58 +0800 Subject: [PATCH 01/14] restore: remove tiflash replica before restore Signed-off-by: 5kbpers --- pkg/restore/client.go | 19 +++++++ pkg/restore/db.go | 25 +++++++-- pkg/restore/split.go | 107 ++++++++++++++++++++++++++++++++------ pkg/restore/split_test.go | 2 +- pkg/restore/util.go | 7 ++- 5 files changed, 138 insertions(+), 22 deletions(-) diff --git a/pkg/restore/client.go b/pkg/restore/client.go index ba409ec32..feda7d578 100644 --- a/pkg/restore/client.go +++ b/pkg/restore/client.go @@ -240,6 +240,25 @@ func (rc *Client) CreateTables( return rewriteRules, newTables, nil } +// GetTiFlashStores returns an id list of tiflash stores. +func (rc *Client) GetTiFlashStores() ([]uint64, error) { + stores, err := rc.pdClient.GetAllStores(rc.ctx) + if err != nil { + return nil, err + } + + tiflashStores := make([]uint64, 0) + for _, store := range stores { + for _, label := range store.GetLabels() { + if label.GetKey() == "engine" && label.GetValue() == "tiflash" { + tiflashStores = append(tiflashStores, store.GetId()) + } + } + } + + return tiflashStores, nil +} + // ExecDDLs executes the queries of the ddl jobs. func (rc *Client) ExecDDLs(ddlJobs []*model.Job) error { // Sort the ddl jobs by schema version in ascending order. diff --git a/pkg/restore/db.go b/pkg/restore/db.go index 7251b9f24..b6c35673f 100644 --- a/pkg/restore/db.go +++ b/pkg/restore/db.go @@ -80,13 +80,13 @@ func (db *DB) CreateDatabase(ctx context.Context, schema *model.DBInfo) error { // CreateTable executes a CREATE TABLE SQL. func (db *DB) CreateTable(ctx context.Context, table *utils.Table) error { - schema := table.Info - createSQL, err := db.se.ShowCreateTable(schema, newIDAllocator(schema.AutoIncID)) + tableInfo := table.Info + createSQL, err := db.se.ShowCreateTable(tableInfo, newIDAllocator(tableInfo.AutoIncID)) if err != nil { log.Error( "build create table SQL failed", zap.Stringer("db", table.Db.Name), - zap.Stringer("table", schema.Name), + zap.Stringer("table", tableInfo.Name), zap.Error(err)) return errors.Trace(err) } @@ -115,8 +115,8 @@ func (db *DB) CreateTable(ctx context.Context, table *utils.Table) error { } alterAutoIncIDSQL := fmt.Sprintf( "alter table %s auto_increment = %d", - utils.EncloseName(schema.Name.O), - schema.AutoIncID) + utils.EncloseName(tableInfo.Name.O), + tableInfo.AutoIncID) err = db.se.Execute(ctx, alterAutoIncIDSQL) if err != nil { log.Error("alter AutoIncID failed", @@ -125,6 +125,21 @@ func (db *DB) CreateTable(ctx context.Context, table *utils.Table) error { zap.Stringer("table", table.Info.Name), zap.Error(err)) } + + // TODO: remove this after tiflash supports restore + removeTiFlashSQL := fmt.Sprintf( + "alter table %s set tiflash replica 0", + utils.EncloseName(tableInfo.Name.O), + ) + err = db.se.Execute(ctx, removeTiFlashSQL) + if err != nil { + log.Error("remove tiflash replica failed", + zap.String("query", removeTiFlashSQL), + zap.Stringer("db", table.Db.Name), + zap.Stringer("table", table.Info.Name), + zap.Error(err)) + } + return errors.Trace(err) } diff --git a/pkg/restore/split.go b/pkg/restore/split.go index dc0bab80a..4cad599d9 100644 --- a/pkg/restore/split.go +++ b/pkg/restore/split.go @@ -32,8 +32,11 @@ const ( ScatterWaitMaxRetryTimes = 64 ScatterWaitInterval = 50 * time.Millisecond ScatterMaxWaitInterval = time.Second - ScatterWaitUpperInterval = 180 * time.Second + + RejectStoreCheckRetryTimes = 64 + RejectStoreCheckInterval = 100 * time.Millisecond + RejectStoreMaxCheckInterval = 2 * time.Second ) // RegionSplitter is a executor of region split by rules. @@ -60,6 +63,7 @@ func (rs *RegionSplitter) Split( ctx context.Context, ranges []rtree.Range, rewriteRules *RewriteRules, + rejectStores []uint64, onSplit OnSplitFunc, ) error { if len(ranges) == 0 { @@ -67,9 +71,9 @@ func (rs *RegionSplitter) Split( } startTime := time.Now() // Sort the range for getting the min and max key of the ranges - sortedRanges, err := sortRanges(ranges, rewriteRules) - if err != nil { - return errors.Trace(err) + sortedRanges, errSortRange := sortRanges(ranges, rewriteRules) + if errSortRange != nil { + return errors.Trace(errSortRange) } minKey := codec.EncodeBytes([]byte{}, sortedRanges[0].StartKey) maxKey := codec.EncodeBytes([]byte{}, sortedRanges[len(sortedRanges)-1].EndKey) @@ -91,12 +95,14 @@ func (rs *RegionSplitter) Split( } interval := SplitRetryInterval scatterRegions := make([]*RegionInfo, 0) + allRegions := make([]*RegionInfo, 0) SplitRegions: for i := 0; i < SplitRetryTimes; i++ { - regions, err1 := paginateScanRegion(ctx, rs.client, minKey, maxKey, scanRegionPaginationLimit) - if err1 != nil { - return errors.Trace(err1) + regions, errScan := paginateScanRegion(ctx, rs.client, minKey, maxKey, scanRegionPaginationLimit) + if errScan != nil { + return errors.Trace(errScan) } + allRegions = append(allRegions, regions...) if len(regions) == 0 { log.Warn("cannot scan any region") return nil @@ -109,16 +115,16 @@ SplitRegions: for regionID, keys := range splitKeyMap { var newRegions []*RegionInfo region := regionMap[regionID] - newRegions, err = rs.splitAndScatterRegions(ctx, region, keys) - if err != nil { - if strings.Contains(err.Error(), "no valid key") { + newRegions, errSortRange = rs.splitAndScatterRegions(ctx, region, keys) + if errSortRange != nil { + if strings.Contains(errSortRange.Error(), "no valid key") { for _, key := range keys { log.Error("no valid key", zap.Binary("startKey", region.Region.StartKey), zap.Binary("endKey", region.Region.EndKey), zap.Binary("key", codec.EncodeBytes([]byte{}, key))) } - return errors.Trace(err) + return errors.Trace(errSortRange) } interval = 2 * interval if interval > SplitMaxRetryInterval { @@ -126,7 +132,7 @@ SplitRegions: } time.Sleep(interval) if i > 3 { - log.Warn("splitting regions failed, retry it", zap.Error(err), zap.ByteStrings("keys", keys)) + log.Warn("splitting regions failed, retry it", zap.Error(errSortRange), zap.ByteStrings("keys", keys)) } continue SplitRegions } @@ -136,10 +142,27 @@ SplitRegions: } break } - if err != nil { - return errors.Trace(err) + if errSortRange != nil { + return errors.Trace(errSortRange) } - log.Info("splitting regions done, wait for scattering regions", + if len(rejectStores) > 0 { + startTime = time.Now() + log.Info("start to wait for removing rejected stores", zap.Uint64s("rejectStores", rejectStores)) + storeMap := make(map[uint64]bool) + for _, storeID := range rejectStores { + storeMap[storeID] = true + } + for _, region := range allRegions { + if !rs.waitForRemoveRejectStores(ctx, region, storeMap) { + log.Error("waiting for removing rejected stores failed", + zap.Stringer("region", region.Region)) + return errors.New("waiting for removing rejected stores failed") + } + } + log.Info("waiting for removing rejected stores done", + zap.Int("regions", len(allRegions)), zap.Duration("take", time.Since(startTime))) + } + log.Info("start to wait for scattering regions", zap.Int("regions", len(scatterRegions)), zap.Duration("take", time.Since(startTime))) startTime = time.Now() scatterCount := 0 @@ -192,6 +215,30 @@ func (rs *RegionSplitter) isScatterRegionFinished(ctx context.Context, regionID return ok, nil } +func (rs *RegionSplitter) hasRejectStorePeer( + ctx context.Context, + regionID uint64, + rejectStores map[uint64]bool, +) (bool, error) { + regionInfo, err := rs.client.GetRegionByID(ctx, regionID) + if err != nil { + return false, err + } + if regionInfo == nil { + return false, nil + } + for _, peer := range regionInfo.Region.GetPeers() { + if rejectStores[peer.GetStoreId()] { + return true, nil + } + } + retryTimes := ctx.Value(retryTimes).(int) + if retryTimes > 10 { + log.Warn("get region info", zap.Stringer("region", regionInfo.Region)) + } + return false, nil +} + func (rs *RegionSplitter) waitForSplit(ctx context.Context, regionID uint64) { interval := SplitCheckInterval for i := 0; i < SplitCheckMaxRetryTimes; i++ { @@ -237,6 +284,36 @@ func (rs *RegionSplitter) waitForScatterRegion(ctx context.Context, regionInfo * } } +func (rs *RegionSplitter) waitForRemoveRejectStores( + ctx context.Context, + regionInfo *RegionInfo, + rejectStores map[uint64]bool, +) bool { + interval := RejectStoreCheckInterval + regionID := regionInfo.Region.GetId() + for i := 0; i < RejectStoreCheckRetryTimes; i++ { + ctx1 := context.WithValue(ctx, retryTimes, i) + ok, err := rs.hasRejectStorePeer(ctx1, regionID, rejectStores) + if err != nil { + log.Warn("wait for rejecting store failed", + zap.Stringer("region", regionInfo.Region), + zap.Error(err)) + return false + } + // Do not have any peer in the rejected store, return true + if !ok { + return true + } + interval = 2 * interval + if interval > RejectStoreMaxCheckInterval { + interval = RejectStoreMaxCheckInterval + } + time.Sleep(interval) + } + + return false +} + func (rs *RegionSplitter) splitAndScatterRegions( ctx context.Context, regionInfo *RegionInfo, keys [][]byte, ) ([]*RegionInfo, error) { diff --git a/pkg/restore/split_test.go b/pkg/restore/split_test.go index b21cbf781..8d2306a30 100644 --- a/pkg/restore/split_test.go +++ b/pkg/restore/split_test.go @@ -193,7 +193,7 @@ func (s *testRestoreUtilSuite) TestSplit(c *C) { regionSplitter := NewRegionSplitter(client) ctx := context.Background() - err := regionSplitter.Split(ctx, ranges, rewriteRules, func(key [][]byte) {}) + err := regionSplitter.Split(ctx, ranges, rewriteRules, []uint64{}, func(key [][]byte) {}) if err != nil { c.Assert(err, IsNil, Commentf("split regions failed: %v", err)) } diff --git a/pkg/restore/util.go b/pkg/restore/util.go index 03af2b3c0..0071a214c 100644 --- a/pkg/restore/util.go +++ b/pkg/restore/util.go @@ -327,7 +327,12 @@ func SplitRanges( summary.CollectDuration("split region", elapsed) }() splitter := NewRegionSplitter(NewSplitClient(client.GetPDClient(), client.GetTLSConfig())) - return splitter.Split(ctx, ranges, rewriteRules, func(keys [][]byte) { + tiflashStores, err := client.GetTiFlashStores() + if err != nil { + return errors.Trace(err) + } + + return splitter.Split(ctx, ranges, rewriteRules, tiflashStores, func(keys [][]byte) { for range keys { updateCh <- struct{}{} } From c0e26ca6fe13e8671c67dd5a2cff27168aadf2e6 Mon Sep 17 00:00:00 2001 From: 5kbpers Date: Tue, 17 Mar 2020 13:49:16 +0800 Subject: [PATCH 02/14] rename errSplit variable Signed-off-by: 5kbpers --- pkg/restore/split.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/restore/split.go b/pkg/restore/split.go index 4cad599d9..c8e3c4c6a 100644 --- a/pkg/restore/split.go +++ b/pkg/restore/split.go @@ -71,9 +71,9 @@ func (rs *RegionSplitter) Split( } startTime := time.Now() // Sort the range for getting the min and max key of the ranges - sortedRanges, errSortRange := sortRanges(ranges, rewriteRules) - if errSortRange != nil { - return errors.Trace(errSortRange) + sortedRanges, errSplit := sortRanges(ranges, rewriteRules) + if errSplit != nil { + return errors.Trace(errSplit) } minKey := codec.EncodeBytes([]byte{}, sortedRanges[0].StartKey) maxKey := codec.EncodeBytes([]byte{}, sortedRanges[len(sortedRanges)-1].EndKey) @@ -115,16 +115,16 @@ SplitRegions: for regionID, keys := range splitKeyMap { var newRegions []*RegionInfo region := regionMap[regionID] - newRegions, errSortRange = rs.splitAndScatterRegions(ctx, region, keys) - if errSortRange != nil { - if strings.Contains(errSortRange.Error(), "no valid key") { + newRegions, errSplit = rs.splitAndScatterRegions(ctx, region, keys) + if errSplit != nil { + if strings.Contains(errSplit.Error(), "no valid key") { for _, key := range keys { log.Error("no valid key", zap.Binary("startKey", region.Region.StartKey), zap.Binary("endKey", region.Region.EndKey), zap.Binary("key", codec.EncodeBytes([]byte{}, key))) } - return errors.Trace(errSortRange) + return errors.Trace(errSplit) } interval = 2 * interval if interval > SplitMaxRetryInterval { @@ -132,7 +132,7 @@ SplitRegions: } time.Sleep(interval) if i > 3 { - log.Warn("splitting regions failed, retry it", zap.Error(errSortRange), zap.ByteStrings("keys", keys)) + log.Warn("splitting regions failed, retry it", zap.Error(errSplit), zap.ByteStrings("keys", keys)) } continue SplitRegions } @@ -142,8 +142,8 @@ SplitRegions: } break } - if errSortRange != nil { - return errors.Trace(errSortRange) + if errSplit != nil { + return errors.Trace(errSplit) } if len(rejectStores) > 0 { startTime = time.Now() From 8043a9c6234e8aba2d005cfb6768f539f78787ff Mon Sep 17 00:00:00 2001 From: 5kbpers Date: Tue, 17 Mar 2020 14:32:41 +0800 Subject: [PATCH 03/14] address comments Signed-off-by: 5kbpers --- pkg/conn/conn.go | 4 ++-- pkg/restore/client.go | 4 ++-- pkg/task/restore.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/conn/conn.go b/pkg/conn/conn.go index cdfc78168..63203b70e 100644 --- a/pkg/conn/conn.go +++ b/pkg/conn/conn.go @@ -127,9 +127,9 @@ skipStore: if unexpectedStoreBehavior == SkipTiFlash { continue skipStore } - return nil, errors.Errorf( - "cannot restore to a cluster with active TiFlash stores (store %d at %s)", store.Id, store.Address) } + return nil, errors.Errorf( + "cannot restore to a cluster with active TiFlash stores (store %d at %s)", store.Id, store.Address) } stores[j] = store j++ diff --git a/pkg/restore/client.go b/pkg/restore/client.go index ec2512b19..d40f27b4e 100644 --- a/pkg/restore/client.go +++ b/pkg/restore/client.go @@ -346,7 +346,7 @@ func (rc *Client) ExecDDLs(ddlJobs []*model.Job) error { func (rc *Client) setSpeedLimit() error { if !rc.hasSpeedLimited && rc.rateLimit != 0 { - stores, err := conn.GetAllTiKVStores(rc.ctx, rc.pdClient, conn.ErrorOnTiFlash) + stores, err := conn.GetAllTiKVStores(rc.ctx, rc.pdClient, conn.SkipTiFlash) if err != nil { return err } @@ -486,7 +486,7 @@ func (rc *Client) SwitchToNormalMode(ctx context.Context) error { } func (rc *Client) switchTiKVMode(ctx context.Context, mode import_sstpb.SwitchMode) error { - stores, err := conn.GetAllTiKVStores(ctx, rc.pdClient, conn.ErrorOnTiFlash) + stores, err := conn.GetAllTiKVStores(ctx, rc.pdClient, conn.SkipTiFlash) if err != nil { return errors.Trace(err) } diff --git a/pkg/task/restore.go b/pkg/task/restore.go index f2143764c..451aa67b0 100644 --- a/pkg/task/restore.go +++ b/pkg/task/restore.go @@ -75,7 +75,7 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf ctx, cancel := context.WithCancel(c) defer cancel() - mgr, err := newMgr(ctx, g, cfg.PD, cfg.TLS, conn.ErrorOnTiFlash) + mgr, err := newMgr(ctx, g, cfg.PD, cfg.TLS, conn.SkipTiFlash) if err != nil { return err } From 410c84402161670abc1279cf366214a5fa632f25 Mon Sep 17 00:00:00 2001 From: 5kbpers Date: Tue, 17 Mar 2020 16:47:02 +0800 Subject: [PATCH 04/14] check replica count by region info Signed-off-by: 5kbpers --- pkg/restore/client.go | 46 +++++++++++++++++++++++++++++++++++++++++++ pkg/restore/db.go | 14 ------------- pkg/task/restore.go | 4 ++++ pkg/utils/schema.go | 13 ++++++------ 4 files changed, 57 insertions(+), 20 deletions(-) diff --git a/pkg/restore/client.go b/pkg/restore/client.go index d40f27b4e..ba18cfd3c 100644 --- a/pkg/restore/client.go +++ b/pkg/restore/client.go @@ -305,6 +305,52 @@ func (rc *Client) CreateTables( return rewriteRules, newTables, nil } +// RemoveTiFlashReplica removes all the tiflash replica of a table +func (rc *Client) RemoveTiFlashReplica(tables []*utils.Table) error { + stores, err := rc.GetTiFlashStores() + if err != nil { + return errors.Trace(err) + } + storeMap := make(map[uint64]bool) + for _, storeID := range stores { + storeMap[storeID] = true + } + for _, table := range tables { + tableInfo := table.Info + prefix := tablecodec.GenTablePrefix(tableInfo.ID) + region, _, err := rc.pdClient.GetRegion(rc.ctx, prefix) + if err != nil { + return errors.Trace(err) + } + var replicaCount int + for _, peer := range region.GetPeers() { + if storeMap[peer.GetStoreId()] { + replicaCount++ + } + } + // TODO: remove this after tiflash supports restore + removeTiFlashSQL := fmt.Sprintf( + "alter table %s set tiflash replica 0", + utils.EncloseName(tableInfo.Name.O), + ) + err = rc.db.se.Execute(rc.ctx, removeTiFlashSQL) + if err != nil { + log.Error("remove tiflash replica failed", + zap.String("query", removeTiFlashSQL), + zap.Stringer("db", table.Db.Name), + zap.Stringer("table", table.Info.Name), + zap.Error(err)) + return err + } else if replicaCount > 0 { + log.Warn("remove tiflash replica done, please recover it manually later", + zap.Stringer("db", table.Db.Name), + zap.Stringer("table", table.Info.Name), + zap.Int("originalReplicaCount", replicaCount)) + } + } + return nil +} + // GetTiFlashStores returns an id list of tiflash stores. func (rc *Client) GetTiFlashStores() ([]uint64, error) { stores, err := rc.pdClient.GetAllStores(rc.ctx) diff --git a/pkg/restore/db.go b/pkg/restore/db.go index d4a3338ca..8f2f27432 100644 --- a/pkg/restore/db.go +++ b/pkg/restore/db.go @@ -130,20 +130,6 @@ func (db *DB) CreateTable(ctx context.Context, table *utils.Table) error { zap.Error(err)) } - // TODO: remove this after tiflash supports restore - removeTiFlashSQL := fmt.Sprintf( - "alter table %s set tiflash replica 0", - utils.EncloseName(tableInfo.Name.O), - ) - err = db.se.Execute(ctx, removeTiFlashSQL) - if err != nil { - log.Error("remove tiflash replica failed", - zap.String("query", removeTiFlashSQL), - zap.Stringer("db", table.Db.Name), - zap.Stringer("table", table.Info.Name), - zap.Error(err)) - } - return errors.Trace(err) } diff --git a/pkg/task/restore.go b/pkg/task/restore.go index 451aa67b0..6e3db252f 100644 --- a/pkg/task/restore.go +++ b/pkg/task/restore.go @@ -138,6 +138,10 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf if err != nil { return err } + err = client.RemoveTiFlashReplica(tables) + if err != nil { + return err + } ranges, err := restore.ValidateFileRanges(files, rewriteRules) if err != nil { diff --git a/pkg/utils/schema.go b/pkg/utils/schema.go index bc22768e5..f39ab7cae 100644 --- a/pkg/utils/schema.go +++ b/pkg/utils/schema.go @@ -22,12 +22,13 @@ const ( // Table wraps the schema and files of a table. type Table struct { - Db *model.DBInfo - Info *model.TableInfo - Crc64Xor uint64 - TotalKvs uint64 - TotalBytes uint64 - Files []*backup.File + Db *model.DBInfo + Info *model.TableInfo + Crc64Xor uint64 + TotalKvs uint64 + TotalBytes uint64 + Files []*backup.File + TiflashReplica int64 } // Database wraps the schema and tables of a database. From 3592c897d4672480c233ff6645be2eac1add8762 Mon Sep 17 00:00:00 2001 From: 5kbpers Date: Tue, 17 Mar 2020 17:03:54 +0800 Subject: [PATCH 05/14] cleanup Signed-off-by: 5kbpers --- pkg/conn/conn.go | 4 ++-- pkg/utils/schema.go | 13 ++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/pkg/conn/conn.go b/pkg/conn/conn.go index 63203b70e..cdfc78168 100644 --- a/pkg/conn/conn.go +++ b/pkg/conn/conn.go @@ -127,9 +127,9 @@ skipStore: if unexpectedStoreBehavior == SkipTiFlash { continue skipStore } + return nil, errors.Errorf( + "cannot restore to a cluster with active TiFlash stores (store %d at %s)", store.Id, store.Address) } - return nil, errors.Errorf( - "cannot restore to a cluster with active TiFlash stores (store %d at %s)", store.Id, store.Address) } stores[j] = store j++ diff --git a/pkg/utils/schema.go b/pkg/utils/schema.go index f39ab7cae..bc22768e5 100644 --- a/pkg/utils/schema.go +++ b/pkg/utils/schema.go @@ -22,13 +22,12 @@ const ( // Table wraps the schema and files of a table. type Table struct { - Db *model.DBInfo - Info *model.TableInfo - Crc64Xor uint64 - TotalKvs uint64 - TotalBytes uint64 - Files []*backup.File - TiflashReplica int64 + Db *model.DBInfo + Info *model.TableInfo + Crc64Xor uint64 + TotalKvs uint64 + TotalBytes uint64 + Files []*backup.File } // Database wraps the schema and tables of a database. From 901f7b6fa61439b47339bd8eabb38c50ede6b9d8 Mon Sep 17 00:00:00 2001 From: 5kbpers Date: Tue, 17 Mar 2020 20:45:54 +0800 Subject: [PATCH 06/14] save tiflash replica count to backupmeta Signed-off-by: 5kbpers --- cmd/restore.go | 64 ++++++++++++++++++++++++ go.mod | 3 +- go.sum | 5 ++ pkg/restore/backoff.go | 10 ++-- pkg/restore/client.go | 107 ++++++++++++++++++++++++++++------------- pkg/restore/db.go | 26 ++++++++++ pkg/task/restore.go | 16 +++++- pkg/utils/pd.go | 94 ++++++++++++++++++++++++++++++++++++ pkg/utils/schema.go | 13 ++--- pkg/utils/tso.go | 49 ------------------- 10 files changed, 292 insertions(+), 95 deletions(-) create mode 100644 pkg/utils/pd.go delete mode 100644 pkg/utils/tso.go diff --git a/cmd/restore.go b/cmd/restore.go index 6353719f2..591d1f713 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -3,10 +3,16 @@ package cmd import ( + "context" + "fmt" + + "github.com/pingcap/errors" + pd "github.com/pingcap/pd/v4/client" "github.com/pingcap/tidb/session" "github.com/spf13/cobra" "github.com/pingcap/br/pkg/gluetikv" + "github.com/pingcap/br/pkg/restore" "github.com/pingcap/br/pkg/summary" "github.com/pingcap/br/pkg/task" "github.com/pingcap/br/pkg/utils" @@ -30,6 +36,52 @@ func runRestoreRawCommand(command *cobra.Command, cmdName string) error { return task.RunRestoreRaw(GetDefaultContext(), gluetikv.Glue{}, cmdName, &cfg) } +func runRestoreTiflashReplicaCommand(command *cobra.Command) error { + cfg := task.RestoreConfig{Config: task.Config{LogProgress: HasLogFile()}} + if err := cfg.ParseFromFlags(command.Flags()); err != nil { + return err + } + + _, _, backupMeta, err := task.ReadBackupMeta(context.Background(), &cfg.Config) + if err != nil { + return err + } + + dbs, err := utils.LoadBackupTables(backupMeta) + if err != nil { + return errors.Trace(err) + } + + securityOption := pd.SecurityOption{} + tlsConfig := cfg.TLS + if tlsConfig.IsEnabled() { + securityOption.CAPath = tlsConfig.CA + securityOption.CertPath = tlsConfig.Cert + securityOption.KeyPath = tlsConfig.Key + } + + store, err := tidbGlue.Open(fmt.Sprintf("tikv://%s?disableGC=true", cfg.PD), securityOption) + if err != nil { + return err + } + db, err := restore.NewDB(tidbGlue, store) + if err != nil { + return err + } + + for _, d := range dbs { + for _, t := range d.Tables { + if t.TiFlashReplicas > 0 { + err := db.AlterTiflashReplica(context.Background(), t, t.TiFlashReplicas) + if err != nil { + return errors.Trace(err) + } + } + } + } + return nil +} + // NewRestoreCommand returns a restore subcommand func NewRestoreCommand() *cobra.Command { command := &cobra.Command{ @@ -96,6 +148,18 @@ func newTableRestoreCommand() *cobra.Command { return command } +func newTiflashReplicaRestoreCommand() *cobra.Command { + command := &cobra.Command{ + Use: "tiflash-replica", + Short: "restore the tiflash replica before the last restore, it must only be used after the last restore failed", + RunE: func(cmd *cobra.Command, _ []string) error { + return runRestoreTiflashReplicaCommand(cmd) + }, + } + task.DefineTableFlags(command) + return command +} + func newRawRestoreCommand() *cobra.Command { command := &cobra.Command{ Use: "raw", diff --git a/go.mod b/go.mod index 50b81eed8..662731e83 100644 --- a/go.mod +++ b/go.mod @@ -20,9 +20,10 @@ require ( github.com/onsi/gomega v1.8.1 // indirect github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011 - github.com/pingcap/kvproto v0.0.0-20200228095611-2cf9a243b8d5 + github.com/pingcap/kvproto v0.0.0-20200317112120-78042b285b75 github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd github.com/pingcap/parser v0.0.0-20200305120128-bde9faa0df84 + github.com/pingcap/pd v2.1.19+incompatible github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3 github.com/pingcap/tidb v1.1.0-beta.0.20200310133602-7c39e5e5e0bc github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible diff --git a/go.sum b/go.sum index 8c5ae9cff..8b02231c7 100644 --- a/go.sum +++ b/go.sum @@ -363,12 +363,16 @@ github.com/pingcap/kvproto v0.0.0-20200214064158-62d31900d88e/go.mod h1:IOdRDPLy github.com/pingcap/kvproto v0.0.0-20200221034943-a2aa1d1e20a8/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200228095611-2cf9a243b8d5 h1:knEvP4R5v5b2T107/Q6VzB0C8/6T7NXB/V7Vl1FtQsg= github.com/pingcap/kvproto v0.0.0-20200228095611-2cf9a243b8d5/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20200317112120-78042b285b75 h1:DB3NTM0ilba/6sW+vccdEnP10bVvrVunDwWvRa0hSKc= +github.com/pingcap/kvproto v0.0.0-20200317112120-78042b285b75/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM= github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/parser v0.0.0-20200305120128-bde9faa0df84 h1:u5FOwUw9muF8mBTZVV1dQhoAKiEo2Ci54CxN9XchEEY= github.com/pingcap/parser v0.0.0-20200305120128-bde9faa0df84/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= +github.com/pingcap/pd v2.1.19+incompatible h1:N/8HOd5yptSJZ1LdBa1bcvYOH9YCetvX4cg5fsGf9+c= +github.com/pingcap/pd v2.1.19+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3 h1:Yrp99FnjHAEuDrSBql2l0IqCtJX7KwJbTsD5hIArkvk= github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3/go.mod h1:25GfNw6+Jcr9kca5rtmTb4gKCJ4jOpow2zV2S9Dgafs= github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= @@ -486,6 +490,7 @@ github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljT github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3 h1:ZsIlNwu/G0zbChIZaWOeZ2TPGNmKMt46jZLXi3e8LFc= github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= diff --git a/pkg/restore/backoff.go b/pkg/restore/backoff.go index ae5cddd66..21048dd13 100644 --- a/pkg/restore/backoff.go +++ b/pkg/restore/backoff.go @@ -90,21 +90,21 @@ func (bo *importerBackoffer) Attempt() int { return bo.attempt } -type resetTSBackoffer struct { +type pdReqBackoffer struct { attempt int delayTime time.Duration maxDelayTime time.Duration } -func newResetTSBackoffer() utils.Backoffer { - return &resetTSBackoffer{ +func newPDReqBackoffer() utils.Backoffer { + return &pdReqBackoffer{ attempt: resetTsRetryTime, delayTime: resetTSWaitInterval, maxDelayTime: resetTSMaxWaitInterval, } } -func (bo *resetTSBackoffer) NextBackoff(err error) time.Duration { +func (bo *pdReqBackoffer) NextBackoff(err error) time.Duration { bo.delayTime = 2 * bo.delayTime bo.attempt-- if bo.delayTime > bo.maxDelayTime { @@ -113,6 +113,6 @@ func (bo *resetTSBackoffer) NextBackoff(err error) time.Duration { return bo.delayTime } -func (bo *resetTSBackoffer) Attempt() int { +func (bo *pdReqBackoffer) Attempt() int { return bo.attempt } diff --git a/pkg/restore/client.go b/pkg/restore/client.go index ba18cfd3c..dbfdd6e75 100644 --- a/pkg/restore/client.go +++ b/pkg/restore/client.go @@ -15,6 +15,7 @@ import ( "sync" "time" + "github.com/gogo/protobuf/proto" "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/backup" "github.com/pingcap/kvproto/pkg/import_sstpb" @@ -37,6 +38,7 @@ import ( "github.com/pingcap/br/pkg/checksum" "github.com/pingcap/br/pkg/conn" "github.com/pingcap/br/pkg/glue" + "github.com/pingcap/br/pkg/storage" "github.com/pingcap/br/pkg/summary" "github.com/pingcap/br/pkg/utils" ) @@ -65,6 +67,9 @@ type Client struct { hasSpeedLimited bool restoreStores []uint64 + + storage storage.ExternalStorage + backend *backup.StorageBackend } // NewRestoreClient returns a new RestoreClient @@ -97,6 +102,17 @@ func (rc *Client) SetRateLimit(rateLimit uint64) { rc.rateLimit = rateLimit } +// SetStorage set ExternalStorage for client +func (rc *Client) SetStorage(ctx context.Context, backend *backup.StorageBackend, sendCreds bool) error { + var err error + rc.storage, err = storage.Create(ctx, backend, sendCreds) + if err != nil { + return err + } + rc.backend = backend + return nil +} + // GetPDClient returns a pd client. func (rc *Client) GetPDClient() pd.Client { return rc.pdClient @@ -233,7 +249,20 @@ func (rc *Client) ResetTS(pdAddrs []string) error { return utils.WithRetry(rc.ctx, func() error { idx := i % len(pdAddrs) return utils.ResetTS(pdAddrs[idx], restoreTS, rc.tlsConf) - }, newResetTSBackoffer()) + }, newPDReqBackoffer()) +} + +// GetPlacementRules return the current placement rules +func (rc *Client) GetPlacementRules(pdAddrs []string) ([]placement.Rule, error) { + var placementRules []placement.Rule + i := 0 + errRetry := utils.WithRetry(rc.ctx, func() error { + idx := i % len(pdAddrs) + var err error + placementRules, err = utils.GetPlacementRules(pdAddrs[idx], rc.tlsConf) + return err + }, newPDReqBackoffer()) + return placementRules, errRetry } // GetDatabases returns all databases. @@ -305,48 +334,60 @@ func (rc *Client) CreateTables( return rewriteRules, newTables, nil } -// RemoveTiFlashReplica removes all the tiflash replica of a table -func (rc *Client) RemoveTiFlashReplica(tables []*utils.Table) error { - stores, err := rc.GetTiFlashStores() +// RemoveTiFlashReplica removes all the tiflash replicas of a table +// TODO: remove this after tiflash supports restore +func (rc *Client) RemoveTiFlashReplica(tables []*utils.Table, placementRules []placement.Rule) error { + schemas := make([]*backup.Schema, 0, len(tables)) + for _, table := range tables { + if rule := utils.SearchPlacementRule(table.Info.ID, placementRules, placement.Learner); rule != nil { + table.TiFlashReplicas = rule.Count + } + tableData, err := json.Marshal(table.Info) + if err != nil { + return errors.Trace(err) + } + dbData, err := json.Marshal(table.Db) + if err != nil { + return errors.Trace(err) + } + schemas = append(schemas, &backup.Schema{ + Db: dbData, + Table: tableData, + TiflashReplicas: uint32(table.TiFlashReplicas), + }) + } + + rc.backupMeta.Schemas = schemas + backupMetaData, err := proto.Marshal(rc.backupMeta) if err != nil { return errors.Trace(err) } - storeMap := make(map[uint64]bool) - for _, storeID := range stores { - storeMap[storeID] = true + backendURL := storage.FormatBackendURL(rc.backend) + log.Info("update backup meta", zap.Stringer("path", &backendURL)) + err = rc.storage.Write(rc.ctx, utils.MetaFile, backupMetaData) + if err != nil { + return errors.Trace(err) } + for _, table := range tables { - tableInfo := table.Info - prefix := tablecodec.GenTablePrefix(tableInfo.ID) - region, _, err := rc.pdClient.GetRegion(rc.ctx, prefix) + err = rc.db.AlterTiflashReplica(rc.ctx, table, 0) if err != nil { return errors.Trace(err) } - var replicaCount int - for _, peer := range region.GetPeers() { - if storeMap[peer.GetStoreId()] { - replicaCount++ + } + return nil +} + +// RecoverTiFlashReplica recovers all the tiflash replicas of a table +// TODO: remove this after tiflash supports restore +func (rc *Client) RecoverTiFlashReplica(tables []*utils.Table) error { + for _, table := range tables { + if table.TiFlashReplicas > 0 { + err := rc.db.AlterTiflashReplica(rc.ctx, table, table.TiFlashReplicas) + if err != nil { + return errors.Trace(err) } } - // TODO: remove this after tiflash supports restore - removeTiFlashSQL := fmt.Sprintf( - "alter table %s set tiflash replica 0", - utils.EncloseName(tableInfo.Name.O), - ) - err = rc.db.se.Execute(rc.ctx, removeTiFlashSQL) - if err != nil { - log.Error("remove tiflash replica failed", - zap.String("query", removeTiFlashSQL), - zap.Stringer("db", table.Db.Name), - zap.Stringer("table", table.Info.Name), - zap.Error(err)) - return err - } else if replicaCount > 0 { - log.Warn("remove tiflash replica done, please recover it manually later", - zap.Stringer("db", table.Db.Name), - zap.Stringer("table", table.Info.Name), - zap.Int("originalReplicaCount", replicaCount)) - } } return nil } diff --git a/pkg/restore/db.go b/pkg/restore/db.go index 8f2f27432..d7deda036 100644 --- a/pkg/restore/db.go +++ b/pkg/restore/db.go @@ -133,6 +133,32 @@ func (db *DB) CreateTable(ctx context.Context, table *utils.Table) error { return errors.Trace(err) } +// AlterTiflashReplica alters the replica count of tiflash +func (db *DB) AlterTiflashReplica(ctx context.Context, table *utils.Table, count int) error { + alterTiFlashSQL := fmt.Sprintf( + "alter table %s set tiflash replica %d", + utils.EncloseName(table.Info.Name.O), + count, + ) + err := db.se.Execute(ctx, alterTiFlashSQL) + if err != nil { + log.Error("alter tiflash replica failed", + zap.String("query", alterTiFlashSQL), + zap.Stringer("db", table.Db.Name), + zap.Stringer("table", table.Info.Name), + zap.Error(err)) + return err + } else if table.TiFlashReplicas > 0 { + log.Warn("alter tiflash replica done", + zap.Stringer("db", table.Db.Name), + zap.Stringer("table", table.Info.Name), + zap.Int("originalReplicaCount", table.TiFlashReplicas), + zap.Int("replicaCount", count)) + + } + return nil +} + // Close closes the connection func (db *DB) Close() { db.se.Close() diff --git a/pkg/task/restore.go b/pkg/task/restore.go index 6e3db252f..29c5f86d3 100644 --- a/pkg/task/restore.go +++ b/pkg/task/restore.go @@ -17,6 +17,7 @@ import ( "github.com/pingcap/br/pkg/glue" "github.com/pingcap/br/pkg/restore" "github.com/pingcap/br/pkg/rtree" + "github.com/pingcap/br/pkg/storage" "github.com/pingcap/br/pkg/summary" "github.com/pingcap/br/pkg/utils" ) @@ -87,6 +88,13 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf } defer client.Close() + u, err := storage.ParseBackend(cfg.Storage, &cfg.BackendOptions) + if err != nil { + return err + } + if err = client.SetStorage(ctx, u, cfg.SendCreds); err != nil { + return err + } client.SetRateLimit(cfg.RateLimit) client.SetConcurrency(uint(cfg.Concurrency)) if cfg.Online { @@ -138,10 +146,16 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf if err != nil { return err } - err = client.RemoveTiFlashReplica(tables) + placementRules, err := client.GetPlacementRules(cfg.PD) if err != nil { return err } + err = client.RemoveTiFlashReplica(tables, placementRules) + if err != nil { + return err + } + + defer client.RecoverTiFlashReplica(tables) ranges, err := restore.ValidateFileRanges(files, rewriteRules) if err != nil { diff --git a/pkg/utils/pd.go b/pkg/utils/pd.go new file mode 100644 index 000000000..d9d4cfc4a --- /dev/null +++ b/pkg/utils/pd.go @@ -0,0 +1,94 @@ +// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. + +package utils + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/pd/v4/server/schedule/placement" + "github.com/pingcap/tidb/tablecodec" +) + +const ( + resetTSURL = "/pd/api/v1/admin/reset-ts" + placementRuleURL = "/pd/api/v1/config/rules" +) + +// ResetTS resets the timestamp of PD to a bigger value +func ResetTS(pdAddr string, ts uint64, tlsConf *tls.Config) error { + req, err := json.Marshal(struct { + TSO string `json:"tso,omitempty"` + }{TSO: fmt.Sprintf("%d", ts)}) + if err != nil { + return err + } + cli := &http.Client{Timeout: 30 * time.Second} + prefix := "http://" + if tlsConf != nil { + prefix = "https://" + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = tlsConf + cli.Transport = transport + } + reqURL := prefix + pdAddr + resetTSURL + resp, err := cli.Post(reqURL, "application/json", strings.NewReader(string(req))) + if err != nil { + return errors.Trace(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 && resp.StatusCode != 403 { + buf := new(bytes.Buffer) + _, _ = buf.ReadFrom(resp.Body) + return errors.Errorf("pd resets TS failed: req=%v, resp=%v, err=%v", string(req), buf.String(), err) + } + return nil +} + +// GetPlacementRules return the current placement rules +func GetPlacementRules(pdAddr string, tlsConf *tls.Config) ([]placement.Rule, error) { + cli := &http.Client{Timeout: 30 * time.Second} + prefix := "http://" + if tlsConf != nil { + prefix = "https://" + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = tlsConf + cli.Transport = transport + } + reqURL := prefix + pdAddr + placementRuleURL + resp, err := cli.Get(reqURL) + if err != nil { + return nil, errors.Trace(err) + } + defer resp.Body.Close() + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(resp.Body) + if err != nil { + return nil, errors.Trace(err) + } + if resp.StatusCode != 200 { + return nil, errors.Errorf("get placement rules failed: resp=%v, err=%v", buf.String(), err) + } + var rules []placement.Rule + err = json.Unmarshal(buf.Bytes(), &rules) + if err != nil { + return nil, errors.Trace(err) + } + return rules, nil +} + +// SearchPlacementRule returns the placement rule matched to the table or nil +func SearchPlacementRule(tableID int64, placementRules []placement.Rule, role placement.PeerRoleType) *placement.Rule { + for _, rule := range placementRules { + if rule.Role == role && tableID == tablecodec.DecodeTableID(rule.StartKey) { + return &rule + } + } + return nil +} diff --git a/pkg/utils/schema.go b/pkg/utils/schema.go index bc22768e5..5fa80cebe 100644 --- a/pkg/utils/schema.go +++ b/pkg/utils/schema.go @@ -22,12 +22,13 @@ const ( // Table wraps the schema and files of a table. type Table struct { - Db *model.DBInfo - Info *model.TableInfo - Crc64Xor uint64 - TotalKvs uint64 - TotalBytes uint64 - Files []*backup.File + Db *model.DBInfo + Info *model.TableInfo + Crc64Xor uint64 + TotalKvs uint64 + TotalBytes uint64 + Files []*backup.File + TiFlashReplicas int } // Database wraps the schema and tables of a database. diff --git a/pkg/utils/tso.go b/pkg/utils/tso.go deleted file mode 100644 index c90cd3575..000000000 --- a/pkg/utils/tso.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. - -package utils - -import ( - "bytes" - "crypto/tls" - "encoding/json" - "fmt" - "net/http" - "strings" - "time" - - "github.com/pingcap/errors" -) - -const ( - resetTSURL = "/pd/api/v1/admin/reset-ts" -) - -// ResetTS resets the timestamp of PD to a bigger value -func ResetTS(pdAddr string, ts uint64, tlsConf *tls.Config) error { - req, err := json.Marshal(struct { - TSO string `json:"tso,omitempty"` - }{TSO: fmt.Sprintf("%d", ts)}) - if err != nil { - return err - } - cli := &http.Client{Timeout: 30 * time.Second} - prefix := "http://" - if tlsConf != nil { - prefix = "https://" - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.TLSClientConfig = tlsConf - cli.Transport = transport - } - reqURL := prefix + pdAddr + resetTSURL - resp, err := cli.Post(reqURL, "application/json", strings.NewReader(string(req))) - if err != nil { - return errors.Trace(err) - } - defer resp.Body.Close() - if resp.StatusCode != 200 && resp.StatusCode != 403 { - buf := new(bytes.Buffer) - _, err := buf.ReadFrom(resp.Body) - return errors.Errorf("pd resets TS failed: req=%v, resp=%v, err=%v", string(req), buf.String(), err) - } - return nil -} From a604cf5bbfbbcce19dc750080ebf3cb9e63830b3 Mon Sep 17 00:00:00 2001 From: 5kbpers Date: Tue, 17 Mar 2020 20:59:22 +0800 Subject: [PATCH 07/14] fix save crcxor Signed-off-by: 5kbpers --- pkg/restore/client.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/restore/client.go b/pkg/restore/client.go index dbfdd6e75..40343d2b4 100644 --- a/pkg/restore/client.go +++ b/pkg/restore/client.go @@ -353,10 +353,14 @@ func (rc *Client) RemoveTiFlashReplica(tables []*utils.Table, placementRules []p schemas = append(schemas, &backup.Schema{ Db: dbData, Table: tableData, + Crc64Xor: table.Crc64Xor, + TotalKvs: table.TotalKvs, + TotalBytes: table.TotalBytes, TiflashReplicas: uint32(table.TiFlashReplicas), }) } + // Update backup meta rc.backupMeta.Schemas = schemas backupMetaData, err := proto.Marshal(rc.backupMeta) if err != nil { From 44055d1e5f33e428d33e20ee92ae53cdc62de74f Mon Sep 17 00:00:00 2001 From: 5kbpers Date: Tue, 17 Mar 2020 21:58:56 +0800 Subject: [PATCH 08/14] fix decode the key of placement rule Signed-off-by: 5kbpers --- cmd/restore.go | 17 ++++++++++++----- go.mod | 2 +- go.sum | 25 +++++++++++++++++++++++++ pkg/restore/db.go | 11 ++++++++++- pkg/utils/pd.go | 12 +++++++++++- pkg/utils/schema.go | 13 +++++++------ 6 files changed, 66 insertions(+), 14 deletions(-) diff --git a/cmd/restore.go b/cmd/restore.go index 591d1f713..40467f8c9 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -3,13 +3,15 @@ package cmd import ( - "context" "fmt" + "strings" "github.com/pingcap/errors" + "github.com/pingcap/log" pd "github.com/pingcap/pd/v4/client" "github.com/pingcap/tidb/session" "github.com/spf13/cobra" + "go.uber.org/zap" "github.com/pingcap/br/pkg/gluetikv" "github.com/pingcap/br/pkg/restore" @@ -42,7 +44,7 @@ func runRestoreTiflashReplicaCommand(command *cobra.Command) error { return err } - _, _, backupMeta, err := task.ReadBackupMeta(context.Background(), &cfg.Config) + _, _, backupMeta, err := task.ReadBackupMeta(GetDefaultContext(), &cfg.Config) if err != nil { return err } @@ -60,7 +62,11 @@ func runRestoreTiflashReplicaCommand(command *cobra.Command) error { securityOption.KeyPath = tlsConfig.Key } - store, err := tidbGlue.Open(fmt.Sprintf("tikv://%s?disableGC=true", cfg.PD), securityOption) + pdAddress := strings.Join(cfg.PD, ",") + if len(pdAddress) == 0 { + return errors.New("pd address can not be empty") + } + store, err := tidbGlue.Open(fmt.Sprintf("tikv://%s?disableGC=true", pdAddress), securityOption) if err != nil { return err } @@ -71,8 +77,9 @@ func runRestoreTiflashReplicaCommand(command *cobra.Command) error { for _, d := range dbs { for _, t := range d.Tables { + log.Info("get table", zap.Stringer("name", t.Info.Name), zap.Int("replica", t.TiFlashReplicas)) if t.TiFlashReplicas > 0 { - err := db.AlterTiflashReplica(context.Background(), t, t.TiFlashReplicas) + err := db.AlterTiflashReplica(GetDefaultContext(), t, t.TiFlashReplicas) if err != nil { return errors.Trace(err) } @@ -107,6 +114,7 @@ func NewRestoreCommand() *cobra.Command { newDbRestoreCommand(), newTableRestoreCommand(), newRawRestoreCommand(), + newTiflashReplicaRestoreCommand(), ) task.DefineRestoreFlags(command.PersistentFlags()) @@ -156,7 +164,6 @@ func newTiflashReplicaRestoreCommand() *cobra.Command { return runRestoreTiflashReplicaCommand(cmd) }, } - task.DefineTableFlags(command) return command } diff --git a/go.mod b/go.mod index 662731e83..318b452a0 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285 // indirect - github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect + github.com/tikv/client-go v0.0.0-20200110101306-a3ebdb020c83 go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 go.opencensus.io v0.22.2 // indirect go.uber.org/zap v1.14.0 diff --git a/go.sum b/go.sum index 8b02231c7..fab530b1c 100644 --- a/go.sum +++ b/go.sum @@ -61,7 +61,9 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -70,6 +72,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK4aZY8jz+DAF0FHjI51BXjjSwCzS1Dk= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190212144455-93d5ec2c7f76/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -163,6 +166,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -203,16 +207,19 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.8.1/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.12.1 h1:zCy2xE9ablevUOrUZc3Dl72Dt+ya2FNAvC2yLYMHzi4= @@ -309,6 +316,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808 h1:pmpDGKLw4n82EtrNiLqB+xSz/JQwFOaZuMALYUHwX5s= github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk= +github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdcNTgsos+vFzULLwyElndwn+5c= github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI= @@ -356,8 +365,11 @@ github.com/pingcap/failpoint v0.0.0-20200210140405-f8f9fb234798 h1:6DMbRqPI1qzQ8 github.com/pingcap/failpoint v0.0.0-20200210140405-f8f9fb234798/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d h1:rCmRK0lCRrHMUbS99BKFYhK9YxJDNw0xB033cQbYo0s= github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d/go.mod h1:fMRU1BA1y+r89AxUoaAar4JjrhUkVDt0o0Np6V8XbDQ= +github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3/go.mod h1:DazNTg0PTldtpsQiT9I5tVJwV1onHMKBBgXzmJUlMns= +github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= +github.com/pingcap/kvproto v0.0.0-20190305055742-ab7debc182d9/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/kvproto v0.0.0-20200214064158-62d31900d88e/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200221034943-a2aa1d1e20a8/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= @@ -371,6 +383,7 @@ github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfE github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/parser v0.0.0-20200305120128-bde9faa0df84 h1:u5FOwUw9muF8mBTZVV1dQhoAKiEo2Ci54CxN9XchEEY= github.com/pingcap/parser v0.0.0-20200305120128-bde9faa0df84/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= +github.com/pingcap/pd v2.1.5+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= github.com/pingcap/pd v2.1.19+incompatible h1:N/8HOd5yptSJZ1LdBa1bcvYOH9YCetvX4cg5fsGf9+c= github.com/pingcap/pd v2.1.19+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3 h1:Yrp99FnjHAEuDrSBql2l0IqCtJX7KwJbTsD5hIArkvk= @@ -395,6 +408,7 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -402,11 +416,14 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -428,6 +445,7 @@ github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca h1:3fECS8atRjByiji github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= @@ -472,6 +490,8 @@ github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfK github.com/tidwall/gjson v1.3.5/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tikv/client-go v0.0.0-20200110101306-a3ebdb020c83 h1:IfRSAmMavarPgSj0Rr4iUVJ+w6sxF1a6/cTUXZl5yZY= +github.com/tikv/client-go v0.0.0-20200110101306-a3ebdb020c83/go.mod h1:K0NcdVNrXDq92YPLytsrAwRMyuXi7GZCO6dXNH7OzQc= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 h1:lYIiVDtZnyTWlNwiAxLj0bbpTcx1BWCFhXjfsvmPdNc= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -483,15 +503,18 @@ github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1 github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3 h1:ZsIlNwu/G0zbChIZaWOeZ2TPGNmKMt46jZLXi3e8LFc= github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= +github.com/unrolled/render v1.0.0/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/negroni v0.3.0 h1:PaXOb61mWeZJxc1Ji2xJjpVg9QfPo0rrB+lHyBxGNSU= @@ -501,6 +524,7 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yookoala/realpath v1.0.0 h1:7OA9pj4FZd+oZDsyvXWQvjn5oBdcHRTV44PpdMSuImQ= github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= @@ -582,6 +606,7 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/pkg/restore/db.go b/pkg/restore/db.go index d7deda036..9ce3b10c7 100644 --- a/pkg/restore/db.go +++ b/pkg/restore/db.go @@ -135,12 +135,21 @@ func (db *DB) CreateTable(ctx context.Context, table *utils.Table) error { // AlterTiflashReplica alters the replica count of tiflash func (db *DB) AlterTiflashReplica(ctx context.Context, table *utils.Table, count int) error { + switchDbSQL := fmt.Sprintf("use %s;", utils.EncloseName(table.Db.Name.O)) + err := db.se.Execute(ctx, switchDbSQL) + if err != nil { + log.Error("switch db failed", + zap.String("SQL", switchDbSQL), + zap.Stringer("db", table.Db.Name), + zap.Error(err)) + return errors.Trace(err) + } alterTiFlashSQL := fmt.Sprintf( "alter table %s set tiflash replica %d", utils.EncloseName(table.Info.Name.O), count, ) - err := db.se.Execute(ctx, alterTiFlashSQL) + err = db.se.Execute(ctx, alterTiFlashSQL) if err != nil { log.Error("alter tiflash replica failed", zap.String("query", alterTiFlashSQL), diff --git a/pkg/utils/pd.go b/pkg/utils/pd.go index d9d4cfc4a..b0d4cdc4c 100644 --- a/pkg/utils/pd.go +++ b/pkg/utils/pd.go @@ -5,6 +5,7 @@ package utils import ( "bytes" "crypto/tls" + "encoding/hex" "encoding/json" "fmt" "net/http" @@ -14,6 +15,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/pd/v4/server/schedule/placement" "github.com/pingcap/tidb/tablecodec" + "github.com/tikv/client-go/codec" ) const ( @@ -86,7 +88,15 @@ func GetPlacementRules(pdAddr string, tlsConf *tls.Config) ([]placement.Rule, er // SearchPlacementRule returns the placement rule matched to the table or nil func SearchPlacementRule(tableID int64, placementRules []placement.Rule, role placement.PeerRoleType) *placement.Rule { for _, rule := range placementRules { - if rule.Role == role && tableID == tablecodec.DecodeTableID(rule.StartKey) { + key, err := hex.DecodeString(rule.StartKeyHex) + if err != nil { + continue + } + _, decoded, err := codec.DecodeBytes(key) + if err != nil { + continue + } + if rule.Role == role && tableID == tablecodec.DecodeTableID(decoded) { return &rule } } diff --git a/pkg/utils/schema.go b/pkg/utils/schema.go index 5fa80cebe..d18f84399 100644 --- a/pkg/utils/schema.go +++ b/pkg/utils/schema.go @@ -93,12 +93,13 @@ func LoadBackupTables(meta *backup.BackupMeta) (map[string]*Database, error) { } } table := &Table{ - Db: dbInfo, - Info: tableInfo, - Crc64Xor: schema.Crc64Xor, - TotalKvs: schema.TotalKvs, - TotalBytes: schema.TotalBytes, - Files: tableFiles, + Db: dbInfo, + Info: tableInfo, + Crc64Xor: schema.Crc64Xor, + TotalKvs: schema.TotalKvs, + TotalBytes: schema.TotalBytes, + Files: tableFiles, + TiFlashReplicas: int(schema.TiflashReplicas), } db.Tables = append(db.Tables, table) } From 65e47ff197856d4f3ae009d2795ed55a3a283d35 Mon Sep 17 00:00:00 2001 From: 5kbpers Date: Tue, 17 Mar 2020 22:06:34 +0800 Subject: [PATCH 09/14] address lint Signed-off-by: 5kbpers --- go.mod | 1 - go.sum | 3 +-- pkg/task/restore.go | 4 +++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 318b452a0..bfe619934 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/pingcap/kvproto v0.0.0-20200317112120-78042b285b75 github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd github.com/pingcap/parser v0.0.0-20200305120128-bde9faa0df84 - github.com/pingcap/pd v2.1.19+incompatible github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3 github.com/pingcap/tidb v1.1.0-beta.0.20200310133602-7c39e5e5e0bc github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible diff --git a/go.sum b/go.sum index fab530b1c..63ab8bd4d 100644 --- a/go.sum +++ b/go.sum @@ -383,9 +383,8 @@ github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfE github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/parser v0.0.0-20200305120128-bde9faa0df84 h1:u5FOwUw9muF8mBTZVV1dQhoAKiEo2Ci54CxN9XchEEY= github.com/pingcap/parser v0.0.0-20200305120128-bde9faa0df84/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= +github.com/pingcap/pd v2.1.5+incompatible h1:vOLV2tSQdRjjmxaTXtJULoC94dYQOd+6fzn2yChODHc= github.com/pingcap/pd v2.1.5+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= -github.com/pingcap/pd v2.1.19+incompatible h1:N/8HOd5yptSJZ1LdBa1bcvYOH9YCetvX4cg5fsGf9+c= -github.com/pingcap/pd v2.1.19+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3 h1:Yrp99FnjHAEuDrSBql2l0IqCtJX7KwJbTsD5hIArkvk= github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3/go.mod h1:25GfNw6+Jcr9kca5rtmTb4gKCJ4jOpow2zV2S9Dgafs= github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= diff --git a/pkg/task/restore.go b/pkg/task/restore.go index 29c5f86d3..9a964dfef 100644 --- a/pkg/task/restore.go +++ b/pkg/task/restore.go @@ -155,7 +155,9 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf return err } - defer client.RecoverTiFlashReplica(tables) + defer func() { + _ = client.RecoverTiFlashReplica(tables) + }() ranges, err := restore.ValidateFileRanges(files, rewriteRules) if err != nil { From 9a98ffc5052dc9c27a6ba46714ec7d7aed813aaa Mon Sep 17 00:00:00 2001 From: 5kbpers Date: Tue, 17 Mar 2020 22:29:24 +0800 Subject: [PATCH 10/14] address comments Signed-off-by: 5kbpers --- pkg/conn/conn.go | 13 +++++++++++-- pkg/conn/conn_test.go | 12 ++++++++++++ pkg/restore/client.go | 27 +++++---------------------- pkg/restore/split.go | 10 +++------- pkg/restore/split_test.go | 2 +- pkg/restore/util.go | 9 +++++++-- pkg/utils/pd.go | 5 ++++- 7 files changed, 43 insertions(+), 35 deletions(-) diff --git a/pkg/conn/conn.go b/pkg/conn/conn.go index cdfc78168..5e3d2fb24 100644 --- a/pkg/conn/conn.go +++ b/pkg/conn/conn.go @@ -103,6 +103,9 @@ const ( // SkipTiFlash causes GetAllTiKVStores to skip the store when it is found to // be a TiFlash node. SkipTiFlash UnexpectedStoreBehavior = 1 + // TiFlashOnly caused GetAllTiKVStores to skip the store which is not a + // TiFlash node. + TiFlashOnly UnexpectedStoreBehavior = 2 ) // GetAllTiKVStores returns all TiKV stores registered to the PD client. The @@ -122,15 +125,21 @@ func GetAllTiKVStores( j := 0 skipStore: for _, store := range stores { + var isTiFlash bool for _, label := range store.Labels { if label.Key == "engine" && label.Value == "tiflash" { if unexpectedStoreBehavior == SkipTiFlash { continue skipStore + } else if unexpectedStoreBehavior == ErrorOnTiFlash { + return nil, errors.Errorf( + "cannot restore to a cluster with active TiFlash stores (store %d at %s)", store.Id, store.Address) } - return nil, errors.Errorf( - "cannot restore to a cluster with active TiFlash stores (store %d at %s)", store.Id, store.Address) + isTiFlash = true } } + if !isTiFlash && unexpectedStoreBehavior == TiFlashOnly { + continue skipStore + } stores[j] = store j++ } diff --git a/pkg/conn/conn_test.go b/pkg/conn/conn_test.go index 572798e23..fa769389d 100644 --- a/pkg/conn/conn_test.go +++ b/pkg/conn/conn_test.go @@ -221,6 +221,18 @@ func (s *testClientSuite) TestGetAllTiKVStores(c *C) { unexpectedStoreBehavior: ErrorOnTiFlash, expectedError: "cannot restore to a cluster with active TiFlash stores.*", }, + { + stores: []*metapb.Store{ + {Id: 1}, + {Id: 2, Labels: []*metapb.StoreLabel{{Key: "engine", Value: "tiflash"}}}, + {Id: 3}, + {Id: 4, Labels: []*metapb.StoreLabel{{Key: "engine", Value: "tikv"}}}, + {Id: 5, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tikv"}, {Key: "engine", Value: "tiflash"}}}, + {Id: 6, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tiflash"}, {Key: "engine", Value: "tikv"}}}, + }, + unexpectedStoreBehavior: TiFlashOnly, + expectedStores: map[uint64]int{2: 1, 5: 1}, + }, } for _, testCase := range testCases { diff --git a/pkg/restore/client.go b/pkg/restore/client.go index 40343d2b4..242702036 100644 --- a/pkg/restore/client.go +++ b/pkg/restore/client.go @@ -374,9 +374,11 @@ func (rc *Client) RemoveTiFlashReplica(tables []*utils.Table, placementRules []p } for _, table := range tables { - err = rc.db.AlterTiflashReplica(rc.ctx, table, 0) - if err != nil { - return errors.Trace(err) + if table.TiFlashReplicas > 0 { + err = rc.db.AlterTiflashReplica(rc.ctx, table, 0) + if err != nil { + return errors.Trace(err) + } } } return nil @@ -396,25 +398,6 @@ func (rc *Client) RecoverTiFlashReplica(tables []*utils.Table) error { return nil } -// GetTiFlashStores returns an id list of tiflash stores. -func (rc *Client) GetTiFlashStores() ([]uint64, error) { - stores, err := rc.pdClient.GetAllStores(rc.ctx) - if err != nil { - return nil, err - } - - tiflashStores := make([]uint64, 0) - for _, store := range stores { - for _, label := range store.GetLabels() { - if label.GetKey() == "engine" && label.GetValue() == "tiflash" { - tiflashStores = append(tiflashStores, store.GetId()) - } - } - } - - return tiflashStores, nil -} - // ExecDDLs executes the queries of the ddl jobs. func (rc *Client) ExecDDLs(ddlJobs []*model.Job) error { // Sort the ddl jobs by schema version in ascending order. diff --git a/pkg/restore/split.go b/pkg/restore/split.go index c8e3c4c6a..03153097a 100644 --- a/pkg/restore/split.go +++ b/pkg/restore/split.go @@ -63,7 +63,7 @@ func (rs *RegionSplitter) Split( ctx context.Context, ranges []rtree.Range, rewriteRules *RewriteRules, - rejectStores []uint64, + rejectStores map[uint64]bool, onSplit OnSplitFunc, ) error { if len(ranges) == 0 { @@ -147,13 +147,9 @@ SplitRegions: } if len(rejectStores) > 0 { startTime = time.Now() - log.Info("start to wait for removing rejected stores", zap.Uint64s("rejectStores", rejectStores)) - storeMap := make(map[uint64]bool) - for _, storeID := range rejectStores { - storeMap[storeID] = true - } + log.Info("start to wait for removing rejected stores", zap.Reflect("rejectStores", rejectStores)) for _, region := range allRegions { - if !rs.waitForRemoveRejectStores(ctx, region, storeMap) { + if !rs.waitForRemoveRejectStores(ctx, region, rejectStores) { log.Error("waiting for removing rejected stores failed", zap.Stringer("region", region.Region)) return errors.New("waiting for removing rejected stores failed") diff --git a/pkg/restore/split_test.go b/pkg/restore/split_test.go index 8d2306a30..06dab1cf1 100644 --- a/pkg/restore/split_test.go +++ b/pkg/restore/split_test.go @@ -193,7 +193,7 @@ func (s *testRestoreUtilSuite) TestSplit(c *C) { regionSplitter := NewRegionSplitter(client) ctx := context.Background() - err := regionSplitter.Split(ctx, ranges, rewriteRules, []uint64{}, func(key [][]byte) {}) + err := regionSplitter.Split(ctx, ranges, rewriteRules, map[uint64]bool{}, func(key [][]byte) {}) if err != nil { c.Assert(err, IsNil, Commentf("split regions failed: %v", err)) } diff --git a/pkg/restore/util.go b/pkg/restore/util.go index 0071a214c..c49c07994 100644 --- a/pkg/restore/util.go +++ b/pkg/restore/util.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/tidb/util/codec" "go.uber.org/zap" + "github.com/pingcap/br/pkg/conn" "github.com/pingcap/br/pkg/rtree" "github.com/pingcap/br/pkg/summary" ) @@ -327,12 +328,16 @@ func SplitRanges( summary.CollectDuration("split region", elapsed) }() splitter := NewRegionSplitter(NewSplitClient(client.GetPDClient(), client.GetTLSConfig())) - tiflashStores, err := client.GetTiFlashStores() + tiflashStores, err := conn.GetAllTiKVStores(ctx, client.GetPDClient(), conn.TiFlashOnly) if err != nil { return errors.Trace(err) } + storeMap := make(map[uint64]bool) + for _, store := range tiflashStores { + storeMap[store.GetId()] = true + } - return splitter.Split(ctx, ranges, rewriteRules, tiflashStores, func(keys [][]byte) { + return splitter.Split(ctx, ranges, rewriteRules, storeMap, func(keys [][]byte) { for range keys { updateCh <- struct{}{} } diff --git a/pkg/utils/pd.go b/pkg/utils/pd.go index b0d4cdc4c..33acb01d2 100644 --- a/pkg/utils/pd.go +++ b/pkg/utils/pd.go @@ -74,8 +74,11 @@ func GetPlacementRules(pdAddr string, tlsConf *tls.Config) ([]placement.Rule, er if err != nil { return nil, errors.Trace(err) } + if resp.StatusCode == 412 { + return []placement.Rule{}, nil + } if resp.StatusCode != 200 { - return nil, errors.Errorf("get placement rules failed: resp=%v, err=%v", buf.String(), err) + return nil, errors.Errorf("get placement rules failed: resp=%v, err=%v, code=%d", buf.String(), err, resp.StatusCode) } var rules []placement.Rule err = json.Unmarshal(buf.Bytes(), &rules) From 63c1d0f810eb11c0a333937a2ec7861f69f3800c Mon Sep 17 00:00:00 2001 From: 5kbpers Date: Tue, 17 Mar 2020 23:47:20 +0800 Subject: [PATCH 11/14] close domain after restoring tiflash-replica Signed-off-by: 5kbpers --- cmd/restore.go | 56 +++------------------------------------------ pkg/task/restore.go | 48 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 53 deletions(-) diff --git a/cmd/restore.go b/cmd/restore.go index 40467f8c9..bc74bea84 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -3,18 +3,10 @@ package cmd import ( - "fmt" - "strings" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - pd "github.com/pingcap/pd/v4/client" "github.com/pingcap/tidb/session" "github.com/spf13/cobra" - "go.uber.org/zap" "github.com/pingcap/br/pkg/gluetikv" - "github.com/pingcap/br/pkg/restore" "github.com/pingcap/br/pkg/summary" "github.com/pingcap/br/pkg/task" "github.com/pingcap/br/pkg/utils" @@ -38,55 +30,13 @@ func runRestoreRawCommand(command *cobra.Command, cmdName string) error { return task.RunRestoreRaw(GetDefaultContext(), gluetikv.Glue{}, cmdName, &cfg) } -func runRestoreTiflashReplicaCommand(command *cobra.Command) error { +func runRestoreTiflashReplicaCommand(command *cobra.Command, cmdName string) error { cfg := task.RestoreConfig{Config: task.Config{LogProgress: HasLogFile()}} if err := cfg.ParseFromFlags(command.Flags()); err != nil { return err } - _, _, backupMeta, err := task.ReadBackupMeta(GetDefaultContext(), &cfg.Config) - if err != nil { - return err - } - - dbs, err := utils.LoadBackupTables(backupMeta) - if err != nil { - return errors.Trace(err) - } - - securityOption := pd.SecurityOption{} - tlsConfig := cfg.TLS - if tlsConfig.IsEnabled() { - securityOption.CAPath = tlsConfig.CA - securityOption.CertPath = tlsConfig.Cert - securityOption.KeyPath = tlsConfig.Key - } - - pdAddress := strings.Join(cfg.PD, ",") - if len(pdAddress) == 0 { - return errors.New("pd address can not be empty") - } - store, err := tidbGlue.Open(fmt.Sprintf("tikv://%s?disableGC=true", pdAddress), securityOption) - if err != nil { - return err - } - db, err := restore.NewDB(tidbGlue, store) - if err != nil { - return err - } - - for _, d := range dbs { - for _, t := range d.Tables { - log.Info("get table", zap.Stringer("name", t.Info.Name), zap.Int("replica", t.TiFlashReplicas)) - if t.TiFlashReplicas > 0 { - err := db.AlterTiflashReplica(GetDefaultContext(), t, t.TiFlashReplicas) - if err != nil { - return errors.Trace(err) - } - } - } - } - return nil + return task.RunRestoreTiflashReplica(GetDefaultContext(), tidbGlue, cmdName, &cfg) } // NewRestoreCommand returns a restore subcommand @@ -161,7 +111,7 @@ func newTiflashReplicaRestoreCommand() *cobra.Command { Use: "tiflash-replica", Short: "restore the tiflash replica before the last restore, it must only be used after the last restore failed", RunE: func(cmd *cobra.Command, _ []string) error { - return runRestoreTiflashReplicaCommand(cmd) + return runRestoreTiflashReplicaCommand(cmd, "Restore TiFlash Replica") }, } return command diff --git a/pkg/task/restore.go b/pkg/task/restore.go index 9a964dfef..e738544d8 100644 --- a/pkg/task/restore.go +++ b/pkg/task/restore.go @@ -369,3 +369,51 @@ func splitPostWork(ctx context.Context, client *restore.Client, tables []*model. } return nil } + +// RunRestoreTiflashReplica restores the replica of tiflash saved in the last restore. +func RunRestoreTiflashReplica(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConfig) error { + defer summary.Summary(cmdName) + ctx, cancel := context.WithCancel(c) + defer cancel() + + mgr, err := newMgr(ctx, g, cfg.PD, cfg.TLS, conn.SkipTiFlash) + if err != nil { + return err + } + defer mgr.Close() + + // Load backupmeta + _, _, backupMeta, err := ReadBackupMeta(c, &cfg.Config) + if err != nil { + return err + } + dbs, err := utils.LoadBackupTables(backupMeta) + if err != nil { + return err + } + se, err := restore.NewDB(g, mgr.GetTiKV()) + if err != nil { + return err + } + + tables := make([]*utils.Table, 0) + for _, db := range dbs { + tables = append(tables, db.Tables...) + } + updateCh := utils.StartProgress( + ctx, "Checksum", int64(len(tables)), !cfg.LogProgress) + for _, t := range tables { + log.Info("get table", zap.Stringer("name", t.Info.Name), + zap.Int("replica", t.TiFlashReplicas)) + if t.TiFlashReplicas > 0 { + err := se.AlterTiflashReplica(ctx, t, t.TiFlashReplicas) + if err != nil { + return err + } + updateCh <- struct{}{} + } + } + summary.CollectInt("recover tables", len(tables)) + + return nil +} From 158eea49e0a9bb2609d168fb48bf755591e159ad Mon Sep 17 00:00:00 2001 From: 5kbpers Date: Wed, 18 Mar 2020 11:16:11 +0800 Subject: [PATCH 12/14] address comments Signed-off-by: 5kbpers --- pkg/restore/client.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/restore/client.go b/pkg/restore/client.go index 242702036..47dfd1462 100644 --- a/pkg/restore/client.go +++ b/pkg/restore/client.go @@ -248,6 +248,7 @@ func (rc *Client) ResetTS(pdAddrs []string) error { i := 0 return utils.WithRetry(rc.ctx, func() error { idx := i % len(pdAddrs) + i++ return utils.ResetTS(pdAddrs[idx], restoreTS, rc.tlsConf) }, newPDReqBackoffer()) } @@ -257,8 +258,9 @@ func (rc *Client) GetPlacementRules(pdAddrs []string) ([]placement.Rule, error) var placementRules []placement.Rule i := 0 errRetry := utils.WithRetry(rc.ctx, func() error { - idx := i % len(pdAddrs) var err error + idx := i % len(pdAddrs) + i++ placementRules, err = utils.GetPlacementRules(pdAddrs[idx], rc.tlsConf) return err }, newPDReqBackoffer()) From 5cb4c0e72c77d15ce745393a55ad35080eeccd01 Mon Sep 17 00:00:00 2001 From: 5kbpers Date: Wed, 18 Mar 2020 11:49:17 +0800 Subject: [PATCH 13/14] address comments Signed-off-by: 5kbpers --- cmd/validate.go | 6 +++--- go.mod | 3 ++- go.sum | 24 ------------------------ pkg/conn/conn.go | 24 ++++++++++++------------ pkg/conn/conn_test.go | 38 +++++++++++++++++++------------------- pkg/restore/client.go | 28 ++++++++++++++++------------ pkg/task/common.go | 7 ++++--- pkg/task/restore.go | 6 +++--- pkg/task/restore_raw.go | 2 +- pkg/utils/pd.go | 8 ++++---- pkg/utils/schema.go | 2 ++ 11 files changed, 66 insertions(+), 82 deletions(-) diff --git a/cmd/validate.go b/cmd/validate.go index baf30200f..386a7bb47 100644 --- a/cmd/validate.go +++ b/cmd/validate.go @@ -63,7 +63,7 @@ func newCheckSumCommand() *cobra.Command { return err } - _, s, backupMeta, err := task.ReadBackupMeta(ctx, &cfg) + _, s, backupMeta, err := task.ReadBackupMeta(ctx, utils.MetaFile, &cfg) if err != nil { return err } @@ -151,7 +151,7 @@ func newBackupMetaCommand() *cobra.Command { if err = cfg.ParseFromFlags(cmd.Flags()); err != nil { return err } - _, _, backupMeta, err := task.ReadBackupMeta(ctx, &cfg) + _, _, backupMeta, err := task.ReadBackupMeta(ctx, utils.MetaFile, &cfg) if err != nil { log.Error("read backupmeta failed", zap.Error(err)) return err @@ -242,7 +242,7 @@ func decodeBackupMetaCommand() *cobra.Command { if err := cfg.ParseFromFlags(cmd.Flags()); err != nil { return err } - _, s, backupMeta, err := task.ReadBackupMeta(ctx, &cfg) + _, s, backupMeta, err := task.ReadBackupMeta(ctx, utils.MetaFile, &cfg) if err != nil { return err } diff --git a/go.mod b/go.mod index bfe619934..94f4022f9 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/google/uuid v1.1.1 github.com/klauspost/cpuid v1.2.0 // indirect github.com/mattn/go-runewidth v0.0.7 // indirect + github.com/montanaflynn/stats v0.5.0 // indirect github.com/onsi/ginkgo v1.11.0 // indirect github.com/onsi/gomega v1.8.1 // indirect github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 @@ -33,7 +34,7 @@ require ( github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285 // indirect - github.com/tikv/client-go v0.0.0-20200110101306-a3ebdb020c83 + github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 go.opencensus.io v0.22.2 // indirect go.uber.org/zap v1.14.0 diff --git a/go.sum b/go.sum index 63ab8bd4d..31fd50bcc 100644 --- a/go.sum +++ b/go.sum @@ -61,9 +61,7 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -72,7 +70,6 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK4aZY8jz+DAF0FHjI51BXjjSwCzS1Dk= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190212144455-93d5ec2c7f76/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -166,7 +163,6 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -207,19 +203,16 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.8.1/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.12.1 h1:zCy2xE9ablevUOrUZc3Dl72Dt+ya2FNAvC2yLYMHzi4= @@ -365,11 +358,8 @@ github.com/pingcap/failpoint v0.0.0-20200210140405-f8f9fb234798 h1:6DMbRqPI1qzQ8 github.com/pingcap/failpoint v0.0.0-20200210140405-f8f9fb234798/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d h1:rCmRK0lCRrHMUbS99BKFYhK9YxJDNw0xB033cQbYo0s= github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d/go.mod h1:fMRU1BA1y+r89AxUoaAar4JjrhUkVDt0o0Np6V8XbDQ= -github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3/go.mod h1:DazNTg0PTldtpsQiT9I5tVJwV1onHMKBBgXzmJUlMns= -github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= -github.com/pingcap/kvproto v0.0.0-20190305055742-ab7debc182d9/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/kvproto v0.0.0-20200214064158-62d31900d88e/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200221034943-a2aa1d1e20a8/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= @@ -383,8 +373,6 @@ github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfE github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/parser v0.0.0-20200305120128-bde9faa0df84 h1:u5FOwUw9muF8mBTZVV1dQhoAKiEo2Ci54CxN9XchEEY= github.com/pingcap/parser v0.0.0-20200305120128-bde9faa0df84/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4= -github.com/pingcap/pd v2.1.5+incompatible h1:vOLV2tSQdRjjmxaTXtJULoC94dYQOd+6fzn2yChODHc= -github.com/pingcap/pd v2.1.5+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3 h1:Yrp99FnjHAEuDrSBql2l0IqCtJX7KwJbTsD5hIArkvk= github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3/go.mod h1:25GfNw6+Jcr9kca5rtmTb4gKCJ4jOpow2zV2S9Dgafs= github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= @@ -407,7 +395,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -415,14 +402,11 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -444,7 +428,6 @@ github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca h1:3fECS8atRjByiji github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= @@ -489,8 +472,6 @@ github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfK github.com/tidwall/gjson v1.3.5/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tikv/client-go v0.0.0-20200110101306-a3ebdb020c83 h1:IfRSAmMavarPgSj0Rr4iUVJ+w6sxF1a6/cTUXZl5yZY= -github.com/tikv/client-go v0.0.0-20200110101306-a3ebdb020c83/go.mod h1:K0NcdVNrXDq92YPLytsrAwRMyuXi7GZCO6dXNH7OzQc= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 h1:lYIiVDtZnyTWlNwiAxLj0bbpTcx1BWCFhXjfsvmPdNc= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -502,18 +483,15 @@ github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1 github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3 h1:ZsIlNwu/G0zbChIZaWOeZ2TPGNmKMt46jZLXi3e8LFc= github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= -github.com/unrolled/render v1.0.0/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/negroni v0.3.0 h1:PaXOb61mWeZJxc1Ji2xJjpVg9QfPo0rrB+lHyBxGNSU= @@ -523,7 +501,6 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yookoala/realpath v1.0.0 h1:7OA9pj4FZd+oZDsyvXWQvjn5oBdcHRTV44PpdMSuImQ= github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= @@ -605,7 +582,6 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/pkg/conn/conn.go b/pkg/conn/conn.go index 5e3d2fb24..c7c4a9c9e 100644 --- a/pkg/conn/conn.go +++ b/pkg/conn/conn.go @@ -92,20 +92,20 @@ func pdRequest( return r, nil } -// UnexpectedStoreBehavior is the action to do in GetAllTiKVStores when a -// non-TiKV store (e.g. TiFlash store) is found. -type UnexpectedStoreBehavior uint8 +// StoreBehavior is the action to do in GetAllTiKVStores when a non-TiKV +// store (e.g. TiFlash store) is found. +type StoreBehavior uint8 const ( // ErrorOnTiFlash causes GetAllTiKVStores to return error when the store is // found to be a TiFlash node. - ErrorOnTiFlash UnexpectedStoreBehavior = 0 + ErrorOnTiFlash StoreBehavior = 0 // SkipTiFlash causes GetAllTiKVStores to skip the store when it is found to // be a TiFlash node. - SkipTiFlash UnexpectedStoreBehavior = 1 + SkipTiFlash StoreBehavior = 1 // TiFlashOnly caused GetAllTiKVStores to skip the store which is not a // TiFlash node. - TiFlashOnly UnexpectedStoreBehavior = 2 + TiFlashOnly StoreBehavior = 2 ) // GetAllTiKVStores returns all TiKV stores registered to the PD client. The @@ -113,7 +113,7 @@ const ( func GetAllTiKVStores( ctx context.Context, pdClient pd.Client, - unexpectedStoreBehavior UnexpectedStoreBehavior, + storeBehavior StoreBehavior, ) ([]*metapb.Store, error) { // get all live stores. stores, err := pdClient.GetAllStores(ctx, pd.WithExcludeTombstone()) @@ -128,16 +128,16 @@ skipStore: var isTiFlash bool for _, label := range store.Labels { if label.Key == "engine" && label.Value == "tiflash" { - if unexpectedStoreBehavior == SkipTiFlash { + if storeBehavior == SkipTiFlash { continue skipStore - } else if unexpectedStoreBehavior == ErrorOnTiFlash { + } else if storeBehavior == ErrorOnTiFlash { return nil, errors.Errorf( "cannot restore to a cluster with active TiFlash stores (store %d at %s)", store.Id, store.Address) } isTiFlash = true } } - if !isTiFlash && unexpectedStoreBehavior == TiFlashOnly { + if !isTiFlash && storeBehavior == TiFlashOnly { continue skipStore } stores[j] = store @@ -154,7 +154,7 @@ func NewMgr( storage tikv.Storage, tlsConf *tls.Config, securityOption pd.SecurityOption, - unexpectedStoreBehavior UnexpectedStoreBehavior, + storeBehavior StoreBehavior, ) (*Mgr, error) { addrs := strings.Split(pdAddrs, ",") @@ -199,7 +199,7 @@ func NewMgr( log.Info("new mgr", zap.String("pdAddrs", pdAddrs)) // Check live tikv. - stores, err := GetAllTiKVStores(ctx, pdClient, unexpectedStoreBehavior) + stores, err := GetAllTiKVStores(ctx, pdClient, storeBehavior) if err != nil { log.Error("fail to get store", zap.Error(err)) return nil, err diff --git a/pkg/conn/conn_test.go b/pkg/conn/conn_test.go index fa769389d..26278035b 100644 --- a/pkg/conn/conn_test.go +++ b/pkg/conn/conn_test.go @@ -162,40 +162,40 @@ func (fpdc fakePDClient) GetAllStores(context.Context, ...pd.GetStoreOption) ([] func (s *testClientSuite) TestGetAllTiKVStores(c *C) { testCases := []struct { - stores []*metapb.Store - unexpectedStoreBehavior UnexpectedStoreBehavior - expectedStores map[uint64]int - expectedError string + stores []*metapb.Store + storeBehavior StoreBehavior + expectedStores map[uint64]int + expectedError string }{ { stores: []*metapb.Store{ {Id: 1}, }, - unexpectedStoreBehavior: SkipTiFlash, - expectedStores: map[uint64]int{1: 1}, + storeBehavior: SkipTiFlash, + expectedStores: map[uint64]int{1: 1}, }, { stores: []*metapb.Store{ {Id: 1}, }, - unexpectedStoreBehavior: ErrorOnTiFlash, - expectedStores: map[uint64]int{1: 1}, + storeBehavior: ErrorOnTiFlash, + expectedStores: map[uint64]int{1: 1}, }, { stores: []*metapb.Store{ {Id: 1}, {Id: 2, Labels: []*metapb.StoreLabel{{Key: "engine", Value: "tiflash"}}}, }, - unexpectedStoreBehavior: SkipTiFlash, - expectedStores: map[uint64]int{1: 1}, + storeBehavior: SkipTiFlash, + expectedStores: map[uint64]int{1: 1}, }, { stores: []*metapb.Store{ {Id: 1}, {Id: 2, Labels: []*metapb.StoreLabel{{Key: "engine", Value: "tiflash"}}}, }, - unexpectedStoreBehavior: ErrorOnTiFlash, - expectedError: "cannot restore to a cluster with active TiFlash stores.*", + storeBehavior: ErrorOnTiFlash, + expectedError: "cannot restore to a cluster with active TiFlash stores.*", }, { stores: []*metapb.Store{ @@ -206,8 +206,8 @@ func (s *testClientSuite) TestGetAllTiKVStores(c *C) { {Id: 5, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tikv"}, {Key: "engine", Value: "tiflash"}}}, {Id: 6, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tiflash"}, {Key: "engine", Value: "tikv"}}}, }, - unexpectedStoreBehavior: SkipTiFlash, - expectedStores: map[uint64]int{1: 1, 3: 1, 4: 1, 6: 1}, + storeBehavior: SkipTiFlash, + expectedStores: map[uint64]int{1: 1, 3: 1, 4: 1, 6: 1}, }, { stores: []*metapb.Store{ @@ -218,8 +218,8 @@ func (s *testClientSuite) TestGetAllTiKVStores(c *C) { {Id: 5, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tikv"}, {Key: "engine", Value: "tiflash"}}}, {Id: 6, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tiflash"}, {Key: "engine", Value: "tikv"}}}, }, - unexpectedStoreBehavior: ErrorOnTiFlash, - expectedError: "cannot restore to a cluster with active TiFlash stores.*", + storeBehavior: ErrorOnTiFlash, + expectedError: "cannot restore to a cluster with active TiFlash stores.*", }, { stores: []*metapb.Store{ @@ -230,14 +230,14 @@ func (s *testClientSuite) TestGetAllTiKVStores(c *C) { {Id: 5, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tikv"}, {Key: "engine", Value: "tiflash"}}}, {Id: 6, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tiflash"}, {Key: "engine", Value: "tikv"}}}, }, - unexpectedStoreBehavior: TiFlashOnly, - expectedStores: map[uint64]int{2: 1, 5: 1}, + storeBehavior: TiFlashOnly, + expectedStores: map[uint64]int{2: 1, 5: 1}, }, } for _, testCase := range testCases { pdClient := fakePDClient{stores: testCase.stores} - stores, err := GetAllTiKVStores(context.Background(), pdClient, testCase.unexpectedStoreBehavior) + stores, err := GetAllTiKVStores(context.Background(), pdClient, testCase.storeBehavior) if len(testCase.expectedError) != 0 { c.Assert(err, ErrorMatches, testCase.expectedError) continue diff --git a/pkg/restore/client.go b/pkg/restore/client.go index 47dfd1462..2453f2974 100644 --- a/pkg/restore/client.go +++ b/pkg/restore/client.go @@ -340,9 +340,11 @@ func (rc *Client) CreateTables( // TODO: remove this after tiflash supports restore func (rc *Client) RemoveTiFlashReplica(tables []*utils.Table, placementRules []placement.Rule) error { schemas := make([]*backup.Schema, 0, len(tables)) + var updateReplica bool for _, table := range tables { if rule := utils.SearchPlacementRule(table.Info.ID, placementRules, placement.Learner); rule != nil { table.TiFlashReplicas = rule.Count + updateReplica = true } tableData, err := json.Marshal(table.Info) if err != nil { @@ -362,22 +364,24 @@ func (rc *Client) RemoveTiFlashReplica(tables []*utils.Table, placementRules []p }) } - // Update backup meta - rc.backupMeta.Schemas = schemas - backupMetaData, err := proto.Marshal(rc.backupMeta) - if err != nil { - return errors.Trace(err) - } - backendURL := storage.FormatBackendURL(rc.backend) - log.Info("update backup meta", zap.Stringer("path", &backendURL)) - err = rc.storage.Write(rc.ctx, utils.MetaFile, backupMetaData) - if err != nil { - return errors.Trace(err) + if updateReplica { + // Update backup meta + rc.backupMeta.Schemas = schemas + backupMetaData, err := proto.Marshal(rc.backupMeta) + if err != nil { + return errors.Trace(err) + } + backendURL := storage.FormatBackendURL(rc.backend) + log.Info("update backup meta", zap.Stringer("path", &backendURL)) + err = rc.storage.Write(rc.ctx, utils.SavedMetaFile, backupMetaData) + if err != nil { + return errors.Trace(err) + } } for _, table := range tables { if table.TiFlashReplicas > 0 { - err = rc.db.AlterTiflashReplica(rc.ctx, table, 0) + err := rc.db.AlterTiflashReplica(rc.ctx, table, 0) if err != nil { return errors.Trace(err) } diff --git a/pkg/task/common.go b/pkg/task/common.go index 859c4206d..61186abe1 100644 --- a/pkg/task/common.go +++ b/pkg/task/common.go @@ -213,7 +213,7 @@ func newMgr( g glue.Glue, pds []string, tlsConfig TLSConfig, - unexpectedStoreBehavior conn.UnexpectedStoreBehavior, + storeBehavior conn.StoreBehavior, ) (*conn.Mgr, error) { var ( tlsConf *tls.Config @@ -240,7 +240,7 @@ func newMgr( if err != nil { return nil, err } - return conn.NewMgr(ctx, g, pdAddress, store.(tikv.Storage), tlsConf, securityOption, unexpectedStoreBehavior) + return conn.NewMgr(ctx, g, pdAddress, store.(tikv.Storage), tlsConf, securityOption, storeBehavior) } // GetStorage gets the storage backend from the config. @@ -262,13 +262,14 @@ func GetStorage( // ReadBackupMeta reads the backupmeta file from the storage. func ReadBackupMeta( ctx context.Context, + fileName string, cfg *Config, ) (*backup.StorageBackend, storage.ExternalStorage, *backup.BackupMeta, error) { u, s, err := GetStorage(ctx, cfg) if err != nil { return nil, nil, nil, err } - metaData, err := s.Read(ctx, utils.MetaFile) + metaData, err := s.Read(ctx, fileName) if err != nil { return nil, nil, nil, errors.Annotate(err, "load backupmeta failed") } diff --git a/pkg/task/restore.go b/pkg/task/restore.go index e738544d8..5646746a1 100644 --- a/pkg/task/restore.go +++ b/pkg/task/restore.go @@ -107,7 +107,7 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf defer summary.Summary(cmdName) - u, _, backupMeta, err := ReadBackupMeta(ctx, &cfg.Config) + u, _, backupMeta, err := ReadBackupMeta(ctx, utils.MetaFile, &cfg.Config) if err != nil { return err } @@ -382,8 +382,8 @@ func RunRestoreTiflashReplica(c context.Context, g glue.Glue, cmdName string, cf } defer mgr.Close() - // Load backupmeta - _, _, backupMeta, err := ReadBackupMeta(c, &cfg.Config) + // Load saved backupmeta + _, _, backupMeta, err := ReadBackupMeta(ctx, utils.SavedMetaFile, &cfg.Config) if err != nil { return err } diff --git a/pkg/task/restore_raw.go b/pkg/task/restore_raw.go index 8511003a1..3d60df1cd 100644 --- a/pkg/task/restore_raw.go +++ b/pkg/task/restore_raw.go @@ -69,7 +69,7 @@ func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreR defer summary.Summary(cmdName) - u, _, backupMeta, err := ReadBackupMeta(ctx, &cfg.Config) + u, _, backupMeta, err := ReadBackupMeta(ctx, utils.MetaFile, &cfg.Config) if err != nil { return err } diff --git a/pkg/utils/pd.go b/pkg/utils/pd.go index 33acb01d2..7a65a2ac4 100644 --- a/pkg/utils/pd.go +++ b/pkg/utils/pd.go @@ -13,9 +13,9 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/pd/v4/pkg/codec" "github.com/pingcap/pd/v4/server/schedule/placement" "github.com/pingcap/tidb/tablecodec" - "github.com/tikv/client-go/codec" ) const ( @@ -45,7 +45,7 @@ func ResetTS(pdAddr string, ts uint64, tlsConf *tls.Config) error { return errors.Trace(err) } defer resp.Body.Close() - if resp.StatusCode != 200 && resp.StatusCode != 403 { + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusForbidden { buf := new(bytes.Buffer) _, _ = buf.ReadFrom(resp.Body) return errors.Errorf("pd resets TS failed: req=%v, resp=%v, err=%v", string(req), buf.String(), err) @@ -74,10 +74,10 @@ func GetPlacementRules(pdAddr string, tlsConf *tls.Config) ([]placement.Rule, er if err != nil { return nil, errors.Trace(err) } - if resp.StatusCode == 412 { + if resp.StatusCode == http.StatusPreconditionFailed { return []placement.Rule{}, nil } - if resp.StatusCode != 200 { + if resp.StatusCode != http.StatusOK { return nil, errors.Errorf("get placement rules failed: resp=%v, err=%v, code=%d", buf.String(), err, resp.StatusCode) } var rules []placement.Rule diff --git a/pkg/utils/schema.go b/pkg/utils/schema.go index d18f84399..5ac439e36 100644 --- a/pkg/utils/schema.go +++ b/pkg/utils/schema.go @@ -18,6 +18,8 @@ const ( MetaFile = "backupmeta" // MetaJSONFile represents backup meta json file name MetaJSONFile = "backupmeta.json" + // SavedMetaFile represents saved meta file name for recovering later + SavedMetaFile = "backupmeta.bak" ) // Table wraps the schema and files of a table. From 338c4cfb8702f425f0bf6d7c76732eb7c72d8627 Mon Sep 17 00:00:00 2001 From: 5kbpers <20279863+5kbpers@users.noreply.github.com> Date: Wed, 18 Mar 2020 12:31:56 +0800 Subject: [PATCH 14/14] Update pkg/task/restore.go Co-Authored-By: 3pointer --- pkg/task/restore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/task/restore.go b/pkg/task/restore.go index 5646746a1..a4598baa9 100644 --- a/pkg/task/restore.go +++ b/pkg/task/restore.go @@ -401,7 +401,7 @@ func RunRestoreTiflashReplica(c context.Context, g glue.Glue, cmdName string, cf tables = append(tables, db.Tables...) } updateCh := utils.StartProgress( - ctx, "Checksum", int64(len(tables)), !cfg.LogProgress) + ctx, "RecoverTiflashReplica", int64(len(tables)), !cfg.LogProgress) for _, t := range tables { log.Info("get table", zap.Stringer("name", t.Info.Name), zap.Int("replica", t.TiFlashReplicas))