retry when pd is not boostrapped (pingcap#43)

Signed-off-by: David <[email protected]> Signed-off-by: David <[email protected]>
iosmanthus · Sep 5, 2022 · 53a5c96 · 53a5c96
1 parent 910a86e
commit 53a5c96
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 2 deletions.
diff --git a/domain/infosync/info.go b/domain/infosync/info.go
@@ -32,6 +32,7 @@ import (
 	"github.com/coreos/go-semver/semver"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/failpoint"
+	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/tidb/config"
 	"github.com/pingcap/tidb/ddl/label"
 	"github.com/pingcap/tidb/ddl/placement"
@@ -44,6 +45,7 @@ import (
 	"github.com/pingcap/tidb/parser/terror"
 	"github.com/pingcap/tidb/sessionctx/binloginfo"
 	"github.com/pingcap/tidb/sessionctx/variable"
+	store2 "github.com/pingcap/tidb/store"
 	"github.com/pingcap/tidb/store/helper"
 	"github.com/pingcap/tidb/types"
 	util2 "github.com/pingcap/tidb/util"
@@ -446,7 +448,14 @@ func removeVAndHash(v string) string {
 func CheckTiKVVersion(store kv.Storage, minVersion semver.Version) error {
 	if store, ok := store.(kv.StorageWithPD); ok {
 		pdClient := store.GetPDClient()
-		stores, err := pdClient.GetAllStores(context.Background(), pd.WithExcludeTombstone())
+
+		var stores []*metapb.Store
+		var err error
+		err = util2.RunWithRetry(util2.DefaultMaxRetries, util2.RetryInterval, func() (bool, error) {
+			stores, err = pdClient.GetAllStores(context.Background(), pd.WithExcludeTombstone())
+			return store2.IsNotBootstrappedError(err), err
+		})
+
 		if err != nil {
 			return err
 		}

diff --git a/store/store.go b/store/store.go
@@ -20,6 +20,7 @@ import (
 	"sync"
 
 	"github.com/pingcap/errors"
+	"github.com/pingcap/kvproto/pkg/pdpb"
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/util"
 	"github.com/pingcap/tidb/util/logutil"
@@ -73,7 +74,7 @@ func newStoreWithRetry(path string, maxRetries int) (kv.Storage, error) {
 	err = util.RunWithRetry(maxRetries, util.RetryInterval, func() (bool, error) {
 		logutil.BgLogger().Info("new store", zap.String("path", path))
 		s, err = d.Open(path)
-		return kv.IsTxnRetryableError(err), err
+		return kv.IsTxnRetryableError(err) || IsNotBootstrappedError(err), err
 	})
 
 	if err == nil {
@@ -90,3 +91,11 @@ func loadDriver(name string) (kv.Driver, bool) {
 	d, ok := stores[name]
 	return d, ok
 }
+
+// IsNotBootstrappedError returns true if the error is pd not bootstrapped error.
+func IsNotBootstrappedError(err error) bool {
+	if err == nil {
+		return false
+	}
+	return strings.Contains(err.Error(), pdpb.ErrorType_NOT_BOOTSTRAPPED.String())
+}