diff --git a/Makefile b/Makefile index 4dbb16b49..e3624fcce 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,10 @@ testcover: tools failpoint-enable -debug \ -- -coverpkg=./... || ( make failpoint-disable && exit 1 ) -integration_test: build build_for_integration_test +integration_test: bins build build_for_integration_test + tests/run.sh + +bins: @which bin/tidb-server @which bin/tikv-server @which bin/pd-server @@ -52,7 +55,9 @@ integration_test: build build_for_integration_test @which bin/go-ycsb @which bin/minio @which bin/br - tests/run.sh + @which bin/tiflash + @which bin/libtiflash_proxy.so + if [ ! -d bin/flash_cluster_manager ]; then echo "flash_cluster_manager not exist"; exit 1; fi tools: @echo "install tools..." diff --git a/pkg/conn/conn.go b/pkg/conn/conn.go index e9802a673..c604cd7b5 100644 --- a/pkg/conn/conn.go +++ b/pkg/conn/conn.go @@ -321,9 +321,13 @@ func (mgr *Mgr) getGrpcConnLocked(ctx context.Context, storeID uint64) (*grpc.Cl keepAliveTimeout := 3 bfConf := backoff.DefaultConfig bfConf.MaxDelay = time.Second * 3 + addr := store.GetPeerAddress() + if addr == "" { + addr = store.GetAddress() + } conn, err := grpc.DialContext( ctx, - store.GetAddress(), + addr, opt, grpc.WithConnectParams(grpc.ConnectParams{Backoff: bfConf}), grpc.WithKeepaliveParams(keepalive.ClientParameters{ diff --git a/pkg/restore/import.go b/pkg/restore/import.go index 66d8a29a9..f3d1ebaa7 100644 --- a/pkg/restore/import.go +++ b/pkg/restore/import.go @@ -119,7 +119,11 @@ func (ic *importClient) getImportClient( if ic.tlsConf != nil { opt = grpc.WithTransportCredentials(credentials.NewTLS(ic.tlsConf)) } - conn, err := grpc.Dial(store.GetAddress(), opt) + addr := store.GetPeerAddress() + if addr == "" { + addr = store.GetAddress() + } + conn, err := grpc.Dial(addr, opt) if err != nil { return nil, err } diff --git a/pkg/task/common.go b/pkg/task/common.go index 90617721a..27340670c 100644 --- a/pkg/task/common.go +++ b/pkg/task/common.go @@ -49,6 +49,7 @@ const ( flagRateLimitUnit = "ratelimit-unit" flagConcurrency = "concurrency" flagChecksum = "checksum" + flagRemoveTiFlash = "remove-tiflash" flagCheckRequirement = "check-requirements" ) @@ -92,6 +93,7 @@ type Config struct { // LogProgress is true means the progress bar is printed to the log instead of stdout. LogProgress bool `json:"log-progress" toml:"log-progress"` + RemoveTiFlash bool `json:"remove-tiflash" toml:"remove-tiflash"` CaseSensitive bool `json:"case-sensitive" toml:"case-sensitive"` CheckRequirements bool `json:"check-requirements" toml:"check-requirements"` Filter filter.Rules `json:"black-white-list" toml:"black-white-list"` @@ -108,6 +110,8 @@ func DefineCommonFlags(flags *pflag.FlagSet) { flags.Uint64(flagRateLimit, 0, "The rate limit of the task, MB/s per node") flags.Bool(flagChecksum, true, "Run checksum at end of task") + flags.Bool(flagRemoveTiFlash, true, + "Remove TiFlash replicas before backup or restore, for unsupported versions of TiFlash") // Default concurrency is different for backup and restore. // Leave it 0 and let them adjust the value. @@ -193,6 +197,11 @@ func (cfg *Config) ParseFromFlags(flags *pflag.FlagSet) error { } cfg.RateLimit = rateLimit * rateLimitUnit + cfg.RemoveTiFlash, err = flags.GetBool(flagRemoveTiFlash) + if err != nil { + return errors.Trace(err) + } + if dbFlag := flags.Lookup(flagDatabase); dbFlag != nil { db := escapeFilterName(dbFlag.Value.String()) if len(db) == 0 { diff --git a/pkg/task/restore.go b/pkg/task/restore.go index db424452f..eb624565e 100644 --- a/pkg/task/restore.go +++ b/pkg/task/restore.go @@ -82,6 +82,10 @@ func (cfg *RestoreConfig) ParseFromFlags(flags *pflag.FlagSet) error { if err != nil { return errors.Trace(err) } + cfg.RemoveTiFlash, err = flags.GetBool(flagRemoveTiFlash) + if err != nil { + return errors.Trace(err) + } err = cfg.Config.ParseFromFlags(flags) if err != nil { return errors.Trace(err) @@ -195,20 +199,6 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf return nil } - placementRules, err := client.GetPlacementRules(cfg.PD) - if err != nil { - return err - } - - err = client.RemoveTiFlashReplica(tables, newTables, placementRules) - if err != nil { - return err - } - - defer func() { - _ = client.RecoverTiFlashReplica(tables) - }() - ranges, err := restore.ValidateFileRanges(files, rewriteRules) if err != nil { return err @@ -255,19 +245,31 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf } // Restore sst files in batch. - batchSize := int(cfg.Concurrency) - if batchSize < defaultRestoreConcurrency { - batchSize = defaultRestoreConcurrency - } - batchSize = utils.MinInt(batchSize, maxRestoreBatchSizeLimit) + batchSize := utils.ClampInt(int(cfg.Concurrency), defaultRestoreConcurrency, maxRestoreBatchSizeLimit) - tiflashStores, err := conn.GetAllTiKVStores(ctx, client.GetPDClient(), conn.TiFlashOnly) - if err != nil { - return errors.Trace(err) - } rejectStoreMap := make(map[uint64]bool) - for _, store := range tiflashStores { - rejectStoreMap[store.GetId()] = true + if cfg.RemoveTiFlash { + placementRules, err := client.GetPlacementRules(cfg.PD) + if err != nil { + return err + } + + err = client.RemoveTiFlashReplica(tables, newTables, placementRules) + if err != nil { + return err + } + + defer func() { + _ = client.RecoverTiFlashReplica(tables) + }() + + tiflashStores, err := conn.GetAllTiKVStores(ctx, client.GetPDClient(), conn.TiFlashOnly) + if err != nil { + return errors.Trace(err) + } + for _, store := range tiflashStores { + rejectStoreMap[store.GetId()] = true + } } for { diff --git a/pkg/utils/math.go b/pkg/utils/math.go index 00c8dcc4b..42615ff48 100644 --- a/pkg/utils/math.go +++ b/pkg/utils/math.go @@ -2,11 +2,38 @@ package utils -// MinInt choice smaller integer from its two arguments. -func MinInt(x, y int) int { - if x < y { - return x +import ( + "github.com/pingcap/log" + "go.uber.org/zap" +) + +// MinInt choice smallest integer from its arguments. +func MinInt(x int, xs ...int) int { + min := x + for _, n := range xs { + if n < min { + min = n + } + } + return min +} + +// MaxInt choice biggest integer from its arguments. +func MaxInt(x int, xs ...int) int { + max := x + for _, n := range xs { + if n > max { + max = n + } + } + return max +} + +// ClampInt restrict a value to a certain interval. +func ClampInt(n, min, max int) int { + if min > max { + log.Error("clamping integer with min > max", zap.Int("min", min), zap.Int("max", max)) } - return y + return MinInt(max, MaxInt(min, n)) } diff --git a/pkg/utils/math_test.go b/pkg/utils/math_test.go index 90c8e6bed..91c3300ad 100644 --- a/pkg/utils/math_test.go +++ b/pkg/utils/math_test.go @@ -13,5 +13,21 @@ var _ = Suite(&testMathSuite{}) func (*testMathSuite) TestMinInt(c *C) { c.Assert(MinInt(1, 2), Equals, 1) c.Assert(MinInt(2, 1), Equals, 1) + c.Assert(MinInt(4, 2, 1, 3), Equals, 1) c.Assert(MinInt(1, 1), Equals, 1) } + +func (*testMathSuite) TestMaxInt(c *C) { + c.Assert(MaxInt(1, 2), Equals, 2) + c.Assert(MaxInt(2, 1), Equals, 2) + c.Assert(MaxInt(4, 2, 1, 3), Equals, 4) + c.Assert(MaxInt(1, 1), Equals, 1) +} + +func (*testMathSuite) TestClampInt(c *C) { + c.Assert(ClampInt(100, 1, 3), Equals, 3) + c.Assert(ClampInt(2, 1, 3), Equals, 2) + c.Assert(ClampInt(0, 1, 3), Equals, 1) + c.Assert(ClampInt(0, 1, 1), Equals, 1) + c.Assert(ClampInt(100, 1, 1), Equals, 1) +} diff --git a/tests/README.md b/tests/README.md index fbb018505..c0a34c014 100644 --- a/tests/README.md +++ b/tests/README.md @@ -5,16 +5,19 @@ programs. ## Preparations -1. The following 6 executables must be copied or linked into these locations: +1. The following 7 executables must be copied or linked into these locations: * `bin/tidb-server` * `bin/tikv-server` * `bin/pd-server` * `bin/pd-ctl` * `bin/go-ycsb` * `bin/minio` + * `bin/tiflash` The versions must be ≥2.1.0 as usual. + What's more, there must be dynamic link library for TiFlash, see make target `bin` to learn more. You can install most of dependencies by running `download_tools.sh`. + 2. The following programs must be installed: * `mysql` (the CLI client) @@ -31,7 +34,7 @@ Make sure the path is `br/` Run `make integration_test` to execute the integration tests. This command will 1. Build `br` -2. Check that all 6 required executables and `br` executable exist +2. Check that all 7 required executables and `br` executable exist 3. Execute `tests/run.sh` If the first two steps are done before, you could also run `tests/run.sh` directly. diff --git a/tests/_utils/make_tiflash_config b/tests/_utils/make_tiflash_config new file mode 100755 index 000000000..c235f0444 --- /dev/null +++ b/tests/_utils/make_tiflash_config @@ -0,0 +1,89 @@ +#!/bin/sh + +cat > tests/config/tiflash-learner.toml < tests/config/tiflash.toml </dev/null 2>&1; do + i=$((i+1)) + if [ "$i" -gt 20 ]; then + echo "failed to start tiflash" + exit 1 + fi + echo "TiFlash seems doesn't started, retrying..." + sleep 3 + done +} + start_services_withTLS() { stop_services diff --git a/tests/br_rawkv/run.sh b/tests/br_rawkv/run.sh index f57e76827..3746552b0 100644 --- a/tests/br_rawkv/run.sh +++ b/tests/br_rawkv/run.sh @@ -15,6 +15,10 @@ set -eu +# restart service without tiflash +source $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../_utils/run_services +start_services "tests" --no-tiflash + BACKUP_DIR="raw_backup" checksum() { diff --git a/tests/br_tiflash/run.sh b/tests/br_tiflash/run.sh new file mode 100644 index 000000000..aa307ce91 --- /dev/null +++ b/tests/br_tiflash/run.sh @@ -0,0 +1,66 @@ +#!/bin/sh +# +# Copyright 2020 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu +DB="${TEST_NAME}_DATABASE" +RECORD_COUNT=1000 + + +run_sql "CREATE DATABASE $DB" + +run_sql "CREATE TABLE $DB.kv(k varchar(256) primary key, v int)" +run_sql "ALTER TABLE $DB.kv SET TIFLASH REPLICA 1" + +stmt="INSERT INTO $DB.kv(k, v) VALUES ('1-record', 1)" +for i in $(seq 2 $RECORD_COUNT); do + stmt="$stmt,('$i-record', $i)" +done +run_sql "$stmt" + +i=0 +while ! [ $(run_sql "select * from information_schema.tiflash_replica" | grep "PROGRESS" | sed "s/[^0-9]//g") -eq 1 ]; do + i=$(( i + 1 )) + echo "Waiting for TiFlash synchronizing [$i]." + if [ $i -gt 20 ]; then + echo "Failed to sync data to tiflash." + exit 1 + fi + sleep 5 +done + +rm -rf "/${TEST_DIR}/$DB" +run_br backup full -s "local://$TEST_DIR/$DB" --pd $PD_ADDR + +run_sql "DROP DATABASE $DB" +run_br restore full -s "local://$TEST_DIR/$DB" --pd $PD_ADDR + +AFTER_BR_COUNT=`run_sql "SELECT count(*) FROM $DB.kv;" | sed -n "s/[^0-9]//g;/^[0-9]*$/p" | tail -n1` +if [ $AFTER_BR_COUNT -ne $RECORD_COUNT ]; then + echo "failed to restore, before: $RECORD_COUNT; after: $AFTER_BR_COUNT" + exit 1 +fi + +# backup again, but don't remove tiflash replicas. +run_br backup full -s "local://$TEST_DIR/$DB/with-tiflash" --pd $PD_ADDR --remove-tiflash=false +run_sql "DROP DATABASE $DB" +run_br restore full -s "local://$TEST_DIR/$DB/with-tiflash" --pd $PD_ADDR --remove-tiflash=false + +AFTER_BR_COUNT=`run_sql "SELECT count(*) FROM $DB.kv;" | sed -n "s/[^0-9]//g;/^[0-9]*$/p" | tail -n1` +if [ $AFTER_BR_COUNT -ne $RECORD_COUNT ]; then + echo "failed to restore, before: $RECORD_COUNT; after: $AFTER_BR_COUNT" + exit 1 +fi + +echo "TEST $TEST_NAME passed!" \ No newline at end of file diff --git a/tests/config/pd.toml b/tests/config/pd.toml new file mode 100644 index 000000000..7399ac9e9 --- /dev/null +++ b/tests/config/pd.toml @@ -0,0 +1,5 @@ +lease = 360 +tso-save-interval = "360s" + +[replication] +enable-placement-rules = true diff --git a/tests/download_tools.sh b/tests/download_tools.sh index fb8a71604..d4ec2f3d8 100755 --- a/tests/download_tools.sh +++ b/tests/download_tools.sh @@ -31,6 +31,18 @@ for COMPONENT in tidb-server pd-server tikv-server pd-ctl; do fi done +if [ ! -e "$BIN/tiflash" ]; then + echo "Downloading nightly Tiflash..." + curl -L -f -o "$BIN/tiflash.tar.gz" "https://download.pingcap.org/tiflash-nightly-linux-amd64.tar.gz" + tar -xf "$BIN/tiflash.tar.gz" -C "$BIN/" + rm "$BIN/tiflash.tar.gz" + mkdir "$BIN"/flash_cluster_manager + mv "$BIN"/tiflash-nightly-linux-amd64/flash_cluster_manager/* "$BIN/flash_cluster_manager" + rmdir "$BIN/"tiflash-nightly-linux-amd64/flash_cluster_manager + mv "$BIN"/tiflash-nightly-linux-amd64/* "$BIN/" + rmdir "$BIN/"tiflash-nightly-linux-amd64 +fi + if [ -n "$MISSING_TIDB_COMPONENTS" ]; then echo "Downloading latest TiDB bundle..." curl -L -f -o "$BIN/tidb.tar.gz" "https://download.pingcap.org/tidb-nightly-linux-amd64.tar.gz"