diff --git a/Makefile b/Makefile index 266a7076c..2ecde9627 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,16 @@ test: @export log_level=error; \ $(GOTEST) -cover $(PACKAGES) +integration_test: build + @which bin/tidb-server + @which bin/tikv-server + @which bin/pd-server + @which bin/sync_diff_inspector + @which bin/mydumper + @which bin/loader + @which bin/importer + tests/run.sh + fmt: go fmt ./... @goimports -w $(FILES) diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..63b0b2367 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,36 @@ + + +This folder contains all tests which relies on external service such as TiDB. + +## Preparations + +1. The following seven executables must be copied or linked into these locations: + + - `bin/pd-server` + - `bin/tikv-server` + - `bin/tidb-server` + - `bin/sync_diff_inspector` + - `bin/mydumper` + - `bin/loader` + - `bin/importer` + +2. The following programs must be installed: + + - `mysql`(the CLI client) + - `mysqladmin` + +3. The user executing the tests must have permission to create the folder + + `/tmp/tidb_tools_test`. All test artifacts will be written into this folder. + +## Running + +Run `make integration_test` to execute the integration tests. This command will + +1. Build binaries. +2. Check that all executables exist. +3. Execute `tests/run.sh` + +If the first two steps are done before, you could also run `tests/run.sh` directly. + +The scrip will find out all `tests/*/run.sh` and run it. diff --git a/tests/_utils/check_contains b/tests/_utils/check_contains new file mode 100755 index 000000000..93e7970b7 --- /dev/null +++ b/tests/_utils/check_contains @@ -0,0 +1,15 @@ +#!/bin/sh + +# argument 1 is the string need grep +# argument 2 is the filename + +set -eu +OUT_DIR=/tmp/tidb_tools_test + +if ! grep -Fq "$1" "$2"; then + echo "TEST FAILED: '$2' DOES NOT CONTAIN '$1'" + echo "____________________________________" + cat "$2" + echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" + exit 1 +fi diff --git a/tests/_utils/check_db_status b/tests/_utils/check_db_status new file mode 100755 index 000000000..f3ab66ef5 --- /dev/null +++ b/tests/_utils/check_db_status @@ -0,0 +1,20 @@ +#!/bin/bash + +# argument 1 is the host +# argument 2 is the port +# argument 3 is the database service's name + +for i in {1..20} +do + if mysqladmin -h "$1" -P "$2" -u root --default-character-set utf8 ping > /dev/null 2>&1 + then + echo "$3 is alive" + exit 0 + fi + + echo "$3 is not alive, will try again" + sleep 2 +done + +echo "$3 is not alive" +exit 2 diff --git a/tests/importer_integration_test.sh b/tests/importer/run.sh similarity index 98% rename from tests/importer_integration_test.sh rename to tests/importer/run.sh index 8eeaff184..4a7ee182d 100644 --- a/tests/importer_integration_test.sh +++ b/tests/importer/run.sh @@ -1,7 +1,5 @@ #!/bin/bash -source ./util.sh - TEST_DATABASE_NAME=checker_test IMPORT_EXEC="../bin/importer -c 1 -h ${MYSQL_HOST} -P ${MYSQL_PORT} -D ${TEST_DATABASE_NAME}" MYSQL_EXEC="mysql -h ${MYSQL_HOST} -P ${MYSQL_PORT} -u root" diff --git a/tests/run.sh b/tests/run.sh new file mode 100755 index 000000000..388e65f58 --- /dev/null +++ b/tests/run.sh @@ -0,0 +1,99 @@ +#!/bin/sh + +set -eu + +OUT_DIR=/tmp/tidb_tools_test + +mkdir -p $OUT_DIR || true +# to the dir of this script +cd "$(dirname "$0")" + +pwd=$(pwd) + +export PATH=$PATH:$pwd/_utils +export PATH=$PATH:$(dirname $pwd)/bin + +rm -rf $OUT_DIR || true + +stop_services() { + killall -9 tikv-server || true + killall -9 pd-server || true + killall -9 tidb-server || true +} + +start_services() { + stop_services + + echo "Starting PD..." + pd-server \ + --client-urls http://127.0.0.1:2379 \ + --log-file "$OUT_DIR/pd.log" \ + --data-dir "$OUT_DIR/pd" & + # wait until PD is online... + while ! curl -o /dev/null -sf http://127.0.0.1:2379/pd/api/v1/version; do + sleep 1 + done + + # Tries to limit the max number of open files under the system limit + cat - > "$OUT_DIR/tikv-config.toml" <>>\033[0m" diff --git a/tests/sync_diff_inspector/config_base.toml b/tests/sync_diff_inspector/config_base.toml new file mode 100644 index 000000000..5051a19b4 --- /dev/null +++ b/tests/sync_diff_inspector/config_base.toml @@ -0,0 +1,50 @@ +# diff Configuration. + +log-level = "debug" + +# for example, the whole data is [1...100] +# we can split these data to [1...10], [11...20], ..., [91...100] +# the [1...10] is a chunk, and it's chunk size is 10 +# size of the split chunk +chunk-size = 1000 + +# how many goroutines are created to check data +check-thread-count = 4 + +# sampling check percent, for example 10 means only check 10% data +sample-percent = 100 + +# calculate the data's checksum, and compare data by checksum. +# set false if want to comapre the data directly +use-checksum = true + +# set true will continue check from the latest checkpoint +use-checkpoint = false + +# the name of the file which saves sqls used to fix different data. +fix-sql-file = "/tmp/tidb_tools_test/sync_diff_inspector/fix.sql" + +# use this tidb's statistics information to split chunk +tidb-instance-id = "target-1" + +# tables need to check. +[[check-tables]] +# schema name in target database. +schema = "diff_test" + +# table list which need check in target database. +tables = ["test"] + +[[source-db]] +host = "127.0.0.1" +port = 4001 +user = "root" +password = "" +instance-id = "source-1" + +[target-db] +host = "127.0.0.1" +port = 4000 +user = "root" +password = "" +instance-id = "target-1" diff --git a/tests/sync_diff_inspector/run.sh b/tests/sync_diff_inspector/run.sh new file mode 100644 index 000000000..000ba7a71 --- /dev/null +++ b/tests/sync_diff_inspector/run.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +set -e + +cd "$(dirname "$0")" + +OUT_DIR=/tmp/tidb_tools_test/sync_diff_inspector + +mkdir $OUT_DIR || true + +echo "use importer to generate test data" +mysql -uroot -h 127.0.0.1 -P 4000 -e "create database if not exists diff_test" +importer -t "create table diff_test.test(a int, b varchar(10), c float, d datetime, primary key(a));" -c 10 -n 10000 -P 4000 -h 127.0.0.1 -D diff_test -b 1000 + +echo "dump data and then load to tidb" +mydumper --host 127.0.0.1 --port 4000 --user root --outputdir $OUT_DIR/dump_diff -B diff_test -T test + +loader -h 127.0.0.1 -P 4001 -u root -d $OUT_DIR/dump_diff + +echo "use sync_diff_inspector to compare data" + +sync_diff_inspector --config=./config_base.toml > $OUT_DIR/diff.log + +check_contains "test pass!!!" $OUT_DIR/diff.log + +for script in ./*/run.sh; do + test_name="$(basename "$(dirname "$script")")" + echo "---------------------------------------" + echo "Running test $script..." + echo "---------------------------------------" + cp config_base.toml $test_name/ + sh "$script" +done \ No newline at end of file diff --git a/tests/sync_diff_inspector/shard/config.toml b/tests/sync_diff_inspector/shard/config.toml new file mode 100644 index 000000000..6eb6e8bf4 --- /dev/null +++ b/tests/sync_diff_inspector/shard/config.toml @@ -0,0 +1,69 @@ +# diff Configuration. + +log-level = "debug" + +# for example, the whole data is [1...100] +# we can split these data to [1...10], [11...20], ..., [91...100] +# the [1...10] is a chunk, and it's chunk size is 10 +# size of the split chunk +chunk-size = 1000 + +# how many goroutines are created to check data +check-thread-count = 4 + +# sampling check percent, for example 10 means only check 10% data +sample-percent = 100 + +# calculate the data's checksum, and compare data by checksum. +use-checksum = true + +# the name of the file which saves sqls used to fix different data +fix-sql-file = "/tmp/tidb_tools_test/sync_diff_inspector/fix.sql" + +# tables need to check. +[[check-tables]] +# schema name in target database. +schema = "diff_test" + +# table list which need check in target database. +# in sharding mode, you must set config for every table in table-config, otherwise will not check the table. +tables = ["test"] + + +# schema and table in check-tables must be contained in check-tables. +# a example for sharding tables. +[[table-config]] +# target schema name. +schema = "diff_test" + +# target table name. +table = "test" + +# set true if comparing sharding tables with target table +is-sharding = true + +# source tables. +[[table-config.source-tables]] +instance-id = "source-1" +schema = "diff_test" +table = "shard_test1" + +[[table-config.source-tables]] +instance-id = "source-1" +schema = "diff_test" +table = "shard_test2" + + +[[source-db]] +host = "127.0.0.1" +port = 4001 +user = "root" +password = "" +instance-id = "source-1" + +[target-db] +host = "127.0.0.1" +port = 4000 +user = "root" +password = "" +instance-id = "target-1" \ No newline at end of file diff --git a/tests/sync_diff_inspector/shard/run.sh b/tests/sync_diff_inspector/shard/run.sh new file mode 100644 index 000000000..7d3ce4f9f --- /dev/null +++ b/tests/sync_diff_inspector/shard/run.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +set -e + +cd "$(dirname "$0")" + +OUT_DIR=/tmp/tidb_tools_test/sync_diff_inspector + +echo "generate data to sharding tables" +mysql -uroot -h 127.0.0.1 -P 4001 -e "create table diff_test.shard_test1(a int, b varchar(10), c float, d datetime, primary key(a));" +mysql -uroot -h 127.0.0.1 -P 4001 -e "create table diff_test.shard_test2(a int, b varchar(10), c float, d datetime, primary key(a));" + +# each table only have part of data +mysql -uroot -h 127.0.0.1 -P 4001 -e "insert into diff_test.shard_test1 (a, b, c, d) SELECT a, b, c, d FROM diff_test.test WHERE a%2=0" +mysql -uroot -h 127.0.0.1 -P 4001 -e "insert into diff_test.shard_test2 (a, b, c, d) SELECT a, b, c, d FROM diff_test.test WHERE a%2=1" + + +echo "compare sharding tables with one table in downstream, check result should be pass" +sync_diff_inspector --config=./config.toml > $OUT_DIR/shard_diff.log +check_contains "test pass!!!" $OUT_DIR/shard_diff.log + +echo "update data in one shard table, and data should not be equal" +mysql -uroot -h 127.0.0.1 -P 4001 -e "update diff_test.shard_test1 set b = 'abc' limit 1" +sync_diff_inspector --config=./config.toml > $OUT_DIR/shard_diff.log || true +check_contains "sourceDB don't equal targetDB" $OUT_DIR/shard_diff.log + +echo "shard test passed" \ No newline at end of file diff --git a/tests/sync_diff_inspector/snapshot/run.sh b/tests/sync_diff_inspector/snapshot/run.sh new file mode 100644 index 000000000..fa8ee8112 --- /dev/null +++ b/tests/sync_diff_inspector/snapshot/run.sh @@ -0,0 +1,34 @@ + +#!/bin/sh + +set -e + +cd "$(dirname "$0")" + +OUT_DIR=/tmp/tidb_tools_test/sync_diff_inspector + +mysql -uroot -h 127.0.0.1 -P 4000 -e "show master status" > $OUT_DIR/ts.log +cat $OUT_DIR/ts.log +ts=`grep -oE "[0-9]+" $OUT_DIR/ts.log` +echo "get ts $ts" + +echo "delete one data, diff should not passed" +mysql -uroot -h 127.0.0.1 -P 4000 -e "delete from diff_test.test limit 1" + +sync_diff_inspector --config=./config_base.toml > $OUT_DIR/snapshot_diff.log || true +check_contains "sourceDB don't equal targetDB" $OUT_DIR/snapshot_diff.log +# fix.sql will be empty after check below, so backup it +cp $OUT_DIR/fix.sql $OUT_DIR/fix.sql.bak + +echo "use snapshot compare data, data should be equal" +cp config_base.toml config.toml +echo "snapshot = \"$ts\"" >> config.toml +sync_diff_inspector --config=./config.toml > $OUT_DIR/snapshot_diff.log +check_contains "test pass!!!" $OUT_DIR/snapshot_diff.log + +echo "execute fix.sql and use base config, and then compare data, data should be equal" +cat $OUT_DIR/fix.sql.bak | mysql -uroot -h127.0.0.1 -P 4000 +sync_diff_inspector --config=./config_base.toml > $OUT_DIR/snapshot_diff.log +check_contains "test pass!!!" $OUT_DIR/snapshot_diff.log + +echo "snapshot test passed" \ No newline at end of file diff --git a/tests/util.sh b/tests/util.sh deleted file mode 100755 index a5d7d2e05..000000000 --- a/tests/util.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -check_db_status() { - while true - do - if mysqladmin -h "$1" -P "$2" -u root ping > /dev/null 2>&1 - then - break - fi - sleep 1 - done - echo "$3 is alive" -} \ No newline at end of file