diff: add integration test

Makefile

@@ -50,6 +50,16 @@ test:
 	@export log_level=error; \
 	$(GOTEST) -cover $(PACKAGES)
 
+integration_test: build
+	@which bin/tidb-server
+	@which bin/tikv-server
+	@which bin/pd-server
+	@which bin/sync_diff_inspector
+	@which bin/mydumper
+	@which bin/loader
+	@which bin/importer
+	tests/run.sh
+
 fmt:
 	go fmt ./...
 	@goimports -w $(FILES)

tests/README.md

@@ -0,0 +1,36 @@
+
+
+This folder contains all tests which relies on external service such as TiDB.
+
+## Preparations
+
+1. The following three executables must be copied or linked into these locations:
+
+   - `bin/pd-server`
+   - `bin/tikv-server`
+   - `bin/tidb-server`
+   - `bin/sync_diff_inspector`
+   - `bin/mydumper`
+   - `bin/loader`
+   - `bin/importer`
+
+2. The following programs must be installed:
+
+   - `mysql`(the CLI client)
+   - `mysqladmin`
+
+3. The user executing the tests must have permission to create the folder
+
+   `/tmp/tidb_tools_test`. All test artifacts will be written into this folder.
+
+## Running
+
+Run `make integration_test` to execute the integration tests. This command will
+
+1. Build binaries.
+2. Check that all executables exist.
+3. Execute `tests/run.sh`
+
+If the first two steps are done before, you could also run `tests/run.sh directly.
+
+The scrip will find out all `tests/*/run.sh` and run it.

tests/_utils/check_contains

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+# argument 1 is the string need grep
+# argument 2 is the filename
+
+set -eu
+OUT_DIR=/tmp/tidb_tools_test
+
+if ! grep -Fq "$1" "$2"; then
+    echo "TEST FAILED: '$2' DOES NOT CONTAIN '$1'"
+    echo "____________________________________"
+    cat "$2"
+    echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
+    exit 1
+fi

tests/_utils/check_db_status

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# argument 1 is the host
+# argument 2 is the port
+# argument 3 is the database service's name
+
+for i in {1..20}
+do
+    if mysqladmin -h "$1" -P "$2" -u root --default-character-set utf8 ping > /dev/null 2>&1
+    then
+        echo "$3 is alive"
+        exit 0
+    fi
+
+    echo "$3 is not alive, will try again"
+    sleep 2
+done
+
+echo "$3 is not alive"
+exit 2

tests/importer_integration_test.sh → tests/importer/run.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-source ./util.sh
-
 TEST_DATABASE_NAME=checker_test
 IMPORT_EXEC="../bin/importer -c 1 -h ${MYSQL_HOST} -P ${MYSQL_PORT} -D ${TEST_DATABASE_NAME}"
 MYSQL_EXEC="mysql -h ${MYSQL_HOST} -P ${MYSQL_PORT} -u root"

tests/run.sh

@@ -0,0 +1,99 @@
+#!/bin/sh
+
+set -eu
+
+OUT_DIR=/tmp/tidb_tools_test
+
+mkdir -p $OUT_DIR || true
+# to the dir of this script
+cd "$(dirname "$0")"
+
+pwd=$(pwd)
+
+export PATH=$PATH:$pwd/_utils
+export PATH=$PATH:$(dirname $pwd)/bin
+
+rm -rf $OUT_DIR || true
+
+stop_services() {
+    killall -9 tikv-server || true
+    killall -9 pd-server || true
+    killall -9 tidb-server || true
+}
+
+start_services() {
+    stop_services
+
+    echo "Starting PD..."
+    pd-server \
+        --client-urls http://127.0.0.1:2379 \
+        --log-file "$OUT_DIR/pd.log" \
+        --data-dir "$OUT_DIR/pd" &
+    # wait until PD is online...
+    while ! curl -o /dev/null -sf http://127.0.0.1:2379/pd/api/v1/version; do
+        sleep 1
+    done
+
+    # Tries to limit the max number of open files under the system limit
+    cat - > "$OUT_DIR/tikv-config.toml" <<EOF
+[rocksdb]
+max-open-files = 4096
+[raftdb]
+max-open-files = 4096
+[raftstore]
+# true (default value) for high reliability, this can prevent data loss when power failure.
+sync-log = false
+EOF
+
+    echo "Starting TiKV..."
+    tikv-server \
+        --pd 127.0.0.1:2379 \
+        -A 127.0.0.1:20160 \
+        --log-file "$OUT_DIR/tikv.log" \
+        -C "$OUT_DIR/tikv-config.toml" \
+        -s "$OUT_DIR/tikv" &
+    sleep 2
+
+    echo "Starting TiDB..."
+    tidb-server \
+        -P 4000 \
+        --store tikv \
+        --path 127.0.0.1:2379 \
+        --log-file "$OUT_DIR/tidb.log" &
+
+    echo "Verifying TiDB is started..."
+    check_db_status "127.0.0.1" 4000 "tidb"
+
+    echo "Starting Upstream TiDB..."
+    tidb-server \
+        -P 4001 \
+        --path=$OUT_DIR/tidb \
+        --status=20080 \
+        --log-file "$OUT_DIR/down_tidb.log" &
+
+    echo "Verifying Upstream TiDB is started..."
+    check_db_status "127.0.0.1" 4001 "tidb"
+}
+
+trap stop_services EXIT
+start_services
+
+# set to the case name you want to run only for debug
+do_case=""
+
+for script in ./*/run.sh; do
+    test_name="$(basename "$(dirname "$script")")"
+    if [[ $do_case != "" && $test_name != $do_case ]]; then
+        continue
+    fi
+    echo "*******************************************"
+    echo "Running test $script..."
+    echo "*******************************************"
+    PATH="$pwd/../bin:$pwd/_utils:$PATH" \
+    OUT_DIR=$OUT_DIR \
+    TEST_NAME=$test_name \
+    sh "$script"
+done
+
+# with color
+echo "\033[0;36m<<< Run all test success >>>\033[0m"

tests/sync_diff_inspector/config_base.toml

@@ -0,0 +1,50 @@
+# diff Configuration.
+
+log-level = "debug"
+
+# for example, the whole data is [1...100]
+# we can split these data to [1...10], [11...20], ..., [91...100]
+# the [1...10] is a chunk, and it's chunk size is 10
+# size of the split chunk
+chunk-size = 1000
+
+# how many goroutines are created to check data
+check-thread-count = 4
+
+# sampling check percent, for example 10 means only check 10% data
+sample-percent = 100
+
+# calculate the data's checksum, and compare data by checksum.
+# set false if want to comapre the data directly
+use-checksum = true
+
+# set true will continue check from the latest checkpoint
+use-checkpoint = false
+
+# the name of the file which saves sqls used to fix different data.
+fix-sql-file = "/tmp/tidb_tools_test/sync_diff_inspector/fix.sql"
+
+# use this tidb's statistics information to split chunk
+tidb-instance-id = "target-1"
+
+# tables need to check.
+[[check-tables]]
+# schema name in target database.
+schema = "diff_test"
+
+# table list which need check in target database.
+tables = ["test"]
+
+[[source-db]]
+host = "127.0.0.1"
+port = 4001
+user = "root"
+password = ""
+instance-id = "source-1"
+
+[target-db]
+host = "127.0.0.1"
+port = 4000
+user = "root"
+password = ""
+instance-id = "target-1"

tests/sync_diff_inspector/run.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+
+OUT_DIR=/tmp/tidb_tools_test/sync_diff_inspector
+
+mkdir $OUT_DIR || true
+
+echo "use importer to generate test data"
+mysql -uroot -h 127.0.0.1 -P 4000 -e "create database if not exists diff_test"
+importer -t "create table diff_test.test(a int, b varchar(10), c float, d datetime, primary key(a));" -c 10 -n 10000 -P 4000 -h 127.0.0.1 -D diff_test -b 1000
+
+echo "dump data and then load to tidb"
+mydumper --host 127.0.0.1 --port 4000 --user root --outputdir $OUT_DIR/dump_diff -B diff_test -T test
+
+loader -h 127.0.0.1 -P 4001 -u root -d $OUT_DIR/dump_diff
+
+echo "use sync_diff_inspector to compare data"
+
+sync_diff_inspector --config=./config_base.toml > $OUT_DIR/diff.log
+
+check_contains "test pass!!!" $OUT_DIR/diff.log
+
+for script in ./*/run.sh; do
+    test_name="$(basename "$(dirname "$script")")"
+    echo "---------------------------------------"
+    echo "Running test $script..."
+    echo "---------------------------------------"
+    cp config_base.toml $test_name/
+    sh "$script"
+done

tests/sync_diff_inspector/shard/config.toml

@@ -0,0 +1,69 @@
+# diff Configuration.
+
+log-level = "debug"
+
+# for example, the whole data is [1...100]
+# we can split these data to [1...10], [11...20], ..., [91...100]
+# the [1...10] is a chunk, and it's chunk size is 10
+# size of the split chunk
+chunk-size = 1000
+
+# how many goroutines are created to check data
+check-thread-count = 4
+
+# sampling check percent, for example 10 means only check 10% data
+sample-percent = 100
+
+# calculate the data's checksum, and compare data by checksum.
+use-checksum = true
+
+# the name of the file which saves sqls used to fix different data
+fix-sql-file = "/tmp/tidb_tools_test/sync_diff_inspector/fix.sql"
+
+# tables need to check.
+[[check-tables]]
+# schema name in target database.
+schema = "diff_test"
+
+# table list which need check in target database. 
+# in sharding mode, you must set config for every table in table-config, otherwise will not check the table.
+tables = ["test"]
+
+
+# schema and table in check-tables must be contained in check-tables.
+# a example for sharding tables.
+[[table-config]]
+# target schema name.
+schema = "diff_test"
+
+# target table name.
+table = "test"
+
+# set true if comparing sharding tables with target table
+is-sharding = true
+
+# source tables.
+[[table-config.source-tables]]
+instance-id = "source-1"
+schema = "diff_test"
+table  = "shard_test1"
+
+[[table-config.source-tables]]
+instance-id = "source-1"
+schema = "diff_test"
+table  = "shard_test2"
+
+
+[[source-db]]
+host = "127.0.0.1"
+port = 4001
+user = "root"
+password = ""
+instance-id = "source-1"
+
+[target-db]
+host = "127.0.0.1"
+port = 4000
+user = "root"
+password = ""
+instance-id = "target-1"

tests/sync_diff_inspector/shard/run.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+
+OUT_DIR=/tmp/tidb_tools_test/sync_diff_inspector
+
+echo "generate data to sharding tables"
+mysql -uroot -h 127.0.0.1 -P 4001 -e "create table diff_test.shard_test1(a int, b varchar(10), c float, d datetime, primary key(a));"
+mysql -uroot -h 127.0.0.1 -P 4001 -e "create table diff_test.shard_test2(a int, b varchar(10), c float, d datetime, primary key(a));"
+
+# each table only have part of data
+mysql -uroot -h 127.0.0.1 -P 4001 -e "insert into diff_test.shard_test1 (a, b, c, d) SELECT a, b, c, d FROM diff_test.test WHERE a%2=0"
+mysql -uroot -h 127.0.0.1 -P 4001 -e "insert into diff_test.shard_test2 (a, b, c, d) SELECT a, b, c, d FROM diff_test.test WHERE a%2=1"
+
+
+echo "compare sharding tables with one table in downstream, check result should be pass"
+sync_diff_inspector --config=./config.toml > $OUT_DIR/shard_diff.log
+check_contains "test pass!!!" $OUT_DIR/shard_diff.log
+
+echo "update data in one shard table, and data should not be equal"
+mysql -uroot -h 127.0.0.1 -P 4001 -e "update diff_test.shard_test1 set b = 'abc' limit 1"
+sync_diff_inspector --config=./config.toml > $OUT_DIR/shard_diff.log || true
+check_contains "sourceDB don't equal targetDB" $OUT_DIR/shard_diff.log
+
+echo "shard test passed"