lightning: move tests out of br (#52289)

ref #52182
pingcap · Apr 2, 2024 · a57ebeb · a57ebeb
1 parent 76abd8b
commit a57ebeb
Show file tree

Hide file tree

Showing 853 changed files with 686 additions and 142 deletions.
diff --git a/.github/licenserc.yml b/.github/licenserc.yml
@@ -45,6 +45,7 @@ header:
     - "pkg/tidb-binlog/proto/go-binlog/secondary_binlog.pb.go"
     - "**/*.sql"
     - "**/*.csv"
+    - "**/*.CSV"
     - "**/*.parquet"
     - "**/*.zst"
     - ".bazelversion"
@@ -63,4 +64,5 @@ header:
     - "pkg/lightning/manual/manual_nocgo.go"
     - "pkg/lightning/mydump/bytes.go"
     - "pkg/lightning/mydump/examples/metadata"
+    - "tests/_utils/config/"
   comment: on-failure
diff --git a/Makefile b/Makefile
@@ -329,26 +329,35 @@ build_lightning:
 build_lightning-ctl:
 	CGO_ENABLED=1 $(GOBUILD) $(RACE_FLAG) -ldflags '$(LDFLAGS) $(CHECK_FLAG)' -o $(LIGHTNING_CTL_BIN) ./lightning/cmd/tidb-lightning-ctl
 
-build_for_br_integration_test:
+build_for_lightning_integration_test:
 	@make failpoint-enable
 	($(GOTEST) -c -cover -covermode=count \
-		-coverpkg=github.com/pingcap/tidb/br/... \
-		-o $(BR_BIN).test \
-		github.com/pingcap/tidb/br/cmd/br && \
-	$(GOTEST) -c -cover -covermode=count \
 		-coverpkg=github.com/pingcap/tidb/lightning/...,github.com/pingcap/tidb/pkg/lightning/... \
 		-o $(LIGHTNING_BIN).test \
 		github.com/pingcap/tidb/lightning/cmd/tidb-lightning && \
 	$(GOTEST) -c -cover -covermode=count \
 		-coverpkg=github.com/pingcap/tidb/lightning/...,github.com/pingcap/tidb/pkg/lightning/... \
 		-o $(LIGHTNING_CTL_BIN).test \
 		github.com/pingcap/tidb/lightning/cmd/tidb-lightning-ctl && \
+	$(GOBUILD) $(RACE_FLAG) -o bin/fake-oauth tools/fake-oauth/main.go && \
+	$(GOBUILD) $(RACE_FLAG) -o bin/parquet_gen tools/gen-parquet/main.go \
+	) || (make failpoint-disable && exit 1)
+	@make failpoint-disable
+
+lightning_integration_test: build_lightning build_for_lightning_integration_test
+	lightning/tests/run.sh
+
+build_for_br_integration_test:
+	@make failpoint-enable
+	($(GOTEST) -c -cover -covermode=count \
+		-coverpkg=github.com/pingcap/tidb/br/... \
+		-o $(BR_BIN).test \
+		github.com/pingcap/tidb/br/cmd/br && \
 	$(GOBUILD) $(RACE_FLAG) -o bin/locker br/tests/br_key_locked/*.go && \
 	$(GOBUILD) $(RACE_FLAG) -o bin/gc br/tests/br_z_gc_safepoint/*.go && \
-	$(GOBUILD) $(RACE_FLAG) -o bin/oauth br/tests/br_gcs/*.go && \
+	$(GOBUILD) $(RACE_FLAG) -o bin/fake-oauth tools/fake-oauth/main.go && \
 	$(GOBUILD) $(RACE_FLAG) -o bin/rawkv br/tests/br_rawkv/*.go && \
-	$(GOBUILD) $(RACE_FLAG) -o bin/txnkv br/tests/br_txn/*.go && \
-	$(GOBUILD) $(RACE_FLAG) -o bin/parquet_gen br/tests/lightning_checkpoint_parquet/*.go \
+	$(GOBUILD) $(RACE_FLAG) -o bin/txnkv br/tests/br_txn/*.go \
 	) || (make failpoint-disable && exit 1)
 	@make failpoint-disable
 

diff --git a/br/tests/README.md b/br/tests/README.md
@@ -89,8 +89,8 @@ After executing the tests, run `make br_coverage` to get a coverage report at
 
 1. New integration tests can be written as shell scripts in `tests/TEST_NAME/run.sh`.
 The script should exit with a nonzero error code on failure.
-2. Add TEST_NAME to existing group in [run_group_br_tests.sh](./run_group_br_tests.sh)(Recommended) or [run_group_lightning_tests.sh](./run_group_lightning_tests.sh)(Recommended), or add a new group for it.
-3. If you add a new group, the name of the new group must be added to CI [br-integration-test](https://github.com/PingCAP-QE/ci/blob/main/pipelines/pingcap/tidb/latest/pull_br_integration_test.groovy) or [lightning-integration-test](https://github.com/PingCAP-QE/ci/blob/main/pipelines/pingcap/tidb/latest/pull_lightning_integration_test.groovy).
+2. Add TEST_NAME to existing group in [run_group_br_tests.sh](./run_group_br_tests.sh)(Recommended), or add a new group for it.
+3. If you add a new group, the name of the new group must be added to CI [br-integration-test](https://github.com/PingCAP-QE/ci/blob/main/pipelines/pingcap/tidb/latest/pull_br_integration_test.groovy).
 
 Several convenient commands are provided:
 

diff --git a/br/tests/br_charset_gbk/run.sh b/br/tests/br_charset_gbk/run.sh
@@ -19,7 +19,7 @@ check_cluster_version 5 4 0 'new collation' || { echo 'TiDB does not support new
 set -eu
 
 cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-. $cur/../_utils/run_services
+. $UTILS_DIR/run_services
 
 # restart cluster with new collation enabled
 start_services --tidb-cfg $cur/tidb-new-collation.toml

diff --git a/br/tests/br_check_new_collocation_enable/run.sh b/br/tests/br_check_new_collocation_enable/run.sh
@@ -18,7 +18,7 @@ set -eu
 DB="$TEST_NAME"
 
 cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source $cur/../_utils/run_services
+source $UTILS_DIR/run_services
 
 PROGRESS_FILE="$TEST_DIR/progress_unit_file"
 rm -rf $PROGRESS_FILE

diff --git a/br/tests/br_full_cluster_restore/run.sh b/br/tests/br_full_cluster_restore/run.sh
@@ -8,7 +8,7 @@ backup_dir=$TEST_DIR/keep_${TEST_NAME}
 incr_backup_dir=${backup_dir}_inc
 res_file="$TEST_DIR/sql_res.$TEST_NAME.txt"
 br_log_file=$TEST_DIR/br.log
-source $CUR/../_utils/run_services
+source $UTILS_DIR/run_services
 
 function run_sql_as() {
 	user=$1

diff --git a/br/tests/br_gcs/run.sh b/br/tests/br_gcs/run.sh
@@ -37,11 +37,11 @@ while ! curl -o /dev/null -v -s "http://$GCS_HOST:$GCS_PORT/"; do
 done
 
 # start oauth server
-bin/oauth &
+bin/fake-oauth &
 
 stop_gcs() {
     killall -9 fake-gcs-server || true
-    killall -9 oauth || true
+    killall -9 fake-oauth || true
 }
 trap stop_gcs EXIT
 

diff --git a/br/tests/br_incompatible_tidb_config/run.sh b/br/tests/br_incompatible_tidb_config/run.sh
@@ -17,7 +17,7 @@
 set -eux
 
 cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source $cur/../_utils/run_services
+source $UTILS_DIR/run_services
 
 DB="$TEST_NAME"
 

diff --git a/br/tests/br_rawkv/run.sh b/br/tests/br_rawkv/run.sh
@@ -17,7 +17,7 @@
 set -eux
 
 # restart service without tiflash
-source $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../_utils/run_services
+source $UTILS_DIR/run_services
 start_services --no-tiflash
 
 BACKUP_DIR=$TEST_DIR/"raw_backup"

diff --git a/br/tests/br_txn/run.sh b/br/tests/br_txn/run.sh
@@ -17,7 +17,7 @@
 set -eux
 
 # restart service without tiflash
-source $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../_utils/run_services
+source $UTILS_DIR/run_services
 start_services --no-tiflash
 
 BACKUP_DIR=$TEST_DIR/"txn_backup"

diff --git a/br/tests/config/root.cert b/br/tests/config/root.cert
diff --git a/br/tests/config/root.key b/br/tests/config/root.key
diff --git a/br/tests/lightning_checksum_mismatch/run.sh b/br/tests/lightning_checksum_mismatch/run.sh
diff --git a/br/tests/lightning_cmdline_override/run.sh b/br/tests/lightning_cmdline_override/run.sh
diff --git a/br/tests/lightning_new_collation/tidb-new-collation.toml b/br/tests/lightning_new_collation/tidb-new-collation.toml
diff --git a/br/tests/lightning_parquet/data/test/test.customer/_SUCCESS b/br/tests/lightning_parquet/data/test/test.customer/_SUCCESS
diff --git a/br/tests/lightning_parquet/data/test/test.district/_SUCCESS b/br/tests/lightning_parquet/data/test/test.district/_SUCCESS
diff --git a/br/tests/lightning_parquet/data/test/test.history/_SUCCESS b/br/tests/lightning_parquet/data/test/test.history/_SUCCESS
diff --git a/br/tests/lightning_parquet/data/test/test.item/_SUCCESS b/br/tests/lightning_parquet/data/test/test.item/_SUCCESS
diff --git a/br/tests/lightning_parquet/data/test/test.new_order/_SUCCESS b/br/tests/lightning_parquet/data/test/test.new_order/_SUCCESS
diff --git a/br/tests/lightning_parquet/data/test/test.order_line/_SUCCESS b/br/tests/lightning_parquet/data/test/test.order_line/_SUCCESS
diff --git a/br/tests/lightning_parquet/data/test/test.orders/_SUCCESS b/br/tests/lightning_parquet/data/test/test.orders/_SUCCESS
diff --git a/br/tests/lightning_parquet/data/test/test.stock/_SUCCESS b/br/tests/lightning_parquet/data/test/test.stock/_SUCCESS
diff --git a/br/tests/lightning_parquet/data/test/test.warehouse/_SUCCESS b/br/tests/lightning_parquet/data/test/test.warehouse/_SUCCESS
diff --git a/br/tests/lightning_routes_panic/run.sh b/br/tests/lightning_routes_panic/run.sh
diff --git a/br/tests/run.sh b/br/tests/run.sh
@@ -16,11 +16,16 @@
 
 set -eu
 CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-export PATH="$PATH:$CUR/../../bin:$CUR/../bin:$CUR/_utils"
+export UTILS_DIR="$CUR/../../tests/_utils"
+export PATH="$PATH:$CUR/../../bin:$CUR/../bin:$UTILS_DIR"
 export TEST_DIR=/tmp/backup_restore_test
 export COV_DIR="/tmp/group_cover"
 mkdir -p $COV_DIR || true
-source $CUR/_utils/run_services
+export TIDB_CONFIG="$CUR/config/tidb.toml"
+export TIKV_CONFIG="$CUR/config/tikv.toml"
+export PD_CONFIG="$CUR/config/pd.toml"
+export TESTS_ROOT="$CUR"
+source $UTILS_DIR/run_services
 
 # Create COV_DIR if not exists
 if [ -d "$COV_DIR" ]; then

diff --git a/lightning/README.md b/lightning/README.md
@@ -25,9 +25,7 @@ When TiDB-Lightning is built successfully, you can find binary in the `bin` dire
 
 ## Running tests
 
-See [this document](../br/tests/README.md) for how to run integration tests.
-
-Due to history reasons, BR and TiDB-Lightning share the same test directory. You can find the test cases in the [`tests`](../br/tests) directory.
+See [this document](../lightning/tests/README.md) for how to run integration tests.
 
 ## Quick start
 See [Quick Start for TiDB Lightning](https://docs.pingcap.com/tidb/stable/get-started-with-tidb-lightning).

diff --git a/lightning/tests/README.md b/lightning/tests/README.md
@@ -0,0 +1,64 @@
+# Integration tests
+
+This folder contains all tests which relies on external processes such as TiDB.
+
+## Preparations
+
+1. The following executables must be copied or linked into these locations:
+
+    * `bin/tidb-server`
+    * `bin/tikv-server`
+    * `bin/pd-server`
+    * `bin/pd-ctl`
+    * `bin/minio`
+    * `bin/mc`
+    * `bin/tiflash`
+
+    The versions must be ≥2.1.0.
+
+    What's more, there must be dynamic link library for TiFlash, see make target `bin` to learn more.
+    You can install most of dependencies by running `download_tools.sh`.
+
+2. `make build_for_lightning_integration_test`
+
+3. The following programs must be installed:
+
+    * `mysql` (the CLI client)
+    * `curl`
+    * `openssl`
+    * `wget`
+    * `lsof`
+
+4. The user executing the tests must have permission to create the folder
+    `/tmp/lightning_test`. All test artifacts will be written into this folder.
+
+## Running
+
+Run `make lightning_integration_test` to execute all the integration tests.
+- Logs will be written into `/tmp/lightning_test` directory.
+
+Run `tests/run.sh --debug` to pause immediately after all servers are started.
+
+If you only want to run some tests, you can use:
+```shell
+TEST_NAME="lightning_gcs lightning_view" lightning/tests/run.sh
+```
+
+Case names are separated by spaces.
+
+## Writing new tests
+
+1. New integration tests can be written as shell scripts in `tests/TEST_NAME/run.sh`.
+    - `TEST_NAME` should start with `lightning_`.
+    - The script should exit with a nonzero error code on failure.
+2. Add TEST_NAME to existing group in [run_group_lightning_tests.sh](./run_group_lightning_tests.sh)(Recommended), or add a new group for it.
+3. If you add a new group, the name of the new group must be added to CI  [lightning-integration-test](https://github.com/PingCAP-QE/ci/blob/main/pipelines/pingcap/tidb/latest/pull_lightning_integration_test.groovy).
+
+Several convenient commands are provided in [utils](../../tests/_utils/):
+
+* `run_sql <SQL>` — Executes an SQL query on the TiDB database
+* `run_lightning [CONFIG]` — Starts `tidb-lightning` using `tests/TEST_NAME/CONFIG.toml`
+* `check_contains <TEXT>` — Checks if the previous `run_sql` result contains the given text
+    (in `-E` format)
+* `check_not_contains <TEXT>` — Checks if the previous `run_sql` result does not contain the given
+    text (in `-E` format)
diff --git a/lightning/tests/config/pd.toml b/lightning/tests/config/pd.toml
@@ -0,0 +1,10 @@
+lease = 360
+tso-save-interval = "360s"
+
+[replication]
+enable-placement-rules = true
+
+[security]
+cacert-path = "/tmp/lightning_test/certs/ca.pem"
+cert-path = "/tmp/lightning_test/certs/pd.pem"
+key-path = "/tmp/lightning_test/certs/pd.key"
diff --git a/lightning/tests/config/tidb.toml b/lightning/tests/config/tidb.toml
@@ -0,0 +1,19 @@
+# config of tidb
+
+# Schema lease duration
+# There are lot of ddl in the tests, setting this
+# to 360s to test whther BR is gracefully shutdown.
+lease = "360s"
+[security]
+ssl-ca = "/tmp/lightning_test/certs/ca.pem"
+ssl-cert = "/tmp/lightning_test/certs/tidb.pem"
+ssl-key = "/tmp/lightning_test/certs/tidb.key"
+cluster-ssl-ca = "/tmp/lightning_test/certs/ca.pem"
+cluster-ssl-cert = "/tmp/lightning_test/certs/tidb.pem"
+cluster-ssl-key = "/tmp/lightning_test/certs/tidb.key"
+
+# experimental section controls the features that are still experimental: their semantics,
+# interfaces are subject to change, using these features in the production environment is not recommended.
+[experimental]
+# enable creating expression index.
+allow-expression-index = true
diff --git a/br/tests/config/tikv-multi-rocksdb.toml → ...ning/tests/config/tikv-multi-rocksdb.toml b/br/tests/config/tikv-multi-rocksdb.toml → ...ning/tests/config/tikv-multi-rocksdb.toml
@@ -1,7 +1,7 @@
 # config of tikv
 [storage]
 reserve-space = "1KB"
-data-dir = "/tmp/backup_restore_test/tikv1/"
+data-dir = "/tmp/lightning_test/tikv1/"
 engine = "partitioned-raft-kv"
 
 [coprocessor]
@@ -24,6 +24,6 @@ pd-heartbeat-tick-interval = "1s"
 hibernate-regions-compatible=false
 
 [security]
-ca-path = "/tmp/backup_restore_test/certs/ca.pem"
-cert-path = "/tmp/backup_restore_test/certs/tikv.pem"
-key-path = "/tmp/backup_restore_test/certs/tikv.key"
+ca-path = "/tmp/lightning_test/certs/ca.pem"
+cert-path = "/tmp/lightning_test/certs/tikv.pem"
+key-path = "/tmp/lightning_test/certs/tikv.key"
diff --git a/lightning/tests/config/tikv.toml b/lightning/tests/config/tikv.toml
@@ -0,0 +1,35 @@
+# config of tikv
+[storage]
+reserve-space = "1KB"
+data-dir = "/tmp/lightning_test/tikv1/"
+
+[coprocessor]
+region-max-keys = 100
+region-split-keys = 60
+
+[rocksdb]
+max-open-files = 4096
+[raftdb]
+max-open-files = 4096
+
+[raftstore]
+# true (default value) for high reliability, this can prevent data loss when power failure.
+sync-log = false
+capacity = "10GB"
+# Speed up TiKV region heartbeat
+pd-heartbeat-tick-interval = "1s"
+
+[cdc]
+hibernate-regions-compatible=false
+
+[security]
+ca-path = "/tmp/lightning_test/certs/ca.pem"
+cert-path = "/tmp/lightning_test/certs/tikv.pem"
+key-path = "/tmp/lightning_test/certs/tikv.key"
+
+[security.encryption]
+data-encryption-method = "aes256-ctr"
+
+[security.encryption.master-key]
+type = "file"
+path = "/tmp/lightning_test/master-key-file"