diff --git a/.github/workflows/integration-test-compile-br.yml b/.github/workflows/integration-test-compile-br.yml
index 0dd0fc1688cdc..2ee3f3c6f1c73 100644
--- a/.github/workflows/integration-test-compile-br.yml
+++ b/.github/workflows/integration-test-compile-br.yml
@@ -51,10 +51,10 @@ jobs:
os: [macos-latest, ubuntu-latest, windows-latest]
steps:
- - uses: actions/checkout@v2.1.0
+ - uses: actions/checkout@v4
- name: Set up Go
- uses: actions/setup-go@v3
+ uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
@@ -66,10 +66,10 @@ jobs:
name: Compile for FreeBSD job
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2.1.0
+ - uses: actions/checkout@v4
- name: Set up Go
- uses: actions/setup-go@v3
+ uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
diff --git a/.github/workflows/integration-test-dumpling.yml b/.github/workflows/integration-test-dumpling.yml
index 14bae6e6b7115..55d65a47dfb9f 100644
--- a/.github/workflows/integration-test-dumpling.yml
+++ b/.github/workflows/integration-test-dumpling.yml
@@ -52,6 +52,7 @@ jobs:
- 5.7.35
- 8.0.22
- 8.0.26
+ - 8.0.37
runs-on: ubuntu-latest
timeout-minutes: 15
services:
@@ -64,13 +65,13 @@ jobs:
options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
steps:
- name: "checkout repository"
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- name: "set up golang"
- uses: actions/setup-go@v3
+ uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
- name: "try to use build cache"
- uses: actions/cache@v3
+ uses: actions/cache@v4
with:
path: |
~/.cache/go-build
diff --git a/DEPS.bzl b/DEPS.bzl
index 5330ae4d36b62..92c40c92209a8 100644
--- a/DEPS.bzl
+++ b/DEPS.bzl
@@ -21,13 +21,13 @@ def go_deps():
name = "cc_mvdan_unparam",
build_file_proto_mode = "disable_global",
importpath = "mvdan.cc/unparam",
- sha256 = "e2d0554cad489ddd9a5e06425139ddb223447007682b11fc6fe75fb98d194f38",
- strip_prefix = "mvdan.cc/unparam@v0.0.0-20240104100049-c549a3470d14",
+ sha256 = "25aeecd0b84daff70d162df59a3e231591252ab5a048bcf40c1aa38afdc5ef81",
+ strip_prefix = "mvdan.cc/unparam@v0.0.0-20240427195214-063aff900ca1",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/mvdan.cc/unparam/cc_mvdan_unparam-v0.0.0-20240104100049-c549a3470d14.zip",
- "http://ats.apps.svc/gomod/mvdan.cc/unparam/cc_mvdan_unparam-v0.0.0-20240104100049-c549a3470d14.zip",
- "https://cache.hawkingrei.com/gomod/mvdan.cc/unparam/cc_mvdan_unparam-v0.0.0-20240104100049-c549a3470d14.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/mvdan.cc/unparam/cc_mvdan_unparam-v0.0.0-20240104100049-c549a3470d14.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/mvdan.cc/unparam/cc_mvdan_unparam-v0.0.0-20240427195214-063aff900ca1.zip",
+ "http://ats.apps.svc/gomod/mvdan.cc/unparam/cc_mvdan_unparam-v0.0.0-20240427195214-063aff900ca1.zip",
+ "https://cache.hawkingrei.com/gomod/mvdan.cc/unparam/cc_mvdan_unparam-v0.0.0-20240427195214-063aff900ca1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/mvdan.cc/unparam/cc_mvdan_unparam-v0.0.0-20240427195214-063aff900ca1.zip",
],
)
go_repository(
@@ -73,13 +73,13 @@ def go_deps():
name = "com_github_4meepo_tagalign",
build_file_proto_mode = "disable_global",
importpath = "github.com/4meepo/tagalign",
- sha256 = "7787f1327d989f71ad57d73d86a771a4e5376268991c29e4361da0a69ddecfdf",
- strip_prefix = "github.com/4meepo/tagalign@v1.3.3",
+ sha256 = "1d1264ebe9e8034ccf3dd0f0a237ca9c14e16265592115ac684ac2f2216f9dff",
+ strip_prefix = "github.com/4meepo/tagalign@v1.3.4",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/4meepo/tagalign/com_github_4meepo_tagalign-v1.3.3.zip",
- "http://ats.apps.svc/gomod/github.com/4meepo/tagalign/com_github_4meepo_tagalign-v1.3.3.zip",
- "https://cache.hawkingrei.com/gomod/github.com/4meepo/tagalign/com_github_4meepo_tagalign-v1.3.3.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/4meepo/tagalign/com_github_4meepo_tagalign-v1.3.3.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/4meepo/tagalign/com_github_4meepo_tagalign-v1.3.4.zip",
+ "http://ats.apps.svc/gomod/github.com/4meepo/tagalign/com_github_4meepo_tagalign-v1.3.4.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/4meepo/tagalign/com_github_4meepo_tagalign-v1.3.4.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/4meepo/tagalign/com_github_4meepo_tagalign-v1.3.4.zip",
],
)
go_repository(
@@ -121,17 +121,43 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/aclements/go-moremath/com_github_aclements_go_moremath-v0.0.0-20210112150236-f10218a38794.zip",
],
)
+ go_repository(
+ name = "com_github_ajstarks_deck",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/ajstarks/deck",
+ sha256 = "68bad2e38bf5b01e6bbd7b9bbdba35da94dac72bc4ba41f8ea5fe92aa836a3c3",
+ strip_prefix = "github.com/ajstarks/deck@v0.0.0-20200831202436-30c9fc6549a9",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/ajstarks/deck/com_github_ajstarks_deck-v0.0.0-20200831202436-30c9fc6549a9.zip",
+ "http://ats.apps.svc/gomod/github.com/ajstarks/deck/com_github_ajstarks_deck-v0.0.0-20200831202436-30c9fc6549a9.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/ajstarks/deck/com_github_ajstarks_deck-v0.0.0-20200831202436-30c9fc6549a9.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ajstarks/deck/com_github_ajstarks_deck-v0.0.0-20200831202436-30c9fc6549a9.zip",
+ ],
+ )
+ go_repository(
+ name = "com_github_ajstarks_deck_generate",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/ajstarks/deck/generate",
+ sha256 = "dce1cbc4cb42ac26512dd0bccf997baeea99fb4595cd419a28e8566d2d7c7ba8",
+ strip_prefix = "github.com/ajstarks/deck/generate@v0.0.0-20210309230005-c3f852c02e19",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/ajstarks/deck/generate/com_github_ajstarks_deck_generate-v0.0.0-20210309230005-c3f852c02e19.zip",
+ "http://ats.apps.svc/gomod/github.com/ajstarks/deck/generate/com_github_ajstarks_deck_generate-v0.0.0-20210309230005-c3f852c02e19.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/ajstarks/deck/generate/com_github_ajstarks_deck_generate-v0.0.0-20210309230005-c3f852c02e19.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ajstarks/deck/generate/com_github_ajstarks_deck_generate-v0.0.0-20210309230005-c3f852c02e19.zip",
+ ],
+ )
go_repository(
name = "com_github_ajstarks_svgo",
build_file_proto_mode = "disable_global",
importpath = "github.com/ajstarks/svgo",
- sha256 = "cdf6900823539ab02e7f6e0edcdb4c12b3dcec97068a350e564ff622132ae7fc",
- strip_prefix = "github.com/ajstarks/svgo@v0.0.0-20180226025133-644b8db467af",
+ sha256 = "e25b5dbb6cc86d2a0b5db08aad757c534681c2cecb30d84746e09c661cbd7c6f",
+ strip_prefix = "github.com/ajstarks/svgo@v0.0.0-20211024235047-1546f124cd8b",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/ajstarks/svgo/com_github_ajstarks_svgo-v0.0.0-20180226025133-644b8db467af.zip",
- "http://ats.apps.svc/gomod/github.com/ajstarks/svgo/com_github_ajstarks_svgo-v0.0.0-20180226025133-644b8db467af.zip",
- "https://cache.hawkingrei.com/gomod/github.com/ajstarks/svgo/com_github_ajstarks_svgo-v0.0.0-20180226025133-644b8db467af.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ajstarks/svgo/com_github_ajstarks_svgo-v0.0.0-20180226025133-644b8db467af.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/ajstarks/svgo/com_github_ajstarks_svgo-v0.0.0-20211024235047-1546f124cd8b.zip",
+ "http://ats.apps.svc/gomod/github.com/ajstarks/svgo/com_github_ajstarks_svgo-v0.0.0-20211024235047-1546f124cd8b.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/ajstarks/svgo/com_github_ajstarks_svgo-v0.0.0-20211024235047-1546f124cd8b.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ajstarks/svgo/com_github_ajstarks_svgo-v0.0.0-20211024235047-1546f124cd8b.zip",
],
)
go_repository(
@@ -268,26 +294,26 @@ def go_deps():
name = "com_github_antonboom_errname",
build_file_proto_mode = "disable_global",
importpath = "github.com/Antonboom/errname",
- sha256 = "b153fac1a78bbeb14a1e2898307bdbb683dd62bd0f3e7a2a99610b9865081438",
- strip_prefix = "github.com/Antonboom/errname@v0.1.12",
+ sha256 = "457abd15ce39dd2ff61b7d976d38c153982183f5cab00ee13f7ad67a71bfef9f",
+ strip_prefix = "github.com/Antonboom/errname@v0.1.13",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/Antonboom/errname/com_github_antonboom_errname-v0.1.12.zip",
- "http://ats.apps.svc/gomod/github.com/Antonboom/errname/com_github_antonboom_errname-v0.1.12.zip",
- "https://cache.hawkingrei.com/gomod/github.com/Antonboom/errname/com_github_antonboom_errname-v0.1.12.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Antonboom/errname/com_github_antonboom_errname-v0.1.12.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/Antonboom/errname/com_github_antonboom_errname-v0.1.13.zip",
+ "http://ats.apps.svc/gomod/github.com/Antonboom/errname/com_github_antonboom_errname-v0.1.13.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/Antonboom/errname/com_github_antonboom_errname-v0.1.13.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Antonboom/errname/com_github_antonboom_errname-v0.1.13.zip",
],
)
go_repository(
name = "com_github_antonboom_nilnil",
build_file_proto_mode = "disable_global",
importpath = "github.com/Antonboom/nilnil",
- sha256 = "f82e42fd3a6601252d881b50ba52d91131a981e3835cc160c55c4735734feade",
- strip_prefix = "github.com/Antonboom/nilnil@v0.1.7",
+ sha256 = "dbcecf12148c670e385a6a07b68360cdb6d66103a539352e7b7eaae5b2bc0253",
+ strip_prefix = "github.com/Antonboom/nilnil@v0.1.8",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/Antonboom/nilnil/com_github_antonboom_nilnil-v0.1.7.zip",
- "http://ats.apps.svc/gomod/github.com/Antonboom/nilnil/com_github_antonboom_nilnil-v0.1.7.zip",
- "https://cache.hawkingrei.com/gomod/github.com/Antonboom/nilnil/com_github_antonboom_nilnil-v0.1.7.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Antonboom/nilnil/com_github_antonboom_nilnil-v0.1.7.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/Antonboom/nilnil/com_github_antonboom_nilnil-v0.1.8.zip",
+ "http://ats.apps.svc/gomod/github.com/Antonboom/nilnil/com_github_antonboom_nilnil-v0.1.8.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/Antonboom/nilnil/com_github_antonboom_nilnil-v0.1.8.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Antonboom/nilnil/com_github_antonboom_nilnil-v0.1.8.zip",
],
)
go_repository(
@@ -303,6 +329,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/Antonboom/testifylint/com_github_antonboom_testifylint-v1.2.0.zip",
],
)
+ go_repository(
+ name = "com_github_apache_arrow_go_v12",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/apache/arrow/go/v12",
+ sha256 = "5eb05ed9c2c5e164503b00912b7b2456400578de29e7e8a8956a41acd861ab5b",
+ strip_prefix = "github.com/apache/arrow/go/v12@v12.0.1",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/apache/arrow/go/v12/com_github_apache_arrow_go_v12-v12.0.1.zip",
+ "http://ats.apps.svc/gomod/github.com/apache/arrow/go/v12/com_github_apache_arrow_go_v12-v12.0.1.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/apache/arrow/go/v12/com_github_apache_arrow_go_v12-v12.0.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/apache/arrow/go/v12/com_github_apache_arrow_go_v12-v12.0.1.zip",
+ ],
+ )
go_repository(
name = "com_github_apache_skywalking_eyes",
build_file_proto_mode = "disable_global",
@@ -667,6 +706,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/bombsimon/wsl/v4/com_github_bombsimon_wsl_v4-v4.2.1.zip",
],
)
+ go_repository(
+ name = "com_github_boombuler_barcode",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/boombuler/barcode",
+ sha256 = "812c5beeaa87864227f9d92a9ae71792dc0e6302a33737a91aabe1e511cde42b",
+ strip_prefix = "github.com/boombuler/barcode@v1.0.1",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/boombuler/barcode/com_github_boombuler_barcode-v1.0.1.zip",
+ "http://ats.apps.svc/gomod/github.com/boombuler/barcode/com_github_boombuler_barcode-v1.0.1.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/boombuler/barcode/com_github_boombuler_barcode-v1.0.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/boombuler/barcode/com_github_boombuler_barcode-v1.0.1.zip",
+ ],
+ )
go_repository(
name = "com_github_breml_bidichk",
build_file_proto_mode = "disable_global",
@@ -736,13 +788,13 @@ def go_deps():
name = "com_github_butuzov_mirror",
build_file_proto_mode = "disable_global",
importpath = "github.com/butuzov/mirror",
- sha256 = "6ac5d5075646123f8d7b0f3659087c50e6762b06606497ad05fadc3a8f196c06",
- strip_prefix = "github.com/butuzov/mirror@v1.1.0",
+ sha256 = "81803d1cfc8f32392dcbd24649f43813a9ff560f50c178aa370e1edb2c8fcf41",
+ strip_prefix = "github.com/butuzov/mirror@v1.2.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/butuzov/mirror/com_github_butuzov_mirror-v1.1.0.zip",
- "http://ats.apps.svc/gomod/github.com/butuzov/mirror/com_github_butuzov_mirror-v1.1.0.zip",
- "https://cache.hawkingrei.com/gomod/github.com/butuzov/mirror/com_github_butuzov_mirror-v1.1.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/butuzov/mirror/com_github_butuzov_mirror-v1.1.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/butuzov/mirror/com_github_butuzov_mirror-v1.2.0.zip",
+ "http://ats.apps.svc/gomod/github.com/butuzov/mirror/com_github_butuzov_mirror-v1.2.0.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/butuzov/mirror/com_github_butuzov_mirror-v1.2.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/butuzov/mirror/com_github_butuzov_mirror-v1.2.0.zip",
],
)
go_repository(
@@ -983,13 +1035,13 @@ def go_deps():
name = "com_github_ckaznocha_intrange",
build_file_proto_mode = "disable_global",
importpath = "github.com/ckaznocha/intrange",
- sha256 = "b19bafbea253ee13186cd0edb806d76417195061b1df593fbbd6bfac0710470b",
- strip_prefix = "github.com/ckaznocha/intrange@v0.1.1",
+ sha256 = "46655c00818508557f76eaed9da3f5b5497ae34fcfd99c3dd2179bedaeac6dc9",
+ strip_prefix = "github.com/ckaznocha/intrange@v0.1.2",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/ckaznocha/intrange/com_github_ckaznocha_intrange-v0.1.1.zip",
- "http://ats.apps.svc/gomod/github.com/ckaznocha/intrange/com_github_ckaznocha_intrange-v0.1.1.zip",
- "https://cache.hawkingrei.com/gomod/github.com/ckaznocha/intrange/com_github_ckaznocha_intrange-v0.1.1.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ckaznocha/intrange/com_github_ckaznocha_intrange-v0.1.1.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/ckaznocha/intrange/com_github_ckaznocha_intrange-v0.1.2.zip",
+ "http://ats.apps.svc/gomod/github.com/ckaznocha/intrange/com_github_ckaznocha_intrange-v0.1.2.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/ckaznocha/intrange/com_github_ckaznocha_intrange-v0.1.2.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ckaznocha/intrange/com_github_ckaznocha_intrange-v0.1.2.zip",
],
)
go_repository(
@@ -1048,26 +1100,26 @@ def go_deps():
name = "com_github_cncf_udpa_go",
build_file_proto_mode = "disable_global",
importpath = "github.com/cncf/udpa/go",
- sha256 = "f2a2fee0b2024946ddd3b7ec5cd06a6d318cdb8421a8d5afff4c2fd69f1e74a7",
- strip_prefix = "github.com/cncf/udpa/go@v0.0.0-20191209042840-269d4d468f6f",
+ sha256 = "a449fa94e58117a79c17577e39f72f695c4876f74cbd9142d512278192ca90aa",
+ strip_prefix = "github.com/cncf/udpa/go@v0.0.0-20210930031921-04548b0d99d4",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/cncf/udpa/go/com_github_cncf_udpa_go-v0.0.0-20191209042840-269d4d468f6f.zip",
- "http://ats.apps.svc/gomod/github.com/cncf/udpa/go/com_github_cncf_udpa_go-v0.0.0-20191209042840-269d4d468f6f.zip",
- "https://cache.hawkingrei.com/gomod/github.com/cncf/udpa/go/com_github_cncf_udpa_go-v0.0.0-20191209042840-269d4d468f6f.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/cncf/udpa/go/com_github_cncf_udpa_go-v0.0.0-20191209042840-269d4d468f6f.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/cncf/udpa/go/com_github_cncf_udpa_go-v0.0.0-20210930031921-04548b0d99d4.zip",
+ "http://ats.apps.svc/gomod/github.com/cncf/udpa/go/com_github_cncf_udpa_go-v0.0.0-20210930031921-04548b0d99d4.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/cncf/udpa/go/com_github_cncf_udpa_go-v0.0.0-20210930031921-04548b0d99d4.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/cncf/udpa/go/com_github_cncf_udpa_go-v0.0.0-20210930031921-04548b0d99d4.zip",
],
)
go_repository(
name = "com_github_cncf_xds_go",
build_file_proto_mode = "disable_global",
importpath = "github.com/cncf/xds/go",
- sha256 = "ab0d2fd980b15a582708a728cf8080ebb88778e59f3003b67c6aafaa9ad0f447",
- strip_prefix = "github.com/cncf/xds/go@v0.0.0-20231128003011-0fa0005c9caa",
+ sha256 = "7395d4a588bcabf822f2347b647b66853a14a98088dd1ea0582cfa7a241c4234",
+ strip_prefix = "github.com/cncf/xds/go@v0.0.0-20240318125728-8a4994d93e50",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/cncf/xds/go/com_github_cncf_xds_go-v0.0.0-20231128003011-0fa0005c9caa.zip",
- "http://ats.apps.svc/gomod/github.com/cncf/xds/go/com_github_cncf_xds_go-v0.0.0-20231128003011-0fa0005c9caa.zip",
- "https://cache.hawkingrei.com/gomod/github.com/cncf/xds/go/com_github_cncf_xds_go-v0.0.0-20231128003011-0fa0005c9caa.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/cncf/xds/go/com_github_cncf_xds_go-v0.0.0-20231128003011-0fa0005c9caa.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/cncf/xds/go/com_github_cncf_xds_go-v0.0.0-20240318125728-8a4994d93e50.zip",
+ "http://ats.apps.svc/gomod/github.com/cncf/xds/go/com_github_cncf_xds_go-v0.0.0-20240318125728-8a4994d93e50.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/cncf/xds/go/com_github_cncf_xds_go-v0.0.0-20240318125728-8a4994d93e50.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/cncf/xds/go/com_github_cncf_xds_go-v0.0.0-20240318125728-8a4994d93e50.zip",
],
)
go_repository(
@@ -1343,6 +1395,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/creack/pty/com_github_creack_pty-v1.1.11.zip",
],
)
+ go_repository(
+ name = "com_github_crocmagnon_fatcontext",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/Crocmagnon/fatcontext",
+ sha256 = "cedbb10954148c7e528493c73770925de8cfe1602d78f51e348f115d6f045acc",
+ strip_prefix = "github.com/Crocmagnon/fatcontext@v0.2.2",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/Crocmagnon/fatcontext/com_github_crocmagnon_fatcontext-v0.2.2.zip",
+ "http://ats.apps.svc/gomod/github.com/Crocmagnon/fatcontext/com_github_crocmagnon_fatcontext-v0.2.2.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/Crocmagnon/fatcontext/com_github_crocmagnon_fatcontext-v0.2.2.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Crocmagnon/fatcontext/com_github_crocmagnon_fatcontext-v0.2.2.zip",
+ ],
+ )
go_repository(
name = "com_github_curioswitch_go_reassign",
build_file_proto_mode = "disable_global",
@@ -1399,13 +1464,13 @@ def go_deps():
name = "com_github_daixiang0_gci",
build_file_proto_mode = "disable_global",
importpath = "github.com/daixiang0/gci",
- sha256 = "a52365d71b265b0951b796beeee92013a615f32955bf54a6685861a0533068e4",
- strip_prefix = "github.com/daixiang0/gci@v0.12.3",
+ sha256 = "912505818a9c36b0090a7a4c786bc8ae090e375f3e3c0244770fc36491cb3b1f",
+ strip_prefix = "github.com/daixiang0/gci@v0.13.4",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/daixiang0/gci/com_github_daixiang0_gci-v0.12.3.zip",
- "http://ats.apps.svc/gomod/github.com/daixiang0/gci/com_github_daixiang0_gci-v0.12.3.zip",
- "https://cache.hawkingrei.com/gomod/github.com/daixiang0/gci/com_github_daixiang0_gci-v0.12.3.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/daixiang0/gci/com_github_daixiang0_gci-v0.12.3.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/daixiang0/gci/com_github_daixiang0_gci-v0.13.4.zip",
+ "http://ats.apps.svc/gomod/github.com/daixiang0/gci/com_github_daixiang0_gci-v0.13.4.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/daixiang0/gci/com_github_daixiang0_gci-v0.13.4.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/daixiang0/gci/com_github_daixiang0_gci-v0.13.4.zip",
],
)
go_repository(
@@ -1629,6 +1694,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/docker/go-units/com_github_docker_go_units-v0.5.0.zip",
],
)
+ go_repository(
+ name = "com_github_docopt_docopt_go",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/docopt/docopt-go",
+ sha256 = "00aad861d150c62598ca4fb01cfbe15c2eefb5186df7e5d4a59286dcf09556c8",
+ strip_prefix = "github.com/docopt/docopt-go@v0.0.0-20180111231733-ee0de3bc6815",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/docopt/docopt-go/com_github_docopt_docopt_go-v0.0.0-20180111231733-ee0de3bc6815.zip",
+ "http://ats.apps.svc/gomod/github.com/docopt/docopt-go/com_github_docopt_docopt_go-v0.0.0-20180111231733-ee0de3bc6815.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/docopt/docopt-go/com_github_docopt_docopt_go-v0.0.0-20180111231733-ee0de3bc6815.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/docopt/docopt-go/com_github_docopt_docopt_go-v0.0.0-20180111231733-ee0de3bc6815.zip",
+ ],
+ )
go_repository(
name = "com_github_dolthub_maphash",
build_file_proto_mode = "disable_global",
@@ -1919,13 +1997,13 @@ def go_deps():
name = "com_github_firefart_nonamedreturns",
build_file_proto_mode = "disable_global",
importpath = "github.com/firefart/nonamedreturns",
- sha256 = "293f84c4e1737d2558e1d289f9ca6f7ca851276fb204bee9a21664da4ddd9cac",
- strip_prefix = "github.com/firefart/nonamedreturns@v1.0.4",
+ sha256 = "f5ceb22b6e64ba3ffd2fec61b1f0daf0a84e6811c608625c92c597d81b4674d6",
+ strip_prefix = "github.com/firefart/nonamedreturns@v1.0.5",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/firefart/nonamedreturns/com_github_firefart_nonamedreturns-v1.0.4.zip",
- "http://ats.apps.svc/gomod/github.com/firefart/nonamedreturns/com_github_firefart_nonamedreturns-v1.0.4.zip",
- "https://cache.hawkingrei.com/gomod/github.com/firefart/nonamedreturns/com_github_firefart_nonamedreturns-v1.0.4.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/firefart/nonamedreturns/com_github_firefart_nonamedreturns-v1.0.4.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/firefart/nonamedreturns/com_github_firefart_nonamedreturns-v1.0.5.zip",
+ "http://ats.apps.svc/gomod/github.com/firefart/nonamedreturns/com_github_firefart_nonamedreturns-v1.0.5.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/firefart/nonamedreturns/com_github_firefart_nonamedreturns-v1.0.5.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/firefart/nonamedreturns/com_github_firefart_nonamedreturns-v1.0.5.zip",
],
)
go_repository(
@@ -1945,13 +2023,13 @@ def go_deps():
name = "com_github_fogleman_gg",
build_file_proto_mode = "disable_global",
importpath = "github.com/fogleman/gg",
- sha256 = "75b657490d88ac3bad9af07ec4acfe57a995944c50eeb1f167467cf82ff814c5",
- strip_prefix = "github.com/fogleman/gg@v1.2.1-0.20190220221249-0403632d5b90",
+ sha256 = "792f7a3ea9eea31b7947dabaf9d5a307389245069078e4bf435d76cb0505439c",
+ strip_prefix = "github.com/fogleman/gg@v1.3.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/fogleman/gg/com_github_fogleman_gg-v1.2.1-0.20190220221249-0403632d5b90.zip",
- "http://ats.apps.svc/gomod/github.com/fogleman/gg/com_github_fogleman_gg-v1.2.1-0.20190220221249-0403632d5b90.zip",
- "https://cache.hawkingrei.com/gomod/github.com/fogleman/gg/com_github_fogleman_gg-v1.2.1-0.20190220221249-0403632d5b90.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/fogleman/gg/com_github_fogleman_gg-v1.2.1-0.20190220221249-0403632d5b90.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/fogleman/gg/com_github_fogleman_gg-v1.3.0.zip",
+ "http://ats.apps.svc/gomod/github.com/fogleman/gg/com_github_fogleman_gg-v1.3.0.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/fogleman/gg/com_github_fogleman_gg-v1.3.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/fogleman/gg/com_github_fogleman_gg-v1.3.0.zip",
],
)
go_repository(
@@ -2075,13 +2153,13 @@ def go_deps():
name = "com_github_ghostiam_protogetter",
build_file_proto_mode = "disable_global",
importpath = "github.com/ghostiam/protogetter",
- sha256 = "943e161a511da743cd1a555981b33ac5e6c33fa86c74cc66dc79324c23c77830",
- strip_prefix = "github.com/ghostiam/protogetter@v0.3.5",
+ sha256 = "67aa66a7346aefbe6ebd3e1c4b9fdd0fb54af284c82b8788c7d15b8a9a5abfcf",
+ strip_prefix = "github.com/ghostiam/protogetter@v0.3.6",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/ghostiam/protogetter/com_github_ghostiam_protogetter-v0.3.5.zip",
- "http://ats.apps.svc/gomod/github.com/ghostiam/protogetter/com_github_ghostiam_protogetter-v0.3.5.zip",
- "https://cache.hawkingrei.com/gomod/github.com/ghostiam/protogetter/com_github_ghostiam_protogetter-v0.3.5.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ghostiam/protogetter/com_github_ghostiam_protogetter-v0.3.5.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/ghostiam/protogetter/com_github_ghostiam_protogetter-v0.3.6.zip",
+ "http://ats.apps.svc/gomod/github.com/ghostiam/protogetter/com_github_ghostiam_protogetter-v0.3.6.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/ghostiam/protogetter/com_github_ghostiam_protogetter-v0.3.6.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ghostiam/protogetter/com_github_ghostiam_protogetter-v0.3.6.zip",
],
)
go_repository(
@@ -2127,13 +2205,13 @@ def go_deps():
name = "com_github_go_critic_go_critic",
build_file_proto_mode = "disable_global",
importpath = "github.com/go-critic/go-critic",
- sha256 = "56fa9e4568110cf091dedc417641cd2badc4e239b92092d73886d7d5310e0903",
- strip_prefix = "github.com/go-critic/go-critic@v0.11.2",
+ sha256 = "c02b35b050f81e7db73d0256695f4afc9d740270b023f86c1594e78f670be4f7",
+ strip_prefix = "github.com/go-critic/go-critic@v0.11.3",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-critic/go-critic/com_github_go_critic_go_critic-v0.11.2.zip",
- "http://ats.apps.svc/gomod/github.com/go-critic/go-critic/com_github_go_critic_go_critic-v0.11.2.zip",
- "https://cache.hawkingrei.com/gomod/github.com/go-critic/go-critic/com_github_go_critic_go_critic-v0.11.2.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-critic/go-critic/com_github_go_critic_go_critic-v0.11.2.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-critic/go-critic/com_github_go_critic_go_critic-v0.11.3.zip",
+ "http://ats.apps.svc/gomod/github.com/go-critic/go-critic/com_github_go_critic_go_critic-v0.11.3.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/go-critic/go-critic/com_github_go_critic_go_critic-v0.11.3.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-critic/go-critic/com_github_go_critic_go_critic-v0.11.3.zip",
],
)
go_repository(
@@ -2149,6 +2227,58 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-errors/errors/com_github_go_errors_errors-v1.4.2.zip",
],
)
+ go_repository(
+ name = "com_github_go_fonts_dejavu",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/go-fonts/dejavu",
+ sha256 = "c2094ce49cfc24b7b7a041e54d924e311322b73a8e56db28ff179fcd403b4111",
+ strip_prefix = "github.com/go-fonts/dejavu@v0.1.0",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-fonts/dejavu/com_github_go_fonts_dejavu-v0.1.0.zip",
+ "http://ats.apps.svc/gomod/github.com/go-fonts/dejavu/com_github_go_fonts_dejavu-v0.1.0.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/go-fonts/dejavu/com_github_go_fonts_dejavu-v0.1.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-fonts/dejavu/com_github_go_fonts_dejavu-v0.1.0.zip",
+ ],
+ )
+ go_repository(
+ name = "com_github_go_fonts_latin_modern",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/go-fonts/latin-modern",
+ sha256 = "037085a80ad108287e772d064d64bb72deb62514de84ef610506bc079f330ec0",
+ strip_prefix = "github.com/go-fonts/latin-modern@v0.2.0",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-fonts/latin-modern/com_github_go_fonts_latin_modern-v0.2.0.zip",
+ "http://ats.apps.svc/gomod/github.com/go-fonts/latin-modern/com_github_go_fonts_latin_modern-v0.2.0.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/go-fonts/latin-modern/com_github_go_fonts_latin_modern-v0.2.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-fonts/latin-modern/com_github_go_fonts_latin_modern-v0.2.0.zip",
+ ],
+ )
+ go_repository(
+ name = "com_github_go_fonts_liberation",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/go-fonts/liberation",
+ sha256 = "bd7561251c221fe0fd8cd4c361b062a5796f6f3a1096968b8fecdd61eb82d8fe",
+ strip_prefix = "github.com/go-fonts/liberation@v0.2.0",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-fonts/liberation/com_github_go_fonts_liberation-v0.2.0.zip",
+ "http://ats.apps.svc/gomod/github.com/go-fonts/liberation/com_github_go_fonts_liberation-v0.2.0.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/go-fonts/liberation/com_github_go_fonts_liberation-v0.2.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-fonts/liberation/com_github_go_fonts_liberation-v0.2.0.zip",
+ ],
+ )
+ go_repository(
+ name = "com_github_go_fonts_stix",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/go-fonts/stix",
+ sha256 = "51ea5a38b9fda7854af60f280dbd8b40a3e5b5a48eb00d3f8d4e43de3f514ecf",
+ strip_prefix = "github.com/go-fonts/stix@v0.1.0",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-fonts/stix/com_github_go_fonts_stix-v0.1.0.zip",
+ "http://ats.apps.svc/gomod/github.com/go-fonts/stix/com_github_go_fonts_stix-v0.1.0.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/go-fonts/stix/com_github_go_fonts_stix-v0.1.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-fonts/stix/com_github_go_fonts_stix-v0.1.0.zip",
+ ],
+ )
go_repository(
name = "com_github_go_gl_glfw",
build_file_proto_mode = "disable_global",
@@ -2201,6 +2331,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-kit/log/com_github_go_kit_log-v0.2.1.zip",
],
)
+ go_repository(
+ name = "com_github_go_latex_latex",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/go-latex/latex",
+ sha256 = "c58be686b31679ad0a51a5d70e60df92fb4bb50a16727caa58b4a67b33f16509",
+ strip_prefix = "github.com/go-latex/latex@v0.0.0-20210823091927-c0d11ff05a81",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-latex/latex/com_github_go_latex_latex-v0.0.0-20210823091927-c0d11ff05a81.zip",
+ "http://ats.apps.svc/gomod/github.com/go-latex/latex/com_github_go_latex_latex-v0.0.0-20210823091927-c0d11ff05a81.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/go-latex/latex/com_github_go_latex_latex-v0.0.0-20210823091927-c0d11ff05a81.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-latex/latex/com_github_go_latex_latex-v0.0.0-20210823091927-c0d11ff05a81.zip",
+ ],
+ )
go_repository(
name = "com_github_go_ldap_ldap_v3",
build_file_proto_mode = "disable_global",
@@ -2396,6 +2539,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-openapi/validate/com_github_go_openapi_validate-v0.22.1.zip",
],
)
+ go_repository(
+ name = "com_github_go_pdf_fpdf",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/go-pdf/fpdf",
+ sha256 = "03a6909fc346ac972b008b77585ac3954d76b416c33b4b64dc22c5f35f0e1edb",
+ strip_prefix = "github.com/go-pdf/fpdf@v0.6.0",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-pdf/fpdf/com_github_go_pdf_fpdf-v0.6.0.zip",
+ "http://ats.apps.svc/gomod/github.com/go-pdf/fpdf/com_github_go_pdf_fpdf-v0.6.0.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/go-pdf/fpdf/com_github_go_pdf_fpdf-v0.6.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-pdf/fpdf/com_github_go_pdf_fpdf-v0.6.0.zip",
+ ],
+ )
go_repository(
name = "com_github_go_playground_locales",
build_file_proto_mode = "disable_global",
@@ -2682,6 +2838,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/goccy/go-json/com_github_goccy_go_json-v0.10.2.zip",
],
)
+ go_repository(
+ name = "com_github_goccy_go_reflect",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/goccy/go-reflect",
+ sha256 = "d5d5b55be60c40d1ecfbd13a7e89c3fb5363e8b7cd07e2827f7e987944c41458",
+ strip_prefix = "github.com/goccy/go-reflect@v1.2.0",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/goccy/go-reflect/com_github_goccy_go_reflect-v1.2.0.zip",
+ "http://ats.apps.svc/gomod/github.com/goccy/go-reflect/com_github_goccy_go_reflect-v1.2.0.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/goccy/go-reflect/com_github_goccy_go_reflect-v1.2.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/goccy/go-reflect/com_github_goccy_go_reflect-v1.2.0.zip",
+ ],
+ )
go_repository(
name = "com_github_godbus_dbus_v5",
build_file_proto_mode = "disable_global",
@@ -2855,10 +3024,6 @@ def go_deps():
name = "com_github_golang_protobuf",
build_file_proto_mode = "disable_global",
importpath = "github.com/golang/protobuf",
- patch_args = ["-p1"],
- patches = [
- "//build/patches:com_github_golang_protobuf.patch",
- ],
sha256 = "9a2f43d3eac8ceda506ebbeb4f229254b87235ce90346692a0e233614182190b",
strip_prefix = "github.com/golang/protobuf@v1.5.4",
urls = [
@@ -2911,13 +3076,13 @@ def go_deps():
name = "com_github_golangci_golangci_lint",
build_file_proto_mode = "disable_global",
importpath = "github.com/golangci/golangci-lint",
- sha256 = "9356e7a1047287af7ef44d2b1b45965f03a44d7e4fa2cdfe99c71393e8131173",
- strip_prefix = "github.com/golangci/golangci-lint@v1.57.2",
+ sha256 = "40069eff1b500ea7c5531dbeba0296ace42e09bf1cbf7e11d6169c22395dffa1",
+ strip_prefix = "github.com/golangci/golangci-lint@v1.58.1",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/golangci/golangci-lint/com_github_golangci_golangci_lint-v1.57.2.zip",
- "http://ats.apps.svc/gomod/github.com/golangci/golangci-lint/com_github_golangci_golangci_lint-v1.57.2.zip",
- "https://cache.hawkingrei.com/gomod/github.com/golangci/golangci-lint/com_github_golangci_golangci_lint-v1.57.2.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/golangci/golangci-lint/com_github_golangci_golangci_lint-v1.57.2.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/golangci/golangci-lint/com_github_golangci_golangci_lint-v1.58.1.zip",
+ "http://ats.apps.svc/gomod/github.com/golangci/golangci-lint/com_github_golangci_golangci_lint-v1.58.1.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/golangci/golangci-lint/com_github_golangci_golangci_lint-v1.58.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/golangci/golangci-lint/com_github_golangci_golangci_lint-v1.58.1.zip",
],
)
go_repository(
@@ -2937,13 +3102,26 @@ def go_deps():
name = "com_github_golangci_misspell",
build_file_proto_mode = "disable_global",
importpath = "github.com/golangci/misspell",
- sha256 = "5792fe3dd490249e6288020ff82c72a716ebaf52a8e99fe787b908423587fba3",
- strip_prefix = "github.com/golangci/misspell@v0.4.1",
+ sha256 = "945083642b79fd34a825945173e37456bc0a25e6eda0817c21419cc7a81c3183",
+ strip_prefix = "github.com/golangci/misspell@v0.5.1",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/golangci/misspell/com_github_golangci_misspell-v0.5.1.zip",
+ "http://ats.apps.svc/gomod/github.com/golangci/misspell/com_github_golangci_misspell-v0.5.1.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/golangci/misspell/com_github_golangci_misspell-v0.5.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/golangci/misspell/com_github_golangci_misspell-v0.5.1.zip",
+ ],
+ )
+ go_repository(
+ name = "com_github_golangci_modinfo",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/golangci/modinfo",
+ sha256 = "52b2fa1dfa55b4a08335599825ede662b1f4c8c78a0df0c9500fb204f0502c3c",
+ strip_prefix = "github.com/golangci/modinfo@v0.3.4",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/golangci/misspell/com_github_golangci_misspell-v0.4.1.zip",
- "http://ats.apps.svc/gomod/github.com/golangci/misspell/com_github_golangci_misspell-v0.4.1.zip",
- "https://cache.hawkingrei.com/gomod/github.com/golangci/misspell/com_github_golangci_misspell-v0.4.1.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/golangci/misspell/com_github_golangci_misspell-v0.4.1.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/golangci/modinfo/com_github_golangci_modinfo-v0.3.4.zip",
+ "http://ats.apps.svc/gomod/github.com/golangci/modinfo/com_github_golangci_modinfo-v0.3.4.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/golangci/modinfo/com_github_golangci_modinfo-v0.3.4.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/golangci/modinfo/com_github_golangci_modinfo-v0.3.4.zip",
],
)
go_repository(
@@ -2976,13 +3154,13 @@ def go_deps():
name = "com_github_golangci_revgrep",
build_file_proto_mode = "disable_global",
importpath = "github.com/golangci/revgrep",
- sha256 = "0806458bba9e33b3a3566e5f246ed4be0f356695606c5987974effbc9081f750",
- strip_prefix = "github.com/golangci/revgrep@v0.5.2",
+ sha256 = "3a38e449d5e26494ecd074671e74a9a0f18f49911f7aec8b088a3dbdf5d0d3b4",
+ strip_prefix = "github.com/golangci/revgrep@v0.5.3",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/golangci/revgrep/com_github_golangci_revgrep-v0.5.2.zip",
- "http://ats.apps.svc/gomod/github.com/golangci/revgrep/com_github_golangci_revgrep-v0.5.2.zip",
- "https://cache.hawkingrei.com/gomod/github.com/golangci/revgrep/com_github_golangci_revgrep-v0.5.2.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/golangci/revgrep/com_github_golangci_revgrep-v0.5.2.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/golangci/revgrep/com_github_golangci_revgrep-v0.5.3.zip",
+ "http://ats.apps.svc/gomod/github.com/golangci/revgrep/com_github_golangci_revgrep-v0.5.3.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/golangci/revgrep/com_github_golangci_revgrep-v0.5.3.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/golangci/revgrep/com_github_golangci_revgrep-v0.5.3.zip",
],
)
go_repository(
@@ -3011,6 +3189,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/google/btree/com_github_google_btree-v1.1.2.zip",
],
)
+ go_repository(
+ name = "com_github_google_flatbuffers",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/google/flatbuffers",
+ sha256 = "0c0a4aab1c6029141d655bc7fdc07e22dd06f3f64ebbf7a2250b870ef7aac7ee",
+ strip_prefix = "github.com/google/flatbuffers@v2.0.8+incompatible",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/google/flatbuffers/com_github_google_flatbuffers-v2.0.8+incompatible.zip",
+ "http://ats.apps.svc/gomod/github.com/google/flatbuffers/com_github_google_flatbuffers-v2.0.8+incompatible.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/google/flatbuffers/com_github_google_flatbuffers-v2.0.8+incompatible.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/google/flatbuffers/com_github_google_flatbuffers-v2.0.8+incompatible.zip",
+ ],
+ )
go_repository(
name = "com_github_google_gnostic_models",
build_file_proto_mode = "disable_global",
@@ -3223,13 +3414,13 @@ def go_deps():
name = "com_github_googleapis_gax_go_v2",
build_file_proto_mode = "disable_global",
importpath = "github.com/googleapis/gax-go/v2",
- sha256 = "10ad5944b8bcce3f2cb9a215a0dda163de5b1f092e61b74a4e162d1eb8f7f7a2",
- strip_prefix = "github.com/googleapis/gax-go/v2@v2.12.0",
+ sha256 = "2509958273e5988a7b0442ecc06a99f292061d1fe0df05c4a858ab408f120764",
+ strip_prefix = "github.com/googleapis/gax-go/v2@v2.12.2",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/googleapis/gax-go/v2/com_github_googleapis_gax_go_v2-v2.12.0.zip",
- "http://ats.apps.svc/gomod/github.com/googleapis/gax-go/v2/com_github_googleapis_gax_go_v2-v2.12.0.zip",
- "https://cache.hawkingrei.com/gomod/github.com/googleapis/gax-go/v2/com_github_googleapis_gax_go_v2-v2.12.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/googleapis/gax-go/v2/com_github_googleapis_gax_go_v2-v2.12.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/googleapis/gax-go/v2/com_github_googleapis_gax_go_v2-v2.12.2.zip",
+ "http://ats.apps.svc/gomod/github.com/googleapis/gax-go/v2/com_github_googleapis_gax_go_v2-v2.12.2.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/googleapis/gax-go/v2/com_github_googleapis_gax_go_v2-v2.12.2.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/googleapis/gax-go/v2/com_github_googleapis_gax_go_v2-v2.12.2.zip",
],
)
go_repository(
@@ -4021,13 +4212,13 @@ def go_deps():
name = "com_github_jjti_go_spancheck",
build_file_proto_mode = "disable_global",
importpath = "github.com/jjti/go-spancheck",
- sha256 = "d1de833aeb7fc0bf8b8761fe3cfeef42b66c8fa479cb0aababd024840da2d7d6",
- strip_prefix = "github.com/jjti/go-spancheck@v0.5.3",
+ sha256 = "d8a4115c878fd382f434eb9cf74183cfa91b42b4f83a213f4d45499d852db49e",
+ strip_prefix = "github.com/jjti/go-spancheck@v0.6.1",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/jjti/go-spancheck/com_github_jjti_go_spancheck-v0.5.3.zip",
- "http://ats.apps.svc/gomod/github.com/jjti/go-spancheck/com_github_jjti_go_spancheck-v0.5.3.zip",
- "https://cache.hawkingrei.com/gomod/github.com/jjti/go-spancheck/com_github_jjti_go_spancheck-v0.5.3.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/jjti/go-spancheck/com_github_jjti_go_spancheck-v0.5.3.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/jjti/go-spancheck/com_github_jjti_go_spancheck-v0.6.1.zip",
+ "http://ats.apps.svc/gomod/github.com/jjti/go-spancheck/com_github_jjti_go_spancheck-v0.6.1.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/jjti/go-spancheck/com_github_jjti_go_spancheck-v0.6.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/jjti/go-spancheck/com_github_jjti_go_spancheck-v0.6.1.zip",
],
)
go_repository(
@@ -4069,6 +4260,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/johannesboyne/gofakes3/com_github_johannesboyne_gofakes3-v0.0.0-20230506070712-04da935ef877.zip",
],
)
+ go_repository(
+ name = "com_github_johncgriffin_overflow",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/JohnCGriffin/overflow",
+ sha256 = "8ad4da840214861386d243127290666cc54eb914d1f4a8856523481876af2a09",
+ strip_prefix = "github.com/JohnCGriffin/overflow@v0.0.0-20211019200055-46fa312c352c",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/JohnCGriffin/overflow/com_github_johncgriffin_overflow-v0.0.0-20211019200055-46fa312c352c.zip",
+ "http://ats.apps.svc/gomod/github.com/JohnCGriffin/overflow/com_github_johncgriffin_overflow-v0.0.0-20211019200055-46fa312c352c.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/JohnCGriffin/overflow/com_github_johncgriffin_overflow-v0.0.0-20211019200055-46fa312c352c.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/JohnCGriffin/overflow/com_github_johncgriffin_overflow-v0.0.0-20211019200055-46fa312c352c.zip",
+ ],
+ )
go_repository(
name = "com_github_joho_sqltocsv",
build_file_proto_mode = "disable_global",
@@ -4203,13 +4407,13 @@ def go_deps():
name = "com_github_karamaru_alpha_copyloopvar",
build_file_proto_mode = "disable_global",
importpath = "github.com/karamaru-alpha/copyloopvar",
- sha256 = "4deae3d4f7026a0244a030c41c62e343442f6b7b9e18defb69b75f970f629e48",
- strip_prefix = "github.com/karamaru-alpha/copyloopvar@v1.0.10",
+ sha256 = "60c387926b32870d5f5acc818ec648a3bde2f95acfd37151e073466cf7cafd86",
+ strip_prefix = "github.com/karamaru-alpha/copyloopvar@v1.1.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/karamaru-alpha/copyloopvar/com_github_karamaru_alpha_copyloopvar-v1.0.10.zip",
- "http://ats.apps.svc/gomod/github.com/karamaru-alpha/copyloopvar/com_github_karamaru_alpha_copyloopvar-v1.0.10.zip",
- "https://cache.hawkingrei.com/gomod/github.com/karamaru-alpha/copyloopvar/com_github_karamaru_alpha_copyloopvar-v1.0.10.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/karamaru-alpha/copyloopvar/com_github_karamaru_alpha_copyloopvar-v1.0.10.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/karamaru-alpha/copyloopvar/com_github_karamaru_alpha_copyloopvar-v1.1.0.zip",
+ "http://ats.apps.svc/gomod/github.com/karamaru-alpha/copyloopvar/com_github_karamaru_alpha_copyloopvar-v1.1.0.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/karamaru-alpha/copyloopvar/com_github_karamaru_alpha_copyloopvar-v1.1.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/karamaru-alpha/copyloopvar/com_github_karamaru_alpha_copyloopvar-v1.1.0.zip",
],
)
go_repository(
@@ -4290,6 +4494,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/kataras/tunnel/com_github_kataras_tunnel-v0.0.4.zip",
],
)
+ go_repository(
+ name = "com_github_kballard_go_shellquote",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/kballard/go-shellquote",
+ sha256 = "ae4cb7b097dc4eb0c248dff00ed3bbf0f36984c4162ad1d615266084e58bd6cc",
+ strip_prefix = "github.com/kballard/go-shellquote@v0.0.0-20180428030007-95032a82bc51",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/kballard/go-shellquote/com_github_kballard_go_shellquote-v0.0.0-20180428030007-95032a82bc51.zip",
+ "http://ats.apps.svc/gomod/github.com/kballard/go-shellquote/com_github_kballard_go_shellquote-v0.0.0-20180428030007-95032a82bc51.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/kballard/go-shellquote/com_github_kballard_go_shellquote-v0.0.0-20180428030007-95032a82bc51.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/kballard/go-shellquote/com_github_kballard_go_shellquote-v0.0.0-20180428030007-95032a82bc51.zip",
+ ],
+ )
go_repository(
name = "com_github_kimmachinegun_automemlimit",
build_file_proto_mode = "disable_global",
@@ -4346,6 +4563,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/kkHAIKE/contextcheck/com_github_kkhaike_contextcheck-v1.1.5.zip",
],
)
+ go_repository(
+ name = "com_github_klauspost_asmfmt",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/klauspost/asmfmt",
+ sha256 = "fa6a350a8677a77e0dbf3664c6baf23aab5c0b60a64b8f3c00299da5d279021f",
+ strip_prefix = "github.com/klauspost/asmfmt@v1.3.2",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/klauspost/asmfmt/com_github_klauspost_asmfmt-v1.3.2.zip",
+ "http://ats.apps.svc/gomod/github.com/klauspost/asmfmt/com_github_klauspost_asmfmt-v1.3.2.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/klauspost/asmfmt/com_github_klauspost_asmfmt-v1.3.2.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/klauspost/asmfmt/com_github_klauspost_asmfmt-v1.3.2.zip",
+ ],
+ )
go_repository(
name = "com_github_klauspost_compress",
build_file_proto_mode = "disable_global",
@@ -4372,6 +4602,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/klauspost/cpuid/com_github_klauspost_cpuid-v1.3.1.zip",
],
)
+ go_repository(
+ name = "com_github_klauspost_cpuid_v2",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/klauspost/cpuid/v2",
+ sha256 = "52c716413296dce2b1698c6cdbc4c53927ce4aee2a60980daf9672e6b6a3b4cb",
+ strip_prefix = "github.com/klauspost/cpuid/v2@v2.0.9",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/klauspost/cpuid/v2/com_github_klauspost_cpuid_v2-v2.0.9.zip",
+ "http://ats.apps.svc/gomod/github.com/klauspost/cpuid/v2/com_github_klauspost_cpuid_v2-v2.0.9.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/klauspost/cpuid/v2/com_github_klauspost_cpuid_v2-v2.0.9.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/klauspost/cpuid/v2/com_github_klauspost_cpuid_v2-v2.0.9.zip",
+ ],
+ )
go_repository(
name = "com_github_kolo_xmlrpc",
build_file_proto_mode = "disable_global",
@@ -4528,6 +4771,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/labstack/gommon/com_github_labstack_gommon-v0.4.0.zip",
],
)
+ go_repository(
+ name = "com_github_lasiar_canonicalheader",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/lasiar/canonicalheader",
+ sha256 = "6185dc1a2c06926f966aca3cd702dc6b931106e415894c6d97b10a8248947c44",
+ strip_prefix = "github.com/lasiar/canonicalheader@v1.0.6",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/lasiar/canonicalheader/com_github_lasiar_canonicalheader-v1.0.6.zip",
+ "http://ats.apps.svc/gomod/github.com/lasiar/canonicalheader/com_github_lasiar_canonicalheader-v1.0.6.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/lasiar/canonicalheader/com_github_lasiar_canonicalheader-v1.0.6.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/lasiar/canonicalheader/com_github_lasiar_canonicalheader-v1.0.6.zip",
+ ],
+ )
go_repository(
name = "com_github_ldez_gomoddirectives",
build_file_proto_mode = "disable_global",
@@ -4571,13 +4827,13 @@ def go_deps():
name = "com_github_leonklingele_grouper",
build_file_proto_mode = "disable_global",
importpath = "github.com/leonklingele/grouper",
- sha256 = "bfc82b49e2d6a73b3df108ae164fc2b72030a16f94de4204d38d96d2bc06fb60",
- strip_prefix = "github.com/leonklingele/grouper@v1.1.1",
+ sha256 = "8e6018a6009b21d0e260a4a7afd281a6b7d97ade749319ae42e493d2cf5a5cbc",
+ strip_prefix = "github.com/leonklingele/grouper@v1.1.2",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/leonklingele/grouper/com_github_leonklingele_grouper-v1.1.1.zip",
- "http://ats.apps.svc/gomod/github.com/leonklingele/grouper/com_github_leonklingele_grouper-v1.1.1.zip",
- "https://cache.hawkingrei.com/gomod/github.com/leonklingele/grouper/com_github_leonklingele_grouper-v1.1.1.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/leonklingele/grouper/com_github_leonklingele_grouper-v1.1.1.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/leonklingele/grouper/com_github_leonklingele_grouper-v1.1.2.zip",
+ "http://ats.apps.svc/gomod/github.com/leonklingele/grouper/com_github_leonklingele_grouper-v1.1.2.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/leonklingele/grouper/com_github_leonklingele_grouper-v1.1.2.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/leonklingele/grouper/com_github_leonklingele_grouper-v1.1.2.zip",
],
)
go_repository(
@@ -4805,13 +5061,13 @@ def go_deps():
name = "com_github_masterminds_semver_v3",
build_file_proto_mode = "disable_global",
importpath = "github.com/Masterminds/semver/v3",
- sha256 = "0a46c7403dfeda09b0821e851f8e1cec8f1ea4276281e42ea399da5bc5bf0704",
- strip_prefix = "github.com/Masterminds/semver/v3@v3.1.1",
+ sha256 = "d3e3b1dae669d44d9f92a314e02c3b2bbff2c5b2463f650cdbb7340f413e854b",
+ strip_prefix = "github.com/Masterminds/semver/v3@v3.2.1",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/Masterminds/semver/v3/com_github_masterminds_semver_v3-v3.1.1.zip",
- "http://ats.apps.svc/gomod/github.com/Masterminds/semver/v3/com_github_masterminds_semver_v3-v3.1.1.zip",
- "https://cache.hawkingrei.com/gomod/github.com/Masterminds/semver/v3/com_github_masterminds_semver_v3-v3.1.1.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Masterminds/semver/v3/com_github_masterminds_semver_v3-v3.1.1.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/Masterminds/semver/v3/com_github_masterminds_semver_v3-v3.2.1.zip",
+ "http://ats.apps.svc/gomod/github.com/Masterminds/semver/v3/com_github_masterminds_semver_v3-v3.2.1.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/Masterminds/semver/v3/com_github_masterminds_semver_v3-v3.2.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/Masterminds/semver/v3/com_github_masterminds_semver_v3-v3.2.1.zip",
],
)
go_repository(
@@ -4879,6 +5135,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/mattn/go-runewidth/com_github_mattn_go_runewidth-v0.0.15.zip",
],
)
+ go_repository(
+ name = "com_github_mattn_go_sqlite3",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/mattn/go-sqlite3",
+ sha256 = "0114d2df439ddeb03eef49a4bf2cc8fb69665c0d76494463cafa7d189a16e0f9",
+ strip_prefix = "github.com/mattn/go-sqlite3@v1.14.15",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/mattn/go-sqlite3/com_github_mattn_go_sqlite3-v1.14.15.zip",
+ "http://ats.apps.svc/gomod/github.com/mattn/go-sqlite3/com_github_mattn_go_sqlite3-v1.14.15.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/mattn/go-sqlite3/com_github_mattn_go_sqlite3-v1.14.15.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/mattn/go-sqlite3/com_github_mattn_go_sqlite3-v1.14.15.zip",
+ ],
+ )
go_repository(
name = "com_github_matttproud_golang_protobuf_extensions",
build_file_proto_mode = "disable_global",
@@ -4957,6 +5226,32 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/miekg/dns/com_github_miekg_dns-v1.1.58.zip",
],
)
+ go_repository(
+ name = "com_github_minio_asm2plan9s",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/minio/asm2plan9s",
+ sha256 = "39a2e28284764fd5423247d7469875046d0c8c4c2773333abf1c544197e9d946",
+ strip_prefix = "github.com/minio/asm2plan9s@v0.0.0-20200509001527-cdd76441f9d8",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/minio/asm2plan9s/com_github_minio_asm2plan9s-v0.0.0-20200509001527-cdd76441f9d8.zip",
+ "http://ats.apps.svc/gomod/github.com/minio/asm2plan9s/com_github_minio_asm2plan9s-v0.0.0-20200509001527-cdd76441f9d8.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/minio/asm2plan9s/com_github_minio_asm2plan9s-v0.0.0-20200509001527-cdd76441f9d8.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/minio/asm2plan9s/com_github_minio_asm2plan9s-v0.0.0-20200509001527-cdd76441f9d8.zip",
+ ],
+ )
+ go_repository(
+ name = "com_github_minio_c2goasm",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/minio/c2goasm",
+ sha256 = "04367ddf0fc5cd0f293e2c4f1acefb131b572539d88b5804d92efc905eb718b5",
+ strip_prefix = "github.com/minio/c2goasm@v0.0.0-20190812172519-36a3d3bbc4f3",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/minio/c2goasm/com_github_minio_c2goasm-v0.0.0-20190812172519-36a3d3bbc4f3.zip",
+ "http://ats.apps.svc/gomod/github.com/minio/c2goasm/com_github_minio_c2goasm-v0.0.0-20190812172519-36a3d3bbc4f3.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/minio/c2goasm/com_github_minio_c2goasm-v0.0.0-20190812172519-36a3d3bbc4f3.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/minio/c2goasm/com_github_minio_c2goasm-v0.0.0-20190812172519-36a3d3bbc4f3.zip",
+ ],
+ )
go_repository(
name = "com_github_mitchellh_copystructure",
build_file_proto_mode = "disable_global",
@@ -5533,13 +5828,13 @@ def go_deps():
name = "com_github_pelletier_go_toml_v2",
build_file_proto_mode = "disable_global",
importpath = "github.com/pelletier/go-toml/v2",
- sha256 = "09630910d93bb6f0021807aa3c655d35f54915eb2a9955e54e61f176863bfe8d",
- strip_prefix = "github.com/pelletier/go-toml/v2@v2.2.0",
+ sha256 = "8d724e35b485503810f866bca278d518e731713441e380634f6b33c27aefdf3e",
+ strip_prefix = "github.com/pelletier/go-toml/v2@v2.2.2",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/pelletier/go-toml/v2/com_github_pelletier_go_toml_v2-v2.2.0.zip",
- "http://ats.apps.svc/gomod/github.com/pelletier/go-toml/v2/com_github_pelletier_go_toml_v2-v2.2.0.zip",
- "https://cache.hawkingrei.com/gomod/github.com/pelletier/go-toml/v2/com_github_pelletier_go_toml_v2-v2.2.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pelletier/go-toml/v2/com_github_pelletier_go_toml_v2-v2.2.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/pelletier/go-toml/v2/com_github_pelletier_go_toml_v2-v2.2.2.zip",
+ "http://ats.apps.svc/gomod/github.com/pelletier/go-toml/v2/com_github_pelletier_go_toml_v2-v2.2.2.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/pelletier/go-toml/v2/com_github_pelletier_go_toml_v2-v2.2.2.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pelletier/go-toml/v2/com_github_pelletier_go_toml_v2-v2.2.2.zip",
],
)
go_repository(
@@ -5568,6 +5863,32 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/phayes/freeport/com_github_phayes_freeport-v0.0.0-20180830031419-95f893ade6f2.zip",
],
)
+ go_repository(
+ name = "com_github_phpdave11_gofpdf",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/phpdave11/gofpdf",
+ sha256 = "4db05258f281b40d8a17392fd71648779ea758a9aa506a8d1346ded737ede43f",
+ strip_prefix = "github.com/phpdave11/gofpdf@v1.4.2",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/phpdave11/gofpdf/com_github_phpdave11_gofpdf-v1.4.2.zip",
+ "http://ats.apps.svc/gomod/github.com/phpdave11/gofpdf/com_github_phpdave11_gofpdf-v1.4.2.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/phpdave11/gofpdf/com_github_phpdave11_gofpdf-v1.4.2.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/phpdave11/gofpdf/com_github_phpdave11_gofpdf-v1.4.2.zip",
+ ],
+ )
+ go_repository(
+ name = "com_github_phpdave11_gofpdi",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/phpdave11/gofpdi",
+ sha256 = "09b728136cf290f4ee87aa47b60f2f9df2b3f4f64119ff10f12319bc3438b58d",
+ strip_prefix = "github.com/phpdave11/gofpdi@v1.0.13",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/phpdave11/gofpdi/com_github_phpdave11_gofpdi-v1.0.13.zip",
+ "http://ats.apps.svc/gomod/github.com/phpdave11/gofpdi/com_github_phpdave11_gofpdi-v1.0.13.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/phpdave11/gofpdi/com_github_phpdave11_gofpdi-v1.0.13.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/phpdave11/gofpdi/com_github_phpdave11_gofpdi-v1.0.13.zip",
+ ],
+ )
go_repository(
name = "com_github_pierrec_lz4",
build_file_proto_mode = "disable_global",
@@ -5581,6 +5902,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/pierrec/lz4/com_github_pierrec_lz4-v2.6.1+incompatible.zip",
],
)
+ go_repository(
+ name = "com_github_pierrec_lz4_v4",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/pierrec/lz4/v4",
+ sha256 = "5dadfc447d593c4a8a75520b9f048142d725e4d966d48883ece2380c16081900",
+ strip_prefix = "github.com/pierrec/lz4/v4@v4.1.15",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/pierrec/lz4/v4/com_github_pierrec_lz4_v4-v4.1.15.zip",
+ "http://ats.apps.svc/gomod/github.com/pierrec/lz4/v4/com_github_pierrec_lz4_v4-v4.1.15.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/pierrec/lz4/v4/com_github_pierrec_lz4_v4-v4.1.15.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pierrec/lz4/v4/com_github_pierrec_lz4_v4-v4.1.15.zip",
+ ],
+ )
go_repository(
name = "com_github_pingcap_badger",
build_file_proto_mode = "disable_global",
@@ -5611,13 +5945,13 @@ def go_deps():
name = "com_github_pingcap_failpoint",
build_file_proto_mode = "disable_global",
importpath = "github.com/pingcap/failpoint",
- sha256 = "ea37b4dddfbccaaed9b313f9f1099dfbf00d36d768a8416d6d175cbe2c8b1254",
- strip_prefix = "github.com/pingcap/failpoint@v0.0.0-20220801062533-2eaa32854a6c",
+ sha256 = "7ba8f68be702b49ac26b9b7cd14fc7ce0aaf9c2cc4b35d9a80eac7f52a9a0d76",
+ strip_prefix = "github.com/pingcap/failpoint@v0.0.0-20240527053858-9b3b6e34194a",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/failpoint/com_github_pingcap_failpoint-v0.0.0-20220801062533-2eaa32854a6c.zip",
- "http://ats.apps.svc/gomod/github.com/pingcap/failpoint/com_github_pingcap_failpoint-v0.0.0-20220801062533-2eaa32854a6c.zip",
- "https://cache.hawkingrei.com/gomod/github.com/pingcap/failpoint/com_github_pingcap_failpoint-v0.0.0-20220801062533-2eaa32854a6c.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/failpoint/com_github_pingcap_failpoint-v0.0.0-20220801062533-2eaa32854a6c.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/failpoint/com_github_pingcap_failpoint-v0.0.0-20240527053858-9b3b6e34194a.zip",
+ "http://ats.apps.svc/gomod/github.com/pingcap/failpoint/com_github_pingcap_failpoint-v0.0.0-20240527053858-9b3b6e34194a.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/pingcap/failpoint/com_github_pingcap_failpoint-v0.0.0-20240527053858-9b3b6e34194a.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/failpoint/com_github_pingcap_failpoint-v0.0.0-20240527053858-9b3b6e34194a.zip",
],
)
go_repository(
@@ -5650,13 +5984,13 @@ def go_deps():
name = "com_github_pingcap_kvproto",
build_file_proto_mode = "disable_global",
importpath = "github.com/pingcap/kvproto",
- sha256 = "07dff29e9848e79f36ac8dcd0d5b48bbfbb2796308702451afb862accb79fedb",
- strip_prefix = "github.com/pingcap/kvproto@v0.0.0-20240227073058-929ab83f9754",
+ sha256 = "ab79db554bdb20f7ebb44cd88cac36f3bf56a9d087912eda252fe40c7c828a55",
+ strip_prefix = "github.com/pingcap/kvproto@v0.0.0-20240513094934-d9297553c900",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240227073058-929ab83f9754.zip",
- "http://ats.apps.svc/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240227073058-929ab83f9754.zip",
- "https://cache.hawkingrei.com/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240227073058-929ab83f9754.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240227073058-929ab83f9754.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240513094934-d9297553c900.zip",
+ "http://ats.apps.svc/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240513094934-d9297553c900.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240513094934-d9297553c900.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240513094934-d9297553c900.zip",
],
)
go_repository(
@@ -5780,13 +6114,13 @@ def go_deps():
name = "com_github_polyfloyd_go_errorlint",
build_file_proto_mode = "disable_global",
importpath = "github.com/polyfloyd/go-errorlint",
- sha256 = "e8e25aa733cc13a4c1a4301ffc2f657043bff25bec5e907f5dbfa5c33e32733e",
- strip_prefix = "github.com/polyfloyd/go-errorlint@v1.4.8",
+ sha256 = "bac1e1f85fcbbb7f62f3893247cbe88e0c7cc087ab86e06ee39e61ac9781c3dd",
+ strip_prefix = "github.com/polyfloyd/go-errorlint@v1.5.1",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/polyfloyd/go-errorlint/com_github_polyfloyd_go_errorlint-v1.4.8.zip",
- "http://ats.apps.svc/gomod/github.com/polyfloyd/go-errorlint/com_github_polyfloyd_go_errorlint-v1.4.8.zip",
- "https://cache.hawkingrei.com/gomod/github.com/polyfloyd/go-errorlint/com_github_polyfloyd_go_errorlint-v1.4.8.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/polyfloyd/go-errorlint/com_github_polyfloyd_go_errorlint-v1.4.8.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/polyfloyd/go-errorlint/com_github_polyfloyd_go_errorlint-v1.5.1.zip",
+ "http://ats.apps.svc/gomod/github.com/polyfloyd/go-errorlint/com_github_polyfloyd_go_errorlint-v1.5.1.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/polyfloyd/go-errorlint/com_github_polyfloyd_go_errorlint-v1.5.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/polyfloyd/go-errorlint/com_github_polyfloyd_go_errorlint-v1.5.1.zip",
],
)
go_repository(
@@ -5910,13 +6244,13 @@ def go_deps():
name = "com_github_prometheus_procfs",
build_file_proto_mode = "disable_global",
importpath = "github.com/prometheus/procfs",
- sha256 = "6fe923fc0ca170a60524d1031cf9ee634cb2eba16798edcbe1ed02e591994cfa",
- strip_prefix = "github.com/prometheus/procfs@v0.13.0",
+ sha256 = "d31ad13f1ae121d842ff0f243d029c247e68710edab8a358d6366a67b7feaa6d",
+ strip_prefix = "github.com/prometheus/procfs@v0.15.1",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/prometheus/procfs/com_github_prometheus_procfs-v0.13.0.zip",
- "http://ats.apps.svc/gomod/github.com/prometheus/procfs/com_github_prometheus_procfs-v0.13.0.zip",
- "https://cache.hawkingrei.com/gomod/github.com/prometheus/procfs/com_github_prometheus_procfs-v0.13.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/prometheus/procfs/com_github_prometheus_procfs-v0.13.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/prometheus/procfs/com_github_prometheus_procfs-v0.15.1.zip",
+ "http://ats.apps.svc/gomod/github.com/prometheus/procfs/com_github_prometheus_procfs-v0.15.1.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/prometheus/procfs/com_github_prometheus_procfs-v0.15.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/prometheus/procfs/com_github_prometheus_procfs-v0.15.1.zip",
],
)
go_repository(
@@ -5932,6 +6266,32 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/prometheus/prometheus/com_github_prometheus_prometheus-v0.50.1.zip",
],
)
+ go_repository(
+ name = "com_github_qri_io_jsonpointer",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/qri-io/jsonpointer",
+ sha256 = "6870d4b9fc5ac8efb9226447975fecfb07241133e23c7e661f5aac1a3088f338",
+ strip_prefix = "github.com/qri-io/jsonpointer@v0.1.1",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/qri-io/jsonpointer/com_github_qri_io_jsonpointer-v0.1.1.zip",
+ "http://ats.apps.svc/gomod/github.com/qri-io/jsonpointer/com_github_qri_io_jsonpointer-v0.1.1.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/qri-io/jsonpointer/com_github_qri_io_jsonpointer-v0.1.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/qri-io/jsonpointer/com_github_qri_io_jsonpointer-v0.1.1.zip",
+ ],
+ )
+ go_repository(
+ name = "com_github_qri_io_jsonschema",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/qri-io/jsonschema",
+ sha256 = "51305cc45fd383b24de94e2eb421ffba8d83679520c18348842c4255025c5940",
+ strip_prefix = "github.com/qri-io/jsonschema@v0.2.1",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/qri-io/jsonschema/com_github_qri_io_jsonschema-v0.2.1.zip",
+ "http://ats.apps.svc/gomod/github.com/qri-io/jsonschema/com_github_qri_io_jsonschema-v0.2.1.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/qri-io/jsonschema/com_github_qri_io_jsonschema-v0.2.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/qri-io/jsonschema/com_github_qri_io_jsonschema-v0.2.1.zip",
+ ],
+ )
go_repository(
name = "com_github_quasilyte_go_ruleguard",
build_file_proto_mode = "disable_global",
@@ -6101,17 +6461,30 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/russross/blackfriday/v2/com_github_russross_blackfriday_v2-v2.1.0.zip",
],
)
+ go_repository(
+ name = "com_github_ruudk_golang_pdf417",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/ruudk/golang-pdf417",
+ sha256 = "f0006c0f60789da76c1b3fef73bb63f5581744fbe3ab5973ec718b40c6822f69",
+ strip_prefix = "github.com/ruudk/golang-pdf417@v0.0.0-20201230142125-a7e3863a1245",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/ruudk/golang-pdf417/com_github_ruudk_golang_pdf417-v0.0.0-20201230142125-a7e3863a1245.zip",
+ "http://ats.apps.svc/gomod/github.com/ruudk/golang-pdf417/com_github_ruudk_golang_pdf417-v0.0.0-20201230142125-a7e3863a1245.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/ruudk/golang-pdf417/com_github_ruudk_golang_pdf417-v0.0.0-20201230142125-a7e3863a1245.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ruudk/golang-pdf417/com_github_ruudk_golang_pdf417-v0.0.0-20201230142125-a7e3863a1245.zip",
+ ],
+ )
go_repository(
name = "com_github_ryancurrah_gomodguard",
build_file_proto_mode = "disable_global",
importpath = "github.com/ryancurrah/gomodguard",
- sha256 = "34294558448d4a468aef35f716464a30db07c26630314d795800295c62d69cfb",
- strip_prefix = "github.com/ryancurrah/gomodguard@v1.3.1",
+ sha256 = "03930d9ec32fe7f11777dc352ef034f4c21bdb0d2915cc21777dca1a8e180a06",
+ strip_prefix = "github.com/ryancurrah/gomodguard@v1.3.2",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/ryancurrah/gomodguard/com_github_ryancurrah_gomodguard-v1.3.1.zip",
- "http://ats.apps.svc/gomod/github.com/ryancurrah/gomodguard/com_github_ryancurrah_gomodguard-v1.3.1.zip",
- "https://cache.hawkingrei.com/gomod/github.com/ryancurrah/gomodguard/com_github_ryancurrah_gomodguard-v1.3.1.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ryancurrah/gomodguard/com_github_ryancurrah_gomodguard-v1.3.1.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/ryancurrah/gomodguard/com_github_ryancurrah_gomodguard-v1.3.2.zip",
+ "http://ats.apps.svc/gomod/github.com/ryancurrah/gomodguard/com_github_ryancurrah_gomodguard-v1.3.2.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/ryancurrah/gomodguard/com_github_ryancurrah_gomodguard-v1.3.2.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ryancurrah/gomodguard/com_github_ryancurrah_gomodguard-v1.3.2.zip",
],
)
go_repository(
@@ -6300,13 +6673,13 @@ def go_deps():
name = "com_github_shirou_gopsutil_v3",
build_file_proto_mode = "disable_global",
importpath = "github.com/shirou/gopsutil/v3",
- sha256 = "bf63d0877101d8ab85d39576448c838e11691e27906036bbab374680ac5921a4",
- strip_prefix = "github.com/shirou/gopsutil/v3@v3.24.2",
+ sha256 = "2bb2f820942ced0f23a05b929a6b7ffbac7e69b5bd2d3bea70d37e40a4d47e12",
+ strip_prefix = "github.com/shirou/gopsutil/v3@v3.24.4",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/shirou/gopsutil/v3/com_github_shirou_gopsutil_v3-v3.24.2.zip",
- "http://ats.apps.svc/gomod/github.com/shirou/gopsutil/v3/com_github_shirou_gopsutil_v3-v3.24.2.zip",
- "https://cache.hawkingrei.com/gomod/github.com/shirou/gopsutil/v3/com_github_shirou_gopsutil_v3-v3.24.2.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/shirou/gopsutil/v3/com_github_shirou_gopsutil_v3-v3.24.2.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/shirou/gopsutil/v3/com_github_shirou_gopsutil_v3-v3.24.4.zip",
+ "http://ats.apps.svc/gomod/github.com/shirou/gopsutil/v3/com_github_shirou_gopsutil_v3-v3.24.4.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/shirou/gopsutil/v3/com_github_shirou_gopsutil_v3-v3.24.4.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/shirou/gopsutil/v3/com_github_shirou_gopsutil_v3-v3.24.4.zip",
],
)
go_repository(
@@ -6807,26 +7180,26 @@ def go_deps():
name = "com_github_tikv_client_go_v2",
build_file_proto_mode = "disable_global",
importpath = "github.com/tikv/client-go/v2",
- sha256 = "1caf3b1821ac27db865e6200e6a629afe54f3fcdaeff557b159d93e5c7eaaef1",
- strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20240430145241-6cb0704fce51",
+ sha256 = "0e79a43d532d91c3a3fe0fb95bf8ce2aaced3db3e5f3282cceb6facc000ad03f",
+ strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20240531102121-cb580bc4ea29",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240430145241-6cb0704fce51.zip",
- "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240430145241-6cb0704fce51.zip",
- "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240430145241-6cb0704fce51.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240430145241-6cb0704fce51.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240531102121-cb580bc4ea29.zip",
+ "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240531102121-cb580bc4ea29.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240531102121-cb580bc4ea29.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240531102121-cb580bc4ea29.zip",
],
)
go_repository(
name = "com_github_tikv_pd_client",
build_file_proto_mode = "disable_global",
importpath = "github.com/tikv/pd/client",
- sha256 = "88e41e2f075159a2ceec954c549ced6e25ad98a9cad5e5879a65eda62d74a260",
- strip_prefix = "github.com/tikv/pd/client@v0.0.0-20240430080403-1679dbca25b3",
+ sha256 = "634cdbdda95680c4bc4c5fb16e1d70461e17a1584ff77362197e0ae41d66018e",
+ strip_prefix = "github.com/tikv/pd/client@v0.0.0-20240531082952-199b01792159",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240430080403-1679dbca25b3.zip",
- "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240430080403-1679dbca25b3.zip",
- "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240430080403-1679dbca25b3.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240430080403-1679dbca25b3.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240531082952-199b01792159.zip",
+ "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240531082952-199b01792159.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240531082952-199b01792159.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240531082952-199b01792159.zip",
],
)
go_repository(
@@ -6989,13 +7362,13 @@ def go_deps():
name = "com_github_ultraware_whitespace",
build_file_proto_mode = "disable_global",
importpath = "github.com/ultraware/whitespace",
- sha256 = "43c2ee4bae63b133741fd18677473fd5ff652c673f4137f3bebd85f34e9903fa",
- strip_prefix = "github.com/ultraware/whitespace@v0.1.0",
+ sha256 = "a305b3b4f28ce7e4600849e65df750f6056ca8abf468f774a22d44902a76b984",
+ strip_prefix = "github.com/ultraware/whitespace@v0.1.1",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/ultraware/whitespace/com_github_ultraware_whitespace-v0.1.0.zip",
- "http://ats.apps.svc/gomod/github.com/ultraware/whitespace/com_github_ultraware_whitespace-v0.1.0.zip",
- "https://cache.hawkingrei.com/gomod/github.com/ultraware/whitespace/com_github_ultraware_whitespace-v0.1.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ultraware/whitespace/com_github_ultraware_whitespace-v0.1.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/ultraware/whitespace/com_github_ultraware_whitespace-v0.1.1.zip",
+ "http://ats.apps.svc/gomod/github.com/ultraware/whitespace/com_github_ultraware_whitespace-v0.1.1.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/ultraware/whitespace/com_github_ultraware_whitespace-v0.1.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ultraware/whitespace/com_github_ultraware_whitespace-v0.1.1.zip",
],
)
go_repository(
@@ -7223,13 +7596,13 @@ def go_deps():
name = "com_github_xitongsys_parquet_go",
build_file_proto_mode = "disable_global",
importpath = "github.com/xitongsys/parquet-go",
- sha256 = "12f897439857389593402023bd18c9c4d21c19011d6a014910528a3ff3fad28e",
- strip_prefix = "github.com/xitongsys/parquet-go@v1.5.5-0.20201110004701-b09c49d6d457",
+ sha256 = "5b9473cce95cf094d398348fd394002b656ae1363bb5c33c1338fcdcd57e1b33",
+ strip_prefix = "github.com/xitongsys/parquet-go@v1.6.3-0.20240520233950-75e935fc3e17",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/xitongsys/parquet-go/com_github_xitongsys_parquet_go-v1.5.5-0.20201110004701-b09c49d6d457.zip",
- "http://ats.apps.svc/gomod/github.com/xitongsys/parquet-go/com_github_xitongsys_parquet_go-v1.5.5-0.20201110004701-b09c49d6d457.zip",
- "https://cache.hawkingrei.com/gomod/github.com/xitongsys/parquet-go/com_github_xitongsys_parquet_go-v1.5.5-0.20201110004701-b09c49d6d457.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/xitongsys/parquet-go/com_github_xitongsys_parquet_go-v1.5.5-0.20201110004701-b09c49d6d457.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/xitongsys/parquet-go/com_github_xitongsys_parquet_go-v1.6.3-0.20240520233950-75e935fc3e17.zip",
+ "http://ats.apps.svc/gomod/github.com/xitongsys/parquet-go/com_github_xitongsys_parquet_go-v1.6.3-0.20240520233950-75e935fc3e17.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/xitongsys/parquet-go/com_github_xitongsys_parquet_go-v1.6.3-0.20240520233950-75e935fc3e17.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/xitongsys/parquet-go/com_github_xitongsys_parquet_go-v1.6.3-0.20240520233950-75e935fc3e17.zip",
],
)
go_repository(
@@ -7275,13 +7648,13 @@ def go_deps():
name = "com_github_yeya24_promlinter",
build_file_proto_mode = "disable_global",
importpath = "github.com/yeya24/promlinter",
- sha256 = "4cb02c5b5875f37d89ca8a908911f6943784e5348eedd2d7096d6d2e8e263f8c",
- strip_prefix = "github.com/yeya24/promlinter@v0.2.0",
+ sha256 = "1e93a590632c1c0cd43b37c48754084b013656d6597f420c01786deeca91c275",
+ strip_prefix = "github.com/yeya24/promlinter@v0.3.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/yeya24/promlinter/com_github_yeya24_promlinter-v0.2.0.zip",
- "http://ats.apps.svc/gomod/github.com/yeya24/promlinter/com_github_yeya24_promlinter-v0.2.0.zip",
- "https://cache.hawkingrei.com/gomod/github.com/yeya24/promlinter/com_github_yeya24_promlinter-v0.2.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/yeya24/promlinter/com_github_yeya24_promlinter-v0.2.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/yeya24/promlinter/com_github_yeya24_promlinter-v0.3.0.zip",
+ "http://ats.apps.svc/gomod/github.com/yeya24/promlinter/com_github_yeya24_promlinter-v0.3.0.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/yeya24/promlinter/com_github_yeya24_promlinter-v0.3.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/yeya24/promlinter/com_github_yeya24_promlinter-v0.3.0.zip",
],
)
go_repository(
@@ -7336,30 +7709,56 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/yusufpapurcu/wmi/com_github_yusufpapurcu_wmi-v1.2.4.zip",
],
)
+ go_repository(
+ name = "com_github_zeebo_assert",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/zeebo/assert",
+ sha256 = "1f01421d74ff37cb8247988155be9e6877d336029bcd887a1d035fd32d7ab6ae",
+ strip_prefix = "github.com/zeebo/assert@v1.3.0",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/zeebo/assert/com_github_zeebo_assert-v1.3.0.zip",
+ "http://ats.apps.svc/gomod/github.com/zeebo/assert/com_github_zeebo_assert-v1.3.0.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/zeebo/assert/com_github_zeebo_assert-v1.3.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/zeebo/assert/com_github_zeebo_assert-v1.3.0.zip",
+ ],
+ )
+ go_repository(
+ name = "com_github_zeebo_xxh3",
+ build_file_proto_mode = "disable_global",
+ importpath = "github.com/zeebo/xxh3",
+ sha256 = "190e5ef1f672e9321a1580bdd31c6440fde6044ca8168d2b489cf50cdc4f58a6",
+ strip_prefix = "github.com/zeebo/xxh3@v1.0.2",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/zeebo/xxh3/com_github_zeebo_xxh3-v1.0.2.zip",
+ "http://ats.apps.svc/gomod/github.com/zeebo/xxh3/com_github_zeebo_xxh3-v1.0.2.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/zeebo/xxh3/com_github_zeebo_xxh3-v1.0.2.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/zeebo/xxh3/com_github_zeebo_xxh3-v1.0.2.zip",
+ ],
+ )
go_repository(
name = "com_gitlab_bosi_decorder",
build_file_proto_mode = "disable_global",
importpath = "gitlab.com/bosi/decorder",
- sha256 = "ada0aaccd3bee67d4eabb98c83c320f00c65dca36441ed92c5638cf269c60ba6",
- strip_prefix = "gitlab.com/bosi/decorder@v0.4.1",
+ sha256 = "0f7e812510baa3144be12fca66ee6551cc72177fe53d5ca68dfe10cfa3085cd1",
+ strip_prefix = "gitlab.com/bosi/decorder@v0.4.2",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/gitlab.com/bosi/decorder/com_gitlab_bosi_decorder-v0.4.1.zip",
- "http://ats.apps.svc/gomod/gitlab.com/bosi/decorder/com_gitlab_bosi_decorder-v0.4.1.zip",
- "https://cache.hawkingrei.com/gomod/gitlab.com/bosi/decorder/com_gitlab_bosi_decorder-v0.4.1.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/gitlab.com/bosi/decorder/com_gitlab_bosi_decorder-v0.4.1.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/gitlab.com/bosi/decorder/com_gitlab_bosi_decorder-v0.4.2.zip",
+ "http://ats.apps.svc/gomod/gitlab.com/bosi/decorder/com_gitlab_bosi_decorder-v0.4.2.zip",
+ "https://cache.hawkingrei.com/gomod/gitlab.com/bosi/decorder/com_gitlab_bosi_decorder-v0.4.2.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/gitlab.com/bosi/decorder/com_gitlab_bosi_decorder-v0.4.2.zip",
],
)
go_repository(
name = "com_google_cloud_go",
build_file_proto_mode = "disable_global",
importpath = "cloud.google.com/go",
- sha256 = "b64ae080db87dc2827d84b409daaed7f4e42b56391db239e1f41e1bf076c1dd3",
- strip_prefix = "cloud.google.com/go@v0.112.0",
+ sha256 = "e66c48fc4993daca98fe02c5fce704ed42b00d2d553d4e19c3bfcfe3613ef440",
+ strip_prefix = "cloud.google.com/go@v0.112.1",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/cloud.google.com/go/com_google_cloud_go-v0.112.0.zip",
- "http://ats.apps.svc/gomod/cloud.google.com/go/com_google_cloud_go-v0.112.0.zip",
- "https://cache.hawkingrei.com/gomod/cloud.google.com/go/com_google_cloud_go-v0.112.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/cloud.google.com/go/com_google_cloud_go-v0.112.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/cloud.google.com/go/com_google_cloud_go-v0.112.1.zip",
+ "http://ats.apps.svc/gomod/cloud.google.com/go/com_google_cloud_go-v0.112.1.zip",
+ "https://cache.hawkingrei.com/gomod/cloud.google.com/go/com_google_cloud_go-v0.112.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/cloud.google.com/go/com_google_cloud_go-v0.112.1.zip",
],
)
go_repository(
@@ -7678,26 +8077,26 @@ def go_deps():
name = "com_google_cloud_go_compute",
build_file_proto_mode = "disable_global",
importpath = "cloud.google.com/go/compute",
- sha256 = "0cf3d4325e378c92ff90cef3d1b7752682a77f0eaa0b11c092cc3ea32e5ed638",
- strip_prefix = "cloud.google.com/go/compute@v1.24.0",
+ sha256 = "5173a017a15f7874e68752a8116556fe0d7e5e11344dd4265c454467bb651cb8",
+ strip_prefix = "cloud.google.com/go/compute@v1.25.1",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/cloud.google.com/go/compute/com_google_cloud_go_compute-v1.24.0.zip",
- "http://ats.apps.svc/gomod/cloud.google.com/go/compute/com_google_cloud_go_compute-v1.24.0.zip",
- "https://cache.hawkingrei.com/gomod/cloud.google.com/go/compute/com_google_cloud_go_compute-v1.24.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/cloud.google.com/go/compute/com_google_cloud_go_compute-v1.24.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/cloud.google.com/go/compute/com_google_cloud_go_compute-v1.25.1.zip",
+ "http://ats.apps.svc/gomod/cloud.google.com/go/compute/com_google_cloud_go_compute-v1.25.1.zip",
+ "https://cache.hawkingrei.com/gomod/cloud.google.com/go/compute/com_google_cloud_go_compute-v1.25.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/cloud.google.com/go/compute/com_google_cloud_go_compute-v1.25.1.zip",
],
)
go_repository(
name = "com_google_cloud_go_compute_metadata",
build_file_proto_mode = "disable_global",
importpath = "cloud.google.com/go/compute/metadata",
- sha256 = "292864dbd0b1de37a968e285e949885e573384837d81cd3695be5ce2e2391887",
- strip_prefix = "cloud.google.com/go/compute/metadata@v0.2.3",
+ sha256 = "c0ab79c30870c1aa9912fb0fdcb043e0044782825988e40f59401d227976b677",
+ strip_prefix = "cloud.google.com/go/compute/metadata@v0.3.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/cloud.google.com/go/compute/metadata/com_google_cloud_go_compute_metadata-v0.2.3.zip",
- "http://ats.apps.svc/gomod/cloud.google.com/go/compute/metadata/com_google_cloud_go_compute_metadata-v0.2.3.zip",
- "https://cache.hawkingrei.com/gomod/cloud.google.com/go/compute/metadata/com_google_cloud_go_compute_metadata-v0.2.3.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/cloud.google.com/go/compute/metadata/com_google_cloud_go_compute_metadata-v0.2.3.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/cloud.google.com/go/compute/metadata/com_google_cloud_go_compute_metadata-v0.3.0.zip",
+ "http://ats.apps.svc/gomod/cloud.google.com/go/compute/metadata/com_google_cloud_go_compute_metadata-v0.3.0.zip",
+ "https://cache.hawkingrei.com/gomod/cloud.google.com/go/compute/metadata/com_google_cloud_go_compute_metadata-v0.3.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/cloud.google.com/go/compute/metadata/com_google_cloud_go_compute_metadata-v0.3.0.zip",
],
)
go_repository(
@@ -8679,13 +9078,13 @@ def go_deps():
name = "com_google_cloud_go_storage",
build_file_proto_mode = "disable_global",
importpath = "cloud.google.com/go/storage",
- sha256 = "272530d5e205a825b33546e9ac349a66346267a0bc06f006d617d353cdec5525",
- strip_prefix = "cloud.google.com/go/storage@v1.36.0",
+ sha256 = "011944e62d8526015c5c024af08cc4dd1d0d0317f7c5e9af49e062c879b06ea7",
+ strip_prefix = "cloud.google.com/go/storage@v1.38.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/cloud.google.com/go/storage/com_google_cloud_go_storage-v1.36.0.zip",
- "http://ats.apps.svc/gomod/cloud.google.com/go/storage/com_google_cloud_go_storage-v1.36.0.zip",
- "https://cache.hawkingrei.com/gomod/cloud.google.com/go/storage/com_google_cloud_go_storage-v1.36.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/cloud.google.com/go/storage/com_google_cloud_go_storage-v1.36.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/cloud.google.com/go/storage/com_google_cloud_go_storage-v1.38.0.zip",
+ "http://ats.apps.svc/gomod/cloud.google.com/go/storage/com_google_cloud_go_storage-v1.38.0.zip",
+ "https://cache.hawkingrei.com/gomod/cloud.google.com/go/storage/com_google_cloud_go_storage-v1.38.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/cloud.google.com/go/storage/com_google_cloud_go_storage-v1.38.0.zip",
],
)
go_repository(
@@ -8883,6 +9282,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/cloud.google.com/go/workflows/com_google_cloud_go_workflows-v1.12.4.zip",
],
)
+ go_repository(
+ name = "com_lukechampine_uint128",
+ build_file_proto_mode = "disable_global",
+ importpath = "lukechampine.com/uint128",
+ sha256 = "9ff6e9ad553a69fdb961ab2d92f92cda183ef616a6709c15972c2d4bedf33de5",
+ strip_prefix = "lukechampine.com/uint128@v1.2.0",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/lukechampine.com/uint128/com_lukechampine_uint128-v1.2.0.zip",
+ "http://ats.apps.svc/gomod/lukechampine.com/uint128/com_lukechampine_uint128-v1.2.0.zip",
+ "https://cache.hawkingrei.com/gomod/lukechampine.com/uint128/com_lukechampine_uint128-v1.2.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/lukechampine.com/uint128/com_lukechampine_uint128-v1.2.0.zip",
+ ],
+ )
go_repository(
name = "com_shuralyov_dmitri_gpu_mtl",
build_file_proto_mode = "disable_global",
@@ -8935,6 +9347,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/stathat.com/c/consistent/com_stathat_c_consistent-v1.0.0.zip",
],
)
+ go_repository(
+ name = "ht_sr_git_~sbinet_gg",
+ build_file_proto_mode = "disable_global",
+ importpath = "git.sr.ht/~sbinet/gg",
+ sha256 = "435103529c4f24aecf7e4550bc816db2482dda4ee0123d337daba99971a8c498",
+ strip_prefix = "git.sr.ht/~sbinet/gg@v0.3.1",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/git.sr.ht/~sbinet/gg/ht_sr_git_~sbinet_gg-v0.3.1.zip",
+ "http://ats.apps.svc/gomod/git.sr.ht/~sbinet/gg/ht_sr_git_~sbinet_gg-v0.3.1.zip",
+ "https://cache.hawkingrei.com/gomod/git.sr.ht/~sbinet/gg/ht_sr_git_~sbinet_gg-v0.3.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/git.sr.ht/~sbinet/gg/ht_sr_git_~sbinet_gg-v0.3.1.zip",
+ ],
+ )
go_repository(
name = "in_gopkg_check_v1",
build_file_proto_mode = "disable_global",
@@ -9134,10 +9559,6 @@ def go_deps():
name = "io_etcd_go_etcd_api_v3",
build_file_proto_mode = "disable",
importpath = "go.etcd.io/etcd/api/v3",
- patch_args = ["-p2"],
- patches = [
- "//build/patches:io_etcd_go_etcd_api_v3.patch",
- ],
sha256 = "d935e64c70766be57ab28611ef071285f1aed5f62172dd5a2acf5b1aa536684c",
strip_prefix = "go.etcd.io/etcd/api/v3@v3.5.12",
urls = [
@@ -9450,39 +9871,39 @@ def go_deps():
name = "io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc",
build_file_proto_mode = "disable_global",
importpath = "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc",
- sha256 = "efb13c3eb89199b0f677057a238017b06ffceb868ac55f4f649a31309ec0321d",
- strip_prefix = "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.47.0",
+ sha256 = "db8eab928fe25f92268029064d194bd4c1f72d0bf69635c35709ebfbf66b2a55",
+ strip_prefix = "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.49.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc-v0.47.0.zip",
- "http://ats.apps.svc/gomod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc-v0.47.0.zip",
- "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc-v0.47.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc-v0.47.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc-v0.49.0.zip",
+ "http://ats.apps.svc/gomod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc-v0.49.0.zip",
+ "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc-v0.49.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc-v0.49.0.zip",
],
)
go_repository(
name = "io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp",
build_file_proto_mode = "disable_global",
importpath = "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp",
- sha256 = "476b9113a426e31a3802d8371c348ae3334c56acba9fc7228b886096c64647a1",
- strip_prefix = "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.47.0",
+ sha256 = "205c8117aebdc6f6ebab7fbb946d260933716c68a1b2dda8d43ab142b6622b14",
+ strip_prefix = "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.49.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp-v0.47.0.zip",
- "http://ats.apps.svc/gomod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp-v0.47.0.zip",
- "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp-v0.47.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp-v0.47.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp-v0.49.0.zip",
+ "http://ats.apps.svc/gomod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp-v0.49.0.zip",
+ "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp-v0.49.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp-v0.49.0.zip",
],
)
go_repository(
name = "io_opentelemetry_go_otel",
build_file_proto_mode = "disable_global",
importpath = "go.opentelemetry.io/otel",
- sha256 = "6c02668caef05c8221cf2fb43f4f2943de701f3c4fb69096f2f7da523f1c80d2",
- strip_prefix = "go.opentelemetry.io/otel@v1.22.0",
+ sha256 = "4aebfe22b33a77bfab346224c3cd0d2da6cb8992318b104f33e9bf5e533effa5",
+ strip_prefix = "go.opentelemetry.io/otel@v1.24.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/otel/io_opentelemetry_go_otel-v1.22.0.zip",
- "http://ats.apps.svc/gomod/go.opentelemetry.io/otel/io_opentelemetry_go_otel-v1.22.0.zip",
- "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/otel/io_opentelemetry_go_otel-v1.22.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/otel/io_opentelemetry_go_otel-v1.22.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/otel/io_opentelemetry_go_otel-v1.24.0.zip",
+ "http://ats.apps.svc/gomod/go.opentelemetry.io/otel/io_opentelemetry_go_otel-v1.24.0.zip",
+ "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/otel/io_opentelemetry_go_otel-v1.24.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/otel/io_opentelemetry_go_otel-v1.24.0.zip",
],
)
go_repository(
@@ -9528,39 +9949,39 @@ def go_deps():
name = "io_opentelemetry_go_otel_metric",
build_file_proto_mode = "disable_global",
importpath = "go.opentelemetry.io/otel/metric",
- sha256 = "deb750b41631365dbda60fb872dc6901f16817e0b2646307fbfeb4eeb391e02f",
- strip_prefix = "go.opentelemetry.io/otel/metric@v1.22.0",
+ sha256 = "bbe9ee9443b924d0a9f58a9e2a4a7b1c1ab1274f7d41d0db2a243d9c33eca93c",
+ strip_prefix = "go.opentelemetry.io/otel/metric@v1.24.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/otel/metric/io_opentelemetry_go_otel_metric-v1.22.0.zip",
- "http://ats.apps.svc/gomod/go.opentelemetry.io/otel/metric/io_opentelemetry_go_otel_metric-v1.22.0.zip",
- "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/otel/metric/io_opentelemetry_go_otel_metric-v1.22.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/otel/metric/io_opentelemetry_go_otel_metric-v1.22.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/otel/metric/io_opentelemetry_go_otel_metric-v1.24.0.zip",
+ "http://ats.apps.svc/gomod/go.opentelemetry.io/otel/metric/io_opentelemetry_go_otel_metric-v1.24.0.zip",
+ "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/otel/metric/io_opentelemetry_go_otel_metric-v1.24.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/otel/metric/io_opentelemetry_go_otel_metric-v1.24.0.zip",
],
)
go_repository(
name = "io_opentelemetry_go_otel_sdk",
build_file_proto_mode = "disable_global",
importpath = "go.opentelemetry.io/otel/sdk",
- sha256 = "d17065c74f1cbae1c5fff90289919807a859733673ab8491c5f5883475ae4657",
- strip_prefix = "go.opentelemetry.io/otel/sdk@v1.22.0",
+ sha256 = "60b78206b3b5d44c3f9069ac24e87f90e5474b52bbbbf5a0f454cc7e5f5f409a",
+ strip_prefix = "go.opentelemetry.io/otel/sdk@v1.24.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/otel/sdk/io_opentelemetry_go_otel_sdk-v1.22.0.zip",
- "http://ats.apps.svc/gomod/go.opentelemetry.io/otel/sdk/io_opentelemetry_go_otel_sdk-v1.22.0.zip",
- "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/otel/sdk/io_opentelemetry_go_otel_sdk-v1.22.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/otel/sdk/io_opentelemetry_go_otel_sdk-v1.22.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/otel/sdk/io_opentelemetry_go_otel_sdk-v1.24.0.zip",
+ "http://ats.apps.svc/gomod/go.opentelemetry.io/otel/sdk/io_opentelemetry_go_otel_sdk-v1.24.0.zip",
+ "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/otel/sdk/io_opentelemetry_go_otel_sdk-v1.24.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/otel/sdk/io_opentelemetry_go_otel_sdk-v1.24.0.zip",
],
)
go_repository(
name = "io_opentelemetry_go_otel_trace",
build_file_proto_mode = "disable_global",
importpath = "go.opentelemetry.io/otel/trace",
- sha256 = "d63e40f32d614b00bedfa945eae95e2bc5fe867e167cd7dbfe7f90d96fa599d7",
- strip_prefix = "go.opentelemetry.io/otel/trace@v1.22.0",
+ sha256 = "4d561c375a78a5404d154add753f9ee24b30307a233d346bdecdb4adb83ff408",
+ strip_prefix = "go.opentelemetry.io/otel/trace@v1.24.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/otel/trace/io_opentelemetry_go_otel_trace-v1.22.0.zip",
- "http://ats.apps.svc/gomod/go.opentelemetry.io/otel/trace/io_opentelemetry_go_otel_trace-v1.22.0.zip",
- "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/otel/trace/io_opentelemetry_go_otel_trace-v1.22.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/otel/trace/io_opentelemetry_go_otel_trace-v1.22.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/go.opentelemetry.io/otel/trace/io_opentelemetry_go_otel_trace-v1.24.0.zip",
+ "http://ats.apps.svc/gomod/go.opentelemetry.io/otel/trace/io_opentelemetry_go_otel_trace-v1.24.0.zip",
+ "https://cache.hawkingrei.com/gomod/go.opentelemetry.io/otel/trace/io_opentelemetry_go_otel_trace-v1.24.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/go.opentelemetry.io/otel/trace/io_opentelemetry_go_otel_trace-v1.24.0.zip",
],
)
go_repository(
@@ -9641,43 +10062,56 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/go.starlark.net/net_starlark_go-v0.0.0-20210223155950-e043a3d3c984.zip",
],
)
+ go_repository(
+ name = "org_gioui",
+ build_file_proto_mode = "disable_global",
+ importpath = "gioui.org",
+ sha256 = "fcbab2a0ea09ff775c1ff4fa99299d95b94aad496b1ac329e3c7389119168fc0",
+ strip_prefix = "gioui.org@v0.0.0-20210308172011-57750fc8a0a6",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/gioui.org/org_gioui-v0.0.0-20210308172011-57750fc8a0a6.zip",
+ "http://ats.apps.svc/gomod/gioui.org/org_gioui-v0.0.0-20210308172011-57750fc8a0a6.zip",
+ "https://cache.hawkingrei.com/gomod/gioui.org/org_gioui-v0.0.0-20210308172011-57750fc8a0a6.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/gioui.org/org_gioui-v0.0.0-20210308172011-57750fc8a0a6.zip",
+ ],
+ )
go_repository(
name = "org_go_simpler_musttag",
build_file_proto_mode = "disable_global",
importpath = "go-simpler.org/musttag",
- sha256 = "7804335481f59dfda5ef1d9afad9cbfdd6342525fa82c20d6cfe60b0e7eefebb",
- strip_prefix = "go-simpler.org/musttag@v0.9.0",
+ sha256 = "264733af21689a6eb237e9b974d0de9cd3779e42d9c0a08203907563ca380ba0",
+ strip_prefix = "go-simpler.org/musttag@v0.12.1",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/go-simpler.org/musttag/org_go_simpler_musttag-v0.9.0.zip",
- "http://ats.apps.svc/gomod/go-simpler.org/musttag/org_go_simpler_musttag-v0.9.0.zip",
- "https://cache.hawkingrei.com/gomod/go-simpler.org/musttag/org_go_simpler_musttag-v0.9.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/go-simpler.org/musttag/org_go_simpler_musttag-v0.9.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/go-simpler.org/musttag/org_go_simpler_musttag-v0.12.1.zip",
+ "http://ats.apps.svc/gomod/go-simpler.org/musttag/org_go_simpler_musttag-v0.12.1.zip",
+ "https://cache.hawkingrei.com/gomod/go-simpler.org/musttag/org_go_simpler_musttag-v0.12.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/go-simpler.org/musttag/org_go_simpler_musttag-v0.12.1.zip",
],
)
go_repository(
name = "org_go_simpler_sloglint",
build_file_proto_mode = "disable_global",
importpath = "go-simpler.org/sloglint",
- sha256 = "dcc18e96a86fec32194f0bc7fa831051ab91285ea06fd8186cef4cfa99e5f18f",
- strip_prefix = "go-simpler.org/sloglint@v0.5.0",
+ sha256 = "8fbdf0a36af6f51f00881b6f1874e36ccdf9b0e78ab90472e80e8cf8bf7cb978",
+ strip_prefix = "go-simpler.org/sloglint@v0.6.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/go-simpler.org/sloglint/org_go_simpler_sloglint-v0.5.0.zip",
- "http://ats.apps.svc/gomod/go-simpler.org/sloglint/org_go_simpler_sloglint-v0.5.0.zip",
- "https://cache.hawkingrei.com/gomod/go-simpler.org/sloglint/org_go_simpler_sloglint-v0.5.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/go-simpler.org/sloglint/org_go_simpler_sloglint-v0.5.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/go-simpler.org/sloglint/org_go_simpler_sloglint-v0.6.0.zip",
+ "http://ats.apps.svc/gomod/go-simpler.org/sloglint/org_go_simpler_sloglint-v0.6.0.zip",
+ "https://cache.hawkingrei.com/gomod/go-simpler.org/sloglint/org_go_simpler_sloglint-v0.6.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/go-simpler.org/sloglint/org_go_simpler_sloglint-v0.6.0.zip",
],
)
go_repository(
name = "org_golang_google_api",
build_file_proto_mode = "disable_global",
importpath = "google.golang.org/api",
- sha256 = "5eb3019627247e9dfccf458bdad1c4227c7919079bb8f300c7f46612d118e9c7",
- strip_prefix = "google.golang.org/api@v0.162.0",
+ sha256 = "76ed417ddd1b6523cdd856aacb70109e779909ccd59193933b4c44d565eb1722",
+ strip_prefix = "google.golang.org/api@v0.169.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/api/org_golang_google_api-v0.162.0.zip",
- "http://ats.apps.svc/gomod/google.golang.org/api/org_golang_google_api-v0.162.0.zip",
- "https://cache.hawkingrei.com/gomod/google.golang.org/api/org_golang_google_api-v0.162.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/api/org_golang_google_api-v0.162.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/api/org_golang_google_api-v0.169.0.zip",
+ "http://ats.apps.svc/gomod/google.golang.org/api/org_golang_google_api-v0.169.0.zip",
+ "https://cache.hawkingrei.com/gomod/google.golang.org/api/org_golang_google_api-v0.169.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/api/org_golang_google_api-v0.169.0.zip",
],
)
go_repository(
@@ -9710,52 +10144,52 @@ def go_deps():
name = "org_golang_google_genproto_googleapis_api",
build_file_proto_mode = "disable_global",
importpath = "google.golang.org/genproto/googleapis/api",
- sha256 = "1ebe3c1107c126819cd1b27f2eb3966df4fcd434bbe45d5ef2cba514091a8c80",
- strip_prefix = "google.golang.org/genproto/googleapis/api@v0.0.0-20240304212257-790db918fca8",
+ sha256 = "7a24304baa150f3e64521242491823738fa6e9bd4bd85acf6e79c1cd6ebd847f",
+ strip_prefix = "google.golang.org/genproto/googleapis/api@v0.0.0-20240318140521-94a12d6c2237",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/genproto/googleapis/api/org_golang_google_genproto_googleapis_api-v0.0.0-20240304212257-790db918fca8.zip",
- "http://ats.apps.svc/gomod/google.golang.org/genproto/googleapis/api/org_golang_google_genproto_googleapis_api-v0.0.0-20240304212257-790db918fca8.zip",
- "https://cache.hawkingrei.com/gomod/google.golang.org/genproto/googleapis/api/org_golang_google_genproto_googleapis_api-v0.0.0-20240304212257-790db918fca8.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/genproto/googleapis/api/org_golang_google_genproto_googleapis_api-v0.0.0-20240304212257-790db918fca8.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/genproto/googleapis/api/org_golang_google_genproto_googleapis_api-v0.0.0-20240318140521-94a12d6c2237.zip",
+ "http://ats.apps.svc/gomod/google.golang.org/genproto/googleapis/api/org_golang_google_genproto_googleapis_api-v0.0.0-20240318140521-94a12d6c2237.zip",
+ "https://cache.hawkingrei.com/gomod/google.golang.org/genproto/googleapis/api/org_golang_google_genproto_googleapis_api-v0.0.0-20240318140521-94a12d6c2237.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/genproto/googleapis/api/org_golang_google_genproto_googleapis_api-v0.0.0-20240318140521-94a12d6c2237.zip",
],
)
go_repository(
name = "org_golang_google_genproto_googleapis_bytestream",
build_file_proto_mode = "disable_global",
importpath = "google.golang.org/genproto/googleapis/bytestream",
- sha256 = "407e8ddfcc6c5c96a84bc80425139c36f0e12a5134cb5efe52ecc6233644d381",
- strip_prefix = "google.golang.org/genproto/googleapis/bytestream@v0.0.0-20240125205218-1f4bbc51befe",
+ sha256 = "c0d158dbb41d2c85587e9f09d257e7f4ab4f41862891941f0fcfee3e3c0c8b2f",
+ strip_prefix = "google.golang.org/genproto/googleapis/bytestream@v0.0.0-20240304161311-37d4d3c04a78",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/genproto/googleapis/bytestream/org_golang_google_genproto_googleapis_bytestream-v0.0.0-20240125205218-1f4bbc51befe.zip",
- "http://ats.apps.svc/gomod/google.golang.org/genproto/googleapis/bytestream/org_golang_google_genproto_googleapis_bytestream-v0.0.0-20240125205218-1f4bbc51befe.zip",
- "https://cache.hawkingrei.com/gomod/google.golang.org/genproto/googleapis/bytestream/org_golang_google_genproto_googleapis_bytestream-v0.0.0-20240125205218-1f4bbc51befe.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/genproto/googleapis/bytestream/org_golang_google_genproto_googleapis_bytestream-v0.0.0-20240125205218-1f4bbc51befe.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/genproto/googleapis/bytestream/org_golang_google_genproto_googleapis_bytestream-v0.0.0-20240304161311-37d4d3c04a78.zip",
+ "http://ats.apps.svc/gomod/google.golang.org/genproto/googleapis/bytestream/org_golang_google_genproto_googleapis_bytestream-v0.0.0-20240304161311-37d4d3c04a78.zip",
+ "https://cache.hawkingrei.com/gomod/google.golang.org/genproto/googleapis/bytestream/org_golang_google_genproto_googleapis_bytestream-v0.0.0-20240304161311-37d4d3c04a78.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/genproto/googleapis/bytestream/org_golang_google_genproto_googleapis_bytestream-v0.0.0-20240304161311-37d4d3c04a78.zip",
],
)
go_repository(
name = "org_golang_google_genproto_googleapis_rpc",
build_file_proto_mode = "disable_global",
importpath = "google.golang.org/genproto/googleapis/rpc",
- sha256 = "5520055455146fd452a014e10bfa069a5e132f30f8fea905431333dabb24c394",
- strip_prefix = "google.golang.org/genproto/googleapis/rpc@v0.0.0-20240401170217-c3f982113cda",
+ sha256 = "53ce5ee04a9fd853c81fdd00cd06b426ec3212e57ae6d591153ad823243bae8a",
+ strip_prefix = "google.golang.org/genproto/googleapis/rpc@v0.0.0-20240515191416-fc5f0ca64291",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240401170217-c3f982113cda.zip",
- "http://ats.apps.svc/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240401170217-c3f982113cda.zip",
- "https://cache.hawkingrei.com/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240401170217-c3f982113cda.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240401170217-c3f982113cda.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240515191416-fc5f0ca64291.zip",
+ "http://ats.apps.svc/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240515191416-fc5f0ca64291.zip",
+ "https://cache.hawkingrei.com/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240515191416-fc5f0ca64291.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240515191416-fc5f0ca64291.zip",
],
)
go_repository(
name = "org_golang_google_grpc",
build_file_proto_mode = "disable_global",
importpath = "google.golang.org/grpc",
- sha256 = "1d49288986efd05a7b4ac6cee078f84e29e464678c776f25e09efddfba852fd1",
- strip_prefix = "google.golang.org/grpc@v1.63.2",
+ sha256 = "42d45071add08827509bd8ca098804563f638da2bfc44c8335fb95cc99cd96e9",
+ strip_prefix = "google.golang.org/grpc@v1.64.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.63.2.zip",
- "http://ats.apps.svc/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.63.2.zip",
- "https://cache.hawkingrei.com/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.63.2.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.63.2.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.64.0.zip",
+ "http://ats.apps.svc/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.64.0.zip",
+ "https://cache.hawkingrei.com/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.64.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.64.0.zip",
],
)
go_repository(
@@ -9775,26 +10209,26 @@ def go_deps():
name = "org_golang_google_protobuf",
build_file_proto_mode = "disable_global",
importpath = "google.golang.org/protobuf",
- sha256 = "2cc1c98e12903009bd4bf0d5e938a421ca2f88ae87b0fc50004b2c7598b1fd24",
- strip_prefix = "google.golang.org/protobuf@v1.33.0",
+ sha256 = "b5987adbf21a6cbe6463ea795c320b700537d5dbb6f3e2aa3ffaf226cedf476b",
+ strip_prefix = "google.golang.org/protobuf@v1.34.1",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/protobuf/org_golang_google_protobuf-v1.33.0.zip",
- "http://ats.apps.svc/gomod/google.golang.org/protobuf/org_golang_google_protobuf-v1.33.0.zip",
- "https://cache.hawkingrei.com/gomod/google.golang.org/protobuf/org_golang_google_protobuf-v1.33.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/protobuf/org_golang_google_protobuf-v1.33.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/protobuf/org_golang_google_protobuf-v1.34.1.zip",
+ "http://ats.apps.svc/gomod/google.golang.org/protobuf/org_golang_google_protobuf-v1.34.1.zip",
+ "https://cache.hawkingrei.com/gomod/google.golang.org/protobuf/org_golang_google_protobuf-v1.34.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/protobuf/org_golang_google_protobuf-v1.34.1.zip",
],
)
go_repository(
name = "org_golang_x_crypto",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/crypto",
- sha256 = "3148f58c2893286075e7e13e2de19d8e4fd9883debb6d76b45f7929e1d79e53e",
- strip_prefix = "golang.org/x/crypto@v0.22.0",
+ sha256 = "65d22b9f54aef5f7f064900d2ecf8d8b231729aebc46c3b7ca56ff897fb70b57",
+ strip_prefix = "golang.org/x/crypto@v0.23.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.22.0.zip",
- "http://ats.apps.svc/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.22.0.zip",
- "https://cache.hawkingrei.com/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.22.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.22.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.23.0.zip",
+ "http://ats.apps.svc/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.23.0.zip",
+ "https://cache.hawkingrei.com/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.23.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.23.0.zip",
],
)
go_repository(
@@ -9827,13 +10261,13 @@ def go_deps():
name = "org_golang_x_image",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/image",
- sha256 = "4a44b498934a95e8f84e8374530de0cab38d81fcd558898d4880c3c5ce1efe47",
- strip_prefix = "golang.org/x/image@v0.0.0-20190802002840-cff245a6509b",
+ sha256 = "56176a4d4d47910d61df9a77aa66a8469ae79fa18b7f5821c43bef1ef212116d",
+ strip_prefix = "golang.org/x/image@v0.0.0-20220302094943-723b81ca9867",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/image/org_golang_x_image-v0.0.0-20190802002840-cff245a6509b.zip",
- "http://ats.apps.svc/gomod/golang.org/x/image/org_golang_x_image-v0.0.0-20190802002840-cff245a6509b.zip",
- "https://cache.hawkingrei.com/gomod/golang.org/x/image/org_golang_x_image-v0.0.0-20190802002840-cff245a6509b.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/image/org_golang_x_image-v0.0.0-20190802002840-cff245a6509b.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/image/org_golang_x_image-v0.0.0-20220302094943-723b81ca9867.zip",
+ "http://ats.apps.svc/gomod/golang.org/x/image/org_golang_x_image-v0.0.0-20220302094943-723b81ca9867.zip",
+ "https://cache.hawkingrei.com/gomod/golang.org/x/image/org_golang_x_image-v0.0.0-20220302094943-723b81ca9867.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/image/org_golang_x_image-v0.0.0-20220302094943-723b81ca9867.zip",
],
)
go_repository(
@@ -9879,26 +10313,26 @@ def go_deps():
name = "org_golang_x_net",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/net",
- sha256 = "389940dbee4a10516de85368bb1a550d6df814ed1f893db18de8def9168147c7",
- strip_prefix = "golang.org/x/net@v0.24.0",
+ sha256 = "7fd8464681c3011736f2c75beb20f88fff553a17f4f574325bce5ca5dc1fcf83",
+ strip_prefix = "golang.org/x/net@v0.25.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/net/org_golang_x_net-v0.24.0.zip",
- "http://ats.apps.svc/gomod/golang.org/x/net/org_golang_x_net-v0.24.0.zip",
- "https://cache.hawkingrei.com/gomod/golang.org/x/net/org_golang_x_net-v0.24.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/net/org_golang_x_net-v0.24.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/net/org_golang_x_net-v0.25.0.zip",
+ "http://ats.apps.svc/gomod/golang.org/x/net/org_golang_x_net-v0.25.0.zip",
+ "https://cache.hawkingrei.com/gomod/golang.org/x/net/org_golang_x_net-v0.25.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/net/org_golang_x_net-v0.25.0.zip",
],
)
go_repository(
name = "org_golang_x_oauth2",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/oauth2",
- sha256 = "7716d06cf89e1736b74b94584cdcad120c769519f0c41e6d77d0f12657870772",
- strip_prefix = "golang.org/x/oauth2@v0.18.0",
+ sha256 = "48bbe74b766a2a899e9688084dfbf9bda44b2cd0aa98d0245eec058960ec4bdc",
+ strip_prefix = "golang.org/x/oauth2@v0.20.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.18.0.zip",
- "http://ats.apps.svc/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.18.0.zip",
- "https://cache.hawkingrei.com/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.18.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.18.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.20.0.zip",
+ "http://ats.apps.svc/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.20.0.zip",
+ "https://cache.hawkingrei.com/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.20.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/oauth2/org_golang_x_oauth2-v0.20.0.zip",
],
)
go_repository(
@@ -9931,13 +10365,13 @@ def go_deps():
name = "org_golang_x_sys",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/sys",
- sha256 = "f3e06adc66b840da7719fcf496d2916a38317706509fb5beed5932cd8ae5fb6b",
- strip_prefix = "golang.org/x/sys@v0.19.0",
+ sha256 = "3f826b191eab1ebda925feb551d334e37e1b5865d1aa790fade46598811a8b1a",
+ strip_prefix = "golang.org/x/sys@v0.20.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/sys/org_golang_x_sys-v0.19.0.zip",
- "http://ats.apps.svc/gomod/golang.org/x/sys/org_golang_x_sys-v0.19.0.zip",
- "https://cache.hawkingrei.com/gomod/golang.org/x/sys/org_golang_x_sys-v0.19.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/sys/org_golang_x_sys-v0.19.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/sys/org_golang_x_sys-v0.20.0.zip",
+ "http://ats.apps.svc/gomod/golang.org/x/sys/org_golang_x_sys-v0.20.0.zip",
+ "https://cache.hawkingrei.com/gomod/golang.org/x/sys/org_golang_x_sys-v0.20.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/sys/org_golang_x_sys-v0.20.0.zip",
],
)
go_repository(
@@ -9957,26 +10391,26 @@ def go_deps():
name = "org_golang_x_term",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/term",
- sha256 = "893e851afedc40a9c253e611fed322ec6f5a1c56180dd29229ff1cc1bd35dbfb",
- strip_prefix = "golang.org/x/term@v0.19.0",
+ sha256 = "840eacc0ffb306dcb4b0f5bf6e071c91d2e7957fcc604eec4e73c0fc22f2920c",
+ strip_prefix = "golang.org/x/term@v0.20.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/term/org_golang_x_term-v0.19.0.zip",
- "http://ats.apps.svc/gomod/golang.org/x/term/org_golang_x_term-v0.19.0.zip",
- "https://cache.hawkingrei.com/gomod/golang.org/x/term/org_golang_x_term-v0.19.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/term/org_golang_x_term-v0.19.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/term/org_golang_x_term-v0.20.0.zip",
+ "http://ats.apps.svc/gomod/golang.org/x/term/org_golang_x_term-v0.20.0.zip",
+ "https://cache.hawkingrei.com/gomod/golang.org/x/term/org_golang_x_term-v0.20.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/term/org_golang_x_term-v0.20.0.zip",
],
)
go_repository(
name = "org_golang_x_text",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/text",
- sha256 = "b9814897e0e09cd576a7a013f066c7db537a3d538d2e0f60f0caee9bc1b3f4af",
- strip_prefix = "golang.org/x/text@v0.14.0",
+ sha256 = "13faee7e46c8a18c8a28f3eceebf15db6d724b9a108c3c0482a6d2e58ba73a73",
+ strip_prefix = "golang.org/x/text@v0.15.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/text/org_golang_x_text-v0.14.0.zip",
- "http://ats.apps.svc/gomod/golang.org/x/text/org_golang_x_text-v0.14.0.zip",
- "https://cache.hawkingrei.com/gomod/golang.org/x/text/org_golang_x_text-v0.14.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/text/org_golang_x_text-v0.14.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/text/org_golang_x_text-v0.15.0.zip",
+ "http://ats.apps.svc/gomod/golang.org/x/text/org_golang_x_text-v0.15.0.zip",
+ "https://cache.hawkingrei.com/gomod/golang.org/x/text/org_golang_x_text-v0.15.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/text/org_golang_x_text-v0.15.0.zip",
],
)
go_repository(
@@ -9996,13 +10430,13 @@ def go_deps():
name = "org_golang_x_tools",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/tools",
- sha256 = "f9537c85fc51e59299b627c842381f97cabde123f9fc40a0da51eec0d637dbd9",
- strip_prefix = "golang.org/x/tools@v0.20.0",
+ sha256 = "1099b286fba466d61da042e950e7da3cc0373260e95fe116bf61cfb6ec4828a8",
+ strip_prefix = "golang.org/x/tools@v0.21.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/tools/org_golang_x_tools-v0.20.0.zip",
- "http://ats.apps.svc/gomod/golang.org/x/tools/org_golang_x_tools-v0.20.0.zip",
- "https://cache.hawkingrei.com/gomod/golang.org/x/tools/org_golang_x_tools-v0.20.0.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/tools/org_golang_x_tools-v0.20.0.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/tools/org_golang_x_tools-v0.21.0.zip",
+ "http://ats.apps.svc/gomod/golang.org/x/tools/org_golang_x_tools-v0.21.0.zip",
+ "https://cache.hawkingrei.com/gomod/golang.org/x/tools/org_golang_x_tools-v0.21.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/tools/org_golang_x_tools-v0.21.0.zip",
],
)
go_repository(
@@ -10022,13 +10456,13 @@ def go_deps():
name = "org_gonum_v1_gonum",
build_file_proto_mode = "disable_global",
importpath = "gonum.org/v1/gonum",
- sha256 = "57ecefd9c1ab5a40ed9e37e824597e523e85e78022cd8a4fc5533ff785f49863",
- strip_prefix = "gonum.org/v1/gonum@v0.8.2",
+ sha256 = "abdfee15ce7c9d2cd96b66468d3ae28d6054add4efbfc1b15fadfe3613f3d362",
+ strip_prefix = "gonum.org/v1/gonum@v0.11.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/gonum.org/v1/gonum/org_gonum_v1_gonum-v0.8.2.zip",
- "http://ats.apps.svc/gomod/gonum.org/v1/gonum/org_gonum_v1_gonum-v0.8.2.zip",
- "https://cache.hawkingrei.com/gomod/gonum.org/v1/gonum/org_gonum_v1_gonum-v0.8.2.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/gonum.org/v1/gonum/org_gonum_v1_gonum-v0.8.2.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/gonum.org/v1/gonum/org_gonum_v1_gonum-v0.11.0.zip",
+ "http://ats.apps.svc/gomod/gonum.org/v1/gonum/org_gonum_v1_gonum-v0.11.0.zip",
+ "https://cache.hawkingrei.com/gomod/gonum.org/v1/gonum/org_gonum_v1_gonum-v0.11.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/gonum.org/v1/gonum/org_gonum_v1_gonum-v0.11.0.zip",
],
)
go_repository(
@@ -10048,13 +10482,52 @@ def go_deps():
name = "org_gonum_v1_plot",
build_file_proto_mode = "disable_global",
importpath = "gonum.org/v1/plot",
- sha256 = "2d4cadb4bafb5bbfe1f614d7e402c670446fccd154bc4c6b1699e3dffde68ff4",
- strip_prefix = "gonum.org/v1/plot@v0.0.0-20190515093506-e2840ee46a6b",
+ sha256 = "eaa47ad966b3b67325c1f3ae704d566332c573b7cca79016cb4ffe82155aab39",
+ strip_prefix = "gonum.org/v1/plot@v0.10.1",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/gonum.org/v1/plot/org_gonum_v1_plot-v0.10.1.zip",
+ "http://ats.apps.svc/gomod/gonum.org/v1/plot/org_gonum_v1_plot-v0.10.1.zip",
+ "https://cache.hawkingrei.com/gomod/gonum.org/v1/plot/org_gonum_v1_plot-v0.10.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/gonum.org/v1/plot/org_gonum_v1_plot-v0.10.1.zip",
+ ],
+ )
+ go_repository(
+ name = "org_modernc_cc_v3",
+ build_file_proto_mode = "disable_global",
+ importpath = "modernc.org/cc/v3",
+ sha256 = "fe3aeb761e55ce77a95b297321a122b4273aeffe1c08f48fc99310e065211f74",
+ strip_prefix = "modernc.org/cc/v3@v3.40.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/gonum.org/v1/plot/org_gonum_v1_plot-v0.0.0-20190515093506-e2840ee46a6b.zip",
- "http://ats.apps.svc/gomod/gonum.org/v1/plot/org_gonum_v1_plot-v0.0.0-20190515093506-e2840ee46a6b.zip",
- "https://cache.hawkingrei.com/gomod/gonum.org/v1/plot/org_gonum_v1_plot-v0.0.0-20190515093506-e2840ee46a6b.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/gonum.org/v1/plot/org_gonum_v1_plot-v0.0.0-20190515093506-e2840ee46a6b.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/cc/v3/org_modernc_cc_v3-v3.40.0.zip",
+ "http://ats.apps.svc/gomod/modernc.org/cc/v3/org_modernc_cc_v3-v3.40.0.zip",
+ "https://cache.hawkingrei.com/gomod/modernc.org/cc/v3/org_modernc_cc_v3-v3.40.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/cc/v3/org_modernc_cc_v3-v3.40.0.zip",
+ ],
+ )
+ go_repository(
+ name = "org_modernc_ccgo_v3",
+ build_file_proto_mode = "disable_global",
+ importpath = "modernc.org/ccgo/v3",
+ sha256 = "bfc293300cd1ce656ba0ce0cee1f508afec2518bc4214a6b10ccfad6e8e6046e",
+ strip_prefix = "modernc.org/ccgo/v3@v3.16.13",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/ccgo/v3/org_modernc_ccgo_v3-v3.16.13.zip",
+ "http://ats.apps.svc/gomod/modernc.org/ccgo/v3/org_modernc_ccgo_v3-v3.16.13.zip",
+ "https://cache.hawkingrei.com/gomod/modernc.org/ccgo/v3/org_modernc_ccgo_v3-v3.16.13.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/ccgo/v3/org_modernc_ccgo_v3-v3.16.13.zip",
+ ],
+ )
+ go_repository(
+ name = "org_modernc_ccorpus",
+ build_file_proto_mode = "disable_global",
+ importpath = "modernc.org/ccorpus",
+ sha256 = "3831b62a73a379b81ac927e17e3e9ffe2d44ad07c934505e1ae24eea8a26a6d3",
+ strip_prefix = "modernc.org/ccorpus@v1.11.6",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/ccorpus/org_modernc_ccorpus-v1.11.6.zip",
+ "http://ats.apps.svc/gomod/modernc.org/ccorpus/org_modernc_ccorpus-v1.11.6.zip",
+ "https://cache.hawkingrei.com/gomod/modernc.org/ccorpus/org_modernc_ccorpus-v1.11.6.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/ccorpus/org_modernc_ccorpus-v1.11.6.zip",
],
)
go_repository(
@@ -10070,6 +10543,32 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/golex/org_modernc_golex-v1.1.0.zip",
],
)
+ go_repository(
+ name = "org_modernc_httpfs",
+ build_file_proto_mode = "disable_global",
+ importpath = "modernc.org/httpfs",
+ sha256 = "0b5314649c1327a199397eb6fd52b3ce41c9d3bc6dd2a4dea565b5fb87c13f41",
+ strip_prefix = "modernc.org/httpfs@v1.0.6",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/httpfs/org_modernc_httpfs-v1.0.6.zip",
+ "http://ats.apps.svc/gomod/modernc.org/httpfs/org_modernc_httpfs-v1.0.6.zip",
+ "https://cache.hawkingrei.com/gomod/modernc.org/httpfs/org_modernc_httpfs-v1.0.6.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/httpfs/org_modernc_httpfs-v1.0.6.zip",
+ ],
+ )
+ go_repository(
+ name = "org_modernc_libc",
+ build_file_proto_mode = "disable_global",
+ importpath = "modernc.org/libc",
+ sha256 = "5f98bedf9f0663b3b87555904ee41b82fe9d8e9ac5c47c9fac9a42a7fe232313",
+ strip_prefix = "modernc.org/libc@v1.22.2",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/libc/org_modernc_libc-v1.22.2.zip",
+ "http://ats.apps.svc/gomod/modernc.org/libc/org_modernc_libc-v1.22.2.zip",
+ "https://cache.hawkingrei.com/gomod/modernc.org/libc/org_modernc_libc-v1.22.2.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/libc/org_modernc_libc-v1.22.2.zip",
+ ],
+ )
go_repository(
name = "org_modernc_mathutil",
build_file_proto_mode = "disable_global",
@@ -10083,6 +10582,32 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/mathutil/org_modernc_mathutil-v1.6.0.zip",
],
)
+ go_repository(
+ name = "org_modernc_memory",
+ build_file_proto_mode = "disable_global",
+ importpath = "modernc.org/memory",
+ sha256 = "f79e8ada14c36d08817ee2bf6b2103f65c1a61a91b042b59016465869624043c",
+ strip_prefix = "modernc.org/memory@v1.5.0",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/memory/org_modernc_memory-v1.5.0.zip",
+ "http://ats.apps.svc/gomod/modernc.org/memory/org_modernc_memory-v1.5.0.zip",
+ "https://cache.hawkingrei.com/gomod/modernc.org/memory/org_modernc_memory-v1.5.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/memory/org_modernc_memory-v1.5.0.zip",
+ ],
+ )
+ go_repository(
+ name = "org_modernc_opt",
+ build_file_proto_mode = "disable_global",
+ importpath = "modernc.org/opt",
+ sha256 = "294b1b80137cb86292c8893481d545eee90b17b84b6ad1dcb2e6c9bb523a2d9e",
+ strip_prefix = "modernc.org/opt@v0.1.3",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/opt/org_modernc_opt-v0.1.3.zip",
+ "http://ats.apps.svc/gomod/modernc.org/opt/org_modernc_opt-v0.1.3.zip",
+ "https://cache.hawkingrei.com/gomod/modernc.org/opt/org_modernc_opt-v0.1.3.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/opt/org_modernc_opt-v0.1.3.zip",
+ ],
+ )
go_repository(
name = "org_modernc_parser",
build_file_proto_mode = "disable_global",
@@ -10109,6 +10634,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/sortutil/org_modernc_sortutil-v1.2.0.zip",
],
)
+ go_repository(
+ name = "org_modernc_sqlite",
+ build_file_proto_mode = "disable_global",
+ importpath = "modernc.org/sqlite",
+ sha256 = "be0501f87458962a00c8fb07d1f131af010a534cd6ffb654c570be35b9b608d5",
+ strip_prefix = "modernc.org/sqlite@v1.18.2",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/sqlite/org_modernc_sqlite-v1.18.2.zip",
+ "http://ats.apps.svc/gomod/modernc.org/sqlite/org_modernc_sqlite-v1.18.2.zip",
+ "https://cache.hawkingrei.com/gomod/modernc.org/sqlite/org_modernc_sqlite-v1.18.2.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/sqlite/org_modernc_sqlite-v1.18.2.zip",
+ ],
+ )
go_repository(
name = "org_modernc_strutil",
build_file_proto_mode = "disable_global",
@@ -10122,6 +10660,32 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/strutil/org_modernc_strutil-v1.2.0.zip",
],
)
+ go_repository(
+ name = "org_modernc_tcl",
+ build_file_proto_mode = "disable_global",
+ importpath = "modernc.org/tcl",
+ sha256 = "f966db0dd1ccbc7f8d5ac2e752b64c3be343aa3f92215ed98b6f2a51b7abbb64",
+ strip_prefix = "modernc.org/tcl@v1.13.2",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/tcl/org_modernc_tcl-v1.13.2.zip",
+ "http://ats.apps.svc/gomod/modernc.org/tcl/org_modernc_tcl-v1.13.2.zip",
+ "https://cache.hawkingrei.com/gomod/modernc.org/tcl/org_modernc_tcl-v1.13.2.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/tcl/org_modernc_tcl-v1.13.2.zip",
+ ],
+ )
+ go_repository(
+ name = "org_modernc_token",
+ build_file_proto_mode = "disable_global",
+ importpath = "modernc.org/token",
+ sha256 = "3efaa49e9fb10569da9e09e785fa230cd5c0f50fcf605f3b5439dfcd61577c58",
+ strip_prefix = "modernc.org/token@v1.1.0",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/token/org_modernc_token-v1.1.0.zip",
+ "http://ats.apps.svc/gomod/modernc.org/token/org_modernc_token-v1.1.0.zip",
+ "https://cache.hawkingrei.com/gomod/modernc.org/token/org_modernc_token-v1.1.0.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/token/org_modernc_token-v1.1.0.zip",
+ ],
+ )
go_repository(
name = "org_modernc_y",
build_file_proto_mode = "disable_global",
@@ -10135,6 +10699,19 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/y/org_modernc_y-v1.1.0.zip",
],
)
+ go_repository(
+ name = "org_modernc_z",
+ build_file_proto_mode = "disable_global",
+ importpath = "modernc.org/z",
+ sha256 = "5be23ef96669963e52d25b787d71028fff4fe1c468dec20aac59c9512caa2eb7",
+ strip_prefix = "modernc.org/z@v1.5.1",
+ urls = [
+ "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/z/org_modernc_z-v1.5.1.zip",
+ "http://ats.apps.svc/gomod/modernc.org/z/org_modernc_z-v1.5.1.zip",
+ "https://cache.hawkingrei.com/gomod/modernc.org/z/org_modernc_z-v1.5.1.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/z/org_modernc_z-v1.5.1.zip",
+ ],
+ )
go_repository(
name = "org_mongodb_go_mongo_driver",
build_file_proto_mode = "disable_global",
diff --git a/Makefile b/Makefile
index 55e310d304feb..64e1227e20e3a 100644
--- a/Makefile
+++ b/Makefile
@@ -312,7 +312,7 @@ tools/bin/revive:
.PHONY: tools/bin/failpoint-ctl
tools/bin/failpoint-ctl:
- GOBIN=$(shell pwd)/tools/bin $(GO) install github.com/pingcap/failpoint/failpoint-ctl@2eaa328
+ GOBIN=$(shell pwd)/tools/bin $(GO) install github.com/pingcap/failpoint/failpoint-ctl@9b3b6e3
.PHONY: tools/bin/errdoc-gen
tools/bin/errdoc-gen:
diff --git a/OWNERS b/OWNERS
index a1a2af65416c7..609be7f7022b3 100644
--- a/OWNERS
+++ b/OWNERS
@@ -51,6 +51,7 @@ approvers:
- ichn-hu
- imtbkcat
- jackysp
+ - jiyfhust
- joccau
- july2993
- kennytm
@@ -127,7 +128,6 @@ reviewers:
- fengou1
- fzzf678
- iamxy
- - jiyfhust
- JmPotato
- js00070
- lamxTyler
@@ -150,6 +150,7 @@ reviewers:
- tsthght
- TszKitLo40
- xzhangxian1008
+ - yibin87
- zhangjinpeng1987
- zhongzc
- zhuo-zhi
diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES
index 26d44b7e83898..a2a5f1b465999 100644
--- a/OWNERS_ALIASES
+++ b/OWNERS_ALIASES
@@ -68,6 +68,8 @@ aliases:
- gmhdbjd
- lichunzhu
- okJiang
+ - lance6716
+ - tangenta
sig-approvers-infoschema: # approvers for infoschema pkg
- zimulala
- wjhuang2016
diff --git a/README.md b/README.md
index 62b9858d915d5..0f14d17b7824f 100644
--- a/README.md
+++ b/README.md
@@ -47,9 +47,9 @@ TiDB (/’taɪdiːbi:/, "Ti" stands for Titanium) is an open-source distributed
- [Architecture](#architecture)
- [MySQL compatibility](https://docs.pingcap.com/tidb/stable/mysql-compatibility)
-See what TiDB is capable ONLINE at [TiDB Playground](https://play.tidbcloud.com/?utm_source=github&utm_medium=tidb_readme).
+See what TiDB is capable of ONLINE at the [TiDB Playground](https://play.tidbcloud.com/?utm_source=github&utm_medium=tidb_readme).
-For more details and latest updates, see [TiDB documentation](https://docs.pingcap.com/tidb/stable) and [release notes](https://docs.pingcap.com/tidb/dev/release-notes).
+For more details and the latest updates, see the [TiDB documentation](https://docs.pingcap.com/tidb/stable) and [release notes](https://docs.pingcap.com/tidb/dev/release-notes).
For future plans, see the [TiDB roadmap](roadmap.md).
@@ -61,15 +61,15 @@ TiDB Cloud is the fully-managed service of TiDB, currently available on AWS and
Quickly check out TiDB Cloud with [a free trial](https://tidbcloud.com/free-trial).
-See [TiDB Cloud Quick Start Guide](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart).
+See the [TiDB Cloud Quick Start Guide](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart).
### Start with TiDB
See [TiDB Quick Start Guide](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb).
-### Start developing TiDB
+### Start developing with TiDB
-See the [Get Started](https://pingcap.github.io/tidb-dev-guide/get-started/introduction.html) chapter of [TiDB Development Guide](https://pingcap.github.io/tidb-dev-guide/index.html).
+See [TiDB Developer Guide](https://docs.pingcap.com/tidb/stable/dev-guide-overview) and [TiDB Cloud Developer Guide](https://docs.pingcap.com/tidbcloud/dev-guide-overview).
## Community
@@ -96,7 +96,7 @@ Contributions are welcomed and greatly appreciated. You can get started with one
[
](https://github.com/pingcap/tidb-map/blob/master/maps/contribution-map.md#tidb-is-an-open-source-distributed-htap-database-compatible-with-the-mysql-protocol)
-Every contributor is welcome to claim your contribution swag by filling in and submitting this [form](https://forms.pingcap.com/f/tidb-contribution-swag).
+Every contributor is welcome to claim their contribution swag by filling in and submitting this [form](https://forms.pingcap.com/f/tidb-contribution-swag).
diff --git a/WORKSPACE b/WORKSPACE
index 5c81b9ffbb158..e588445f2ce02 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -1,5 +1,26 @@
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+# Required by toolchains_protoc.
+http_archive(
+ name = "platforms",
+ sha256 = "218efe8ee736d26a3572663b374a253c012b716d8af0c07e842e82f238a0a7ee",
+ urls = [
+ "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.10/platforms-0.0.10.tar.gz",
+ "https://github.com/bazelbuild/platforms/releases/download/0.0.10/platforms-0.0.10.tar.gz",
+ ],
+)
+
+http_archive(
+ name = "bazel_features",
+ sha256 = "d7787da289a7fb497352211ad200ec9f698822a9e0757a4976fd9f713ff372b3",
+ strip_prefix = "bazel_features-1.9.1",
+ url = "https://github.com/bazel-contrib/bazel_features/releases/download/v1.9.1/bazel_features-v1.9.1.tar.gz",
+)
+
+load("@bazel_features//:deps.bzl", "bazel_features_deps")
+
+bazel_features_deps()
+
http_archive(
name = "bazel_skylib",
sha256 = "66ffd9315665bfaafc96b52278f57c7e2dd09f5ede279ea6d39b2be471e7e3aa",
@@ -16,23 +37,23 @@ versions.check(minimum_bazel_version = "6.0.0")
http_archive(
name = "io_bazel_rules_go",
- sha256 = "af47f30e9cbd70ae34e49866e201b3f77069abb111183f2c0297e7e74ba6bbc0",
+ sha256 = "33acc4ae0f70502db4b893c9fc1dd7a9bf998c23e7ff2c4517741d4049a976f8",
urls = [
- "http://bazel-cache.pingcap.net:8080/bazelbuild/rules_go/releases/download/v0.47.0/rules_go-v0.47.0.zip",
- "http://ats.apps.svc/bazelbuild/rules_go/releases/download/v0.47.0/rules_go-v0.47.0.zip",
- "https://github.com/bazelbuild/rules_go/releases/download/v0.47.0/rules_go-v0.47.0.zip",
- "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.47.0/rules_go-v0.47.0.zip",
+ "http://bazel-cache.pingcap.net:8080/bazelbuild/rules_go/releases/download/v0.48.0/rules_go-v0.48.0.zip",
+ "http://ats.apps.svc/bazelbuild/rules_go/releases/download/v0.48.0/rules_go-v0.48.0.zip",
+ "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.48.0/rules_go-v0.48.0.zip",
+ "https://github.com/bazelbuild/rules_go/releases/download/v0.48.0/rules_go-v0.48.0.zip",
],
)
http_archive(
name = "bazel_gazelle",
- sha256 = "32938bda16e6700063035479063d9d24c60eda8d79fd4739563f50d331cb3209",
+ sha256 = "d76bf7a60fd8b050444090dfa2837a4eaf9829e1165618ee35dceca5cbdf58d5",
urls = [
- "http://bazel-cache.pingcap.net:8080/bazelbuild/bazel-gazelle/releases/download/v0.35.0/bazel-gazelle-v0.35.0.tar.gz",
- "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.35.0/bazel-gazelle-v0.35.0.tar.gz",
- "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.35.0/bazel-gazelle-v0.35.0.tar.gz",
- "http://ats.apps.svc/bazelbuild/bazel-gazelle/releases/download/v0.35.0/bazel-gazelle-v0.35.0.tar.gz",
+ "http://bazel-cache.pingcap.net:8080/bazelbuild/bazel-gazelle/releases/download/v0.37.0/bazel-gazelle-v0.37.0.tar.gz",
+ "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.37.0/bazel-gazelle-v0.37.0.tar.gz",
+ "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.37.0/bazel-gazelle-v0.37.0.tar.gz",
+ "http://ats.apps.svc/bazelbuild/bazel-gazelle/releases/download/v0.37.0/bazel-gazelle-v0.37.0.tar.gz",
],
)
@@ -64,7 +85,7 @@ go_download_sdk(
"https://mirrors.aliyun.com/golang/{}",
"https://dl.google.com/go/{}",
],
- version = "1.21.9",
+ version = "1.21.10",
)
go_register_toolchains(
@@ -113,16 +134,21 @@ http_archive(
http_archive(
name = "rules_proto",
- sha256 = "aa1ee19226f707d44bee44c720915199c20c84a23318bb0597ed4e5c873ccbd5",
- strip_prefix = "rules_proto-40298556293ae502c66579620a7ce867d5f57311",
+ sha256 = "303e86e722a520f6f326a50b41cfc16b98fe6d1955ce46642a5b7a67c11c0f5d",
+ strip_prefix = "rules_proto-6.0.0",
urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/rules/rules_proto/rules_proto-40298556293ae502c66579620a7ce867d5f57311.tar.gz",
- "http://ats.apps.svc/gomod/gomod/rules/rules_proto/rules_proto-40298556293ae502c66579620a7ce867d5f57311.tar.gz",
- "https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/40298556293ae502c66579620a7ce867d5f57311.tar.gz",
- "https://github.com/bazelbuild/rules_proto/archive/40298556293ae502c66579620a7ce867d5f57311.tar.gz",
+ "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0/rules_proto-6.0.0.tar.gz",
],
)
+load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies")
+
+rules_proto_dependencies()
+
+load("@rules_proto//proto:toolchains.bzl", "rules_proto_toolchains")
+
+rules_proto_toolchains()
+
http_archive(
name = "rules_java",
sha256 = "f5a3e477e579231fca27bf202bb0e8fbe4fc6339d63b38ccb87c2760b533d1c3",
@@ -134,3 +160,10 @@ http_archive(
"https://github.com/bazelbuild/rules_java/archive/981f06c3d2bd10225e85209904090eb7b5fb26bd.tar.gz",
],
)
+
+http_archive(
+ name = "toolchains_protoc",
+ sha256 = "117af61ee2f1b9b014dcac7c9146f374875551abb8a30e51d1b3c5946d25b142",
+ strip_prefix = "toolchains_protoc-0.3.0",
+ url = "https://github.com/aspect-build/toolchains_protoc/releases/download/v0.3.0/toolchains_protoc-v0.3.0.tar.gz",
+)
diff --git a/br/pkg/aws/BUILD.bazel b/br/pkg/aws/BUILD.bazel
index a704d1aab69b4..4fed52b134b97 100644
--- a/br/pkg/aws/BUILD.bazel
+++ b/br/pkg/aws/BUILD.bazel
@@ -29,9 +29,11 @@ go_test(
srcs = ["ebs_test.go"],
embed = [":aws"],
flaky = True,
+ shard_count = 3,
deps = [
"@com_github_aws_aws_sdk_go//aws",
"@com_github_aws_aws_sdk_go//service/ec2",
+ "@com_github_aws_aws_sdk_go//service/ec2/ec2iface",
"@com_github_stretchr_testify//require",
],
)
diff --git a/br/pkg/aws/ebs.go b/br/pkg/aws/ebs.go
index d578868e2408c..fc4169e6fb817 100644
--- a/br/pkg/aws/ebs.go
+++ b/br/pkg/aws/ebs.go
@@ -251,6 +251,9 @@ func (e *EC2Session) WaitSnapshotsCreated(snapIDMap map[string]string, progress
if *s.State == ec2.SnapshotStateCompleted {
log.Info("snapshot completed", zap.String("id", *s.SnapshotId))
totalVolumeSize += *s.VolumeSize
+ } else if *s.State == ec2.SnapshotStateError {
+ log.Error("snapshot failed", zap.String("id", *s.SnapshotId), zap.String("error", (*s.StateMessage)))
+ return 0, errors.Errorf("snapshot %s failed", *s.SnapshotId)
} else {
log.Debug("snapshot creating...", zap.Stringer("snap", s))
uncompletedSnapshots = append(uncompletedSnapshots, s.SnapshotId)
diff --git a/br/pkg/aws/ebs_test.go b/br/pkg/aws/ebs_test.go
index e55ea68c86e04..96fcf358ff952 100644
--- a/br/pkg/aws/ebs_test.go
+++ b/br/pkg/aws/ebs_test.go
@@ -14,10 +14,12 @@
package aws
import (
+ "context"
"testing"
awsapi "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
+ "github.com/aws/aws-sdk-go/service/ec2/ec2iface"
"github.com/stretchr/testify/require"
)
@@ -76,3 +78,127 @@ func TestHandleDescribeVolumesResponse(t *testing.T) {
require.Equal(t, int64(4), createdVolumeSize)
require.Equal(t, 1, len(unfinishedVolumes))
}
+
+type mockEC2 struct {
+ ec2iface.EC2API
+ output ec2.DescribeSnapshotsOutput
+}
+
+func (m mockEC2) DescribeSnapshots(*ec2.DescribeSnapshotsInput) (*ec2.DescribeSnapshotsOutput, error) {
+ return &m.output, nil
+}
+
+func NewMockEc2Session(mock mockEC2) *EC2Session {
+ return &EC2Session{
+ ec2: mock,
+ }
+}
+
+func TestWaitSnapshotsCreated(t *testing.T) {
+ snapIdMap := map[string]string{
+ "vol-1": "snap-1",
+ "vol-2": "snap-2",
+ }
+
+ cases := []struct {
+ desc string
+ snapshotsOutput ec2.DescribeSnapshotsOutput
+ expectedSize int64
+ expectErr bool
+ expectTimeout bool
+ }{
+ {
+ desc: "snapshots are all completed",
+ snapshotsOutput: ec2.DescribeSnapshotsOutput{
+ Snapshots: []*ec2.Snapshot{
+ {
+ SnapshotId: awsapi.String("snap-1"),
+ VolumeSize: awsapi.Int64(1),
+ State: awsapi.String(ec2.SnapshotStateCompleted),
+ },
+ {
+ SnapshotId: awsapi.String("snap-2"),
+ VolumeSize: awsapi.Int64(2),
+ State: awsapi.String(ec2.SnapshotStateCompleted),
+ },
+ },
+ },
+ expectedSize: 3,
+ expectErr: false,
+ },
+ {
+ desc: "snapshot failed",
+ snapshotsOutput: ec2.DescribeSnapshotsOutput{
+ Snapshots: []*ec2.Snapshot{
+ {
+ SnapshotId: awsapi.String("snap-1"),
+ VolumeSize: awsapi.Int64(1),
+ State: awsapi.String(ec2.SnapshotStateCompleted),
+ },
+ {
+ SnapshotId: awsapi.String("snap-2"),
+ State: awsapi.String(ec2.SnapshotStateError),
+ StateMessage: awsapi.String("snapshot failed"),
+ },
+ },
+ },
+ expectedSize: 0,
+ expectErr: true,
+ },
+ {
+ desc: "snapshots pending",
+ snapshotsOutput: ec2.DescribeSnapshotsOutput{
+ Snapshots: []*ec2.Snapshot{
+ {
+ SnapshotId: awsapi.String("snap-1"),
+ VolumeSize: awsapi.Int64(1),
+ State: awsapi.String(ec2.SnapshotStateCompleted),
+ },
+ {
+ SnapshotId: awsapi.String("snap-2"),
+ State: awsapi.String(ec2.SnapshotStatePending),
+ },
+ },
+ },
+ expectTimeout: true,
+ },
+ }
+
+ for _, c := range cases {
+ e := NewMockEc2Session(mockEC2{
+ output: c.snapshotsOutput,
+ })
+
+ if c.expectTimeout {
+ func() {
+ // We wait 5s before checking snapshots
+ ctx, cancel := context.WithTimeout(context.Background(), 6)
+ defer cancel()
+
+ done := make(chan struct{})
+ go func() {
+ _, _ = e.WaitSnapshotsCreated(snapIdMap, nil)
+ done <- struct{}{}
+ }()
+
+ select {
+ case <-done:
+ t.Fatal("WaitSnapshotsCreated should not return before timeout")
+ case <-ctx.Done():
+ require.True(t, true)
+ }
+ }()
+
+ continue
+ }
+
+ size, err := e.WaitSnapshotsCreated(snapIdMap, nil)
+ if c.expectErr {
+ require.Error(t, err)
+ } else {
+ require.NoError(t, err)
+ }
+
+ require.Equal(t, c.expectedSize, size)
+ }
+}
diff --git a/br/pkg/backup/BUILD.bazel b/br/pkg/backup/BUILD.bazel
index 7d7602193fb1e..3a93b3c7b773c 100644
--- a/br/pkg/backup/BUILD.bazel
+++ b/br/pkg/backup/BUILD.bazel
@@ -6,8 +6,8 @@ go_library(
"check.go",
"client.go",
"metrics.go",
- "push.go",
"schema.go",
+ "store.go",
],
importpath = "github.com/pingcap/tidb/br/pkg/backup",
visibility = ["//visibility:public"],
@@ -24,6 +24,7 @@ go_library(
"//br/pkg/storage",
"//br/pkg/summary",
"//br/pkg/utils",
+ "//br/pkg/utils/storewatch",
"//br/pkg/version",
"//pkg/ddl",
"//pkg/distsql",
@@ -66,10 +67,10 @@ go_test(
embed = [":backup"],
flaky = True,
race = "on",
- shard_count = 8,
+ shard_count = 9,
deps = [
"//br/pkg/conn",
- "//br/pkg/gluetidb",
+ "//br/pkg/gluetidb/mock",
"//br/pkg/metautil",
"//br/pkg/mock",
"//br/pkg/pdutil",
diff --git a/br/pkg/backup/client.go b/br/pkg/backup/client.go
index d7b1d40a41d8c..1022865afd7c9 100644
--- a/br/pkg/backup/client.go
+++ b/br/pkg/backup/client.go
@@ -62,9 +62,264 @@ type Checksum struct {
// ProgressUnit represents the unit of progress.
type ProgressUnit string
-type StoreBackupPolicy struct {
- One uint64
- All bool
+type MainBackupLoop struct {
+ BackupSender
+
+ // backup requests for all stores.
+ // the subRanges may changed every round.
+ BackupReq backuppb.BackupRequest
+ // record the whole backup progress in infinite loop.
+ GlobalProgressTree *rtree.ProgressRangeTree
+ ReplicaReadLabel map[string]string
+ StateNotifier chan BackupRetryPolicy
+
+ ProgressCallBack func()
+ GetBackupClientCallBack func(ctx context.Context, storeID uint64) (backuppb.BackupClient, error)
+}
+
+type MainBackupSender struct{}
+
+func (s *MainBackupSender) SendAsync(
+ ctx context.Context,
+ round uint64,
+ storeID uint64,
+ request backuppb.BackupRequest,
+ cli backuppb.BackupClient,
+ respCh chan *ResponseAndStore,
+ StateNotifier chan BackupRetryPolicy,
+) {
+ go func() {
+ defer func() {
+ logutil.CL(ctx).Info("exit store backup goroutine", zap.Uint64("store", storeID))
+ close(respCh)
+ }()
+ err := startBackup(ctx, storeID, request, cli, respCh)
+ if err != nil {
+ // only 2 kinds of errors will occur here.
+ // 1. grpc connection error(already retry inside)
+ // 2. context cancelled outside.
+ if errors.Cause(err) == context.Canceled {
+ logutil.CL(ctx).Info("store backup cancelled",
+ zap.Uint64("round", round),
+ zap.Uint64("storeID", storeID))
+ } else {
+ // otherwise retry backup this store
+ logutil.CL(ctx).Error("store backup failed",
+ zap.Uint64("round", round),
+ zap.Uint64("storeID", storeID), zap.Error(err))
+ select {
+ case <-ctx.Done():
+ case StateNotifier <- BackupRetryPolicy{One: storeID}:
+ }
+ }
+ }
+ }()
+}
+
+// CollectStoreBackupsAsync is the receiver function of all stores backup results.
+func (l *MainBackupLoop) CollectStoreBackupsAsync(
+ ctx context.Context,
+ round uint64,
+ storeBackupChs map[uint64]chan *ResponseAndStore,
+ globalCh chan *ResponseAndStore,
+) {
+ go func() {
+ defer func() {
+ logutil.CL(ctx).Info("exit collect backups goroutine", zap.Uint64("round", round))
+ close(globalCh)
+ }()
+ cases := make([]reflect.SelectCase, 0)
+ for _, ch := range storeBackupChs {
+ cases = append(cases, reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(ch)})
+ }
+
+ remainingProducers := len(cases)
+ logutil.CL(ctx).Info("start wait store backups", zap.Int("remainingProducers", remainingProducers))
+ for remainingProducers > 0 {
+ chosen, value, ok := reflect.Select(cases)
+ if !ok {
+ // The chosen channel has been closed, so zero out the channel to disable the case
+ cases[chosen].Chan = reflect.ValueOf(nil)
+ remainingProducers -= 1
+ continue
+ }
+
+ select {
+ case <-ctx.Done():
+ return
+ case globalCh <- value.Interface().(*ResponseAndStore):
+ }
+ }
+ }()
+}
+
+// infinite loop to backup ranges on all tikv stores
+// if one client grpc disconnected. resend backup request to this store.
+// if new tikv store joined. send backup request to new store.
+// if one tikv store rebooted. consider leader changes, resend backup request to all stores.
+// if one tikv store disconnected. consider leader changes, resend backup request to all stores.a
+func (bc *Client) RunLoop(ctx context.Context, loop *MainBackupLoop) error {
+ // a flag to indicate the backup round
+ // one backup round try to backup all ranges on all tikv stores.
+ // ideally, backup should be finished in one round
+ // unless the cluster state changed or some kv errors occurred.
+ round := uint64(0)
+mainLoop:
+ for {
+ round += 1
+ logutil.CL(ctx).Info("This round of backup starts...", zap.Uint64("round", round))
+ // initialize the error context every round
+ errContext := utils.NewErrorContext("MainBackupLoop", 10)
+
+ // a channel to collect all store backup results
+ globalBackupResultCh := make(chan *ResponseAndStore)
+ // channel slices to receive backup region result from different tikv stores
+ storeBackupResultChMap := make(map[uint64]chan *ResponseAndStore)
+
+ // mainCtx used to control mainLoop
+ // every round need a new context to control the main backup process
+ mainCtx, mainCancel := context.WithCancel(ctx)
+
+ // handleCtx used to control handleLoop
+ // every round has another infinite loop to handle all tikv backup responses
+ // until backup finished, store state changed or error occurred.
+ handleCtx, handleCancel := context.WithCancel(ctx)
+
+ // Compute the left ranges that not backuped yet
+ start := time.Now()
+
+ var inCompleteRanges []rtree.Range
+ select {
+ case <-ctx.Done():
+ // ctx cancal outside
+ handleCancel()
+ mainCancel()
+ return ctx.Err()
+ default:
+ iter := loop.GlobalProgressTree.Iter()
+ inCompleteRanges = iter.GetIncompleteRanges()
+ if len(inCompleteRanges) == 0 {
+ // all range backuped
+ logutil.CL(ctx).Info("This round finished all backup ranges", zap.Uint64("round", round))
+ handleCancel()
+ mainCancel()
+ return nil
+ }
+ }
+
+ logutil.CL(mainCtx).Info("backup ranges", zap.Uint64("round", round),
+ zap.Int("incomplete-ranges", len(inCompleteRanges)), zap.Duration("cost", time.Since(start)))
+
+ loop.BackupReq.SubRanges = getBackupRanges(inCompleteRanges)
+
+ allStores, err := bc.getBackupStores(mainCtx, loop.ReplicaReadLabel)
+ if err != nil {
+ // because we have connectted to pd before.
+ // so this error must be retryable, just make infinite retry here
+ logutil.CL(mainCtx).Error("failed to get backup stores", zap.Uint64("round", round), zap.Error(err))
+ mainCancel()
+ continue mainLoop
+ }
+ for _, store := range allStores {
+ if err = utils.CheckStoreLiveness(store); err != nil {
+ // skip this store in this round.
+ logutil.CL(mainCtx).Warn("store not alive, skip backup it in this round", zap.Uint64("round", round), zap.Error(err))
+ continue
+ }
+ storeID := store.GetId()
+ // reset backup client every round, to get a clean grpc connection.
+ cli, err := loop.GetBackupClientCallBack(mainCtx, storeID)
+ if err != nil {
+ // because the we get store info from pd.
+ // there is no customer setting here, so make infinite retry.
+ logutil.CL(ctx).Error("failed to reset backup client", zap.Uint64("round", round), zap.Uint64("storeID", storeID), zap.Error(err))
+ mainCancel()
+ continue mainLoop
+ }
+ ch := make(chan *ResponseAndStore)
+ storeBackupResultChMap[storeID] = ch
+ loop.SendAsync(mainCtx, round, storeID, loop.BackupReq, cli, ch, loop.StateNotifier)
+ }
+ // infinite loop to collect region backup response to global channel
+ loop.CollectStoreBackupsAsync(handleCtx, round, storeBackupResultChMap, globalBackupResultCh)
+ handleLoop:
+ for {
+ select {
+ case <-ctx.Done():
+ handleCancel()
+ mainCancel()
+ return ctx.Err()
+ case storeBackupInfo := <-loop.StateNotifier:
+ if storeBackupInfo.All {
+ logutil.CL(mainCtx).Info("cluster state changed. restart store backups", zap.Uint64("round", round))
+ // stop current connections
+ handleCancel()
+ mainCancel()
+ // start next round backups
+ continue mainLoop
+ }
+ if storeBackupInfo.One != 0 {
+ storeID := storeBackupInfo.One
+ store, err := bc.mgr.GetPDClient().GetStore(mainCtx, storeID)
+ if err != nil {
+ // cannot get store, maybe store has scaled-in.
+ logutil.CL(mainCtx).Info("cannot get store from pd", zap.Uint64("round", round), zap.Error(err))
+ // try next round
+ handleCancel()
+ mainCancel()
+ continue mainLoop
+ }
+ if err = utils.CheckStoreLiveness(store); err != nil {
+ // skip this store in this round.
+ logutil.CL(mainCtx).Warn("store not alive, skip backup it in this round", zap.Uint64("round", round), zap.Error(err))
+ continue mainLoop
+ }
+ // reset backup client. store address could change but store id remained.
+ cli, err := loop.GetBackupClientCallBack(mainCtx, storeID)
+ if err != nil {
+ logutil.CL(mainCtx).Error("failed to reset backup client", zap.Uint64("round", round), zap.Uint64("storeID", storeID), zap.Error(err))
+ handleCancel()
+ mainCancel()
+ // receive new store info but failed to get backup client.
+ // start next round backups to get all tikv stores and reset all client connections.
+ continue mainLoop
+ }
+
+ // cancel the former collect goroutine
+ handleCancel()
+ ch := make(chan *ResponseAndStore)
+
+ storeBackupResultChMap[storeID] = ch
+ // start backup for this store
+ loop.SendAsync(mainCtx, round, storeID, loop.BackupReq, cli, ch, loop.StateNotifier)
+ // re-create context for new handler loop
+ handleCtx, handleCancel = context.WithCancel(mainCtx)
+ // handleCancel makes the former collect goroutine exits
+ // so we need to re-create a new channel and restart a new collect goroutine.
+ globalBackupResultCh = make(chan *ResponseAndStore)
+ // collect all store backup producer channel result to one channel
+ loop.CollectStoreBackupsAsync(handleCtx, round, storeBackupResultChMap, globalBackupResultCh)
+ }
+ case respAndStore, ok := <-globalBackupResultCh:
+ if !ok {
+ // this round backup finished. break and check incomplete ranges in mainLoop.
+ break handleLoop
+ }
+ err = bc.OnBackupResponse(handleCtx, respAndStore, errContext, loop.GlobalProgressTree)
+ if err != nil {
+ // if error occurred here, stop the backup process
+ // because only 3 kinds of errors will be returned here:
+ // 1. permission denied on tikv store.
+ // 2. parse backup response error.(shouldn't happen in any case)
+ // 3. checkpoint update failed. TODO: should we retry here?
+ handleCancel()
+ mainCancel()
+ return err
+ }
+ loop.ProgressCallBack()
+ }
+ }
+ }
}
// Client is a client instructs TiKV how to do a backup.
@@ -640,7 +895,6 @@ func BuildBackupSchemas(
}
if !hasTable {
- log.Info("backup empty database", zap.Stringer("db", dbInfo.Name))
fn(dbInfo, nil)
}
}
@@ -803,39 +1057,27 @@ func (bc *Client) BackupRanges(
ctx = opentracing.ContextWithSpan(ctx, span1)
}
- stateChan := make(chan StoreBackupPolicy)
- // TODO implement state change watch goroutine @3pointer
- // go func() {
- // // TODO watch changes on cluste state
- // cb := storewatch.MakeCallback(storewatch.WithOnReboot(func(s *metapb.Store) {
- // stateChan <- StoreBackups{All: true}
- // }), storewatch.WithOnDisconnect(func(s *metapb.Store) {
- // stateChan <- StoreBackups{All: true}
- // }), storewatch.WithOnNewStoreRegistered(func(s *metapb.Store) {
- // // only backup for this store
- // stateChan <- StoreBackups{One: s.Id}
- // }))
- // watcher := storewatch.New(bc.mgr.GetPDClient(), cb)
- // tick := time.NewTicker(30 * time.Second)
- // for {
- // select {
- // case <-ctx.Done():
- // return
- // case <-tick.C:
- // err := watcher.Step(ctx)
- // if err != nil {
- // // ignore it
- // }
- // }
- // }
- // }()
-
globalProgressTree, err := bc.BuildProgressRangeTree(ranges)
if err != nil {
return errors.Trace(err)
}
- err = bc.startMainBackupLoop(ctx, &globalProgressTree, replicaReadLabel, request, stateChan, progressCallBack)
+ stateNotifier := make(chan BackupRetryPolicy)
+ ObserveStoreChangesAsync(ctx, stateNotifier, bc.mgr.GetPDClient())
+
+ mainBackupLoop := &MainBackupLoop{
+ BackupSender: &MainBackupSender{},
+ BackupReq: request,
+ GlobalProgressTree: &globalProgressTree,
+ ReplicaReadLabel: replicaReadLabel,
+ StateNotifier: stateNotifier,
+ ProgressCallBack: progressCallBack,
+ // always use reset connection here.
+ // because we need to reset connection when store state changed.
+ GetBackupClientCallBack: bc.mgr.ResetBackupClient,
+ }
+
+ err = bc.RunLoop(ctx, mainBackupLoop)
if err != nil {
return errors.Trace(err)
}
@@ -880,7 +1122,9 @@ func (bc *Client) OnBackupResponse(
resp := r.GetResponse()
storeID := r.GetStoreID()
if resp.GetError() == nil {
+ start := time.Now()
pr, err := globalProgressTree.FindContained(resp.StartKey, resp.EndKey)
+ logutil.CL(ctx).Debug("find the range tree contains response ranges", zap.Duration("take", time.Since(start)))
if err != nil {
logutil.CL(ctx).Error("failed to update the backup response",
zap.Reflect("error", err))
@@ -922,238 +1166,6 @@ func (bc *Client) OnBackupResponse(
return nil
}
-// infinite loop to backup ranges on all tikv stores
-// if one client grpc disconnected. resend backup request to this store.
-// if new tikv store joined. send backup request to new store.
-// if one tikv store rebooted. consider leader changes, resend backup request to all stores.
-// if one tikv store disconnected. consider leader changes, resend backup request to all stores.
-func (bc *Client) startMainBackupLoop(
- ctx context.Context,
- globalProgressTree *rtree.ProgressRangeTree,
- replicaReadLabel map[string]string,
- request backuppb.BackupRequest,
- stateChan chan StoreBackupPolicy,
- progressCallBack func(),
-) error {
- startStoreBackupAsyncFn := func(
- ctx context.Context,
- round uint64,
- storeID uint64,
- request backuppb.BackupRequest,
- cli backuppb.BackupClient,
- respCh chan *ResponseAndStore,
- ) {
- go func() {
- defer func() {
- logutil.CL(ctx).Info("exit store backup goroutine", zap.Uint64("store", storeID))
- close(respCh)
- }()
- err := startStoreBackup(ctx, storeID, request, cli, respCh)
- if err != nil {
- // only 2 kinds of errors will occur here.
- // 1. grpc connection error(already retry inside)
- // 2. context cancelled outside.
- if errors.Cause(err) == context.Canceled {
- logutil.CL(ctx).Info("store backup cancelled",
- zap.Uint64("round", round),
- zap.Uint64("storeID", storeID))
- } else {
- // otherwise retry backup this store
- logutil.CL(ctx).Error("store backup failed",
- zap.Uint64("round", round),
- zap.Uint64("storeID", storeID), zap.Error(err))
- stateChan <- StoreBackupPolicy{One: storeID}
- }
- }
- }()
- }
-
- collectStoreBackupsAsyncFn := func(
- ctx context.Context,
- round uint64,
- storeBackupChs map[uint64]chan *ResponseAndStore,
- globalCh chan *ResponseAndStore,
-
- ) {
- go func() {
- defer func() {
- logutil.CL(ctx).Info("exit collect backups goroutine", zap.Uint64("round", round))
- close(globalCh)
- }()
- cases := make([]reflect.SelectCase, 0)
- for _, ch := range storeBackupChs {
- cases = append(cases, reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(ch)})
- }
-
- remainingProducers := len(cases)
- logutil.CL(ctx).Info("start wait store backups", zap.Int("remainingProducers", remainingProducers))
- for remainingProducers > 0 {
- chosen, value, ok := reflect.Select(cases)
- if !ok {
- // The chosen channel has been closed, so zero out the channel to disable the case
- cases[chosen].Chan = reflect.ValueOf(nil)
- remainingProducers -= 1
- continue
- }
-
- select {
- case <-ctx.Done():
- return
- case globalCh <- value.Interface().(*ResponseAndStore):
- }
- }
- }()
- }
-
- // a flag to indicate the backup round
- // one backup round try to backup all ranges on all tikv stores.
- // ideally, backup should be finished in one round
- // unless the cluster state changed or some kv errors occurred.
- round := uint64(0)
-mainLoop:
- for {
- round += 1
- // initialize the error context every round
- errContext := utils.NewErrorContext("MainBackupLoop", 10)
-
- // a channel to collect all store backup results
- globalBackupResultCh := make(chan *ResponseAndStore)
- // channel slices to receive backup region result from different tikv stores
- storeBackupResultChMap := make(map[uint64]chan *ResponseAndStore)
-
- // mainCtx used to control mainLoop
- // every round need a new context to control the main backup process
- mainCtx, mainCancel := context.WithCancel(ctx)
-
- // handleCtx used to control handleLoop
- // every round has another infinite loop to handle all tikv backup responses
- // until backup finished, store state changed or error occurred.
- handleCtx, handleCancel := context.WithCancel(ctx)
-
- // Compute the left ranges that not backuped yet
- iter := globalProgressTree.Iter()
- inCompleteRanges := iter.GetIncompleteRanges()
- if len(inCompleteRanges) == 0 {
- // all range backuped
- handleCancel()
- mainCancel()
- return nil
- }
-
- logutil.CL(ctx).Info("backup round start...", zap.Uint64("round", round))
-
- request.SubRanges = getBackupRanges(inCompleteRanges)
-
- allStores, err := bc.getBackupStores(mainCtx, replicaReadLabel)
- if err != nil {
- // because we have connectted to pd before.
- // so this error must be retryable, just make infinite retry here
- logutil.CL(mainCtx).Error("failed to get backup stores", zap.Uint64("round", round), zap.Error(err))
- mainCancel()
- continue mainLoop
- }
- for _, store := range allStores {
- if err = utils.CheckStoreLiveness(store); err != nil {
- // skip this store in this round.
- logutil.CL(mainCtx).Warn("store not alive, skip backup it in this round", zap.Uint64("round", round), zap.Error(err))
- continue
- }
- storeID := store.GetId()
- // reset backup client every round, to get a clean grpc connection.
- cli, err := bc.mgr.ResetBackupClient(mainCtx, storeID)
- if err != nil {
- // because the we get store info from pd.
- // there is no customer setting here, so make infinite retry.
- logutil.CL(ctx).Error("failed to reset backup client", zap.Uint64("round", round), zap.Uint64("storeID", storeID), zap.Error(err))
- mainCancel()
- continue mainLoop
- }
- ch := make(chan *ResponseAndStore)
- storeBackupResultChMap[storeID] = ch
- startStoreBackupAsyncFn(mainCtx, round, storeID, request, cli, ch)
- }
- // infinite loop to collect region backup response to global channel
- collectStoreBackupsAsyncFn(handleCtx, round, storeBackupResultChMap, globalBackupResultCh)
- handleLoop:
- for {
- select {
- case <-ctx.Done():
- handleCancel()
- mainCancel()
- return ctx.Err()
- case storeBackupInfo := <-stateChan:
- if storeBackupInfo.All {
- logutil.CL(mainCtx).Info("cluster state changed. restart store backups", zap.Uint64("round", round))
- // stop current connections
- handleCancel()
- mainCancel()
- // start next round backups
- continue mainLoop
- }
- if storeBackupInfo.One != 0 {
- storeID := storeBackupInfo.One
- store, err := bc.mgr.GetPDClient().GetStore(mainCtx, storeID)
- if err != nil {
- // cannot get store, maybe store has scaled-in.
- logutil.CL(mainCtx).Info("cannot get store from pd", zap.Uint64("round", round), zap.Error(err))
- // try next round
- handleCancel()
- mainCancel()
- continue mainLoop
- }
- if err = utils.CheckStoreLiveness(store); err != nil {
- // skip this store in this round.
- logutil.CL(mainCtx).Warn("store not alive, skip backup it in this round", zap.Uint64("round", round), zap.Error(err))
- continue
- }
- // reset backup client. store address could change but store id remained.
- cli, err := bc.mgr.ResetBackupClient(mainCtx, storeID)
- if err != nil {
- logutil.CL(mainCtx).Error("failed to reset backup client", zap.Uint64("round", round), zap.Uint64("storeID", storeID), zap.Error(err))
- handleCancel()
- mainCancel()
- // receive new store info but failed to get backup client.
- // start next round backups to get all tikv stores and reset all client connections.
- continue mainLoop
- }
-
- // cancel the former collect goroutine
- handleCancel()
- ch := make(chan *ResponseAndStore)
-
- storeBackupResultChMap[storeID] = ch
- // start backup for this store
- startStoreBackupAsyncFn(mainCtx, round, storeID, request, cli, ch)
- // re-create context for new handler loop
- handleCtx, handleCancel = context.WithCancel(mainCtx)
- // handleCancel makes the former collect goroutine exits
- // so we need to re-create a new channel and restart a new collect goroutine.
- globalBackupResultCh = make(chan *ResponseAndStore)
- // collect all store backup producer channel result to one channel
- collectStoreBackupsAsyncFn(handleCtx, round, storeBackupResultChMap, globalBackupResultCh)
- }
- case respAndStore, ok := <-globalBackupResultCh:
- if !ok {
- // this round backup finished. break and check incomplete ranges in mainLoop.
- break handleLoop
- }
- err = bc.OnBackupResponse(handleCtx, respAndStore, errContext, globalProgressTree)
- if err != nil {
- // if error occurred here, stop the backup process
- // because only 3 kinds of errors will be returned here:
- // 1. permission denied on tikv store.
- // 2. parse backup response error.(shouldn't happen in any case)
- // 3. checkpoint update failed. TODO: should we retry here?
- handleCancel()
- mainCancel()
- return err
- }
- progressCallBack()
- }
- }
- }
-}
-
func collectRangeFiles(progressRangeTree *rtree.ProgressRangeTree, metaWriter *metautil.MetaWriter) error {
var progressRangeAscendErr error
progressRangeTree.Ascend(func(progressRange *rtree.ProgressRange) bool {
diff --git a/br/pkg/backup/client_test.go b/br/pkg/backup/client_test.go
index 5814ae6dc718a..a74b170ed8b2b 100644
--- a/br/pkg/backup/client_test.go
+++ b/br/pkg/backup/client_test.go
@@ -3,8 +3,12 @@
package backup_test
import (
+ "bytes"
"context"
"encoding/json"
+ "math/rand"
+ "sort"
+ "sync"
"testing"
"time"
@@ -13,7 +17,7 @@ import (
"github.com/pingcap/kvproto/pkg/encryptionpb"
"github.com/pingcap/tidb/br/pkg/backup"
"github.com/pingcap/tidb/br/pkg/conn"
- "github.com/pingcap/tidb/br/pkg/gluetidb"
+ gluemock "github.com/pingcap/tidb/br/pkg/gluetidb/mock"
"github.com/pingcap/tidb/br/pkg/metautil"
"github.com/pingcap/tidb/br/pkg/mock"
"github.com/pingcap/tidb/br/pkg/pdutil"
@@ -35,18 +39,75 @@ type testBackup struct {
mockPDClient pd.Client
mockCluster *testutils.MockCluster
- mockGlue *gluetidb.MockGlue
+ mockGlue *gluemock.MockGlue
backupClient *backup.Client
cluster *mock.Cluster
storage storage.ExternalStorage
}
+// locks used to solve race when mock then behaviour when store drops
+var lock sync.Mutex
+
+// check the connect behaviour in test.
+var connectedStore map[uint64]int
+
+var _ backup.BackupSender = (*mockBackupBackupSender)(nil)
+
+func mockGetBackupClientCallBack(ctx context.Context, storeID uint64) (backuppb.BackupClient, error) {
+ lock.Lock()
+ defer lock.Unlock()
+ connectedStore[storeID] += 1
+ // we don't need connect real tikv in unit test
+ // and we have already mock the backup response in `SendAsync`
+ // so just return nil here
+ return nil, nil
+}
+
+type mockBackupBackupSender struct {
+ backupResponses map[uint64][]*backup.ResponseAndStore
+}
+
+func (m *mockBackupBackupSender) SendAsync(
+ ctx context.Context,
+ round uint64,
+ storeID uint64,
+ request backuppb.BackupRequest,
+ cli backuppb.BackupClient,
+ respCh chan *backup.ResponseAndStore,
+ StateNotifier chan backup.BackupRetryPolicy,
+) {
+ go func() {
+ defer func() {
+ close(respCh)
+ }()
+ lock.Lock()
+ resps := m.backupResponses[storeID]
+ lock.Unlock()
+ for len(resps) == 0 {
+ // store has no response
+ // block for a while and try again.
+ // let this goroutine never return.
+ time.Sleep(100 * time.Millisecond)
+ lock.Lock()
+ resps = m.backupResponses[storeID]
+ lock.Unlock()
+ }
+ for _, r := range resps {
+ select {
+ case <-ctx.Done():
+ return
+ case respCh <- r:
+ }
+ }
+ }()
+}
+
func createBackupSuite(t *testing.T) *testBackup {
tikvClient, mockCluster, pdClient, err := testutils.NewMockTiKV("", nil)
require.NoError(t, err)
s := new(testBackup)
- s.mockGlue = &gluetidb.MockGlue{}
+ s.mockGlue = &gluemock.MockGlue{}
s.mockPDClient = pdClient
s.mockCluster = mockCluster
s.ctx, s.cancel = context.WithCancel(context.Background())
@@ -269,6 +330,309 @@ func TestOnBackupResponse(t *testing.T) {
require.Len(t, incomplete, 0)
}
+func TestMainBackupLoop(t *testing.T) {
+ s := createBackupSuite(t)
+ backgroundCtx := context.Background()
+
+ // test 2 stores backup
+ s.mockCluster.AddStore(1, "127.0.0.1:20160")
+ s.mockCluster.AddStore(2, "127.0.0.1:20161")
+
+ stores, err := s.mockPDClient.GetAllStores(backgroundCtx)
+ require.NoError(t, err)
+ require.Len(t, stores, 2)
+
+ // random generate bytes in range [a, b)
+ genRandBytesFn := func(a, b []byte) ([]byte, error) {
+ n := len(a)
+ result := make([]byte, n)
+ for {
+ _, err := rand.Read(result)
+ if err != nil {
+ return nil, err
+ }
+ if bytes.Compare(result, a) > 0 && bytes.Compare(result, b) < 0 {
+ return result, nil
+ }
+ }
+ }
+ // split each range into limit parts
+ splitRangesFn := func(ranges []rtree.Range, limit int) [][]byte {
+ if len(ranges) == 0 {
+ return nil
+ }
+ res := make([][]byte, 0)
+ res = append(res, ranges[0].StartKey)
+ for i := 0; i < len(ranges); i++ {
+ partRes := make([][]byte, 0)
+ for j := 0; j < limit; j++ {
+ x, err := genRandBytesFn(ranges[i].StartKey, ranges[i].EndKey)
+ require.NoError(t, err)
+ partRes = append(partRes, x)
+ }
+ sort.Slice(partRes, func(i, j int) bool {
+ return bytes.Compare(partRes[i], partRes[j]) < 0
+ })
+ res = append(res, partRes...)
+ }
+ res = append(res, ranges[len(ranges)-1].EndKey)
+ return res
+ }
+
+ // Case #1: normal case
+ ranges := []rtree.Range{
+ {
+ StartKey: []byte("aaa"),
+ EndKey: []byte("zzz"),
+ },
+ }
+ tree, err := s.backupClient.BuildProgressRangeTree(ranges)
+ require.NoError(t, err)
+
+ mockBackupResponses := make(map[uint64][]*backup.ResponseAndStore)
+ splitKeys := splitRangesFn(ranges, 10)
+ for i := 0; i < len(splitKeys)-1; i++ {
+ randStoreID := uint64(rand.Int()%len(stores) + 1)
+ mockBackupResponses[randStoreID] = append(mockBackupResponses[randStoreID], &backup.ResponseAndStore{
+ StoreID: randStoreID,
+ Resp: &backuppb.BackupResponse{
+ StartKey: splitKeys[i],
+ EndKey: splitKeys[i+1],
+ },
+ })
+ }
+ ch := make(chan backup.BackupRetryPolicy)
+ mainLoop := &backup.MainBackupLoop{
+ BackupSender: &mockBackupBackupSender{
+ backupResponses: mockBackupResponses,
+ },
+
+ BackupReq: backuppb.BackupRequest{},
+ GlobalProgressTree: &tree,
+ ReplicaReadLabel: nil,
+ StateNotifier: ch,
+ ProgressCallBack: func() {},
+ GetBackupClientCallBack: mockGetBackupClientCallBack,
+ }
+
+ connectedStore = make(map[uint64]int)
+ require.NoError(t, s.backupClient.RunLoop(backgroundCtx, mainLoop))
+
+ // Case #2: canceled case
+ ranges = []rtree.Range{
+ {
+ StartKey: []byte("aaa"),
+ EndKey: []byte("zzz"),
+ },
+ }
+ tree, err = s.backupClient.BuildProgressRangeTree(ranges)
+ require.NoError(t, err)
+
+ clear(mockBackupResponses)
+ splitKeys = splitRangesFn(ranges, 10)
+ // range is not complete
+ for i := 0; i < len(splitKeys)-2; i++ {
+ randStoreID := uint64(rand.Int()%len(stores) + 1)
+ mockBackupResponses[randStoreID] = append(mockBackupResponses[randStoreID], &backup.ResponseAndStore{
+ StoreID: randStoreID,
+ Resp: &backuppb.BackupResponse{
+ StartKey: splitKeys[i],
+ EndKey: splitKeys[i+1],
+ },
+ })
+ }
+ mainLoop = &backup.MainBackupLoop{
+ BackupSender: &mockBackupBackupSender{
+ backupResponses: mockBackupResponses,
+ },
+
+ BackupReq: backuppb.BackupRequest{},
+ GlobalProgressTree: &tree,
+ ReplicaReadLabel: nil,
+ StateNotifier: ch,
+ ProgressCallBack: func() {},
+ GetBackupClientCallBack: mockGetBackupClientCallBack,
+ }
+
+ // cancel the backup in another goroutine
+ ctx, cancel := context.WithCancel(backgroundCtx)
+ go func() {
+ time.Sleep(500 * time.Millisecond)
+ cancel()
+ }()
+ connectedStore = make(map[uint64]int)
+ require.Error(t, s.backupClient.RunLoop(ctx, mainLoop))
+
+ // Case #3: one store drops and never come back
+ ranges = []rtree.Range{
+ {
+ StartKey: []byte("aaa"),
+ EndKey: []byte("zzz"),
+ },
+ }
+ tree, err = s.backupClient.BuildProgressRangeTree(ranges)
+ require.NoError(t, err)
+
+ clear(mockBackupResponses)
+ splitKeys = splitRangesFn(ranges, 10)
+ for i := 0; i < len(splitKeys)-1; i++ {
+ randStoreID := uint64(rand.Int()%len(stores) + 1)
+ mockBackupResponses[randStoreID] = append(mockBackupResponses[randStoreID], &backup.ResponseAndStore{
+ StoreID: randStoreID,
+ Resp: &backuppb.BackupResponse{
+ StartKey: splitKeys[i],
+ EndKey: splitKeys[i+1],
+ },
+ })
+ }
+ remainStoreID := uint64(1)
+ dropStoreID := uint64(2)
+ s.mockCluster.StopStore(dropStoreID)
+ dropBackupResponses := mockBackupResponses[dropStoreID]
+ lock.Lock()
+ mockBackupResponses[dropStoreID] = nil
+ lock.Unlock()
+
+ mainLoop = &backup.MainBackupLoop{
+ BackupSender: &mockBackupBackupSender{
+ backupResponses: mockBackupResponses,
+ },
+
+ BackupReq: backuppb.BackupRequest{},
+ GlobalProgressTree: &tree,
+ ReplicaReadLabel: nil,
+ StateNotifier: ch,
+ ProgressCallBack: func() {},
+ GetBackupClientCallBack: mockGetBackupClientCallBack,
+ }
+ go func() {
+ // mock region leader balance behaviour.
+ time.Sleep(500 * time.Millisecond)
+ lock.Lock()
+ // store 1 has the range after some while.
+ mockBackupResponses[remainStoreID] = append(mockBackupResponses[remainStoreID], dropBackupResponses...)
+ lock.Unlock()
+ }()
+
+ connectedStore = make(map[uint64]int)
+ // backup can finished until store 1 has response the full ranges.
+ require.NoError(t, s.backupClient.RunLoop(backgroundCtx, mainLoop))
+ // connect to store 1 more than 1 time
+ require.Less(t, 1, connectedStore[remainStoreID])
+ // never connect to a dropped store.
+ require.Equal(t, 0, connectedStore[dropStoreID])
+
+ // Case #4 one store drops and come back soon
+ ranges = []rtree.Range{
+ {
+ StartKey: []byte("aaa"),
+ EndKey: []byte("zzz"),
+ },
+ }
+ tree, err = s.backupClient.BuildProgressRangeTree(ranges)
+ require.NoError(t, err)
+
+ clear(mockBackupResponses)
+ splitKeys = splitRangesFn(ranges, 10)
+ for i := 0; i < len(splitKeys)-1; i++ {
+ randStoreID := uint64(rand.Int()%len(stores) + 1)
+ mockBackupResponses[randStoreID] = append(mockBackupResponses[randStoreID], &backup.ResponseAndStore{
+ StoreID: randStoreID,
+ Resp: &backuppb.BackupResponse{
+ StartKey: splitKeys[i],
+ EndKey: splitKeys[i+1],
+ },
+ })
+ }
+ remainStoreID = uint64(1)
+ dropStoreID = uint64(2)
+ s.mockCluster.StopStore(dropStoreID)
+
+ mainLoop = &backup.MainBackupLoop{
+ BackupSender: &mockBackupBackupSender{
+ backupResponses: mockBackupResponses,
+ },
+
+ BackupReq: backuppb.BackupRequest{},
+ GlobalProgressTree: &tree,
+ ReplicaReadLabel: nil,
+ StateNotifier: ch,
+ ProgressCallBack: func() {},
+ GetBackupClientCallBack: mockGetBackupClientCallBack,
+ }
+ go func() {
+ time.Sleep(500 * time.Millisecond)
+ lock.Lock()
+ s.mockCluster.StartStore(dropStoreID)
+ lock.Unlock()
+ }()
+
+ connectedStore = make(map[uint64]int)
+ // backup can finished until store 1 has response the full ranges.
+ require.NoError(t, s.backupClient.RunLoop(backgroundCtx, mainLoop))
+ // connect to store 1 more than 1 time
+ require.Less(t, 1, connectedStore[remainStoreID])
+ // connect to store 2 once should finished the backup.
+ require.Equal(t, 1, connectedStore[dropStoreID])
+
+ // Case #5 one store drops and watch store back
+ ranges = []rtree.Range{
+ {
+ StartKey: []byte("aaa"),
+ EndKey: []byte("zzz"),
+ },
+ }
+ tree, err = s.backupClient.BuildProgressRangeTree(ranges)
+ require.NoError(t, err)
+
+ clear(mockBackupResponses)
+ splitKeys = splitRangesFn(ranges, 10)
+ for i := 0; i < len(splitKeys)-1; i++ {
+ randStoreID := uint64(rand.Int()%len(stores) + 1)
+ mockBackupResponses[randStoreID] = append(mockBackupResponses[randStoreID], &backup.ResponseAndStore{
+ StoreID: randStoreID,
+ Resp: &backuppb.BackupResponse{
+ StartKey: splitKeys[i],
+ EndKey: splitKeys[i+1],
+ },
+ })
+ }
+ remainStoreID = uint64(1)
+ dropStoreID = uint64(2)
+ // make store 2 never no response.
+ dropBackupResponses = mockBackupResponses[dropStoreID]
+ lock.Lock()
+ mockBackupResponses[dropStoreID] = nil
+ lock.Unlock()
+
+ mainLoop = &backup.MainBackupLoop{
+ BackupSender: &mockBackupBackupSender{
+ backupResponses: mockBackupResponses,
+ },
+
+ BackupReq: backuppb.BackupRequest{},
+ GlobalProgressTree: &tree,
+ ReplicaReadLabel: nil,
+ StateNotifier: ch,
+ ProgressCallBack: func() {},
+ GetBackupClientCallBack: mockGetBackupClientCallBack,
+ }
+ go func() {
+ time.Sleep(500 * time.Millisecond)
+ lock.Lock()
+ mockBackupResponses[dropStoreID] = dropBackupResponses
+ lock.Unlock()
+ }()
+
+ connectedStore = make(map[uint64]int)
+ // backup can finished until store 1 has response the full ranges.
+ require.NoError(t, s.backupClient.RunLoop(backgroundCtx, mainLoop))
+ // connect to store 1 time, and then handle loop blocked until watch store 2 back.
+ require.Equal(t, 1, connectedStore[remainStoreID])
+ // connect to store 2 once should finished the backup.
+ require.Equal(t, 1, connectedStore[dropStoreID])
+}
+
func TestBuildProgressRangeTree(t *testing.T) {
s := createBackupSuite(t)
ranges := []rtree.Range{
diff --git a/br/pkg/backup/push.go b/br/pkg/backup/store.go
similarity index 75%
rename from br/pkg/backup/push.go
rename to br/pkg/backup/store.go
index fc8147ae31f62..4fd150085b141 100644
--- a/br/pkg/backup/push.go
+++ b/br/pkg/backup/store.go
@@ -13,15 +13,34 @@ import (
backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/kvproto/pkg/errorpb"
"github.com/pingcap/kvproto/pkg/kvrpcpb"
+ "github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/log"
"github.com/pingcap/tidb/br/pkg/logutil"
"github.com/pingcap/tidb/br/pkg/rtree"
"github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/br/pkg/utils/storewatch"
+ pd "github.com/tikv/pd/client"
"go.uber.org/zap"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
+type BackupRetryPolicy struct {
+ One uint64
+ All bool
+}
+
+type BackupSender interface {
+ SendAsync(
+ ctx context.Context,
+ round uint64,
+ storeID uint64,
+ request backuppb.BackupRequest,
+ cli backuppb.BackupClient,
+ respCh chan *ResponseAndStore,
+ StateNotifier chan BackupRetryPolicy)
+}
+
type ResponseAndStore struct {
Resp *backuppb.BackupResponse
StoreID uint64
@@ -106,7 +125,7 @@ func doSendBackup(
}
}
-func startStoreBackup(
+func startBackup(
ctx context.Context,
storeID uint64,
backupReq backuppb.BackupRequest,
@@ -184,3 +203,49 @@ func getBackupRanges(ranges []rtree.Range) []*kvrpcpb.KeyRange {
}
return requestRanges
}
+
+func ObserveStoreChangesAsync(ctx context.Context, stateNotifier chan BackupRetryPolicy, pdCli pd.Client) {
+ go func() {
+ sendAll := false
+ newJoinStoresMap := make(map[uint64]struct{})
+ cb := storewatch.MakeCallback(storewatch.WithOnReboot(func(s *metapb.Store) {
+ sendAll = true
+ }), storewatch.WithOnDisconnect(func(s *metapb.Store) {
+ sendAll = true
+ }), storewatch.WithOnNewStoreRegistered(func(s *metapb.Store) {
+ // only backup for this store
+ newJoinStoresMap[s.Id] = struct{}{}
+ }))
+
+ notifyFn := func(ctx context.Context, sendPolicy BackupRetryPolicy) {
+ select {
+ case <-ctx.Done():
+ case stateNotifier <- sendPolicy:
+ }
+ }
+
+ watcher := storewatch.New(pdCli, cb)
+ tick := time.NewTicker(30 * time.Second)
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-tick.C:
+ logutil.CL(ctx).Info("check store changes by tick")
+ err := watcher.Step(ctx)
+ if err != nil {
+ logutil.CL(ctx).Warn("failed to watch store changes, ignore it", zap.Error(err))
+ }
+ if sendAll {
+ notifyFn(ctx, BackupRetryPolicy{All: true})
+ } else if len(newJoinStoresMap) > 0 {
+ for storeID := range newJoinStoresMap {
+ notifyFn(ctx, BackupRetryPolicy{One: storeID})
+ }
+ }
+ sendAll = false
+ clear(newJoinStoresMap)
+ }
+ }
+ }()
+}
diff --git a/br/pkg/conn/BUILD.bazel b/br/pkg/conn/BUILD.bazel
index 8f25055538375..b784d5c080afa 100644
--- a/br/pkg/conn/BUILD.bazel
+++ b/br/pkg/conn/BUILD.bazel
@@ -51,7 +51,7 @@ go_test(
"//br/pkg/config",
"//br/pkg/conn/util",
"//br/pkg/pdutil",
- "//br/pkg/utils",
+ "//br/pkg/utiltest",
"//pkg/testkit/testsetup",
"@com_github_docker_go_units//:go-units",
"@com_github_pingcap_errors//:errors",
diff --git a/br/pkg/conn/conn_test.go b/br/pkg/conn/conn_test.go
index 3d594a3c1216a..e15fc747eb837 100644
--- a/br/pkg/conn/conn_test.go
+++ b/br/pkg/conn/conn_test.go
@@ -1,6 +1,6 @@
// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
-package conn
+package conn_test
import (
"context"
@@ -15,9 +15,10 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/kvproto/pkg/metapb"
kvconfig "github.com/pingcap/tidb/br/pkg/config"
+ "github.com/pingcap/tidb/br/pkg/conn"
"github.com/pingcap/tidb/br/pkg/conn/util"
"github.com/pingcap/tidb/br/pkg/pdutil"
- "github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/br/pkg/utiltest"
"github.com/stretchr/testify/require"
"go.uber.org/multierr"
"google.golang.org/grpc/codes"
@@ -61,11 +62,9 @@ func TestGetAllTiKVStoresWithRetryCancel(t *testing.T) {
},
}
- fpdc := utils.FakePDClient{
- Stores: stores,
- }
+ fpdc := utiltest.NewFakePDClient(stores, false, nil)
- _, err = GetAllTiKVStoresWithRetry(ctx, fpdc, util.SkipTiFlash)
+ _, err = conn.GetAllTiKVStoresWithRetry(ctx, fpdc, util.SkipTiFlash)
require.Error(t, err)
errs := multierr.Errors(err)
require.Equal(t, 1, len(errs))
@@ -109,11 +108,9 @@ func TestGetAllTiKVStoresWithUnknown(t *testing.T) {
},
}
- fpdc := utils.FakePDClient{
- Stores: stores,
- }
+ fpdc := utiltest.NewFakePDClient(stores, false, nil)
- _, err = GetAllTiKVStoresWithRetry(ctx, fpdc, util.SkipTiFlash)
+ _, err = conn.GetAllTiKVStoresWithRetry(ctx, fpdc, util.SkipTiFlash)
require.Error(t, err)
errs := multierr.Errors(err)
require.Equal(t, 1, len(errs))
@@ -167,16 +164,14 @@ func TestCheckStoresAlive(t *testing.T) {
},
}
- fpdc := utils.FakePDClient{
- Stores: stores,
- }
+ fpdc := utiltest.NewFakePDClient(stores, false, nil)
- kvStores, err := GetAllTiKVStoresWithRetry(ctx, fpdc, util.SkipTiFlash)
+ kvStores, err := conn.GetAllTiKVStoresWithRetry(ctx, fpdc, util.SkipTiFlash)
require.NoError(t, err)
require.Len(t, kvStores, 2)
require.Equal(t, stores[2:], kvStores)
- err = checkStoresAlive(ctx, fpdc, util.SkipTiFlash)
+ err = conn.CheckStoresAlive(ctx, fpdc, util.SkipTiFlash)
require.NoError(t, err)
}
@@ -256,7 +251,7 @@ func TestGetAllTiKVStores(t *testing.T) {
}
for _, testCase := range testCases {
- pdClient := utils.FakePDClient{Stores: testCase.stores}
+ pdClient := utiltest.NewFakePDClient(testCase.stores, false, nil)
stores, err := util.GetAllTiKVStores(context.Background(), pdClient, testCase.storeBehavior)
if len(testCase.expectedError) != 0 {
require.Error(t, err)
@@ -275,7 +270,7 @@ func TestGetConnOnCanceledContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
- mgr := &Mgr{PdController: &pdutil.PdController{}}
+ mgr := &conn.Mgr{PdController: &pdutil.PdController{}}
_, err := mgr.GetBackupClient(ctx, 42)
require.Error(t, err)
@@ -309,9 +304,9 @@ func TestGetMergeRegionSizeAndCount(t *testing.T) {
},
content: []string{""},
// no tikv detected in this case
- importNumGoroutines: DefaultImportNumGoroutines,
- regionSplitSize: DefaultMergeRegionSizeBytes,
- regionSplitKeys: DefaultMergeRegionKeyCount,
+ importNumGoroutines: conn.DefaultImportNumGoroutines,
+ regionSplitSize: conn.DefaultMergeRegionSizeBytes,
+ regionSplitKeys: conn.DefaultMergeRegionKeyCount,
},
{
stores: []*metapb.Store{
@@ -342,9 +337,9 @@ func TestGetMergeRegionSizeAndCount(t *testing.T) {
"",
},
// no tikv detected in this case
- importNumGoroutines: DefaultImportNumGoroutines,
- regionSplitSize: DefaultMergeRegionSizeBytes,
- regionSplitKeys: DefaultMergeRegionKeyCount,
+ importNumGoroutines: conn.DefaultImportNumGoroutines,
+ regionSplitSize: conn.DefaultMergeRegionSizeBytes,
+ regionSplitKeys: conn.DefaultMergeRegionKeyCount,
},
{
stores: []*metapb.Store{
@@ -426,7 +421,7 @@ func TestGetMergeRegionSizeAndCount(t *testing.T) {
pctx := context.Background()
for _, ca := range cases {
ctx, cancel := context.WithCancel(pctx)
- pdCli := utils.FakePDClient{Stores: ca.stores}
+ pdCli := utiltest.NewFakePDClient(ca.stores, false, nil)
require.Equal(t, len(ca.content), len(ca.stores))
count := 0
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -448,12 +443,12 @@ func TestGetMergeRegionSizeAndCount(t *testing.T) {
}
httpCli := mockServer.Client()
- mgr := &Mgr{PdController: &pdutil.PdController{}}
+ mgr := &conn.Mgr{PdController: &pdutil.PdController{}}
mgr.PdController.SetPDClient(pdCli)
kvConfigs := &kvconfig.KVConfig{
- ImportGoroutines: kvconfig.ConfigTerm[uint]{Value: DefaultImportNumGoroutines, Modified: false},
- MergeRegionSize: kvconfig.ConfigTerm[uint64]{Value: DefaultMergeRegionSizeBytes, Modified: false},
- MergeRegionKeyCount: kvconfig.ConfigTerm[uint64]{Value: DefaultMergeRegionKeyCount, Modified: false},
+ ImportGoroutines: kvconfig.ConfigTerm[uint]{Value: conn.DefaultImportNumGoroutines, Modified: false},
+ MergeRegionSize: kvconfig.ConfigTerm[uint64]{Value: conn.DefaultMergeRegionSizeBytes, Modified: false},
+ MergeRegionKeyCount: kvconfig.ConfigTerm[uint64]{Value: conn.DefaultMergeRegionKeyCount, Modified: false},
}
mgr.ProcessTiKVConfigs(ctx, kvConfigs, httpCli)
require.EqualValues(t, ca.regionSplitSize, kvConfigs.MergeRegionSize.Value)
@@ -591,7 +586,7 @@ func TestIsLogBackupEnabled(t *testing.T) {
pctx := context.Background()
for _, ca := range cases {
ctx, cancel := context.WithCancel(pctx)
- pdCli := utils.FakePDClient{Stores: ca.stores}
+ pdCli := utiltest.NewFakePDClient(ca.stores, false, nil)
require.Equal(t, len(ca.content), len(ca.stores))
count := 0
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -613,7 +608,7 @@ func TestIsLogBackupEnabled(t *testing.T) {
}
httpCli := mockServer.Client()
- mgr := &Mgr{PdController: &pdutil.PdController{}}
+ mgr := &conn.Mgr{PdController: &pdutil.PdController{}}
mgr.PdController.SetPDClient(pdCli)
enable, err := mgr.IsLogBackupEnabled(ctx, httpCli)
if ca.err {
@@ -655,7 +650,7 @@ func TestHandleTiKVAddress(t *testing.T) {
},
}
for _, ca := range cases {
- addr, err := handleTiKVAddress(ca.store, ca.httpPrefix)
+ addr, err := conn.HandleTiKVAddress(ca.store, ca.httpPrefix)
require.Nil(t, err)
require.Equal(t, ca.result, addr.String())
}
diff --git a/br/pkg/conn/main_test.go b/br/pkg/conn/main_test.go
index 23f0382f2d443..32ef5990c156b 100644
--- a/br/pkg/conn/main_test.go
+++ b/br/pkg/conn/main_test.go
@@ -21,6 +21,11 @@ import (
"go.uber.org/goleak"
)
+var (
+ CheckStoresAlive = checkStoresAlive
+ HandleTiKVAddress = handleTiKVAddress
+)
+
func TestMain(m *testing.M) {
opts := []goleak.Option{
goleak.IgnoreTopFunction("github.com/golang/glog.(*fileSink).flushDaemon"),
diff --git a/br/pkg/gluetidb/glue.go b/br/pkg/gluetidb/glue.go
index 9ffdaf669e86e..4c308d58afaa7 100644
--- a/br/pkg/gluetidb/glue.go
+++ b/br/pkg/gluetidb/glue.go
@@ -235,135 +235,3 @@ func (gs *tidbSession) GetGlobalVariable(name string) (string, error) {
func (gs *tidbSession) showCreatePlacementPolicy(policy *model.PolicyInfo) string {
return executor.ConstructResultOfShowCreatePlacementPolicy(policy)
}
-
-// mockSession is used for test.
-type mockSession struct {
- se sessiontypes.Session
- globalVars map[string]string
-}
-
-// GetSessionCtx implements glue.Glue
-func (s *mockSession) GetSessionCtx() sessionctx.Context {
- return s.se
-}
-
-// Execute implements glue.Session.
-func (s *mockSession) Execute(ctx context.Context, sql string) error {
- return s.ExecuteInternal(ctx, sql)
-}
-
-func (s *mockSession) ExecuteInternal(ctx context.Context, sql string, args ...any) error {
- ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnBR)
- rs, err := s.se.ExecuteInternal(ctx, sql, args...)
- if err != nil {
- return err
- }
- // Some of SQLs (like ADMIN RECOVER INDEX) may lazily take effect
- // when we are polling the result set.
- // At least call `next` once for triggering theirs side effect.
- // (Maybe we'd better drain all returned rows?)
- if rs != nil {
- //nolint: errcheck
- defer rs.Close()
- c := rs.NewChunk(nil)
- if err := rs.Next(ctx, c); err != nil {
- return nil
- }
- }
- return nil
-}
-
-// CreateDatabase implements glue.Session.
-func (*mockSession) CreateDatabase(_ context.Context, _ *model.DBInfo) error {
- log.Fatal("unimplemented CreateDatabase for mock session")
- return nil
-}
-
-// CreatePlacementPolicy implements glue.Session.
-func (*mockSession) CreatePlacementPolicy(_ context.Context, _ *model.PolicyInfo) error {
- log.Fatal("unimplemented CreateDatabase for mock session")
- return nil
-}
-
-// CreateTables implements glue.BatchCreateTableSession.
-func (*mockSession) CreateTables(_ context.Context, _ map[string][]*model.TableInfo,
- _ ...ddl.CreateTableWithInfoConfigurier) error {
- log.Fatal("unimplemented CreateDatabase for mock session")
- return nil
-}
-
-// CreateTable implements glue.Session.
-func (*mockSession) CreateTable(_ context.Context, _ model.CIStr,
- _ *model.TableInfo, _ ...ddl.CreateTableWithInfoConfigurier) error {
- log.Fatal("unimplemented CreateDatabase for mock session")
- return nil
-}
-
-// Close implements glue.Session.
-func (s *mockSession) Close() {
- s.se.Close()
-}
-
-// GetGlobalVariables implements glue.Session.
-func (s *mockSession) GetGlobalVariable(name string) (string, error) {
- if ret, ok := s.globalVars[name]; ok {
- return ret, nil
- }
- return "True", nil
-}
-
-// MockGlue only used for test
-type MockGlue struct {
- se sessiontypes.Session
- GlobalVars map[string]string
-}
-
-func (m *MockGlue) SetSession(se sessiontypes.Session) {
- m.se = se
-}
-
-// GetDomain implements glue.Glue.
-func (*MockGlue) GetDomain(store kv.Storage) (*domain.Domain, error) {
- return nil, nil
-}
-
-// CreateSession implements glue.Glue.
-func (m *MockGlue) CreateSession(store kv.Storage) (glue.Session, error) {
- glueSession := &mockSession{
- se: m.se,
- globalVars: m.GlobalVars,
- }
- return glueSession, nil
-}
-
-// Open implements glue.Glue.
-func (*MockGlue) Open(path string, option pd.SecurityOption) (kv.Storage, error) {
- return nil, nil
-}
-
-// OwnsStorage implements glue.Glue.
-func (*MockGlue) OwnsStorage() bool {
- return true
-}
-
-// StartProgress implements glue.Glue.
-func (*MockGlue) StartProgress(ctx context.Context, cmdName string, total int64, redirectLog bool) glue.Progress {
- return nil
-}
-
-// Record implements glue.Glue.
-func (*MockGlue) Record(name string, value uint64) {
-}
-
-// GetVersion implements glue.Glue.
-func (*MockGlue) GetVersion() string {
- return "mock glue"
-}
-
-// UseOneShotSession implements glue.Glue.
-func (m *MockGlue) UseOneShotSession(store kv.Storage, closeDomain bool, fn func(glue.Session) error) error {
- glueSession := &mockSession{
- se: m.se,
- }
- return fn(glueSession)
-}
diff --git a/br/pkg/gluetidb/mock/BUILD.bazel b/br/pkg/gluetidb/mock/BUILD.bazel
new file mode 100644
index 0000000000000..81dc56ba2b45f
--- /dev/null
+++ b/br/pkg/gluetidb/mock/BUILD.bazel
@@ -0,0 +1,18 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "mock",
+ srcs = ["mock.go"],
+ importpath = "github.com/pingcap/tidb/br/pkg/gluetidb/mock",
+ visibility = ["//visibility:public"],
+ deps = [
+ "//br/pkg/glue",
+ "//pkg/ddl",
+ "//pkg/domain",
+ "//pkg/kv",
+ "//pkg/parser/model",
+ "//pkg/session/types",
+ "//pkg/sessionctx",
+ "@com_github_tikv_pd_client//:client",
+ ],
+)
diff --git a/br/pkg/gluetidb/mock/mock.go b/br/pkg/gluetidb/mock/mock.go
new file mode 100644
index 0000000000000..0fe4615519e4f
--- /dev/null
+++ b/br/pkg/gluetidb/mock/mock.go
@@ -0,0 +1,161 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mock
+
+import (
+ "context"
+ "log"
+
+ "github.com/pingcap/tidb/br/pkg/glue"
+ "github.com/pingcap/tidb/pkg/ddl"
+ "github.com/pingcap/tidb/pkg/domain"
+ "github.com/pingcap/tidb/pkg/kv"
+ "github.com/pingcap/tidb/pkg/parser/model"
+ sessiontypes "github.com/pingcap/tidb/pkg/session/types"
+ "github.com/pingcap/tidb/pkg/sessionctx"
+ pd "github.com/tikv/pd/client"
+)
+
+// mockSession is used for test.
+type mockSession struct {
+ se sessiontypes.Session
+ globalVars map[string]string
+}
+
+// GetSessionCtx implements glue.Glue
+func (s *mockSession) GetSessionCtx() sessionctx.Context {
+ return s.se
+}
+
+// Execute implements glue.Session.
+func (s *mockSession) Execute(ctx context.Context, sql string) error {
+ return s.ExecuteInternal(ctx, sql)
+}
+
+func (s *mockSession) ExecuteInternal(ctx context.Context, sql string, args ...any) error {
+ ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnBR)
+ rs, err := s.se.ExecuteInternal(ctx, sql, args...)
+ if err != nil {
+ return err
+ }
+ // Some of SQLs (like ADMIN RECOVER INDEX) may lazily take effect
+ // when we are polling the result set.
+ // At least call `next` once for triggering theirs side effect.
+ // (Maybe we'd better drain all returned rows?)
+ if rs != nil {
+ //nolint: errcheck
+ defer rs.Close()
+ c := rs.NewChunk(nil)
+ if err := rs.Next(ctx, c); err != nil {
+ return nil
+ }
+ }
+ return nil
+}
+
+// CreateDatabase implements glue.Session.
+func (*mockSession) CreateDatabase(_ context.Context, _ *model.DBInfo) error {
+ log.Fatal("unimplemented CreateDatabase for mock session")
+ return nil
+}
+
+// CreatePlacementPolicy implements glue.Session.
+func (*mockSession) CreatePlacementPolicy(_ context.Context, _ *model.PolicyInfo) error {
+ log.Fatal("unimplemented CreateDatabase for mock session")
+ return nil
+}
+
+// CreateTables implements glue.BatchCreateTableSession.
+func (*mockSession) CreateTables(_ context.Context, _ map[string][]*model.TableInfo,
+ _ ...ddl.CreateTableWithInfoConfigurier) error {
+ log.Fatal("unimplemented CreateDatabase for mock session")
+ return nil
+}
+
+// CreateTable implements glue.Session.
+func (*mockSession) CreateTable(_ context.Context, _ model.CIStr,
+ _ *model.TableInfo, _ ...ddl.CreateTableWithInfoConfigurier) error {
+ log.Fatal("unimplemented CreateDatabase for mock session")
+ return nil
+}
+
+// Close implements glue.Session.
+func (s *mockSession) Close() {
+ s.se.Close()
+}
+
+// GetGlobalVariables implements glue.Session.
+func (s *mockSession) GetGlobalVariable(name string) (string, error) {
+ if ret, ok := s.globalVars[name]; ok {
+ return ret, nil
+ }
+ return "True", nil
+}
+
+// MockGlue only used for test
+type MockGlue struct {
+ se sessiontypes.Session
+ GlobalVars map[string]string
+}
+
+func (m *MockGlue) SetSession(se sessiontypes.Session) {
+ m.se = se
+}
+
+// GetDomain implements glue.Glue.
+func (*MockGlue) GetDomain(store kv.Storage) (*domain.Domain, error) {
+ return nil, nil
+}
+
+// CreateSession implements glue.Glue.
+func (m *MockGlue) CreateSession(store kv.Storage) (glue.Session, error) {
+ glueSession := &mockSession{
+ se: m.se,
+ globalVars: m.GlobalVars,
+ }
+ return glueSession, nil
+}
+
+// Open implements glue.Glue.
+func (*MockGlue) Open(path string, option pd.SecurityOption) (kv.Storage, error) {
+ return nil, nil
+}
+
+// OwnsStorage implements glue.Glue.
+func (*MockGlue) OwnsStorage() bool {
+ return true
+}
+
+// StartProgress implements glue.Glue.
+func (*MockGlue) StartProgress(ctx context.Context, cmdName string, total int64, redirectLog bool) glue.Progress {
+ return nil
+}
+
+// Record implements glue.Glue.
+func (*MockGlue) Record(name string, value uint64) {
+}
+
+// GetVersion implements glue.Glue.
+func (*MockGlue) GetVersion() string {
+ return "mock glue"
+}
+
+// UseOneShotSession implements glue.Glue.
+func (m *MockGlue) UseOneShotSession(store kv.Storage, closeDomain bool, fn func(glue.Session) error) error {
+ glueSession := &mockSession{
+ se: m.se,
+ }
+ return fn(glueSession)
+}
diff --git a/br/pkg/restore/BUILD.bazel b/br/pkg/restore/BUILD.bazel
index cf4baffc33cb9..5823273eaea88 100644
--- a/br/pkg/restore/BUILD.bazel
+++ b/br/pkg/restore/BUILD.bazel
@@ -3,141 +3,47 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "restore",
srcs = [
- "batcher.go",
- "client.go",
- "db.go",
- "logutil.go",
- "pipeline_items.go",
- "systable_restore.go",
- "util.go",
+ "import_mode_switcher.go",
+ "misc.go",
],
importpath = "github.com/pingcap/tidb/br/pkg/restore",
visibility = ["//visibility:public"],
deps = [
- "//br/pkg/checkpoint",
- "//br/pkg/checksum",
"//br/pkg/conn",
"//br/pkg/conn/util",
- "//br/pkg/errors",
- "//br/pkg/glue",
"//br/pkg/logutil",
- "//br/pkg/metautil",
"//br/pkg/pdutil",
- "//br/pkg/restore/file_importer",
- "//br/pkg/restore/ingestrec",
- "//br/pkg/restore/log_restore",
- "//br/pkg/restore/prealloc_table_id",
- "//br/pkg/restore/rawkv",
- "//br/pkg/restore/split",
- "//br/pkg/restore/tiflashrec",
- "//br/pkg/restore/utils",
- "//br/pkg/rtree",
- "//br/pkg/storage",
- "//br/pkg/stream",
- "//br/pkg/summary",
"//br/pkg/utils",
- "//br/pkg/utils/iter",
- "//br/pkg/version",
- "//pkg/bindinfo",
- "//pkg/ddl",
- "//pkg/ddl/util",
"//pkg/domain",
- "//pkg/domain/infosync",
- "//pkg/kv",
- "//pkg/meta",
"//pkg/parser/model",
- "//pkg/parser/mysql",
- "//pkg/sessionctx/variable",
- "//pkg/statistics/handle",
- "//pkg/store/pdtypes",
- "//pkg/tablecodec",
"//pkg/util",
- "//pkg/util/codec",
- "//pkg/util/collate",
- "//pkg/util/engine",
- "//pkg/util/redact",
- "//pkg/util/table-filter",
- "@com_github_fatih_color//:color",
"@com_github_go_sql_driver_mysql//:mysql",
- "@com_github_opentracing_opentracing_go//:opentracing-go",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
- "@com_github_pingcap_kvproto//pkg/brpb",
"@com_github_pingcap_kvproto//pkg/import_sstpb",
- "@com_github_pingcap_kvproto//pkg/metapb",
"@com_github_pingcap_log//:log",
"@com_github_tikv_client_go_v2//oracle",
- "@com_github_tikv_client_go_v2//util",
"@com_github_tikv_pd_client//:client",
- "@com_github_tikv_pd_client//http",
"@org_golang_google_grpc//:grpc",
"@org_golang_google_grpc//backoff",
"@org_golang_google_grpc//credentials",
"@org_golang_google_grpc//credentials/insecure",
- "@org_golang_google_grpc//keepalive",
"@org_golang_x_sync//errgroup",
- "@org_uber_go_multierr//:multierr",
"@org_uber_go_zap//:zap",
- "@org_uber_go_zap//zapcore",
],
)
go_test(
name = "restore_test",
timeout = "short",
- srcs = [
- "batcher_test.go",
- "client_test.go",
- "db_test.go",
- "main_test.go",
- "util_test.go",
- ],
- embed = [":restore"],
+ srcs = ["misc_test.go"],
flaky = True,
race = "off",
- shard_count = 47,
deps = [
- "//br/pkg/backup",
- "//br/pkg/errors",
- "//br/pkg/glue",
- "//br/pkg/gluetidb",
- "//br/pkg/logutil",
- "//br/pkg/metautil",
+ ":restore",
"//br/pkg/mock",
- "//br/pkg/restore/file_importer",
- "//br/pkg/restore/log_restore",
- "//br/pkg/restore/split",
- "//br/pkg/restore/tiflashrec",
- "//br/pkg/restore/utils",
- "//br/pkg/rtree",
- "//br/pkg/storage",
- "//br/pkg/stream",
- "//br/pkg/utils",
- "//br/pkg/utils/iter",
"//pkg/infoschema",
- "//pkg/meta/autoid",
"//pkg/parser/model",
- "//pkg/parser/mysql",
- "//pkg/parser/types",
- "//pkg/session",
- "//pkg/tablecodec",
- "//pkg/testkit",
- "//pkg/testkit/testsetup",
- "//pkg/util/table-filter",
- "@com_github_golang_protobuf//proto",
- "@com_github_pingcap_errors//:errors",
- "@com_github_pingcap_failpoint//:failpoint",
- "@com_github_pingcap_kvproto//pkg/brpb",
- "@com_github_pingcap_kvproto//pkg/encryptionpb",
- "@com_github_pingcap_kvproto//pkg/import_sstpb",
- "@com_github_pingcap_kvproto//pkg/metapb",
- "@com_github_pingcap_log//:log",
- "@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",
- "@com_github_tikv_client_go_v2//oracle",
- "@com_github_tikv_pd_client//:client",
- "@org_golang_google_grpc//keepalive",
- "@org_uber_go_goleak//:goleak",
- "@org_uber_go_zap//:zap",
],
)
diff --git a/br/pkg/restore/client.go b/br/pkg/restore/client.go
deleted file mode 100644
index 45548e22009a0..0000000000000
--- a/br/pkg/restore/client.go
+++ /dev/null
@@ -1,3878 +0,0 @@
-// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
-
-package restore
-
-import (
- "bytes"
- "cmp"
- "context"
- "crypto/tls"
- "encoding/hex"
- "encoding/json"
- "fmt"
- "math"
- "slices"
- "strconv"
- "strings"
- "sync"
- "time"
-
- "github.com/fatih/color"
- "github.com/opentracing/opentracing-go"
- "github.com/pingcap/errors"
- "github.com/pingcap/failpoint"
- backuppb "github.com/pingcap/kvproto/pkg/brpb"
- "github.com/pingcap/kvproto/pkg/import_sstpb"
- "github.com/pingcap/kvproto/pkg/metapb"
- "github.com/pingcap/log"
- "github.com/pingcap/tidb/br/pkg/checkpoint"
- "github.com/pingcap/tidb/br/pkg/checksum"
- "github.com/pingcap/tidb/br/pkg/conn"
- "github.com/pingcap/tidb/br/pkg/conn/util"
- berrors "github.com/pingcap/tidb/br/pkg/errors"
- "github.com/pingcap/tidb/br/pkg/glue"
- "github.com/pingcap/tidb/br/pkg/logutil"
- "github.com/pingcap/tidb/br/pkg/metautil"
- "github.com/pingcap/tidb/br/pkg/pdutil"
- fileimporter "github.com/pingcap/tidb/br/pkg/restore/file_importer"
- "github.com/pingcap/tidb/br/pkg/restore/ingestrec"
- logrestore "github.com/pingcap/tidb/br/pkg/restore/log_restore"
- tidalloc "github.com/pingcap/tidb/br/pkg/restore/prealloc_table_id"
- "github.com/pingcap/tidb/br/pkg/restore/rawkv"
- "github.com/pingcap/tidb/br/pkg/restore/split"
- "github.com/pingcap/tidb/br/pkg/restore/tiflashrec"
- restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
- "github.com/pingcap/tidb/br/pkg/rtree"
- "github.com/pingcap/tidb/br/pkg/storage"
- "github.com/pingcap/tidb/br/pkg/stream"
- "github.com/pingcap/tidb/br/pkg/summary"
- "github.com/pingcap/tidb/br/pkg/utils"
- "github.com/pingcap/tidb/br/pkg/utils/iter"
- "github.com/pingcap/tidb/br/pkg/version"
- ddlutil "github.com/pingcap/tidb/pkg/ddl/util"
- "github.com/pingcap/tidb/pkg/domain"
- "github.com/pingcap/tidb/pkg/domain/infosync"
- "github.com/pingcap/tidb/pkg/kv"
- "github.com/pingcap/tidb/pkg/meta"
- "github.com/pingcap/tidb/pkg/parser/model"
- "github.com/pingcap/tidb/pkg/parser/mysql"
- "github.com/pingcap/tidb/pkg/statistics/handle"
- "github.com/pingcap/tidb/pkg/store/pdtypes"
- "github.com/pingcap/tidb/pkg/tablecodec"
- tidbutil "github.com/pingcap/tidb/pkg/util"
- "github.com/pingcap/tidb/pkg/util/codec"
- "github.com/pingcap/tidb/pkg/util/collate"
- "github.com/pingcap/tidb/pkg/util/engine"
- "github.com/pingcap/tidb/pkg/util/redact"
- filter "github.com/pingcap/tidb/pkg/util/table-filter"
- "github.com/tikv/client-go/v2/oracle"
- kvutil "github.com/tikv/client-go/v2/util"
- pd "github.com/tikv/pd/client"
- pdhttp "github.com/tikv/pd/client/http"
- "go.uber.org/zap"
- "go.uber.org/zap/zapcore"
- "golang.org/x/sync/errgroup"
- "google.golang.org/grpc"
- "google.golang.org/grpc/backoff"
- "google.golang.org/grpc/credentials"
- "google.golang.org/grpc/credentials/insecure"
- "google.golang.org/grpc/keepalive"
-)
-
-// defaultChecksumConcurrency is the default number of the concurrent
-// checksum tasks.
-const defaultChecksumConcurrency = 64
-const defaultDDLConcurrency = 16
-const minBatchDdlSize = 1
-
-const (
- strictPlacementPolicyMode = "STRICT"
- ignorePlacementPolicyMode = "IGNORE"
-
- MetaKVBatchSize = 64 * 1024 * 1024
-)
-
-// tables in this map is restored when fullClusterRestore=true
-var sysPrivilegeTableMap = map[string]string{
- "user": "(user = '%s' and host = '%%')", // since v1.0.0
- "db": "(user = '%s' and host = '%%')", // since v1.0.0
- "tables_priv": "(user = '%s' and host = '%%')", // since v1.0.0
- "columns_priv": "(user = '%s' and host = '%%')", // since v1.0.0
- "default_roles": "(user = '%s' and host = '%%')", // since v3.0.0
- "role_edges": "(to_user = '%s' and to_host = '%%')", // since v3.0.0
- "global_priv": "(user = '%s' and host = '%%')", // since v3.0.8
- "global_grants": "(user = '%s' and host = '%%')", // since v5.0.3
-}
-
-// Client sends requests to restore files.
-type Client struct {
- pdClient pd.Client
- pdHTTPClient pdhttp.Client
- toolClient split.SplitClient
- fileImporter fileimporter.FileImporter
- rawKVClient *rawkv.RawKVBatchClient
- workerPool *tidbutil.WorkerPool
- tlsConf *tls.Config
- keepaliveConf keepalive.ClientParameters
-
- concurrencyPerStore uint
- databases map[string]*metautil.Database
- ddlJobs []*model.Job
-
- // store tables need to rebase info like auto id and random id and so on after create table
- rebasedTablesMap map[UniqueTableName]bool
-
- backupMeta *backuppb.BackupMeta
- // TODO Remove this field or replace it with a []*DB,
- // since https://github.com/pingcap/br/pull/377 needs more DBs to speed up DDL execution.
- // And for now, we must inject a pool of DBs to `Client.GoCreateTables`, otherwise there would be a race condition.
- // This is dirty: why we need DBs from different sources?
- // By replace it with a []*DB, we can remove the dirty parameter of `Client.GoCreateTable`,
- // along with them in some private functions.
- // Before you do it, you can firstly read discussions at
- // https://github.com/pingcap/br/pull/377#discussion_r446594501,
- // this probably isn't as easy as it seems like (however, not hard, too :D)
- db *DB
-
- // use db pool to speed up restoration in BR binary mode.
- dbPool []*DB
- rateLimit uint64
- isOnline bool
- granularity string
- noSchema bool
- hasSpeedLimited bool
-
- restoreStores []uint64
- storeCount int
-
- cipher *backuppb.CipherInfo
- switchModeInterval time.Duration
- switchCh chan struct{}
-
- // statHandler and dom are used for analyze table after restore.
- // it will backup stats with #dump.DumpStatsToJSON
- // and restore stats with #dump.LoadStatsFromJSONNoUpdate
- statsHandler *handle.Handle
- dom *domain.Domain
-
- batchDdlSize uint
-
- // correspond to --tidb-placement-mode config.
- // STRICT(default) means policy related SQL can be executed in tidb.
- // IGNORE means policy related SQL will be ignored.
- policyMode string
-
- // policy name -> policy info
- policyMap *sync.Map
-
- supportPolicy bool
-
- // currentTS is used for rewrite meta kv when restore stream.
- // Can not use `restoreTS` directly, because schema created in `full backup` maybe is new than `restoreTS`.
- currentTS uint64
-
- // clusterID is the cluster id from down-stream cluster.
- clusterID uint64
-
- *logrestore.LogFileManager
-
- // storage for log restore
- storage storage.ExternalStorage
-
- // if fullClusterRestore = true:
- // - if there's system tables in the backup(backup data since br 5.1.0), the cluster should be a fresh cluster
- // without user database or table. and system tables about privileges is restored together with user data.
- // - if there no system tables in the backup(backup data from br < 5.1.0), restore all user data just like
- // previous version did.
- // if fullClusterRestore = false, restore all user data just like previous version did.
- // fullClusterRestore = true when there is no explicit filter setting, and it's full restore or point command
- // with a full backup data.
- // todo: maybe change to an enum
- // this feature is controlled by flag with-sys-table
- fullClusterRestore bool
- // the query to insert rows into table `gc_delete_range`, lack of ts.
- deleteRangeQuery []*stream.PreDelRangeQuery
- deleteRangeQueryCh chan *stream.PreDelRangeQuery
- deleteRangeQueryWaitGroup sync.WaitGroup
-
- // see RestoreCommonConfig.WithSysTable
- withSysTable bool
-
- // the successfully preallocated table IDs.
- preallocedTableIDs *tidalloc.PreallocIDs
-
- // the rewrite mode of the downloaded SST files in TiKV.
- rewriteMode fileimporter.RewriteMode
-
- // checkpoint information for snapshot restore
- checkpointRunner *checkpoint.CheckpointRunner[checkpoint.RestoreKeyType, checkpoint.RestoreValueType]
- checkpointChecksum map[int64]*checkpoint.ChecksumItem
-
- // checkpoint information for log restore
- useCheckpoint bool
-}
-
-// NewRestoreClient returns a new RestoreClient.
-func NewRestoreClient(
- pdClient pd.Client,
- pdHTTPCli pdhttp.Client,
- tlsConf *tls.Config,
- keepaliveConf keepalive.ClientParameters,
-) *Client {
- return &Client{
- pdClient: pdClient,
- pdHTTPClient: pdHTTPCli,
- // toolClient reuse the split.SplitClient to do miscellaneous things. It doesn't
- // call split related functions so set the arguments to arbitrary values.
- toolClient: split.NewClient(pdClient, pdHTTPCli, tlsConf, maxSplitKeysOnce, 3),
- tlsConf: tlsConf,
- keepaliveConf: keepaliveConf,
- switchCh: make(chan struct{}),
- deleteRangeQuery: make([]*stream.PreDelRangeQuery, 0),
- deleteRangeQueryCh: make(chan *stream.PreDelRangeQuery, 10),
- }
-}
-
-// makeDBPool makes a session pool with specficated size by sessionFactory.
-func makeDBPool(size uint, dbFactory func() (*DB, error)) ([]*DB, error) {
- dbPool := make([]*DB, 0, size)
- for i := uint(0); i < size; i++ {
- db, e := dbFactory()
- if e != nil {
- return dbPool, e
- }
- if db != nil {
- dbPool = append(dbPool, db)
- }
- }
- return dbPool, nil
-}
-
-// Init create db connection and domain for storage.
-func (rc *Client) Init(g glue.Glue, store kv.Storage) error {
- // setDB must happen after set PolicyMode.
- // we will use policyMode to set session variables.
- var err error
- rc.db, rc.supportPolicy, err = NewDB(g, store, rc.policyMode)
- if err != nil {
- return errors.Trace(err)
- }
- rc.dom, err = g.GetDomain(store)
- if err != nil {
- return errors.Trace(err)
- }
- // tikv.Glue will return nil, tidb.Glue will return available domain
- if rc.dom != nil {
- rc.statsHandler = rc.dom.StatsHandle()
- }
- // init backupMeta only for passing unit test
- if rc.backupMeta == nil {
- rc.backupMeta = new(backuppb.BackupMeta)
- }
-
- // There are different ways to create session between in binary and in SQL.
- //
- // Maybe allow user modify the DDL concurrency isn't necessary,
- // because executing DDL is really I/O bound (or, algorithm bound?),
- // and we cost most of time at waiting DDL jobs be enqueued.
- // So these jobs won't be faster or slower when machine become faster or slower,
- // hence make it a fixed value would be fine.
- rc.dbPool, err = makeDBPool(defaultDDLConcurrency, func() (*DB, error) {
- db, _, err := NewDB(g, store, rc.policyMode)
- return db, err
- })
- if err != nil {
- log.Warn("create session pool failed, we will send DDLs only by created sessions",
- zap.Error(err),
- zap.Int("sessionCount", len(rc.dbPool)),
- )
- }
- return errors.Trace(err)
-}
-
-// InitCheckpoint initialize the checkpoint status for the cluster. If the cluster is
-// restored for the first time, it will initialize the checkpoint metadata. Otherwrise,
-// it will load checkpoint metadata and checkpoint ranges/checksum from the external
-// storage.
-func (rc *Client) InitCheckpoint(
- ctx context.Context,
- s storage.ExternalStorage,
- taskName string,
- config *pdutil.ClusterConfig,
- checkpointFirstRun bool,
-) (map[int64]map[string]struct{}, *pdutil.ClusterConfig, error) {
- var (
- // checkpoint sets distinguished by range key
- checkpointSetWithTableID = make(map[int64]map[string]struct{})
-
- checkpointClusterConfig *pdutil.ClusterConfig
-
- err error
- )
-
- if !checkpointFirstRun {
- // load the checkpoint since this is not the first time to restore
- meta, err := checkpoint.LoadCheckpointMetadataForRestore(ctx, s, taskName)
- if err != nil {
- return checkpointSetWithTableID, nil, errors.Trace(err)
- }
-
- // The schedulers config is nil, so the restore-schedulers operation is just nil.
- // Then the undo function would use the result undo of `remove schedulers` operation,
- // instead of that in checkpoint meta.
- if meta.SchedulersConfig != nil {
- checkpointClusterConfig = meta.SchedulersConfig
- }
-
- // t1 is the latest time the checkpoint ranges persisted to the external storage.
- t1, err := checkpoint.WalkCheckpointFileForRestore(ctx, s, rc.cipher, taskName, func(tableID int64, rangeKey checkpoint.RestoreValueType) {
- checkpointSet, exists := checkpointSetWithTableID[tableID]
- if !exists {
- checkpointSet = make(map[string]struct{})
- checkpointSetWithTableID[tableID] = checkpointSet
- }
- checkpointSet[rangeKey.RangeKey] = struct{}{}
- })
- if err != nil {
- return checkpointSetWithTableID, nil, errors.Trace(err)
- }
- // t2 is the latest time the checkpoint checksum persisted to the external storage.
- checkpointChecksum, t2, err := checkpoint.LoadCheckpointChecksumForRestore(ctx, s, taskName)
- if err != nil {
- return checkpointSetWithTableID, nil, errors.Trace(err)
- }
- rc.checkpointChecksum = checkpointChecksum
- // use the later time to adjust the summary elapsed time.
- if t1 > t2 {
- summary.AdjustStartTimeToEarlierTime(t1)
- } else {
- summary.AdjustStartTimeToEarlierTime(t2)
- }
- } else {
- // initialize the checkpoint metadata since it is the first time to restore.
- meta := &checkpoint.CheckpointMetadataForRestore{}
- // a nil config means undo function
- if config != nil {
- meta.SchedulersConfig = &pdutil.ClusterConfig{Schedulers: config.Schedulers, ScheduleCfg: config.ScheduleCfg}
- }
- if err = checkpoint.SaveCheckpointMetadataForRestore(ctx, s, meta, taskName); err != nil {
- return checkpointSetWithTableID, nil, errors.Trace(err)
- }
- }
-
- rc.checkpointRunner, err = checkpoint.StartCheckpointRunnerForRestore(ctx, s, rc.cipher, taskName)
- return checkpointSetWithTableID, checkpointClusterConfig, errors.Trace(err)
-}
-
-func (rc *Client) WaitForFinishCheckpoint(ctx context.Context, flush bool) {
- if rc.checkpointRunner != nil {
- rc.checkpointRunner.WaitForFinish(ctx, flush)
- }
-}
-
-func (rc *Client) GetCheckpointRunner() *checkpoint.CheckpointRunner[checkpoint.RestoreKeyType, checkpoint.RestoreValueType] {
- return rc.checkpointRunner
-}
-
-func (rc *Client) StartCheckpointRunnerForLogRestore(ctx context.Context, taskName string) (*checkpoint.CheckpointRunner[checkpoint.LogRestoreKeyType, checkpoint.LogRestoreValueType], error) {
- runner, err := checkpoint.StartCheckpointRunnerForLogRestore(ctx, rc.storage, rc.cipher, taskName)
- return runner, errors.Trace(err)
-}
-
-func (rc *Client) InitCheckpointMetadataForLogRestore(ctx context.Context, taskName string, gcRatio string) (string, error) {
- rc.useCheckpoint = true
-
- // it shows that the user has modified gc-ratio, if `gcRatio` doesn't equal to "1.1".
- // update the `gcRatio` for checkpoint metadata.
- if gcRatio == utils.DefaultGcRatioVal {
- // if the checkpoint metadata exists in the external storage, the restore is not
- // for the first time.
- exists, err := checkpoint.ExistsRestoreCheckpoint(ctx, rc.storage, taskName)
- if err != nil {
- return "", errors.Trace(err)
- }
-
- if exists {
- // load the checkpoint since this is not the first time to restore
- meta, err := checkpoint.LoadCheckpointMetadataForRestore(ctx, rc.storage, taskName)
- if err != nil {
- return "", errors.Trace(err)
- }
-
- log.Info("reuse gc ratio from checkpoint metadata", zap.String("gc-ratio", gcRatio))
- return meta.GcRatio, nil
- }
- }
-
- // initialize the checkpoint metadata since it is the first time to restore.
- log.Info("save gc ratio into checkpoint metadata", zap.String("gc-ratio", gcRatio))
- if err := checkpoint.SaveCheckpointMetadataForRestore(ctx, rc.storage, &checkpoint.CheckpointMetadataForRestore{
- GcRatio: gcRatio,
- }, taskName); err != nil {
- return gcRatio, errors.Trace(err)
- }
-
- return gcRatio, nil
-}
-
-// AllocTableIDs would pre-allocate the table's origin ID if exists, so that the TiKV doesn't need to rewrite the key in
-// the download stage.
-func (rc *Client) AllocTableIDs(ctx context.Context, tables []*metautil.Table) error {
- rc.preallocedTableIDs = tidalloc.New(tables)
- ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnBR)
- err := kv.RunInNewTxn(ctx, rc.GetDomain().Store(), true, func(_ context.Context, txn kv.Transaction) error {
- return rc.preallocedTableIDs.Alloc(meta.NewMeta(txn))
- })
- if err != nil {
- return err
- }
-
- log.Info("registering the table IDs", zap.Stringer("ids", rc.preallocedTableIDs))
- for i := range rc.dbPool {
- rc.dbPool[i].registerPreallocatedIDs(rc.preallocedTableIDs)
- }
- if rc.db != nil {
- rc.db.registerPreallocatedIDs(rc.preallocedTableIDs)
- }
- return nil
-}
-
-// SetPlacementPolicyMode to policy mode.
-func (rc *Client) SetPlacementPolicyMode(withPlacementPolicy string) {
- switch strings.ToUpper(withPlacementPolicy) {
- case strictPlacementPolicyMode:
- rc.policyMode = strictPlacementPolicyMode
- case ignorePlacementPolicyMode:
- rc.policyMode = ignorePlacementPolicyMode
- default:
- rc.policyMode = strictPlacementPolicyMode
- }
- log.Info("set placement policy mode", zap.String("mode", rc.policyMode))
-}
-
-// SetRateLimit to set rateLimit.
-func (rc *Client) SetRateLimit(rateLimit uint64) {
- rc.rateLimit = rateLimit
-}
-
-func (rc *Client) SetCrypter(crypter *backuppb.CipherInfo) {
- rc.cipher = crypter
-}
-
-// SetPolicyMap set policyMap.
-func (rc *Client) SetPolicyMap(p *sync.Map) {
- rc.policyMap = p
-}
-
-// GetClusterID gets the cluster id from down-stream cluster.
-func (rc *Client) GetClusterID(ctx context.Context) uint64 {
- if rc.clusterID <= 0 {
- rc.clusterID = rc.GetPDClient().GetClusterID(ctx)
- }
- return rc.clusterID
-}
-
-// GetPolicyMap set policyMap.
-func (rc *Client) GetPolicyMap() *sync.Map {
- return rc.policyMap
-}
-
-// GetSupportPolicy tells whether target tidb support placement policy.
-func (rc *Client) GetSupportPolicy() bool {
- return rc.supportPolicy
-}
-
-func (rc *Client) GetDomain() *domain.Domain {
- return rc.dom
-}
-
-func (rc *Client) GetStoreCount() int {
- return rc.storeCount
-}
-
-// GetPDClient returns a pd client.
-func (rc *Client) GetPDClient() pd.Client {
- return rc.pdClient
-}
-
-// IsOnline tells if it's a online restore.
-func (rc *Client) IsOnline() bool {
- return rc.isOnline
-}
-
-// SetSwitchModeInterval set switch mode interval for client.
-func (rc *Client) SetSwitchModeInterval(interval time.Duration) {
- rc.switchModeInterval = interval
-}
-
-func (rc *Client) SetBatchDdlSize(batchDdlsize uint) {
- rc.batchDdlSize = batchDdlsize
-}
-
-func (rc *Client) GetBatchDdlSize() uint {
- return rc.batchDdlSize
-}
-
-func (rc *Client) SetRewriteMode(mode fileimporter.RewriteMode) {
- rc.rewriteMode = mode
-}
-
-func (rc *Client) GetRewriteMode() fileimporter.RewriteMode {
- return rc.rewriteMode
-}
-
-func (rc *Client) closeConn() {
- // rc.db can be nil in raw kv mode.
- if rc.db != nil {
- rc.db.Close()
- }
- for _, db := range rc.dbPool {
- db.Close()
- }
-}
-
-// Close a client.
-func (rc *Client) Close() {
- // close the connection, and it must be succeed when in SQL mode.
- rc.closeConn()
-
- if rc.rawKVClient != nil {
- rc.rawKVClient.Close()
- }
-
- if err := rc.fileImporter.Close(); err != nil {
- log.Warn("failed to close file improter")
- }
-
- log.Info("Restore client closed")
-}
-
-func (rc *Client) SetCurrentTS(ts uint64) {
- rc.currentTS = ts
-}
-
-func (rc *Client) SetStorage(ctx context.Context, backend *backuppb.StorageBackend, opts *storage.ExternalStorageOptions) error {
- var err error
- rc.storage, err = storage.New(ctx, backend, opts)
- if err != nil {
- return errors.Trace(err)
- }
- return nil
-}
-
-func (rc *Client) InitClients(ctx context.Context, backend *backuppb.StorageBackend, isRawKvMode bool, isTxnKvMode bool) {
- stores, err := conn.GetAllTiKVStoresWithRetry(ctx, rc.pdClient, util.SkipTiFlash)
- if err != nil {
- log.Fatal("failed to get stores", zap.Error(err))
- }
- concurrencyPerStore := rc.GetConcurrencyPerStore()
- useTokenBucket := false
- if rc.granularity == string(CoarseGrained) {
- // coarse-grained make split & scatter pipeline fast enough
- // so we can use a new token bucket way to speed up download.
- // ToDo remove it when token bucket is stable enough.
- log.Info("use token bucket to control download and ingest flow")
- useTokenBucket = true
- }
-
- var splitClientOpts []split.ClientOptionalParameter
- if isRawKvMode {
- splitClientOpts = append(splitClientOpts, split.WithRawKV())
- }
- metaClient := split.NewClient(rc.pdClient, rc.pdHTTPClient, rc.tlsConf, maxSplitKeysOnce, rc.GetStoreCount()+1, splitClientOpts...)
- importCli := fileimporter.NewImportClient(metaClient, rc.tlsConf, rc.keepaliveConf)
- rc.fileImporter = fileimporter.NewFileImporter(metaClient, importCli, backend, isRawKvMode, isTxnKvMode, stores, rc.rewriteMode, concurrencyPerStore, useTokenBucket)
-}
-
-func (rc *Client) SetRawKVClient(c *rawkv.RawKVBatchClient) {
- rc.rawKVClient = c
-}
-
-func (rc *Client) needLoadSchemas(backupMeta *backuppb.BackupMeta) bool {
- return !(backupMeta.IsRawKv || backupMeta.IsTxnKv)
-}
-
-// InitBackupMeta loads schemas from BackupMeta to initialize RestoreClient.
-func (rc *Client) InitBackupMeta(
- c context.Context,
- backupMeta *backuppb.BackupMeta,
- backend *backuppb.StorageBackend,
- reader *metautil.MetaReader,
- loadStats bool) error {
- if rc.needLoadSchemas(backupMeta) {
- databases, err := metautil.LoadBackupTables(c, reader, loadStats)
- if err != nil {
- return errors.Trace(err)
- }
- rc.databases = databases
-
- var ddlJobs []*model.Job
- // ddls is the bytes of json.Marshal
- ddls, err := reader.ReadDDLs(c)
- if err != nil {
- return errors.Trace(err)
- }
- if len(ddls) != 0 {
- err = json.Unmarshal(ddls, &ddlJobs)
- if err != nil {
- return errors.Trace(err)
- }
- }
- rc.ddlJobs = ddlJobs
- }
- rc.backupMeta = backupMeta
-
- rc.InitClients(c, backend, backupMeta.IsRawKv, backupMeta.IsTxnKv)
- log.Info("load backupmeta", zap.Int("databases", len(rc.databases)), zap.Int("jobs", len(rc.ddlJobs)))
- return rc.fileImporter.CheckMultiIngestSupport(c, rc.pdClient)
-}
-
-// IsRawKvMode checks whether the backup data is in raw kv format, in which case transactional recover is forbidden.
-func (rc *Client) IsRawKvMode() bool {
- return rc.backupMeta.IsRawKv
-}
-
-// GetFilesInRawRange gets all files that are in the given range or intersects with the given range.
-func (rc *Client) GetFilesInRawRange(startKey []byte, endKey []byte, cf string) ([]*backuppb.File, error) {
- if !rc.IsRawKvMode() {
- return nil, errors.Annotate(berrors.ErrRestoreModeMismatch, "the backup data is not in raw kv mode")
- }
-
- for _, rawRange := range rc.backupMeta.RawRanges {
- // First check whether the given range is backup-ed. If not, we cannot perform the restore.
- if rawRange.Cf != cf {
- continue
- }
-
- if (len(rawRange.EndKey) > 0 && bytes.Compare(startKey, rawRange.EndKey) >= 0) ||
- (len(endKey) > 0 && bytes.Compare(rawRange.StartKey, endKey) >= 0) {
- // The restoring range is totally out of the current range. Skip it.
- continue
- }
-
- if bytes.Compare(startKey, rawRange.StartKey) < 0 ||
- utils.CompareEndKey(endKey, rawRange.EndKey) > 0 {
- // Only partial of the restoring range is in the current backup-ed range. So the given range can't be fully
- // restored.
- return nil, errors.Annotatef(berrors.ErrRestoreRangeMismatch,
- "the given range to restore [%s, %s) is not fully covered by the range that was backed up [%s, %s)",
- redact.Key(startKey), redact.Key(endKey), redact.Key(rawRange.StartKey), redact.Key(rawRange.EndKey),
- )
- }
-
- // We have found the range that contains the given range. Find all necessary files.
- files := make([]*backuppb.File, 0)
-
- for _, file := range rc.backupMeta.Files {
- if file.Cf != cf {
- continue
- }
-
- if len(file.EndKey) > 0 && bytes.Compare(file.EndKey, startKey) < 0 {
- // The file is before the range to be restored.
- continue
- }
- if len(endKey) > 0 && bytes.Compare(endKey, file.StartKey) <= 0 {
- // The file is after the range to be restored.
- // The specified endKey is exclusive, so when it equals to a file's startKey, the file is still skipped.
- continue
- }
-
- files = append(files, file)
- }
-
- // There should be at most one backed up range that covers the restoring range.
- return files, nil
- }
-
- return nil, errors.Annotate(berrors.ErrRestoreRangeMismatch, "no backup data in the range")
-}
-
-// SetConcurrency sets the concurrency of dbs tables files.
-func (rc *Client) SetConcurrency(c uint) {
- log.Info("download worker pool", zap.Uint("size", c))
- if rc.granularity == string(CoarseGrained) {
- // we believe 32 is large enough for download worker pool.
- // it won't reach the limit if sst files distribute evenly.
- // when restore memory usage is still too high, we should reduce concurrencyPerStore
- // to sarifice some speed to reduce memory usage.
- count := uint(rc.storeCount) * rc.concurrencyPerStore * 32
- log.Info("download coarse worker pool", zap.Uint("size", count))
- rc.workerPool = tidbutil.NewWorkerPool(count, "file")
- return
- }
- rc.workerPool = tidbutil.NewWorkerPool(c, "file")
-}
-
-// SetConcurrencyPerStore sets the concurrency of download files for each store.
-func (rc *Client) SetConcurrencyPerStore(c uint) {
- log.Info("per-store download worker pool", zap.Uint("size", c))
- rc.concurrencyPerStore = c
-}
-
-func (rc *Client) GetConcurrencyPerStore() uint {
- return rc.concurrencyPerStore
-}
-
-// EnableOnline sets the mode of restore to online.
-func (rc *Client) EnableOnline() {
- rc.isOnline = true
-}
-
-// SetGranularity sets the ganularity of restore pipeline.
-func (rc *Client) SetGranularity(g string) {
- rc.granularity = g
-}
-
-// GetGranularity sets the ganularity of restore pipeline.
-func (rc *Client) GetGranularity() string {
- return rc.granularity
-}
-
-// GetTLSConfig returns the tls config.
-func (rc *Client) GetTLSConfig() *tls.Config {
- return rc.tlsConf
-}
-
-// GetTS gets a new timestamp from PD.
-func (rc *Client) GetTS(ctx context.Context) (uint64, error) {
- p, l, err := rc.pdClient.GetTS(ctx)
- if err != nil {
- return 0, errors.Trace(err)
- }
- restoreTS := oracle.ComposeTS(p, l)
- return restoreTS, nil
-}
-
-// GetTSWithRetry gets a new timestamp with retry from PD.
-func (rc *Client) GetTSWithRetry(ctx context.Context) (uint64, error) {
- var (
- startTS uint64
- getTSErr error
- retry uint
- )
-
- err := utils.WithRetry(ctx, func() error {
- startTS, getTSErr = rc.GetTS(ctx)
- failpoint.Inject("get-ts-error", func(val failpoint.Value) {
- if val.(bool) && retry < 3 {
- getTSErr = errors.Errorf("rpc error: code = Unknown desc = [PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster")
- }
- })
-
- retry++
- if getTSErr != nil {
- log.Warn("failed to get TS, retry it", zap.Uint("retry time", retry), logutil.ShortError(getTSErr))
- }
- return getTSErr
- }, utils.NewPDReqBackoffer())
-
- if err != nil {
- log.Error("failed to get TS", zap.Error(err))
- }
- return startTS, errors.Trace(err)
-}
-
-// ResetTS resets the timestamp of PD to a bigger value.
-func (rc *Client) ResetTS(ctx context.Context, pdCtrl *pdutil.PdController) error {
- restoreTS := rc.backupMeta.GetEndVersion()
- log.Info("reset pd timestamp", zap.Uint64("ts", restoreTS))
- return utils.WithRetry(ctx, func() error {
- return pdCtrl.ResetTS(ctx, restoreTS)
- }, utils.NewPDReqBackoffer())
-}
-
-// GetPlacementRules return the current placement rules.
-func (rc *Client) GetPlacementRules(ctx context.Context, pdAddrs []string) ([]pdtypes.Rule, error) {
- var placementRules []pdtypes.Rule
- i := 0
- errRetry := utils.WithRetry(ctx, func() error {
- var err error
- idx := i % len(pdAddrs)
- i++
- placementRules, err = pdutil.GetPlacementRules(ctx, pdAddrs[idx], rc.tlsConf)
- return errors.Trace(err)
- }, utils.NewPDReqBackoffer())
- return placementRules, errors.Trace(errRetry)
-}
-
-// GetDatabases returns all databases.
-func (rc *Client) GetDatabases() []*metautil.Database {
- dbs := make([]*metautil.Database, 0, len(rc.databases))
- for _, db := range rc.databases {
- dbs = append(dbs, db)
- }
- return dbs
-}
-
-// GetDatabase returns a database by name.
-func (rc *Client) GetDatabase(name string) *metautil.Database {
- return rc.databases[name]
-}
-
-// HasBackedUpSysDB whether we have backed up system tables
-// br backs system tables up since 5.1.0
-func (rc *Client) HasBackedUpSysDB() bool {
- sysDBs := []string{"mysql", "sys"}
- for _, db := range sysDBs {
- temporaryDB := utils.TemporaryDBName(db)
- _, backedUp := rc.databases[temporaryDB.O]
- if backedUp {
- return true
- }
- }
- return false
-}
-
-// GetPlacementPolicies returns policies.
-func (rc *Client) GetPlacementPolicies() (*sync.Map, error) {
- policies := &sync.Map{}
- for _, p := range rc.backupMeta.Policies {
- policyInfo := &model.PolicyInfo{}
- err := json.Unmarshal(p.Info, policyInfo)
- if err != nil {
- return nil, errors.Trace(err)
- }
- policies.Store(policyInfo.Name.L, policyInfo)
- }
- return policies, nil
-}
-
-// GetDDLJobs returns ddl jobs.
-func (rc *Client) GetDDLJobs() []*model.Job {
- return rc.ddlJobs
-}
-
-// GetTableSchema returns the schema of a table from TiDB.
-func (rc *Client) GetTableSchema(
- dom *domain.Domain,
- dbName model.CIStr,
- tableName model.CIStr,
-) (*model.TableInfo, error) {
- info := dom.InfoSchema()
- table, err := info.TableByName(dbName, tableName)
- if err != nil {
- return nil, errors.Trace(err)
- }
- return table.Meta(), nil
-}
-
-// CreatePolicies creates all policies in full restore.
-func (rc *Client) CreatePolicies(ctx context.Context, policyMap *sync.Map) error {
- var err error
- policyMap.Range(func(key, value any) bool {
- e := rc.db.CreatePlacementPolicy(ctx, value.(*model.PolicyInfo))
- if e != nil {
- err = e
- return false
- }
- return true
- })
- return err
-}
-
-// GetDBSchema gets the schema of a db from TiDB cluster
-func (rc *Client) GetDBSchema(dom *domain.Domain, dbName model.CIStr) (*model.DBInfo, bool) {
- info := dom.InfoSchema()
- return info.SchemaByName(dbName)
-}
-
-// CreateDatabases creates databases. If the client has the db pool, it would create it.
-func (rc *Client) CreateDatabases(ctx context.Context, dbs []*metautil.Database) error {
- if rc.IsSkipCreateSQL() {
- log.Info("skip create database")
- return nil
- }
-
- if len(rc.dbPool) == 0 {
- log.Info("create databases sequentially")
- for _, db := range dbs {
- err := rc.createDatabaseWithDBConn(ctx, db.Info, rc.db)
- if err != nil {
- return errors.Trace(err)
- }
- }
- return nil
- }
-
- log.Info("create databases in db pool", zap.Int("pool size", len(rc.dbPool)))
- eg, ectx := errgroup.WithContext(ctx)
- workers := tidbutil.NewWorkerPool(uint(len(rc.dbPool)), "DB DDL workers")
- for _, db_ := range dbs {
- db := db_
- workers.ApplyWithIDInErrorGroup(eg, func(id uint64) error {
- conn := rc.dbPool[id%uint64(len(rc.dbPool))]
- return rc.createDatabaseWithDBConn(ectx, db.Info, conn)
- })
- }
- return eg.Wait()
-}
-
-func (rc *Client) createDatabaseWithDBConn(ctx context.Context, db *model.DBInfo, conn *DB) error {
- log.Info("create database", zap.Stringer("name", db.Name))
-
- if !rc.supportPolicy {
- log.Info("set placementPolicyRef to nil when target tidb not support policy",
- zap.Stringer("database", db.Name))
- db.PlacementPolicyRef = nil
- }
-
- if db.PlacementPolicyRef != nil {
- if err := conn.ensurePlacementPolicy(ctx, db.PlacementPolicyRef.Name, rc.policyMap); err != nil {
- return errors.Trace(err)
- }
- }
-
- return conn.CreateDatabase(ctx, db)
-}
-
-// CreateTables creates multiple tables, and returns their rewrite rules.
-func (rc *Client) CreateTables(
- dom *domain.Domain,
- tables []*metautil.Table,
- newTS uint64,
-) (*restoreutils.RewriteRules, []*model.TableInfo, error) {
- rewriteRules := &restoreutils.RewriteRules{
- Data: make([]*import_sstpb.RewriteRule, 0),
- }
- newTables := make([]*model.TableInfo, 0, len(tables))
- errCh := make(chan error, 1)
- tbMapping := map[string]int{}
- for i, t := range tables {
- tbMapping[t.Info.Name.String()] = i
- }
- dataCh := rc.GoCreateTables(context.TODO(), dom, tables, newTS, errCh)
- for et := range dataCh {
- rules := et.RewriteRule
- rewriteRules.Data = append(rewriteRules.Data, rules.Data...)
- newTables = append(newTables, et.Table)
- }
- // Let's ensure that it won't break the original order.
- slices.SortFunc(newTables, func(i, j *model.TableInfo) int {
- return cmp.Compare(tbMapping[i.Name.String()], tbMapping[j.Name.String()])
- })
-
- select {
- case err, ok := <-errCh:
- if ok {
- return nil, nil, errors.Trace(err)
- }
- default:
- }
- return rewriteRules, newTables, nil
-}
-func (rc *Client) createTables(
- ctx context.Context,
- db *DB,
- dom *domain.Domain,
- tables []*metautil.Table,
- newTS uint64,
-) ([]CreatedTable, error) {
- log.Info("client to create tables")
- if rc.IsSkipCreateSQL() {
- log.Info("skip create table and alter autoIncID")
- } else {
- err := db.CreateTables(ctx, tables, rc.GetRebasedTables(), rc.GetSupportPolicy(), rc.GetPolicyMap())
- if err != nil {
- return nil, errors.Trace(err)
- }
- }
- cts := make([]CreatedTable, 0, len(tables))
- for _, table := range tables {
- newTableInfo, err := rc.GetTableSchema(dom, table.DB.Name, table.Info.Name)
- if err != nil {
- return nil, errors.Trace(err)
- }
- if newTableInfo.IsCommonHandle != table.Info.IsCommonHandle {
- return nil, errors.Annotatef(berrors.ErrRestoreModeMismatch,
- "Clustered index option mismatch. Restored cluster's @@tidb_enable_clustered_index should be %v (backup table = %v, created table = %v).",
- transferBoolToValue(table.Info.IsCommonHandle),
- table.Info.IsCommonHandle,
- newTableInfo.IsCommonHandle)
- }
- rules := restoreutils.GetRewriteRules(newTableInfo, table.Info, newTS, true)
- ct := CreatedTable{
- RewriteRule: rules,
- Table: newTableInfo,
- OldTable: table,
- }
- log.Debug("new created tables", zap.Any("table", ct))
- cts = append(cts, ct)
- }
- return cts, nil
-}
-
-func (rc *Client) createTable(
- ctx context.Context,
- db *DB,
- dom *domain.Domain,
- table *metautil.Table,
- newTS uint64,
-) (CreatedTable, error) {
- if rc.IsSkipCreateSQL() {
- log.Info("skip create table and alter autoIncID", zap.Stringer("table", table.Info.Name))
- } else {
- err := db.CreateTable(ctx, table, rc.GetRebasedTables(), rc.GetSupportPolicy(), rc.GetPolicyMap())
- if err != nil {
- return CreatedTable{}, errors.Trace(err)
- }
- }
- newTableInfo, err := rc.GetTableSchema(dom, table.DB.Name, table.Info.Name)
- if err != nil {
- return CreatedTable{}, errors.Trace(err)
- }
- if newTableInfo.IsCommonHandle != table.Info.IsCommonHandle {
- return CreatedTable{}, errors.Annotatef(berrors.ErrRestoreModeMismatch,
- "Clustered index option mismatch. Restored cluster's @@tidb_enable_clustered_index should be %v (backup table = %v, created table = %v).",
- transferBoolToValue(table.Info.IsCommonHandle),
- table.Info.IsCommonHandle,
- newTableInfo.IsCommonHandle)
- }
- rules := restoreutils.GetRewriteRules(newTableInfo, table.Info, newTS, true)
- et := CreatedTable{
- RewriteRule: rules,
- Table: newTableInfo,
- OldTable: table,
- }
- return et, nil
-}
-
-// GoCreateTables create tables, and generate their information.
-// this function will use workers as the same number of sessionPool,
-// leave sessionPool nil to send DDLs sequential.
-func (rc *Client) GoCreateTables(
- ctx context.Context,
- dom *domain.Domain,
- tables []*metautil.Table,
- newTS uint64,
- errCh chan<- error,
-) <-chan CreatedTable {
- // Could we have a smaller size of tables?
- log.Info("start create tables")
-
- rc.GenerateRebasedTables(tables)
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("Client.GoCreateTables", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
- outCh := make(chan CreatedTable, len(tables))
- rater := logutil.TraceRateOver(logutil.MetricTableCreatedCounter)
-
- var err error
-
- if rc.batchDdlSize > minBatchDdlSize && len(rc.dbPool) > 0 {
- err = rc.createTablesInWorkerPool(ctx, dom, tables, newTS, outCh)
- if err == nil {
- defer log.Debug("all tables are created")
- close(outCh)
- return outCh
- } else if !utils.FallBack2CreateTable(err) {
- errCh <- err
- close(outCh)
- return outCh
- }
- // fall back to old create table (sequential create table)
- log.Info("fall back to the sequential create table")
- }
-
- createOneTable := func(c context.Context, db *DB, t *metautil.Table) error {
- select {
- case <-c.Done():
- return c.Err()
- default:
- }
- rt, err := rc.createTable(c, db, dom, t, newTS)
- if err != nil {
- log.Error("create table failed",
- zap.Error(err),
- zap.Stringer("db", t.DB.Name),
- zap.Stringer("table", t.Info.Name))
- return errors.Trace(err)
- }
- log.Debug("table created and send to next",
- zap.Int("output chan size", len(outCh)),
- zap.Stringer("table", t.Info.Name),
- zap.Stringer("database", t.DB.Name))
- outCh <- rt
- rater.Inc()
- rater.L().Info("table created",
- zap.Stringer("table", t.Info.Name),
- zap.Stringer("database", t.DB.Name))
- return nil
- }
- go func() {
- defer close(outCh)
- defer log.Debug("all tables are created")
- var err error
- if len(rc.dbPool) > 0 {
- err = rc.createTablesWithDBPool(ctx, createOneTable, tables)
- } else {
- err = rc.createTablesWithSoleDB(ctx, createOneTable, tables)
- }
- if err != nil {
- errCh <- err
- }
- }()
-
- return outCh
-}
-
-func (rc *Client) createTablesWithSoleDB(ctx context.Context,
- createOneTable func(ctx context.Context, db *DB, t *metautil.Table) error,
- tables []*metautil.Table) error {
- for _, t := range tables {
- if err := createOneTable(ctx, rc.db, t); err != nil {
- return errors.Trace(err)
- }
- }
- return nil
-}
-
-func (rc *Client) createTablesWithDBPool(ctx context.Context,
- createOneTable func(ctx context.Context, db *DB, t *metautil.Table) error,
- tables []*metautil.Table) error {
- eg, ectx := errgroup.WithContext(ctx)
- workers := tidbutil.NewWorkerPool(uint(len(rc.dbPool)), "DDL workers")
- for _, t := range tables {
- table := t
- workers.ApplyWithIDInErrorGroup(eg, func(id uint64) error {
- db := rc.dbPool[id%uint64(len(rc.dbPool))]
- return createOneTable(ectx, db, table)
- })
- }
- return eg.Wait()
-}
-
-func (rc *Client) createTablesInWorkerPool(ctx context.Context, dom *domain.Domain, tables []*metautil.Table, newTS uint64, outCh chan<- CreatedTable) error {
- eg, ectx := errgroup.WithContext(ctx)
- rater := logutil.TraceRateOver(logutil.MetricTableCreatedCounter)
- workers := tidbutil.NewWorkerPool(uint(len(rc.dbPool)), "Create Tables Worker")
- numOfTables := len(tables)
-
- for lastSent := 0; lastSent < numOfTables; lastSent += int(rc.batchDdlSize) {
- end := min(lastSent+int(rc.batchDdlSize), len(tables))
- log.Info("create tables", zap.Int("table start", lastSent), zap.Int("table end", end))
-
- tableSlice := tables[lastSent:end]
- workers.ApplyWithIDInErrorGroup(eg, func(id uint64) error {
- db := rc.dbPool[id%uint64(len(rc.dbPool))]
- cts, err := rc.createTables(ectx, db, dom, tableSlice, newTS) // ddl job for [lastSent:i)
- failpoint.Inject("restore-createtables-error", func(val failpoint.Value) {
- if val.(bool) {
- err = errors.New("sample error without extra message")
- }
- })
- if err != nil {
- log.Error("create tables fail", zap.Error(err))
- return err
- }
- for _, ct := range cts {
- log.Debug("table created and send to next",
- zap.Int("output chan size", len(outCh)),
- zap.Stringer("table", ct.OldTable.Info.Name),
- zap.Stringer("database", ct.OldTable.DB.Name))
- outCh <- ct
- rater.Inc()
- rater.L().Info("table created",
- zap.Stringer("table", ct.OldTable.Info.Name),
- zap.Stringer("database", ct.OldTable.DB.Name))
- }
- return err
- })
- }
- return eg.Wait()
-}
-
-// NeedCheckFreshCluster is every time. except restore from a checkpoint or user has not set filter argument.
-func (rc *Client) NeedCheckFreshCluster(ExplicitFilter bool, firstRun bool) bool {
- return rc.IsFull() && !ExplicitFilter && firstRun
-}
-
-// CheckTargetClusterFresh check whether the target cluster is fresh or not
-// if there's no user dbs or tables, we take it as a fresh cluster, although
-// user may have created some users or made other changes.
-func (rc *Client) CheckTargetClusterFresh(ctx context.Context) error {
- log.Info("checking whether target cluster is fresh")
- userDBs := GetExistedUserDBs(rc.dom)
- if len(userDBs) == 0 {
- return nil
- }
-
- const maxPrintCount = 10
- userTableOrDBNames := make([]string, 0, maxPrintCount+1)
- addName := func(name string) bool {
- if len(userTableOrDBNames) == maxPrintCount {
- userTableOrDBNames = append(userTableOrDBNames, "...")
- return false
- }
- userTableOrDBNames = append(userTableOrDBNames, name)
- return true
- }
-outer:
- for _, db := range userDBs {
- if !addName(db.Name.L) {
- break outer
- }
- for _, tbl := range db.Tables {
- if !addName(tbl.Name.L) {
- break outer
- }
- }
- }
- log.Error("not fresh cluster", zap.Strings("user tables", userTableOrDBNames))
- return errors.Annotate(berrors.ErrRestoreNotFreshCluster, "user db/tables: "+strings.Join(userTableOrDBNames, ", "))
-}
-
-func (rc *Client) CheckSysTableCompatibility(dom *domain.Domain, tables []*metautil.Table) error {
- log.Info("checking target cluster system table compatibility with backed up data")
- privilegeTablesInBackup := make([]*metautil.Table, 0)
- for _, table := range tables {
- decodedSysDBName, ok := utils.GetSysDBCIStrName(table.DB.Name)
- if ok && decodedSysDBName.L == mysql.SystemDB && sysPrivilegeTableMap[table.Info.Name.L] != "" {
- privilegeTablesInBackup = append(privilegeTablesInBackup, table)
- }
- }
- sysDB := model.NewCIStr(mysql.SystemDB)
- for _, table := range privilegeTablesInBackup {
- ti, err := rc.GetTableSchema(dom, sysDB, table.Info.Name)
- if err != nil {
- log.Error("missing table on target cluster", zap.Stringer("table", table.Info.Name))
- return errors.Annotate(berrors.ErrRestoreIncompatibleSys, "missed system table: "+table.Info.Name.O)
- }
- backupTi := table.Info
- // skip checking the number of columns in mysql.user table,
- // because higher versions of TiDB may add new columns.
- if len(ti.Columns) != len(backupTi.Columns) && backupTi.Name.L != sysUserTableName {
- log.Error("column count mismatch",
- zap.Stringer("table", table.Info.Name),
- zap.Int("col in cluster", len(ti.Columns)),
- zap.Int("col in backup", len(backupTi.Columns)))
- return errors.Annotatef(berrors.ErrRestoreIncompatibleSys,
- "column count mismatch, table: %s, col in cluster: %d, col in backup: %d",
- table.Info.Name.O, len(ti.Columns), len(backupTi.Columns))
- }
- backupColMap := make(map[string]*model.ColumnInfo)
- for i := range backupTi.Columns {
- col := backupTi.Columns[i]
- backupColMap[col.Name.L] = col
- }
- // order can be different but type must compatible
- for i := range ti.Columns {
- col := ti.Columns[i]
- backupCol := backupColMap[col.Name.L]
- if backupCol == nil {
- // skip when the backed up mysql.user table is missing columns.
- if backupTi.Name.L == sysUserTableName {
- log.Warn("missing column in backup data",
- zap.Stringer("table", table.Info.Name),
- zap.String("col", fmt.Sprintf("%s %s", col.Name, col.FieldType.String())))
- continue
- }
- log.Error("missing column in backup data",
- zap.Stringer("table", table.Info.Name),
- zap.String("col", fmt.Sprintf("%s %s", col.Name, col.FieldType.String())))
- return errors.Annotatef(berrors.ErrRestoreIncompatibleSys,
- "missing column in backup data, table: %s, col: %s %s",
- table.Info.Name.O,
- col.Name, col.FieldType.String())
- }
- if !utils.IsTypeCompatible(backupCol.FieldType, col.FieldType) {
- log.Error("incompatible column",
- zap.Stringer("table", table.Info.Name),
- zap.String("col in cluster", fmt.Sprintf("%s %s", col.Name, col.FieldType.String())),
- zap.String("col in backup", fmt.Sprintf("%s %s", backupCol.Name, backupCol.FieldType.String())))
- return errors.Annotatef(berrors.ErrRestoreIncompatibleSys,
- "incompatible column, table: %s, col in cluster: %s %s, col in backup: %s %s",
- table.Info.Name.O,
- col.Name, col.FieldType.String(),
- backupCol.Name, backupCol.FieldType.String())
- }
- }
-
- if backupTi.Name.L == sysUserTableName {
- // check whether the columns of table in cluster are less than the backup data
- clusterColMap := make(map[string]*model.ColumnInfo)
- for i := range ti.Columns {
- col := ti.Columns[i]
- clusterColMap[col.Name.L] = col
- }
- // order can be different
- for i := range backupTi.Columns {
- col := backupTi.Columns[i]
- clusterCol := clusterColMap[col.Name.L]
- if clusterCol == nil {
- log.Error("missing column in cluster data",
- zap.Stringer("table", table.Info.Name),
- zap.String("col", fmt.Sprintf("%s %s", col.Name, col.FieldType.String())))
- return errors.Annotatef(berrors.ErrRestoreIncompatibleSys,
- "missing column in cluster data, table: %s, col: %s %s",
- table.Info.Name.O,
- col.Name, col.FieldType.String())
- }
- }
- }
- }
- return nil
-}
-
-// ExecDDLs executes the queries of the ddl jobs.
-func (rc *Client) ExecDDLs(ctx context.Context, ddlJobs []*model.Job) error {
- // Sort the ddl jobs by schema version in ascending order.
- slices.SortFunc(ddlJobs, func(i, j *model.Job) int {
- return cmp.Compare(i.BinlogInfo.SchemaVersion, j.BinlogInfo.SchemaVersion)
- })
-
- for _, job := range ddlJobs {
- err := rc.db.ExecDDL(ctx, job)
- if err != nil {
- return errors.Trace(err)
- }
- log.Info("execute ddl query",
- zap.String("db", job.SchemaName),
- zap.String("query", job.Query),
- zap.Int64("historySchemaVersion", job.BinlogInfo.SchemaVersion))
- }
- return nil
-}
-
-// Mock the call of setSpeedLimit function
-func MockCallSetSpeedLimit(ctx context.Context, fakeImportClient fileimporter.ImporterClient, rc *Client, concurrency uint) error {
- rc.SetRateLimit(42)
- rc.SetConcurrency(concurrency)
- rc.hasSpeedLimited = false
- rc.fileImporter = fileimporter.NewFileImporter(nil, fakeImportClient, nil, false, false, nil, rc.rewriteMode, 128, false)
- return rc.setSpeedLimit(ctx, rc.rateLimit)
-}
-
-func (rc *Client) ResetSpeedLimit(ctx context.Context) error {
- rc.hasSpeedLimited = false
- err := rc.setSpeedLimit(ctx, 0)
- if err != nil {
- return errors.Trace(err)
- }
- return nil
-}
-
-func (rc *Client) setSpeedLimit(ctx context.Context, rateLimit uint64) error {
- if !rc.hasSpeedLimited {
- stores, err := util.GetAllTiKVStores(ctx, rc.pdClient, util.SkipTiFlash)
- if err != nil {
- return errors.Trace(err)
- }
-
- eg, ectx := errgroup.WithContext(ctx)
- for _, store := range stores {
- if err := ectx.Err(); err != nil {
- return errors.Trace(err)
- }
-
- finalStore := store
- rc.workerPool.ApplyOnErrorGroup(eg,
- func() error {
- err := rc.fileImporter.SetDownloadSpeedLimit(ectx, finalStore.GetId(), rateLimit)
- if err != nil {
- return errors.Trace(err)
- }
- return nil
- })
- }
-
- if err := eg.Wait(); err != nil {
- return errors.Trace(err)
- }
- rc.hasSpeedLimited = true
- }
- return nil
-}
-
-func getFileRangeKey(f string) string {
- // the backup date file pattern is `{store_id}_{region_id}_{epoch_version}_{key}_{ts}_{cf}.sst`
- // so we need to compare with out the `_{cf}.sst` suffix
- idx := strings.LastIndex(f, "_")
- if idx < 0 {
- panic(fmt.Sprintf("invalid backup data file name: '%s'", f))
- }
-
- return f[:idx]
-}
-
-// isFilesBelongToSameRange check whether two files are belong to the same range with different cf.
-func isFilesBelongToSameRange(f1, f2 string) bool {
- return getFileRangeKey(f1) == getFileRangeKey(f2)
-}
-
-func drainFilesByRange(files []*backuppb.File) ([]*backuppb.File, []*backuppb.File) {
- if len(files) == 0 {
- return nil, nil
- }
- idx := 1
- for idx < len(files) {
- if !isFilesBelongToSameRange(files[idx-1].Name, files[idx].Name) {
- break
- }
- idx++
- }
-
- return files[:idx], files[idx:]
-}
-
-func getGroupFiles(files []*backuppb.File, supportMulti bool) [][]*backuppb.File {
- if len(files) == 0 {
- return nil
- }
- if !supportMulti {
- newFiles := make([][]*backuppb.File, 0, len(files))
- for _, file := range files {
- newFiles = append(newFiles, []*backuppb.File{file})
- }
- return newFiles
- }
- return [][]*backuppb.File{files}
-}
-
-// SplitRanges implements TiKVRestorer.
-func (rc *Client) SplitRanges(ctx context.Context,
- ranges []rtree.Range,
- updateCh glue.Progress,
- isRawKv bool) error {
- return SplitRanges(ctx, rc, ranges, updateCh, isRawKv)
-}
-
-func (rc *Client) WrapLogFilesIterWithSplitHelper(logIter logrestore.LogIter, rules map[int64]*restoreutils.RewriteRules, g glue.Glue, store kv.Storage) (logrestore.LogIter, error) {
- se, err := g.CreateSession(store)
- if err != nil {
- return nil, errors.Trace(err)
- }
- execCtx := se.GetSessionCtx().GetRestrictedSQLExecutor()
- splitSize, splitKeys := utils.GetRegionSplitInfo(execCtx)
- log.Info("get split threshold from tikv config", zap.Uint64("split-size", splitSize), zap.Int64("split-keys", splitKeys))
- client := split.NewClient(rc.GetPDClient(), rc.pdHTTPClient, rc.GetTLSConfig(), maxSplitKeysOnce, 3)
- return logrestore.NewLogFilesIterWithSplitHelper(logIter, rules, client, splitSize, splitKeys), nil
-}
-
-func (rc *Client) generateKvFilesSkipMap(ctx context.Context, downstreamIdset map[int64]struct{}, taskName string) (*restoreutils.LogFilesSkipMap, error) {
- skipMap := restoreutils.NewLogFilesSkipMap()
- t, err := checkpoint.WalkCheckpointFileForRestore(ctx, rc.storage, rc.cipher, taskName, func(groupKey checkpoint.LogRestoreKeyType, off checkpoint.LogRestoreValueMarshaled) {
- for tableID, foffs := range off.Foffs {
- // filter out the checkpoint data of dropped table
- if _, exists := downstreamIdset[tableID]; exists {
- for _, foff := range foffs {
- skipMap.Insert(groupKey, off.Goff, foff)
- }
- }
- }
- })
- if err != nil {
- return nil, errors.Trace(err)
- }
- summary.AdjustStartTimeToEarlierTime(t)
- return skipMap, nil
-}
-
-func (rc *Client) WrapLogFilesIterWithCheckpoint(
- ctx context.Context,
- logIter logrestore.LogIter,
- downstreamIdset map[int64]struct{},
- taskName string,
- updateStats func(kvCount, size uint64),
- onProgress func(),
-) (logrestore.LogIter, error) {
- skipMap, err := rc.generateKvFilesSkipMap(ctx, downstreamIdset, taskName)
- if err != nil {
- return nil, errors.Trace(err)
- }
- return iter.FilterOut(logIter, func(d *logrestore.LogDataFileInfo) bool {
- if skipMap.NeedSkip(d.MetaDataGroupName, d.OffsetInMetaGroup, d.OffsetInMergedGroup) {
- onProgress()
- updateStats(uint64(d.NumberOfEntries), d.Length)
- return true
- }
- return false
- }), nil
-}
-
-// RestoreSSTFiles tries to restore the files.
-func (rc *Client) RestoreSSTFiles(
- ctx context.Context,
- tableIDWithFiles []TableIDWithFiles,
- updateCh glue.Progress,
-) (err error) {
- start := time.Now()
- fileCount := 0
- defer func() {
- elapsed := time.Since(start)
- if err == nil {
- log.Info("Restore files", zap.Duration("take", elapsed), zapTableIDWithFiles(tableIDWithFiles))
- summary.CollectSuccessUnit("files", fileCount, elapsed)
- }
- }()
-
- log.Debug("start to restore files", zap.Int("files", fileCount))
-
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("Client.RestoreSSTFiles", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
-
- eg, ectx := errgroup.WithContext(ctx)
- err = rc.setSpeedLimit(ctx, rc.rateLimit)
- if err != nil {
- return errors.Trace(err)
- }
-
- runner := rc.GetCheckpointRunner()
-
- var rangeFiles []*backuppb.File
- var leftFiles []*backuppb.File
-LOOPFORTABLE:
- for _, tableIDWithFile := range tableIDWithFiles {
- tableID := tableIDWithFile.TableID
- files := tableIDWithFile.Files
- rules := tableIDWithFile.RewriteRules
- fileCount += len(files)
- for rangeFiles, leftFiles = drainFilesByRange(files); len(rangeFiles) != 0; rangeFiles, leftFiles = drainFilesByRange(leftFiles) {
- filesReplica := rangeFiles
- if ectx.Err() != nil {
- log.Warn("Restoring encountered error and already stopped, give up remained files.",
- zap.Int("remained", len(leftFiles)),
- logutil.ShortError(ectx.Err()))
- // We will fetch the error from the errgroup then (If there were).
- // Also note if the parent context has been canceled or something,
- // breaking here directly is also a reasonable behavior.
- break LOOPFORTABLE
- }
- restoreFn := func() error {
- filesGroups := getGroupFiles(filesReplica, rc.fileImporter.SupportMultiIngest)
- for _, filesGroup := range filesGroups {
- if importErr := func(fs []*backuppb.File) (err error) {
- fileStart := time.Now()
- defer func() {
- if err == nil {
- log.Info("import files done", logutil.Files(filesGroup),
- zap.Duration("take", time.Since(fileStart)))
- updateCh.Inc()
- }
- }()
- return rc.fileImporter.ImportSSTFiles(ectx, fs, rules, rc.cipher, rc.dom.Store().GetCodec().GetAPIVersion())
- }(filesGroup); importErr != nil {
- return errors.Trace(importErr)
- }
- }
-
- // the data of this range has been import done
- if runner != nil && len(filesReplica) > 0 {
- rangeKey := getFileRangeKey(filesReplica[0].Name)
- // The checkpoint range shows this ranges of kvs has been restored into
- // the table corresponding to the table-id.
- if err := checkpoint.AppendRangesForRestore(ectx, runner, tableID, rangeKey); err != nil {
- return errors.Trace(err)
- }
- }
- return nil
- }
- if rc.granularity == string(CoarseGrained) {
- rc.fileImporter.WaitUntilUnblock()
- rc.workerPool.ApplyOnErrorGroup(eg, restoreFn)
- } else {
- // if we are not use coarse granularity which means
- // we still pipeline split & scatter regions and import sst files
- // just keep the consistency as before.
- rc.workerPool.ApplyOnErrorGroup(eg, restoreFn)
- }
- }
- }
-
- if err := eg.Wait(); err != nil {
- summary.CollectFailureUnit("file", err)
- log.Error(
- "restore files failed",
- zap.Error(err),
- )
- return errors.Trace(err)
- }
- // Once the parent context canceled and there is no task running in the errgroup,
- // we may break the for loop without error in the errgroup. (Will this happen?)
- // At that time, return the error in the context here.
- return ctx.Err()
-}
-
-func (rc *Client) WaitForFilesRestored(ctx context.Context, files []*backuppb.File, updateCh glue.Progress) error {
- errCh := make(chan error, len(files))
- eg, ectx := errgroup.WithContext(ctx)
- defer close(errCh)
-
- for _, file := range files {
- fileReplica := file
- rc.workerPool.ApplyOnErrorGroup(eg,
- func() error {
- defer func() {
- log.Info("import sst files done", logutil.Files(files))
- updateCh.Inc()
- }()
- return rc.fileImporter.ImportSSTFiles(ectx, []*backuppb.File{fileReplica}, restoreutils.EmptyRewriteRule(), rc.cipher, rc.backupMeta.ApiVersion)
- })
- }
- if err := eg.Wait(); err != nil {
- return errors.Trace(err)
- }
- return nil
-}
-
-// RestoreRaw tries to restore raw keys in the specified range.
-func (rc *Client) RestoreRaw(
- ctx context.Context, startKey []byte, endKey []byte, files []*backuppb.File, updateCh glue.Progress,
-) error {
- start := time.Now()
- defer func() {
- elapsed := time.Since(start)
- log.Info("Restore Raw",
- logutil.Key("startKey", startKey),
- logutil.Key("endKey", endKey),
- zap.Duration("take", elapsed))
- }()
- err := rc.fileImporter.SetRawRange(startKey, endKey)
- if err != nil {
- return errors.Trace(err)
- }
-
- err = rc.WaitForFilesRestored(ctx, files, updateCh)
- if err != nil {
- return errors.Trace(err)
- }
- log.Info(
- "finish to restore raw range",
- logutil.Key("startKey", startKey),
- logutil.Key("endKey", endKey),
- )
- return nil
-}
-
-// SwitchToImportMode switch tikv cluster to import mode.
-func (rc *Client) SwitchToImportMode(ctx context.Context) {
- // tikv automatically switch to normal mode in every 10 minutes
- // so we need ping tikv in less than 10 minute
- go func() {
- tick := time.NewTicker(rc.switchModeInterval)
- defer tick.Stop()
-
- // [important!] switch tikv mode into import at the beginning
- log.Info("switch to import mode at beginning")
- err := rc.switchTiKVMode(ctx, import_sstpb.SwitchMode_Import)
- if err != nil {
- log.Warn("switch to import mode failed", zap.Error(err))
- }
-
- for {
- select {
- case <-ctx.Done():
- return
- case <-tick.C:
- log.Info("switch to import mode")
- err := rc.switchTiKVMode(ctx, import_sstpb.SwitchMode_Import)
- if err != nil {
- log.Warn("switch to import mode failed", zap.Error(err))
- }
- case <-rc.switchCh:
- log.Info("stop automatic switch to import mode")
- return
- }
- }
- }()
-}
-
-// SwitchToNormalMode switch tikv cluster to normal mode.
-func (rc *Client) SwitchToNormalMode(ctx context.Context) error {
- close(rc.switchCh)
- return rc.switchTiKVMode(ctx, import_sstpb.SwitchMode_Normal)
-}
-
-func (rc *Client) switchTiKVMode(ctx context.Context, mode import_sstpb.SwitchMode) error {
- stores, err := util.GetAllTiKVStores(ctx, rc.pdClient, util.SkipTiFlash)
- if err != nil {
- return errors.Trace(err)
- }
- bfConf := backoff.DefaultConfig
- bfConf.MaxDelay = time.Second * 3
-
- eg, ectx := errgroup.WithContext(ctx)
- for _, store := range stores {
- if err := ectx.Err(); err != nil {
- return errors.Trace(err)
- }
-
- finalStore := store
- rc.workerPool.ApplyOnErrorGroup(eg,
- func() error {
- opt := grpc.WithTransportCredentials(insecure.NewCredentials())
- if rc.tlsConf != nil {
- opt = grpc.WithTransportCredentials(credentials.NewTLS(rc.tlsConf))
- }
- gctx, cancel := context.WithTimeout(ectx, time.Second*5)
- connection, err := grpc.DialContext(
- gctx,
- finalStore.GetAddress(),
- opt,
- grpc.WithBlock(),
- grpc.FailOnNonTempDialError(true),
- grpc.WithConnectParams(grpc.ConnectParams{Backoff: bfConf}),
- // we don't need to set keepalive timeout here, because the connection lives
- // at most 5s. (shorter than minimal value for keepalive time!)
- )
- cancel()
- if err != nil {
- return errors.Trace(err)
- }
- client := import_sstpb.NewImportSSTClient(connection)
- _, err = client.SwitchMode(ctx, &import_sstpb.SwitchModeRequest{
- Mode: mode,
- })
- if err != nil {
- return errors.Trace(err)
- }
- err = connection.Close()
- if err != nil {
- log.Error("close grpc connection failed in switch mode", zap.Error(err))
- }
- return nil
- })
- }
-
- if err = eg.Wait(); err != nil {
- return errors.Trace(err)
- }
- return nil
-}
-
-func concurrentHandleTablesCh(
- ctx context.Context,
- inCh <-chan *CreatedTable,
- outCh chan<- *CreatedTable,
- errCh chan<- error,
- workers *tidbutil.WorkerPool,
- processFun func(context.Context, *CreatedTable) error,
- deferFun func()) {
- eg, ectx := errgroup.WithContext(ctx)
- defer func() {
- if err := eg.Wait(); err != nil {
- errCh <- err
- }
- close(outCh)
- deferFun()
- }()
-
- for {
- select {
- // if we use ectx here, maybe canceled will mask real error.
- case <-ctx.Done():
- errCh <- ctx.Err()
- case tbl, ok := <-inCh:
- if !ok {
- return
- }
- cloneTable := tbl
- worker := workers.ApplyWorker()
- eg.Go(func() error {
- defer workers.RecycleWorker(worker)
- err := processFun(ectx, cloneTable)
- if err != nil {
- return err
- }
- outCh <- cloneTable
- return nil
- })
- }
- }
-}
-
-// GoValidateChecksum forks a goroutine to validate checksum after restore.
-// it returns a channel fires a struct{} when all things get done.
-func (rc *Client) GoValidateChecksum(
- ctx context.Context,
- inCh <-chan *CreatedTable,
- kvClient kv.Client,
- errCh chan<- error,
- updateCh glue.Progress,
- concurrency uint,
-) chan *CreatedTable {
- log.Info("Start to validate checksum")
- outCh := DefaultOutputTableChan()
- workers := tidbutil.NewWorkerPool(defaultChecksumConcurrency, "RestoreChecksum")
- go concurrentHandleTablesCh(ctx, inCh, outCh, errCh, workers, func(c context.Context, tbl *CreatedTable) error {
- start := time.Now()
- defer func() {
- elapsed := time.Since(start)
- summary.CollectSuccessUnit("table checksum", 1, elapsed)
- }()
- err := rc.execChecksum(c, tbl, kvClient, concurrency)
- if err != nil {
- return errors.Trace(err)
- }
- updateCh.Inc()
- return nil
- }, func() {
- log.Info("all checksum ended")
- })
- return outCh
-}
-
-func (rc *Client) execChecksum(
- ctx context.Context,
- tbl *CreatedTable,
- kvClient kv.Client,
- concurrency uint,
-) error {
- logger := log.L().With(
- zap.String("db", tbl.OldTable.DB.Name.O),
- zap.String("table", tbl.OldTable.Info.Name.O),
- )
-
- if tbl.OldTable.NoChecksum() {
- logger.Warn("table has no checksum, skipping checksum")
- return nil
- }
-
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("Client.execChecksum", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
-
- item, exists := rc.checkpointChecksum[tbl.Table.ID]
- if !exists {
- startTS, err := rc.GetTSWithRetry(ctx)
- if err != nil {
- return errors.Trace(err)
- }
- exe, err := checksum.NewExecutorBuilder(tbl.Table, startTS).
- SetOldTable(tbl.OldTable).
- SetConcurrency(concurrency).
- SetOldKeyspace(tbl.RewriteRule.OldKeyspace).
- SetNewKeyspace(tbl.RewriteRule.NewKeyspace).
- SetExplicitRequestSourceType(kvutil.ExplicitTypeBR).
- Build()
- if err != nil {
- return errors.Trace(err)
- }
- checksumResp, err := exe.Execute(ctx, kvClient, func() {
- // TODO: update progress here.
- })
- if err != nil {
- return errors.Trace(err)
- }
- item = &checkpoint.ChecksumItem{
- TableID: tbl.Table.ID,
- Crc64xor: checksumResp.Checksum,
- TotalKvs: checksumResp.TotalKvs,
- TotalBytes: checksumResp.TotalBytes,
- }
- if rc.checkpointRunner != nil {
- err = rc.checkpointRunner.FlushChecksumItem(ctx, item)
- if err != nil {
- return errors.Trace(err)
- }
- }
- }
- table := tbl.OldTable
- if item.Crc64xor != table.Crc64Xor ||
- item.TotalKvs != table.TotalKvs ||
- item.TotalBytes != table.TotalBytes {
- logger.Error("failed in validate checksum",
- zap.Uint64("origin tidb crc64", table.Crc64Xor),
- zap.Uint64("calculated crc64", item.Crc64xor),
- zap.Uint64("origin tidb total kvs", table.TotalKvs),
- zap.Uint64("calculated total kvs", item.TotalKvs),
- zap.Uint64("origin tidb total bytes", table.TotalBytes),
- zap.Uint64("calculated total bytes", item.TotalBytes),
- )
- return errors.Annotate(berrors.ErrRestoreChecksumMismatch, "failed to validate checksum")
- }
- logger.Info("success in validate checksum")
- return nil
-}
-
-func (rc *Client) GoUpdateMetaAndLoadStats(
- ctx context.Context,
- s storage.ExternalStorage,
- cipher *backuppb.CipherInfo,
- inCh <-chan *CreatedTable,
- errCh chan<- error,
- statsConcurrency uint,
- loadStats bool,
-) chan *CreatedTable {
- log.Info("Start to update meta then load stats")
- outCh := DefaultOutputTableChan()
- workers := tidbutil.NewWorkerPool(statsConcurrency, "UpdateStats")
-
- go concurrentHandleTablesCh(ctx, inCh, outCh, errCh, workers, func(c context.Context, tbl *CreatedTable) error {
- oldTable := tbl.OldTable
- var statsErr error = nil
- if loadStats && oldTable.Stats != nil {
- log.Info("start loads analyze after validate checksum",
- zap.Int64("old id", oldTable.Info.ID),
- zap.Int64("new id", tbl.Table.ID),
- )
- start := time.Now()
- // NOTICE: skip updating cache after load stats from json
- if statsErr = rc.statsHandler.LoadStatsFromJSONNoUpdate(ctx, rc.dom.InfoSchema(), oldTable.Stats, 0); statsErr != nil {
- log.Error("analyze table failed", zap.Any("table", oldTable.Stats), zap.Error(statsErr))
- }
- log.Info("restore stat done",
- zap.Stringer("table", oldTable.Info.Name),
- zap.Stringer("db", oldTable.DB.Name),
- zap.Duration("cost", time.Since(start)))
- } else if loadStats && len(oldTable.StatsFileIndexes) > 0 {
- log.Info("start to load statistic data for each partition",
- zap.Int64("old id", oldTable.Info.ID),
- zap.Int64("new id", tbl.Table.ID),
- )
- start := time.Now()
- rewriteIDMap := restoreutils.GetTableIDMap(tbl.Table, tbl.OldTable.Info)
- if statsErr = metautil.RestoreStats(ctx, s, cipher, rc.statsHandler, tbl.Table, oldTable.StatsFileIndexes, rewriteIDMap); statsErr != nil {
- log.Error("analyze table failed", zap.Any("table", oldTable.StatsFileIndexes), zap.Error(statsErr))
- }
- log.Info("restore statistic data done",
- zap.Stringer("table", oldTable.Info.Name),
- zap.Stringer("db", oldTable.DB.Name),
- zap.Duration("cost", time.Since(start)))
- }
-
- if statsErr != nil || !loadStats || (oldTable.Stats == nil && len(oldTable.StatsFileIndexes) == 0) {
- // Not need to return err when failed because of update analysis-meta
- log.Info("start update metas", zap.Stringer("table", oldTable.Info.Name), zap.Stringer("db", oldTable.DB.Name))
- // the total kvs contains the index kvs, but the stats meta needs the count of rows
- count := int64(oldTable.TotalKvs / uint64(len(oldTable.Info.Indices)+1))
- if statsErr = rc.statsHandler.SaveMetaToStorage(tbl.Table.ID, count, 0, "br restore"); statsErr != nil {
- log.Error("update stats meta failed", zap.Any("table", tbl.Table), zap.Error(statsErr))
- }
- }
- return nil
- }, func() {
- log.Info("all stats updated")
- })
- return outCh
-}
-
-func (rc *Client) GoWaitTiFlashReady(ctx context.Context, inCh <-chan *CreatedTable, updateCh glue.Progress, errCh chan<- error) chan *CreatedTable {
- log.Info("Start to wait tiflash replica sync")
- outCh := DefaultOutputTableChan()
- workers := tidbutil.NewWorkerPool(4, "WaitForTiflashReady")
- // TODO support tiflash store changes
- tikvStats, err := infosync.GetTiFlashStoresStat(context.Background())
- if err != nil {
- errCh <- err
- }
- tiFlashStores := make(map[int64]pdhttp.StoreInfo)
- for _, store := range tikvStats.Stores {
- if engine.IsTiFlashHTTPResp(&store.Store) {
- tiFlashStores[store.Store.ID] = store
- }
- }
- go concurrentHandleTablesCh(ctx, inCh, outCh, errCh, workers, func(c context.Context, tbl *CreatedTable) error {
- if tbl.Table != nil && tbl.Table.TiFlashReplica == nil {
- log.Info("table has no tiflash replica",
- zap.Stringer("table", tbl.OldTable.Info.Name),
- zap.Stringer("db", tbl.OldTable.DB.Name))
- updateCh.Inc()
- return nil
- }
- if rc.dom == nil {
- // unreachable, current we have initial domain in mgr.
- log.Fatal("unreachable, domain is nil")
- }
- log.Info("table has tiflash replica, start sync..",
- zap.Stringer("table", tbl.OldTable.Info.Name),
- zap.Stringer("db", tbl.OldTable.DB.Name))
- for {
- var progress float64
- if pi := tbl.Table.GetPartitionInfo(); pi != nil && len(pi.Definitions) > 0 {
- for _, p := range pi.Definitions {
- progressOfPartition, err := infosync.MustGetTiFlashProgress(p.ID, tbl.Table.TiFlashReplica.Count, &tiFlashStores)
- if err != nil {
- log.Warn("failed to get progress for tiflash partition replica, retry it",
- zap.Int64("tableID", tbl.Table.ID), zap.Int64("partitionID", p.ID), zap.Error(err))
- time.Sleep(time.Second)
- continue
- }
- progress += progressOfPartition
- }
- progress = progress / float64(len(pi.Definitions))
- } else {
- var err error
- progress, err = infosync.MustGetTiFlashProgress(tbl.Table.ID, tbl.Table.TiFlashReplica.Count, &tiFlashStores)
- if err != nil {
- log.Warn("failed to get progress for tiflash replica, retry it",
- zap.Int64("tableID", tbl.Table.ID), zap.Error(err))
- time.Sleep(time.Second)
- continue
- }
- }
- // check until progress is 1
- if progress == 1 {
- log.Info("tiflash replica synced",
- zap.Stringer("table", tbl.OldTable.Info.Name),
- zap.Stringer("db", tbl.OldTable.DB.Name))
- break
- }
- // just wait for next check
- // tiflash check the progress every 2s
- // we can wait 2.5x times
- time.Sleep(5 * time.Second)
- }
- updateCh.Inc()
- return nil
- }, func() {
- log.Info("all tiflash replica synced")
- })
- return outCh
-}
-
-// called by failpoint, only used for test
-// it would print the checksum result into the log, and
-// the auto-test script records them to compare another
-// cluster's checksum.
-func (rc *Client) FailpointDoChecksumForLogRestore(
- ctx context.Context,
- kvClient kv.Client,
- pdClient pd.Client,
- idrules map[int64]int64,
- rewriteRules map[int64]*restoreutils.RewriteRules,
-) (finalErr error) {
- startTS, err := rc.GetTSWithRetry(ctx)
- if err != nil {
- return errors.Trace(err)
- }
- // set gc safepoint for checksum
- sp := utils.BRServiceSafePoint{
- BackupTS: startTS,
- TTL: utils.DefaultBRGCSafePointTTL,
- ID: utils.MakeSafePointID(),
- }
- cctx, gcSafePointKeeperCancel := context.WithCancel(ctx)
- defer func() {
- log.Info("start to remove gc-safepoint keeper")
- // close the gc safe point keeper at first
- gcSafePointKeeperCancel()
- // set the ttl to 0 to remove the gc-safe-point
- sp.TTL = 0
- if err := utils.UpdateServiceSafePoint(ctx, pdClient, sp); err != nil {
- log.Warn("failed to update service safe point, backup may fail if gc triggered",
- zap.Error(err),
- )
- }
- log.Info("finish removing gc-safepoint keeper")
- }()
- err = utils.StartServiceSafePointKeeper(cctx, pdClient, sp)
- if err != nil {
- return errors.Trace(err)
- }
-
- eg, ectx := errgroup.WithContext(ctx)
- pool := tidbutil.NewWorkerPool(4, "checksum for log restore")
- infoSchema := rc.GetDomain().InfoSchema()
- // downstream id -> upstream id
- reidRules := make(map[int64]int64)
- for upstreamID, downstreamID := range idrules {
- reidRules[downstreamID] = upstreamID
- }
- for upstreamID, downstreamID := range idrules {
- newTable, ok := infoSchema.TableByID(downstreamID)
- if !ok {
- // a dropped table
- continue
- }
- rewriteRule, ok := rewriteRules[upstreamID]
- if !ok {
- continue
- }
- newTableInfo := newTable.Meta()
- var definitions []model.PartitionDefinition
- if newTableInfo.Partition != nil {
- for _, def := range newTableInfo.Partition.Definitions {
- upid, ok := reidRules[def.ID]
- if !ok {
- log.Panic("no rewrite rule for parition table id", zap.Int64("id", def.ID))
- }
- definitions = append(definitions, model.PartitionDefinition{
- ID: upid,
- })
- }
- }
- oldPartition := &model.PartitionInfo{
- Definitions: definitions,
- }
- oldTable := &metautil.Table{
- Info: &model.TableInfo{
- ID: upstreamID,
- Indices: newTableInfo.Indices,
- Partition: oldPartition,
- },
- }
- pool.ApplyOnErrorGroup(eg, func() error {
- exe, err := checksum.NewExecutorBuilder(newTableInfo, startTS).
- SetOldTable(oldTable).
- SetConcurrency(4).
- SetOldKeyspace(rewriteRule.OldKeyspace).
- SetNewKeyspace(rewriteRule.NewKeyspace).
- SetExplicitRequestSourceType(kvutil.ExplicitTypeBR).
- Build()
- if err != nil {
- return errors.Trace(err)
- }
- checksumResp, err := exe.Execute(ectx, kvClient, func() {})
- if err != nil {
- return errors.Trace(err)
- }
- // print to log so that the test script can get final checksum
- log.Info("failpoint checksum completed",
- zap.String("table-name", newTableInfo.Name.O),
- zap.Int64("upstream-id", oldTable.Info.ID),
- zap.Uint64("checksum", checksumResp.Checksum),
- zap.Uint64("total-kvs", checksumResp.TotalKvs),
- zap.Uint64("total-bytes", checksumResp.TotalBytes),
- )
- return nil
- })
- }
-
- return eg.Wait()
-}
-
-const (
- restoreLabelKey = "exclusive"
- restoreLabelValue = "restore"
-)
-
-// LoadRestoreStores loads the stores used to restore data.
-func (rc *Client) LoadRestoreStores(ctx context.Context) error {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("Client.LoadRestoreStores", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
- stores, err := conn.GetAllTiKVStoresWithRetry(ctx, rc.pdClient, util.SkipTiFlash)
-
- if err != nil {
- return errors.Trace(err)
- }
- for _, s := range stores {
- if s.GetState() != metapb.StoreState_Up {
- continue
- }
- rc.storeCount++
- if rc.isOnline {
- for _, l := range s.GetLabels() {
- if l.GetKey() == restoreLabelKey && l.GetValue() == restoreLabelValue {
- rc.restoreStores = append(rc.restoreStores, s.GetId())
- break
- }
- }
- }
- }
- if rc.isOnline {
- log.Info("load restore stores", zap.Uint64s("store-ids", rc.restoreStores))
- }
- return nil
-}
-
-// ResetRestoreLabels removes the exclusive labels of the restore stores.
-func (rc *Client) ResetRestoreLabels(ctx context.Context) error {
- if !rc.isOnline {
- return nil
- }
- log.Info("start resetting store labels")
- return rc.toolClient.SetStoresLabel(ctx, rc.restoreStores, restoreLabelKey, "")
-}
-
-// SetupPlacementRules sets rules for the tables' regions.
-func (rc *Client) SetupPlacementRules(ctx context.Context, tables []*model.TableInfo) error {
- if !rc.isOnline || len(rc.restoreStores) == 0 {
- return nil
- }
- log.Info("start setting placement rules")
- rule, err := rc.toolClient.GetPlacementRule(ctx, "pd", "default")
- if err != nil {
- return errors.Trace(err)
- }
- rule.Index = 100
- rule.Override = true
- rule.LabelConstraints = append(rule.LabelConstraints, pdhttp.LabelConstraint{
- Key: restoreLabelKey,
- Op: "in",
- Values: []string{restoreLabelValue},
- })
- for _, t := range tables {
- rule.ID = rc.getRuleID(t.ID)
- rule.StartKeyHex = hex.EncodeToString(codec.EncodeBytes([]byte{}, tablecodec.EncodeTablePrefix(t.ID)))
- rule.EndKeyHex = hex.EncodeToString(codec.EncodeBytes([]byte{}, tablecodec.EncodeTablePrefix(t.ID+1)))
- err = rc.toolClient.SetPlacementRule(ctx, rule)
- if err != nil {
- return errors.Trace(err)
- }
- }
- log.Info("finish setting placement rules")
- return nil
-}
-
-// WaitPlacementSchedule waits PD to move tables to restore stores.
-func (rc *Client) WaitPlacementSchedule(ctx context.Context, tables []*model.TableInfo) error {
- if !rc.isOnline || len(rc.restoreStores) == 0 {
- return nil
- }
- log.Info("start waiting placement schedule")
- ticker := time.NewTicker(time.Second * 10)
- defer ticker.Stop()
- for {
- select {
- case <-ticker.C:
- ok, progress, err := rc.checkRegions(ctx, tables)
- if err != nil {
- return errors.Trace(err)
- }
- if ok {
- log.Info("finish waiting placement schedule")
- return nil
- }
- log.Info("placement schedule progress: " + progress)
- case <-ctx.Done():
- return ctx.Err()
- }
- }
-}
-
-func (rc *Client) checkRegions(ctx context.Context, tables []*model.TableInfo) (bool, string, error) {
- for i, t := range tables {
- start := codec.EncodeBytes([]byte{}, tablecodec.EncodeTablePrefix(t.ID))
- end := codec.EncodeBytes([]byte{}, tablecodec.EncodeTablePrefix(t.ID+1))
- ok, regionProgress, err := rc.checkRange(ctx, start, end)
- if err != nil {
- return false, "", errors.Trace(err)
- }
- if !ok {
- return false, fmt.Sprintf("table %v/%v, %s", i, len(tables), regionProgress), nil
- }
- }
- return true, "", nil
-}
-
-func (rc *Client) checkRange(ctx context.Context, start, end []byte) (bool, string, error) {
- regions, err := rc.toolClient.ScanRegions(ctx, start, end, -1)
- if err != nil {
- return false, "", errors.Trace(err)
- }
- for i, r := range regions {
- NEXT_PEER:
- for _, p := range r.Region.GetPeers() {
- for _, storeID := range rc.restoreStores {
- if p.GetStoreId() == storeID {
- continue NEXT_PEER
- }
- }
- return false, fmt.Sprintf("region %v/%v", i, len(regions)), nil
- }
- }
- return true, "", nil
-}
-
-// ResetPlacementRules removes placement rules for tables.
-func (rc *Client) ResetPlacementRules(ctx context.Context, tables []*model.TableInfo) error {
- if !rc.isOnline || len(rc.restoreStores) == 0 {
- return nil
- }
- log.Info("start resetting placement rules")
- var failedTables []int64
- for _, t := range tables {
- err := rc.toolClient.DeletePlacementRule(ctx, "pd", rc.getRuleID(t.ID))
- if err != nil {
- log.Info("failed to delete placement rule for table", zap.Int64("table-id", t.ID))
- failedTables = append(failedTables, t.ID)
- }
- }
- if len(failedTables) > 0 {
- return errors.Annotatef(berrors.ErrPDInvalidResponse, "failed to delete placement rules for tables %v", failedTables)
- }
- return nil
-}
-
-func (rc *Client) getRuleID(tableID int64) string {
- return "restore-t" + strconv.FormatInt(tableID, 10)
-}
-
-// IsFull returns whether this backup is full.
-func (rc *Client) IsFull() bool {
- failpoint.Inject("mock-incr-backup-data", func() {
- failpoint.Return(false)
- })
- return !rc.IsIncremental()
-}
-
-// IsIncremental returns whether this backup is incremental.
-func (rc *Client) IsIncremental() bool {
- return !(rc.backupMeta.StartVersion == rc.backupMeta.EndVersion ||
- rc.backupMeta.StartVersion == 0)
-}
-
-// EnableSkipCreateSQL sets switch of skip create schema and tables.
-func (rc *Client) EnableSkipCreateSQL() {
- rc.noSchema = true
-}
-
-// IsSkipCreateSQL returns whether we need skip create schema and tables in restore.
-func (rc *Client) IsSkipCreateSQL() bool {
- return rc.noSchema
-}
-
-// GenerateRebasedTables generate a map[UniqueTableName]bool to represent tables that haven't updated table info.
-// there are two situations:
-// 1. tables that already exists in the restored cluster.
-// 2. tables that are created by executing ddl jobs.
-// so, only tables in incremental restoration will be added to the map
-func (rc *Client) GenerateRebasedTables(tables []*metautil.Table) {
- if !rc.IsIncremental() {
- // in full restoration, all tables are created by Session.CreateTable, and all tables' info is updated.
- rc.rebasedTablesMap = make(map[UniqueTableName]bool)
- return
- }
-
- rc.rebasedTablesMap = make(map[UniqueTableName]bool, len(tables))
- for _, table := range tables {
- rc.rebasedTablesMap[UniqueTableName{DB: table.DB.Name.String(), Table: table.Info.Name.String()}] = true
- }
-}
-
-// GetRebasedTables returns tables that may need to be rebase auto increment id or auto random id
-func (rc *Client) GetRebasedTables() map[UniqueTableName]bool {
- return rc.rebasedTablesMap
-}
-
-func (rc *Client) getTiFlashNodeCount(ctx context.Context) (uint64, error) {
- tiFlashStores, err := util.GetAllTiKVStores(ctx, rc.pdClient, util.TiFlashOnly)
- if err != nil {
- return 0, errors.Trace(err)
- }
- return uint64(len(tiFlashStores)), nil
-}
-
-// PreCheckTableTiFlashReplica checks whether TiFlash replica is less than TiFlash node.
-func (rc *Client) PreCheckTableTiFlashReplica(
- ctx context.Context,
- tables []*metautil.Table,
- recorder *tiflashrec.TiFlashRecorder,
-) error {
- tiFlashStoreCount, err := rc.getTiFlashNodeCount(ctx)
- if err != nil {
- return err
- }
- for _, table := range tables {
- if table.Info.TiFlashReplica != nil {
- // we should not set available to true. because we cannot guarantee the raft log lag of tiflash when restore finished.
- // just let tiflash ticker set it by checking lag of all related regions.
- table.Info.TiFlashReplica.Available = false
- table.Info.TiFlashReplica.AvailablePartitionIDs = nil
- if recorder != nil {
- recorder.AddTable(table.Info.ID, *table.Info.TiFlashReplica)
- log.Info("record tiflash replica for table, to reset it by ddl later",
- zap.Stringer("db", table.DB.Name),
- zap.Stringer("table", table.Info.Name),
- )
- table.Info.TiFlashReplica = nil
- } else if table.Info.TiFlashReplica.Count > tiFlashStoreCount {
- // we cannot satisfy TiFlash replica in restore cluster. so we should
- // set TiFlashReplica to unavailable in tableInfo, to avoid TiDB cannot sense TiFlash and make plan to TiFlash
- // see details at https://github.com/pingcap/br/issues/931
- // TODO maybe set table.Info.TiFlashReplica.Count to tiFlashStoreCount, but we need more tests about it.
- log.Warn("table does not satisfy tiflash replica requirements, set tiflash replcia to unavailable",
- zap.Stringer("db", table.DB.Name),
- zap.Stringer("table", table.Info.Name),
- zap.Uint64("expect tiflash replica", table.Info.TiFlashReplica.Count),
- zap.Uint64("actual tiflash store", tiFlashStoreCount),
- )
- table.Info.TiFlashReplica = nil
- }
- }
- }
- return nil
-}
-
-// PreCheckTableClusterIndex checks whether backup tables and existed tables have different cluster index options。
-func (rc *Client) PreCheckTableClusterIndex(
- tables []*metautil.Table,
- ddlJobs []*model.Job,
- dom *domain.Domain,
-) error {
- for _, table := range tables {
- oldTableInfo, err := rc.GetTableSchema(dom, table.DB.Name, table.Info.Name)
- // table exists in database
- if err == nil {
- if table.Info.IsCommonHandle != oldTableInfo.IsCommonHandle {
- return errors.Annotatef(berrors.ErrRestoreModeMismatch,
- "Clustered index option mismatch. Restored cluster's @@tidb_enable_clustered_index should be %v (backup table = %v, created table = %v).",
- transferBoolToValue(table.Info.IsCommonHandle),
- table.Info.IsCommonHandle,
- oldTableInfo.IsCommonHandle)
- }
- }
- }
- for _, job := range ddlJobs {
- if job.Type == model.ActionCreateTable {
- tableInfo := job.BinlogInfo.TableInfo
- if tableInfo != nil {
- oldTableInfo, err := rc.GetTableSchema(dom, model.NewCIStr(job.SchemaName), tableInfo.Name)
- // table exists in database
- if err == nil {
- if tableInfo.IsCommonHandle != oldTableInfo.IsCommonHandle {
- return errors.Annotatef(berrors.ErrRestoreModeMismatch,
- "Clustered index option mismatch. Restored cluster's @@tidb_enable_clustered_index should be %v (backup table = %v, created table = %v).",
- transferBoolToValue(tableInfo.IsCommonHandle),
- tableInfo.IsCommonHandle,
- oldTableInfo.IsCommonHandle)
- }
- }
- }
- }
- }
- return nil
-}
-
-func (rc *Client) InstallLogFileManager(ctx context.Context, startTS, restoreTS uint64, metadataDownloadBatchSize uint) error {
- init := logrestore.LogFileManagerInit{
- StartTS: startTS,
- RestoreTS: restoreTS,
- Storage: rc.storage,
-
- MetadataDownloadBatchSize: metadataDownloadBatchSize,
- }
- var err error
- rc.LogFileManager, err = logrestore.CreateLogFileManager(ctx, init)
- if err != nil {
- return err
- }
- return nil
-}
-
-// FixIndex tries to fix a single index.
-func (rc *Client) FixIndex(ctx context.Context, schema, table, index string) error {
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan(fmt.Sprintf("Client.LoadRestoreStores index: %s.%s:%s",
- schema, table, index), opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
-
- sql := fmt.Sprintf("ADMIN RECOVER INDEX %s %s;",
- utils.EncloseDBAndTable(schema, table),
- utils.EncloseName(index))
- log.Debug("Executing fix index sql.", zap.String("sql", sql))
- err := rc.db.se.Execute(ctx, sql)
- if err != nil {
- return errors.Annotatef(err, "failed to execute %s", sql)
- }
- return nil
-}
-
-// FixIndicesOfTables tries to fix the indices of the tables via `ADMIN RECOVERY INDEX`.
-func (rc *Client) FixIndicesOfTables(
- ctx context.Context,
- fullBackupTables map[int64]*metautil.Table,
- onProgress func(),
-) error {
- for _, table := range fullBackupTables {
- if name, ok := utils.GetSysDBName(table.DB.Name); utils.IsSysDB(name) && ok {
- // skip system table for now
- onProgress()
- continue
- }
-
- if err := rc.FixIndicesOfTable(ctx, table.DB.Name.L, table.Info); err != nil {
- return errors.Annotatef(err, "failed to fix index for table %s.%s", table.DB.Name, table.Info.Name)
- }
- onProgress()
- }
-
- return nil
-}
-
-// FixIndicdesOfTable tries to fix the indices of the table via `ADMIN RECOVERY INDEX`.
-func (rc *Client) FixIndicesOfTable(ctx context.Context, schema string, table *model.TableInfo) error {
- tableName := table.Name.L
- // NOTE: Maybe we can create multi sessions and restore indices concurrently?
- for _, index := range table.Indices {
- start := time.Now()
- if err := rc.FixIndex(ctx, schema, tableName, index.Name.L); err != nil {
- return errors.Annotatef(err, "failed to fix index %s", index.Name)
- }
-
- log.Info("Fix index done.", zap.Stringer("take", time.Since(start)),
- zap.String("table", schema+"."+tableName),
- zap.Stringer("index", index.Name))
- }
- return nil
-}
-
-type FilesInRegion struct {
- defaultSize uint64
- defaultKVCount int64
- writeSize uint64
- writeKVCount int64
-
- defaultFiles []*logrestore.LogDataFileInfo
- writeFiles []*logrestore.LogDataFileInfo
- deleteFiles []*logrestore.LogDataFileInfo
-}
-
-type FilesInTable struct {
- regionMapFiles map[int64]*FilesInRegion
-}
-
-func ApplyKVFilesWithBatchMethod(
- ctx context.Context,
- logIter logrestore.LogIter,
- batchCount int,
- batchSize uint64,
- applyFunc func(files []*logrestore.LogDataFileInfo, kvCount int64, size uint64),
- applyWg *sync.WaitGroup,
-) error {
- var (
- tableMapFiles = make(map[int64]*FilesInTable)
- tmpFiles = make([]*logrestore.LogDataFileInfo, 0, batchCount)
- tmpSize uint64 = 0
- tmpKVCount int64 = 0
- )
- for r := logIter.TryNext(ctx); !r.Finished; r = logIter.TryNext(ctx) {
- if r.Err != nil {
- return r.Err
- }
-
- f := r.Item
- if f.GetType() == backuppb.FileType_Put && f.GetLength() >= batchSize {
- applyFunc([]*logrestore.LogDataFileInfo{f}, f.GetNumberOfEntries(), f.GetLength())
- continue
- }
-
- fit, exist := tableMapFiles[f.TableId]
- if !exist {
- fit = &FilesInTable{
- regionMapFiles: make(map[int64]*FilesInRegion),
- }
- tableMapFiles[f.TableId] = fit
- }
- fs, exist := fit.regionMapFiles[f.RegionId]
- if !exist {
- fs = &FilesInRegion{}
- fit.regionMapFiles[f.RegionId] = fs
- }
-
- if f.GetType() == backuppb.FileType_Delete {
- if fs.defaultFiles == nil {
- fs.deleteFiles = make([]*logrestore.LogDataFileInfo, 0)
- }
- fs.deleteFiles = append(fs.deleteFiles, f)
- } else {
- if f.GetCf() == stream.DefaultCF {
- if fs.defaultFiles == nil {
- fs.defaultFiles = make([]*logrestore.LogDataFileInfo, 0, batchCount)
- }
- fs.defaultFiles = append(fs.defaultFiles, f)
- fs.defaultSize += f.Length
- fs.defaultKVCount += f.GetNumberOfEntries()
- if len(fs.defaultFiles) >= batchCount || fs.defaultSize >= batchSize {
- applyFunc(fs.defaultFiles, fs.defaultKVCount, fs.defaultSize)
- fs.defaultFiles = nil
- fs.defaultSize = 0
- fs.defaultKVCount = 0
- }
- } else {
- if fs.writeFiles == nil {
- fs.writeFiles = make([]*logrestore.LogDataFileInfo, 0, batchCount)
- }
- fs.writeFiles = append(fs.writeFiles, f)
- fs.writeSize += f.GetLength()
- fs.writeKVCount += f.GetNumberOfEntries()
- if len(fs.writeFiles) >= batchCount || fs.writeSize >= batchSize {
- applyFunc(fs.writeFiles, fs.writeKVCount, fs.writeSize)
- fs.writeFiles = nil
- fs.writeSize = 0
- fs.writeKVCount = 0
- }
- }
- }
- }
-
- for _, fwt := range tableMapFiles {
- for _, fs := range fwt.regionMapFiles {
- if len(fs.defaultFiles) > 0 {
- applyFunc(fs.defaultFiles, fs.defaultKVCount, fs.defaultSize)
- }
- if len(fs.writeFiles) > 0 {
- applyFunc(fs.writeFiles, fs.writeKVCount, fs.writeSize)
- }
- }
- }
-
- applyWg.Wait()
- for _, fwt := range tableMapFiles {
- for _, fs := range fwt.regionMapFiles {
- for _, d := range fs.deleteFiles {
- tmpFiles = append(tmpFiles, d)
- tmpSize += d.GetLength()
- tmpKVCount += d.GetNumberOfEntries()
-
- if len(tmpFiles) >= batchCount || tmpSize >= batchSize {
- applyFunc(tmpFiles, tmpKVCount, tmpSize)
- tmpFiles = make([]*logrestore.LogDataFileInfo, 0, batchCount)
- tmpSize = 0
- tmpKVCount = 0
- }
- }
- if len(tmpFiles) > 0 {
- applyFunc(tmpFiles, tmpKVCount, tmpSize)
- tmpFiles = make([]*logrestore.LogDataFileInfo, 0, batchCount)
- tmpSize = 0
- tmpKVCount = 0
- }
- }
- }
-
- return nil
-}
-
-func ApplyKVFilesWithSingelMethod(
- ctx context.Context,
- files logrestore.LogIter,
- applyFunc func(file []*logrestore.LogDataFileInfo, kvCount int64, size uint64),
- applyWg *sync.WaitGroup,
-) error {
- deleteKVFiles := make([]*logrestore.LogDataFileInfo, 0)
-
- for r := files.TryNext(ctx); !r.Finished; r = files.TryNext(ctx) {
- if r.Err != nil {
- return r.Err
- }
-
- f := r.Item
- if f.GetType() == backuppb.FileType_Delete {
- deleteKVFiles = append(deleteKVFiles, f)
- continue
- }
- applyFunc([]*logrestore.LogDataFileInfo{f}, f.GetNumberOfEntries(), f.GetLength())
- }
-
- applyWg.Wait()
- log.Info("restore delete files", zap.Int("count", len(deleteKVFiles)))
- for _, file := range deleteKVFiles {
- f := file
- applyFunc([]*logrestore.LogDataFileInfo{f}, f.GetNumberOfEntries(), f.GetLength())
- }
-
- return nil
-}
-
-func (rc *Client) RestoreKVFiles(
- ctx context.Context,
- rules map[int64]*restoreutils.RewriteRules,
- idrules map[int64]int64,
- logIter logrestore.LogIter,
- runner *checkpoint.CheckpointRunner[checkpoint.LogRestoreKeyType, checkpoint.LogRestoreValueType],
- pitrBatchCount uint32,
- pitrBatchSize uint32,
- updateStats func(kvCount uint64, size uint64),
- onProgress func(cnt int64),
-) error {
- var (
- err error
- fileCount = 0
- start = time.Now()
- supportBatch = version.CheckPITRSupportBatchKVFiles()
- skipFile = 0
- )
- defer func() {
- if err == nil {
- elapsed := time.Since(start)
- log.Info("Restore KV files", zap.Duration("take", elapsed))
- summary.CollectSuccessUnit("files", fileCount, elapsed)
- }
- }()
-
- if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
- span1 := span.Tracer().StartSpan("Client.RestoreKVFiles", opentracing.ChildOf(span.Context()))
- defer span1.Finish()
- ctx = opentracing.ContextWithSpan(ctx, span1)
- }
-
- var applyWg sync.WaitGroup
- eg, ectx := errgroup.WithContext(ctx)
- applyFunc := func(files []*logrestore.LogDataFileInfo, kvCount int64, size uint64) {
- if len(files) == 0 {
- return
- }
- // get rewrite rule from table id.
- // because the tableID of files is the same.
- rule, ok := rules[files[0].TableId]
- if !ok {
- // TODO handle new created table
- // For this version we do not handle new created table after full backup.
- // in next version we will perform rewrite and restore meta key to restore new created tables.
- // so we can simply skip the file that doesn't have the rule here.
- onProgress(int64(len(files)))
- summary.CollectInt("FileSkip", len(files))
- log.Debug("skip file due to table id not matched", zap.Int64("table-id", files[0].TableId))
- skipFile += len(files)
- } else {
- applyWg.Add(1)
- downstreamId := idrules[files[0].TableId]
- rc.workerPool.ApplyOnErrorGroup(eg, func() (err error) {
- fileStart := time.Now()
- defer applyWg.Done()
- defer func() {
- onProgress(int64(len(files)))
- updateStats(uint64(kvCount), size)
- summary.CollectInt("File", len(files))
-
- if err == nil {
- filenames := make([]string, 0, len(files))
- if runner == nil {
- for _, f := range files {
- filenames = append(filenames, f.Path+", ")
- }
- } else {
- for _, f := range files {
- filenames = append(filenames, f.Path+", ")
- if e := checkpoint.AppendRangeForLogRestore(ectx, runner, f.MetaDataGroupName, downstreamId, f.OffsetInMetaGroup, f.OffsetInMergedGroup); e != nil {
- err = errors.Annotate(e, "failed to append checkpoint data")
- break
- }
- }
- }
- log.Info("import files done", zap.Int("batch-count", len(files)), zap.Uint64("batch-size", size),
- zap.Duration("take", time.Since(fileStart)), zap.Strings("files", filenames))
- }
- }()
-
- return rc.fileImporter.ImportKVFiles(ectx, files, rule, rc.ShiftStartTS, rc.StartTS, rc.RestoreTS, supportBatch)
- })
- }
- }
-
- rc.workerPool.ApplyOnErrorGroup(eg, func() error {
- if supportBatch {
- err = ApplyKVFilesWithBatchMethod(ectx, logIter, int(pitrBatchCount), uint64(pitrBatchSize), applyFunc, &applyWg)
- } else {
- err = ApplyKVFilesWithSingelMethod(ectx, logIter, applyFunc, &applyWg)
- }
- return errors.Trace(err)
- })
-
- if err = eg.Wait(); err != nil {
- summary.CollectFailureUnit("file", err)
- log.Error("restore files failed", zap.Error(err))
- }
-
- log.Info("total skip files due to table id not matched", zap.Int("count", skipFile))
- if skipFile > 0 {
- log.Debug("table id in full backup storage", zap.Any("tables", rules))
- }
-
- return errors.Trace(err)
-}
-
-func (rc *Client) CleanUpKVFiles(
- ctx context.Context,
-) error {
- // Current we only have v1 prefix.
- // In the future, we can add more operation for this interface.
- return rc.fileImporter.ClearFiles(ctx, rc.pdClient, "v1")
-}
-
-func (rc *Client) initSchemasMap(
- ctx context.Context,
- clusterID uint64,
- restoreTS uint64,
-) ([]*backuppb.PitrDBMap, error) {
- filename := metautil.PitrIDMapsFilename(clusterID, restoreTS)
- exist, err := rc.storage.FileExists(ctx, filename)
- if err != nil {
- return nil, errors.Annotatef(err, "failed to check filename:%s ", filename)
- } else if !exist {
- log.Info("pitr id maps isn't existed", zap.String("file", filename))
- return nil, nil
- }
-
- metaData, err := rc.storage.ReadFile(ctx, filename)
- if err != nil {
- return nil, errors.Trace(err)
- }
- backupMeta := &backuppb.BackupMeta{}
- if err = backupMeta.Unmarshal(metaData); err != nil {
- return nil, errors.Trace(err)
- }
-
- return backupMeta.GetDbMaps(), nil
-}
-
-func initFullBackupTables(
- ctx context.Context,
- s storage.ExternalStorage,
- tableFilter filter.Filter,
-) (map[int64]*metautil.Table, error) {
- metaData, err := s.ReadFile(ctx, metautil.MetaFile)
- if err != nil {
- return nil, errors.Trace(err)
- }
- backupMeta := &backuppb.BackupMeta{}
- if err = backupMeta.Unmarshal(metaData); err != nil {
- return nil, errors.Trace(err)
- }
-
- // read full backup databases to get map[table]table.Info
- reader := metautil.NewMetaReader(backupMeta, s, nil)
-
- databases, err := metautil.LoadBackupTables(ctx, reader, false)
- if err != nil {
- return nil, errors.Trace(err)
- }
-
- tables := make(map[int64]*metautil.Table)
- for _, db := range databases {
- dbName := db.Info.Name.O
- if name, ok := utils.GetSysDBName(db.Info.Name); utils.IsSysDB(name) && ok {
- dbName = name
- }
-
- if !tableFilter.MatchSchema(dbName) {
- continue
- }
-
- for _, table := range db.Tables {
- // check this db is empty.
- if table.Info == nil {
- tables[db.Info.ID] = table
- continue
- }
- if !tableFilter.MatchTable(dbName, table.Info.Name.O) {
- continue
- }
- tables[table.Info.ID] = table
- }
- }
-
- return tables, nil
-}
-
-type FullBackupStorageConfig struct {
- Backend *backuppb.StorageBackend
- Opts *storage.ExternalStorageOptions
-}
-
-type InitSchemaConfig struct {
- // required
- IsNewTask bool
- HasFullRestore bool
- TableFilter filter.Filter
-
- // optional
- TiFlashRecorder *tiflashrec.TiFlashRecorder
- FullBackupStorage *FullBackupStorageConfig
-}
-
-// InitSchemasReplaceForDDL gets schemas information Mapping from old schemas to new schemas.
-// It is used to rewrite meta kv-event.
-func (rc *Client) InitSchemasReplaceForDDL(
- ctx context.Context,
- cfg *InitSchemaConfig,
-) (*stream.SchemasReplace, error) {
- var (
- err error
- dbMaps []*backuppb.PitrDBMap
- // the id map doesn't need to construct only when it is not the first execution
- needConstructIdMap bool
-
- dbReplaces = make(map[stream.UpstreamID]*stream.DBReplace)
- )
-
- // not new task, load schemas map from external storage
- if !cfg.IsNewTask {
- log.Info("try to load pitr id maps")
- needConstructIdMap = false
- dbMaps, err = rc.initSchemasMap(ctx, rc.GetClusterID(ctx), rc.RestoreTS)
- if err != nil {
- return nil, errors.Trace(err)
- }
- }
-
- // a new task, but without full snapshot restore, tries to load
- // schemas map whose `restore-ts`` is the task's `start-ts`.
- if len(dbMaps) <= 0 && !cfg.HasFullRestore {
- log.Info("try to load pitr id maps of the previous task", zap.Uint64("start-ts", rc.StartTS))
- needConstructIdMap = true
- dbMaps, err = rc.initSchemasMap(ctx, rc.GetClusterID(ctx), rc.StartTS)
- if err != nil {
- return nil, errors.Trace(err)
- }
- }
-
- if len(dbMaps) <= 0 {
- log.Info("no id maps, build the table replaces from cluster and full backup schemas")
- needConstructIdMap = true
- s, err := storage.New(ctx, cfg.FullBackupStorage.Backend, cfg.FullBackupStorage.Opts)
- if err != nil {
- return nil, errors.Trace(err)
- }
- fullBackupTables, err := initFullBackupTables(ctx, s, cfg.TableFilter)
- if err != nil {
- return nil, errors.Trace(err)
- }
- for _, t := range fullBackupTables {
- dbName, _ := utils.GetSysDBCIStrName(t.DB.Name)
- newDBInfo, exist := rc.GetDBSchema(rc.GetDomain(), dbName)
- if !exist {
- log.Info("db not existed", zap.String("dbname", dbName.String()))
- continue
- }
-
- dbReplace, exist := dbReplaces[t.DB.ID]
- if !exist {
- dbReplace = stream.NewDBReplace(t.DB.Name.O, newDBInfo.ID)
- dbReplaces[t.DB.ID] = dbReplace
- }
-
- if t.Info == nil {
- // If the db is empty, skip it.
- continue
- }
- newTableInfo, err := rc.GetTableSchema(rc.GetDomain(), dbName, t.Info.Name)
- if err != nil {
- log.Info("table not existed", zap.String("tablename", dbName.String()+"."+t.Info.Name.String()))
- continue
- }
-
- dbReplace.TableMap[t.Info.ID] = &stream.TableReplace{
- Name: newTableInfo.Name.O,
- TableID: newTableInfo.ID,
- PartitionMap: restoreutils.GetPartitionIDMap(newTableInfo, t.Info),
- IndexMap: restoreutils.GetIndexIDMap(newTableInfo, t.Info),
- }
- }
- } else {
- dbReplaces = stream.FromSchemaMaps(dbMaps)
- }
-
- for oldDBID, dbReplace := range dbReplaces {
- log.Info("replace info", func() []zapcore.Field {
- fields := make([]zapcore.Field, 0, (len(dbReplace.TableMap)+1)*3)
- fields = append(fields,
- zap.String("dbName", dbReplace.Name),
- zap.Int64("oldID", oldDBID),
- zap.Int64("newID", dbReplace.DbID))
- for oldTableID, tableReplace := range dbReplace.TableMap {
- fields = append(fields,
- zap.String("table", tableReplace.Name),
- zap.Int64("oldID", oldTableID),
- zap.Int64("newID", tableReplace.TableID))
- }
- return fields
- }()...)
- }
-
- rp := stream.NewSchemasReplace(
- dbReplaces, needConstructIdMap, cfg.TiFlashRecorder, rc.currentTS, cfg.TableFilter, rc.GenGlobalID, rc.GenGlobalIDs,
- rc.RecordDeleteRange)
- return rp, nil
-}
-
-func SortMetaKVFiles(files []*backuppb.DataFileInfo) []*backuppb.DataFileInfo {
- slices.SortFunc(files, func(i, j *backuppb.DataFileInfo) int {
- if c := cmp.Compare(i.GetMinTs(), j.GetMinTs()); c != 0 {
- return c
- }
- if c := cmp.Compare(i.GetMaxTs(), j.GetMaxTs()); c != 0 {
- return c
- }
- return cmp.Compare(i.GetResolvedTs(), j.GetResolvedTs())
- })
- return files
-}
-
-// RestoreMetaKVFiles tries to restore files about meta kv-event from stream-backup.
-func (rc *Client) RestoreMetaKVFiles(
- ctx context.Context,
- files []*backuppb.DataFileInfo,
- schemasReplace *stream.SchemasReplace,
- updateStats func(kvCount uint64, size uint64),
- progressInc func(),
-) error {
- filesInWriteCF := make([]*backuppb.DataFileInfo, 0, len(files))
- filesInDefaultCF := make([]*backuppb.DataFileInfo, 0, len(files))
-
- // The k-v events in default CF should be restored firstly. The reason is that:
- // The error of transactions of meta could happen if restore write CF events successfully,
- // but failed to restore default CF events.
- for _, f := range files {
- if f.Cf == stream.WriteCF {
- filesInWriteCF = append(filesInWriteCF, f)
- continue
- }
- if f.Type == backuppb.FileType_Delete {
- // this should happen abnormally.
- // only do some preventive checks here.
- log.Warn("detected delete file of meta key, skip it", zap.Any("file", f))
- continue
- }
- if f.Cf == stream.DefaultCF {
- filesInDefaultCF = append(filesInDefaultCF, f)
- }
- }
- filesInDefaultCF = SortMetaKVFiles(filesInDefaultCF)
- filesInWriteCF = SortMetaKVFiles(filesInWriteCF)
-
- failpoint.Inject("failed-before-id-maps-saved", func(_ failpoint.Value) {
- failpoint.Return(errors.New("failpoint: failed before id maps saved"))
- })
-
- log.Info("start to restore meta files",
- zap.Int("total files", len(files)),
- zap.Int("default files", len(filesInDefaultCF)),
- zap.Int("write files", len(filesInWriteCF)))
-
- if schemasReplace.NeedConstructIdMap() {
- // Preconstruct the map and save it into external storage.
- if err := rc.PreConstructAndSaveIDMap(
- ctx,
- filesInWriteCF,
- filesInDefaultCF,
- schemasReplace,
- ); err != nil {
- return errors.Trace(err)
- }
- }
- failpoint.Inject("failed-after-id-maps-saved", func(_ failpoint.Value) {
- failpoint.Return(errors.New("failpoint: failed after id maps saved"))
- })
-
- // run the rewrite and restore meta-kv into TiKV cluster.
- if err := rc.RestoreMetaKVFilesWithBatchMethod(
- ctx,
- filesInDefaultCF,
- filesInWriteCF,
- schemasReplace,
- updateStats,
- progressInc,
- rc.RestoreBatchMetaKVFiles,
- ); err != nil {
- return errors.Trace(err)
- }
-
- // Update global schema version and report all of TiDBs.
- if err := rc.UpdateSchemaVersion(ctx); err != nil {
- return errors.Trace(err)
- }
- return nil
-}
-
-// PreConstructAndSaveIDMap constructs id mapping and save it.
-func (rc *Client) PreConstructAndSaveIDMap(
- ctx context.Context,
- fsInWriteCF, fsInDefaultCF []*backuppb.DataFileInfo,
- sr *stream.SchemasReplace,
-) error {
- sr.SetPreConstructMapStatus()
-
- if err := rc.constructIDMap(ctx, fsInWriteCF, sr); err != nil {
- return errors.Trace(err)
- }
- if err := rc.constructIDMap(ctx, fsInDefaultCF, sr); err != nil {
- return errors.Trace(err)
- }
-
- if err := rc.SaveIDMap(ctx, sr); err != nil {
- return errors.Trace(err)
- }
- return nil
-}
-
-func (rc *Client) constructIDMap(
- ctx context.Context,
- fs []*backuppb.DataFileInfo,
- sr *stream.SchemasReplace,
-) error {
- for _, f := range fs {
- entries, _, err := rc.ReadAllEntries(ctx, f, math.MaxUint64)
- if err != nil {
- return errors.Trace(err)
- }
-
- for _, entry := range entries {
- if _, err := sr.RewriteKvEntry(&entry.E, f.GetCf()); err != nil {
- return errors.Trace(err)
- }
- }
- }
- return nil
-}
-
-func (rc *Client) RestoreMetaKVFilesWithBatchMethod(
- ctx context.Context,
- defaultFiles []*backuppb.DataFileInfo,
- writeFiles []*backuppb.DataFileInfo,
- schemasReplace *stream.SchemasReplace,
- updateStats func(kvCount uint64, size uint64),
- progressInc func(),
- restoreBatch func(
- ctx context.Context,
- files []*backuppb.DataFileInfo,
- schemasReplace *stream.SchemasReplace,
- kvEntries []*logrestore.KvEntryWithTS,
- filterTS uint64,
- updateStats func(kvCount uint64, size uint64),
- progressInc func(),
- cf string,
- ) ([]*logrestore.KvEntryWithTS, error),
-) error {
- // the average size of each KV is 2560 Bytes
- // kvEntries is kvs left by the previous batch
- const kvSize = 2560
- var (
- rangeMin uint64
- rangeMax uint64
- err error
-
- batchSize uint64 = 0
- defaultIdx int = 0
- writeIdx int = 0
-
- defaultKvEntries = make([]*logrestore.KvEntryWithTS, 0)
- writeKvEntries = make([]*logrestore.KvEntryWithTS, 0)
- )
- // Set restoreKV to SchemaReplace.
- schemasReplace.SetRestoreKVStatus()
-
- for i, f := range defaultFiles {
- if i == 0 {
- rangeMax = f.MaxTs
- rangeMin = f.MinTs
- batchSize = f.Length
- } else {
- if f.MinTs <= rangeMax && batchSize+f.Length <= MetaKVBatchSize {
- rangeMin = min(rangeMin, f.MinTs)
- rangeMax = max(rangeMax, f.MaxTs)
- batchSize += f.Length
- } else {
- // Either f.MinTS > rangeMax or f.MinTs is the filterTs we need.
- // So it is ok to pass f.MinTs as filterTs.
- defaultKvEntries, err = restoreBatch(ctx, defaultFiles[defaultIdx:i], schemasReplace, defaultKvEntries, f.MinTs, updateStats, progressInc, stream.DefaultCF)
- if err != nil {
- return errors.Trace(err)
- }
- defaultIdx = i
- rangeMin = f.MinTs
- rangeMax = f.MaxTs
- // the initial batch size is the size of left kvs and the current file length.
- batchSize = uint64(len(defaultKvEntries)*kvSize) + f.Length
-
- // restore writeCF kv to f.MinTs
- var toWriteIdx int
- for toWriteIdx = writeIdx; toWriteIdx < len(writeFiles); toWriteIdx++ {
- if writeFiles[toWriteIdx].MinTs >= f.MinTs {
- break
- }
- }
- writeKvEntries, err = restoreBatch(ctx, writeFiles[writeIdx:toWriteIdx], schemasReplace, writeKvEntries, f.MinTs, updateStats, progressInc, stream.WriteCF)
- if err != nil {
- return errors.Trace(err)
- }
- writeIdx = toWriteIdx
- }
- }
- }
-
- // restore the left meta kv files and entries
- // Notice: restoreBatch needs to realize the parameter `files` and `kvEntries` might be empty
- // Assert: defaultIdx <= len(defaultFiles) && writeIdx <= len(writeFiles)
- _, err = restoreBatch(ctx, defaultFiles[defaultIdx:], schemasReplace, defaultKvEntries, math.MaxUint64, updateStats, progressInc, stream.DefaultCF)
- if err != nil {
- return errors.Trace(err)
- }
- _, err = restoreBatch(ctx, writeFiles[writeIdx:], schemasReplace, writeKvEntries, math.MaxUint64, updateStats, progressInc, stream.WriteCF)
- if err != nil {
- return errors.Trace(err)
- }
-
- return nil
-}
-
-func (rc *Client) RestoreBatchMetaKVFiles(
- ctx context.Context,
- files []*backuppb.DataFileInfo,
- schemasReplace *stream.SchemasReplace,
- kvEntries []*logrestore.KvEntryWithTS,
- filterTS uint64,
- updateStats func(kvCount uint64, size uint64),
- progressInc func(),
- cf string,
-) ([]*logrestore.KvEntryWithTS, error) {
- nextKvEntries := make([]*logrestore.KvEntryWithTS, 0)
- curKvEntries := make([]*logrestore.KvEntryWithTS, 0)
- if len(files) == 0 && len(kvEntries) == 0 {
- return nextKvEntries, nil
- }
-
- // filter the kv from kvEntries again.
- for _, kv := range kvEntries {
- if kv.Ts < filterTS {
- curKvEntries = append(curKvEntries, kv)
- } else {
- nextKvEntries = append(nextKvEntries, kv)
- }
- }
-
- // read all of entries from files.
- for _, f := range files {
- es, nextEs, err := rc.ReadAllEntries(ctx, f, filterTS)
- if err != nil {
- return nextKvEntries, errors.Trace(err)
- }
-
- curKvEntries = append(curKvEntries, es...)
- nextKvEntries = append(nextKvEntries, nextEs...)
- }
-
- // sort these entries.
- slices.SortFunc(curKvEntries, func(i, j *logrestore.KvEntryWithTS) int {
- return cmp.Compare(i.Ts, j.Ts)
- })
-
- // restore these entries with rawPut() method.
- kvCount, size, err := rc.restoreMetaKvEntries(ctx, schemasReplace, curKvEntries, cf)
- if err != nil {
- return nextKvEntries, errors.Trace(err)
- }
-
- if schemasReplace.IsRestoreKVStatus() {
- updateStats(kvCount, size)
- for i := 0; i < len(files); i++ {
- progressInc()
- }
- }
- return nextKvEntries, nil
-}
-
-func (rc *Client) restoreMetaKvEntries(
- ctx context.Context,
- sr *stream.SchemasReplace,
- entries []*logrestore.KvEntryWithTS,
- columnFamily string,
-) (uint64, uint64, error) {
- var (
- kvCount uint64
- size uint64
- )
-
- rc.rawKVClient.SetColumnFamily(columnFamily)
-
- for _, entry := range entries {
- log.Debug("before rewrte entry", zap.Uint64("key-ts", entry.Ts), zap.Int("key-len", len(entry.E.Key)),
- zap.Int("value-len", len(entry.E.Value)), zap.ByteString("key", entry.E.Key))
-
- newEntry, err := sr.RewriteKvEntry(&entry.E, columnFamily)
- if err != nil {
- log.Error("rewrite txn entry failed", zap.Int("klen", len(entry.E.Key)),
- logutil.Key("txn-key", entry.E.Key))
- return 0, 0, errors.Trace(err)
- } else if newEntry == nil {
- continue
- }
- log.Debug("after rewrite entry", zap.Int("new-key-len", len(newEntry.Key)),
- zap.Int("new-value-len", len(entry.E.Value)), zap.ByteString("new-key", newEntry.Key))
-
- failpoint.Inject("failed-to-restore-metakv", func(_ failpoint.Value) {
- failpoint.Return(0, 0, errors.Errorf("failpoint: failed to restore metakv"))
- })
- if err := rc.rawKVClient.Put(ctx, newEntry.Key, newEntry.Value, entry.Ts); err != nil {
- return 0, 0, errors.Trace(err)
- }
- // for failpoint, we need to flush the cache in rawKVClient every time
- failpoint.Inject("do-not-put-metakv-in-batch", func(_ failpoint.Value) {
- if err := rc.rawKVClient.PutRest(ctx); err != nil {
- failpoint.Return(0, 0, errors.Trace(err))
- }
- })
- kvCount++
- size += uint64(len(newEntry.Key) + len(newEntry.Value))
- }
-
- return kvCount, size, rc.rawKVClient.PutRest(ctx)
-}
-
-func transferBoolToValue(enable bool) string {
- if enable {
- return "ON"
- }
- return "OFF"
-}
-
-// GenGlobalID generates a global id by transaction way.
-func (rc *Client) GenGlobalID(ctx context.Context) (int64, error) {
- var id int64
- storage := rc.GetDomain().Store()
-
- ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnBR)
- err := kv.RunInNewTxn(
- ctx,
- storage,
- true,
- func(ctx context.Context, txn kv.Transaction) error {
- var e error
- t := meta.NewMeta(txn)
- id, e = t.GenGlobalID()
- return e
- })
-
- return id, err
-}
-
-// GenGlobalIDs generates several global ids by transaction way.
-func (rc *Client) GenGlobalIDs(ctx context.Context, n int) ([]int64, error) {
- ids := make([]int64, 0)
- storage := rc.GetDomain().Store()
-
- ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnBR)
- err := kv.RunInNewTxn(
- ctx,
- storage,
- true,
- func(ctx context.Context, txn kv.Transaction) error {
- var e error
- t := meta.NewMeta(txn)
- ids, e = t.GenGlobalIDs(n)
- return e
- })
-
- return ids, err
-}
-
-// UpdateSchemaVersion updates schema version by transaction way.
-func (rc *Client) UpdateSchemaVersion(ctx context.Context) error {
- storage := rc.GetDomain().Store()
- var schemaVersion int64
-
- ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnBR)
- if err := kv.RunInNewTxn(
- ctx,
- storage,
- true,
- func(ctx context.Context, txn kv.Transaction) error {
- t := meta.NewMeta(txn)
- var e error
- // To trigger full-reload instead of diff-reload, we need to increase the schema version
- // by at least `domain.LoadSchemaDiffVersionGapThreshold`.
- schemaVersion, e = t.GenSchemaVersions(64 + domain.LoadSchemaDiffVersionGapThreshold)
- if e != nil {
- return e
- }
- // add the diff key so that the domain won't retry to reload the schemas with `schemaVersion` frequently.
- return t.SetSchemaDiff(&model.SchemaDiff{
- Version: schemaVersion,
- Type: model.ActionNone,
- SchemaID: -1,
- TableID: -1,
- RegenerateSchemaMap: true,
- })
- },
- ); err != nil {
- return errors.Trace(err)
- }
-
- log.Info("update global schema version", zap.Int64("global-schema-version", schemaVersion))
-
- ver := strconv.FormatInt(schemaVersion, 10)
- if err := ddlutil.PutKVToEtcd(
- ctx,
- rc.GetDomain().GetEtcdClient(),
- math.MaxInt,
- ddlutil.DDLGlobalSchemaVersion,
- ver,
- ); err != nil {
- return errors.Annotatef(err, "failed to put global schema verson %v to etcd", ver)
- }
-
- return nil
-}
-
-const (
- alterTableDropIndexSQL = "ALTER TABLE %n.%n DROP INDEX %n"
- alterTableAddIndexFormat = "ALTER TABLE %%n.%%n ADD INDEX %%n(%s)"
- alterTableAddUniqueIndexFormat = "ALTER TABLE %%n.%%n ADD UNIQUE KEY %%n(%s)"
- alterTableAddPrimaryFormat = "ALTER TABLE %%n.%%n ADD PRIMARY KEY (%s) NONCLUSTERED"
-)
-
-func (rc *Client) generateRepairIngestIndexSQLs(
- ctx context.Context,
- ingestRecorder *ingestrec.IngestRecorder,
- allSchema []*model.DBInfo,
- taskName string,
-) ([]checkpoint.CheckpointIngestIndexRepairSQL, bool, error) {
- var sqls []checkpoint.CheckpointIngestIndexRepairSQL
- if rc.useCheckpoint {
- exists, err := checkpoint.ExistsCheckpointIngestIndexRepairSQLs(ctx, rc.storage, taskName)
- if err != nil {
- return sqls, false, errors.Trace(err)
- }
- if exists {
- checkpointSQLs, err := checkpoint.LoadCheckpointIngestIndexRepairSQLs(ctx, rc.storage, taskName)
- if err != nil {
- return sqls, false, errors.Trace(err)
- }
- sqls = checkpointSQLs.SQLs
- log.Info("load ingest index repair sqls from checkpoint", zap.String("category", "ingest"), zap.Reflect("sqls", sqls))
- return sqls, true, nil
- }
- }
-
- ingestRecorder.UpdateIndexInfo(allSchema)
- if err := ingestRecorder.Iterate(func(_, indexID int64, info *ingestrec.IngestIndexInfo) error {
- var (
- addSQL strings.Builder
- addArgs []any = make([]any, 0, 5+len(info.ColumnArgs))
- )
- if info.IsPrimary {
- addSQL.WriteString(fmt.Sprintf(alterTableAddPrimaryFormat, info.ColumnList))
- addArgs = append(addArgs, info.SchemaName.O, info.TableName.O)
- addArgs = append(addArgs, info.ColumnArgs...)
- } else if info.IndexInfo.Unique {
- addSQL.WriteString(fmt.Sprintf(alterTableAddUniqueIndexFormat, info.ColumnList))
- addArgs = append(addArgs, info.SchemaName.O, info.TableName.O, info.IndexInfo.Name.O)
- addArgs = append(addArgs, info.ColumnArgs...)
- } else {
- addSQL.WriteString(fmt.Sprintf(alterTableAddIndexFormat, info.ColumnList))
- addArgs = append(addArgs, info.SchemaName.O, info.TableName.O, info.IndexInfo.Name.O)
- addArgs = append(addArgs, info.ColumnArgs...)
- }
- // USING BTREE/HASH/RTREE
- indexTypeStr := info.IndexInfo.Tp.String()
- if len(indexTypeStr) > 0 {
- addSQL.WriteString(" USING ")
- addSQL.WriteString(indexTypeStr)
- }
-
- // COMMENT [...]
- if len(info.IndexInfo.Comment) > 0 {
- addSQL.WriteString(" COMMENT %?")
- addArgs = append(addArgs, info.IndexInfo.Comment)
- }
-
- if info.IndexInfo.Invisible {
- addSQL.WriteString(" INVISIBLE")
- } else {
- addSQL.WriteString(" VISIBLE")
- }
-
- sqls = append(sqls, checkpoint.CheckpointIngestIndexRepairSQL{
- IndexID: indexID,
- SchemaName: info.SchemaName,
- TableName: info.TableName,
- IndexName: info.IndexInfo.Name.O,
- AddSQL: addSQL.String(),
- AddArgs: addArgs,
- })
-
- return nil
- }); err != nil {
- return sqls, false, errors.Trace(err)
- }
-
- if rc.useCheckpoint && len(sqls) > 0 {
- if err := checkpoint.SaveCheckpointIngestIndexRepairSQLs(ctx, rc.storage, &checkpoint.CheckpointIngestIndexRepairSQLs{
- SQLs: sqls,
- }, taskName); err != nil {
- return sqls, false, errors.Trace(err)
- }
- }
- return sqls, false, nil
-}
-
-// RepairIngestIndex drops the indexes from IngestRecorder and re-add them.
-func (rc *Client) RepairIngestIndex(ctx context.Context, ingestRecorder *ingestrec.IngestRecorder, g glue.Glue, storage kv.Storage, taskName string) error {
- dom, err := g.GetDomain(storage)
- if err != nil {
- return errors.Trace(err)
- }
- info := dom.InfoSchema()
-
- sqls, fromCheckpoint, err := rc.generateRepairIngestIndexSQLs(ctx, ingestRecorder, info.AllSchemas(), taskName)
- if err != nil {
- return errors.Trace(err)
- }
-
- console := glue.GetConsole(g)
-NEXTSQL:
- for _, sql := range sqls {
- progressTitle := fmt.Sprintf("repair ingest index %s for table %s.%s", sql.IndexName, sql.SchemaName, sql.TableName)
-
- tableInfo, err := info.TableByName(sql.SchemaName, sql.TableName)
- if err != nil {
- return errors.Trace(err)
- }
- oldIndexIDFound := false
- if fromCheckpoint {
- for _, idx := range tableInfo.Indices() {
- indexInfo := idx.Meta()
- if indexInfo.ID == sql.IndexID {
- // the original index id is not dropped
- oldIndexIDFound = true
- break
- }
- // what if index's state is not public?
- if indexInfo.Name.O == sql.IndexName {
- // find the same name index, but not the same index id,
- // which means the repaired index id is created
- if _, err := fmt.Fprintf(console.Out(), "%s ... %s\n", progressTitle, color.HiGreenString("SKIPPED DUE TO CHECKPOINT MODE")); err != nil {
- return errors.Trace(err)
- }
- continue NEXTSQL
- }
- }
- }
-
- if err := func(sql checkpoint.CheckpointIngestIndexRepairSQL) error {
- w := console.StartProgressBar(progressTitle, glue.OnlyOneTask)
- defer w.Close()
-
- // TODO: When the TiDB supports the DROP and CREATE the same name index in one SQL,
- // the checkpoint for ingest recorder can be removed and directly use the SQL:
- // ALTER TABLE db.tbl DROP INDEX `i_1`, ADD IDNEX `i_1` ...
- //
- // This SQL is compatible with checkpoint: If one ingest index has been recreated by
- // the SQL, the index's id would be another one. In the next retry execution, BR can
- // not find the ingest index's dropped id so that BR regards it as a dropped index by
- // restored metakv and then skips repairing it.
-
- // only when first execution or old index id is not dropped
- if !fromCheckpoint || oldIndexIDFound {
- if err := rc.db.se.ExecuteInternal(ctx, alterTableDropIndexSQL, sql.SchemaName.O, sql.TableName.O, sql.IndexName); err != nil {
- return errors.Trace(err)
- }
- }
- failpoint.Inject("failed-before-create-ingest-index", func(v failpoint.Value) {
- if v != nil && v.(bool) {
- failpoint.Return(errors.New("failed before create ingest index"))
- }
- })
- // create the repaired index when first execution or not found it
- if err := rc.db.se.ExecuteInternal(ctx, sql.AddSQL, sql.AddArgs...); err != nil {
- return errors.Trace(err)
- }
- w.Inc()
- if err := w.Wait(ctx); err != nil {
- return errors.Trace(err)
- }
- return nil
- }(sql); err != nil {
- return errors.Trace(err)
- }
- }
-
- return nil
-}
-
-func (rc *Client) RecordDeleteRange(sql *stream.PreDelRangeQuery) {
- rc.deleteRangeQueryCh <- sql
-}
-
-// use channel to save the delete-range query to make it thread-safety.
-func (rc *Client) RunGCRowsLoader(ctx context.Context) {
- rc.deleteRangeQueryWaitGroup.Add(1)
-
- go func() {
- defer rc.deleteRangeQueryWaitGroup.Done()
- for {
- select {
- case <-ctx.Done():
- return
- case query, ok := <-rc.deleteRangeQueryCh:
- if !ok {
- return
- }
- rc.deleteRangeQuery = append(rc.deleteRangeQuery, query)
- }
- }
- }()
-}
-
-// InsertGCRows insert the querys into table `gc_delete_range`
-func (rc *Client) InsertGCRows(ctx context.Context) error {
- close(rc.deleteRangeQueryCh)
- rc.deleteRangeQueryWaitGroup.Wait()
- ts, err := rc.GetTSWithRetry(ctx)
- if err != nil {
- return errors.Trace(err)
- }
- jobIDMap := make(map[int64]int64)
- for _, query := range rc.deleteRangeQuery {
- paramsList := make([]any, 0, len(query.ParamsList)*5)
- for _, params := range query.ParamsList {
- newJobID, exists := jobIDMap[params.JobID]
- if !exists {
- newJobID, err = rc.GenGlobalID(ctx)
- if err != nil {
- return errors.Trace(err)
- }
- jobIDMap[params.JobID] = newJobID
- }
- log.Info("insert into the delete range",
- zap.Int64("jobID", newJobID),
- zap.Int64("elemID", params.ElemID),
- zap.String("startKey", params.StartKey),
- zap.String("endKey", params.EndKey),
- zap.Uint64("ts", ts))
- // (job_id, elem_id, start_key, end_key, ts)
- paramsList = append(paramsList, newJobID, params.ElemID, params.StartKey, params.EndKey, ts)
- }
- if len(paramsList) > 0 {
- // trim the ',' behind the query.Sql if exists
- // that's when the rewrite rule of the last table id is not exist
- sql := strings.TrimSuffix(query.Sql, ",")
- if err := rc.db.se.ExecuteInternal(ctx, sql, paramsList...); err != nil {
- return errors.Trace(err)
- }
- }
- }
- return nil
-}
-
-// only for unit test
-func (rc *Client) GetGCRows() []*stream.PreDelRangeQuery {
- close(rc.deleteRangeQueryCh)
- rc.deleteRangeQueryWaitGroup.Wait()
- return rc.deleteRangeQuery
-}
-
-// SaveIDMap saves the id mapping information.
-func (rc *Client) SaveIDMap(
- ctx context.Context,
- sr *stream.SchemasReplace,
-) error {
- idMaps := sr.TidySchemaMaps()
- clusterID := rc.GetClusterID(ctx)
- metaFileName := metautil.PitrIDMapsFilename(clusterID, rc.RestoreTS)
- metaWriter := metautil.NewMetaWriter(rc.storage, metautil.MetaFileSize, false, metaFileName, nil)
- metaWriter.Update(func(m *backuppb.BackupMeta) {
- // save log startTS to backupmeta file
- m.ClusterId = clusterID
- m.DbMaps = idMaps
- })
-
- if err := metaWriter.FlushBackupMeta(ctx); err != nil {
- return errors.Trace(err)
- }
- if rc.useCheckpoint {
- var items map[int64]model.TiFlashReplicaInfo
- if sr.TiflashRecorder != nil {
- items = sr.TiflashRecorder.GetItems()
- }
- log.Info("save checkpoint task info with InLogRestoreAndIdMapPersist status")
- if err := checkpoint.SaveCheckpointTaskInfoForLogRestore(ctx, rc.storage, &checkpoint.CheckpointTaskInfoForLogRestore{
- Progress: checkpoint.InLogRestoreAndIdMapPersist,
- StartTS: rc.StartTS,
- RestoreTS: rc.RestoreTS,
- RewriteTS: rc.currentTS,
- TiFlashItems: items,
- }, rc.GetClusterID(ctx)); err != nil {
- return errors.Trace(err)
- }
- }
- return nil
-}
-
-// InitFullClusterRestore init fullClusterRestore and set SkipGrantTable as needed
-func (rc *Client) InitFullClusterRestore(explicitFilter bool) {
- rc.fullClusterRestore = !explicitFilter && rc.IsFull()
-
- log.Info("full cluster restore", zap.Bool("value", rc.fullClusterRestore))
-}
-
-func (rc *Client) IsFullClusterRestore() bool {
- return rc.fullClusterRestore
-}
-
-func (rc *Client) SetWithSysTable(withSysTable bool) {
- rc.withSysTable = withSysTable
-}
-
-func (rc *Client) ResetTiFlashReplicas(ctx context.Context, g glue.Glue, storage kv.Storage) error {
- dom, err := g.GetDomain(storage)
- if err != nil {
- return errors.Trace(err)
- }
- info := dom.InfoSchema()
- allSchemaName := info.AllSchemaNames()
- recorder := tiflashrec.New()
-
- expectTiFlashStoreCount := uint64(0)
- needTiFlash := false
- for _, s := range allSchemaName {
- for _, t := range info.SchemaTables(s) {
- t := t.Meta()
- if t.TiFlashReplica != nil {
- expectTiFlashStoreCount = max(expectTiFlashStoreCount, t.TiFlashReplica.Count)
- recorder.AddTable(t.ID, *t.TiFlashReplica)
- needTiFlash = true
- }
- }
- }
- if !needTiFlash {
- log.Info("no need to set tiflash replica, since there is no tables enable tiflash replica")
- return nil
- }
- // we wait for ten minutes to wait tiflash starts.
- // since tiflash only starts when set unmark recovery mode finished.
- timeoutCtx, cancel := context.WithTimeout(ctx, 10*time.Minute)
- defer cancel()
- err = utils.WithRetry(timeoutCtx, func() error {
- tiFlashStoreCount, err := rc.getTiFlashNodeCount(ctx)
- log.Info("get tiflash store count for resetting TiFlash Replica",
- zap.Uint64("count", tiFlashStoreCount))
- if err != nil {
- return errors.Trace(err)
- }
- if tiFlashStoreCount < expectTiFlashStoreCount {
- log.Info("still waiting for enough tiflash store start",
- zap.Uint64("expect", expectTiFlashStoreCount),
- zap.Uint64("actual", tiFlashStoreCount),
- )
- return errors.New("tiflash store count is less than expected")
- }
- return nil
- }, &waitTiFlashBackoffer{
- Attempts: 30,
- BaseBackoff: 4 * time.Second,
- })
- if err != nil {
- return err
- }
-
- sqls := recorder.GenerateResetAlterTableDDLs(info)
- log.Info("Generating SQLs for resetting tiflash replica",
- zap.Strings("sqls", sqls))
-
- return g.UseOneShotSession(storage, false, func(se glue.Session) error {
- for _, sql := range sqls {
- if errExec := se.ExecuteInternal(ctx, sql); errExec != nil {
- logutil.WarnTerm("Failed to restore tiflash replica config, you may execute the sql restore it manually.",
- logutil.ShortError(errExec),
- zap.String("sql", sql),
- )
- }
- }
- return nil
- })
-}
-
-// RangeFilterFromIngestRecorder rewrites the table id of items in the ingestRecorder
-// TODO: need to implement the range filter out feature
-func (rc *Client) RangeFilterFromIngestRecorder(recorder *ingestrec.IngestRecorder, rewriteRules map[int64]*restoreutils.RewriteRules) error {
- err := recorder.RewriteTableID(func(tableID int64) (int64, bool, error) {
- rewriteRule, exists := rewriteRules[tableID]
- if !exists {
- // since the table's files will be skipped restoring, here also skips.
- return 0, true, nil
- }
- newTableID := restoreutils.GetRewriteTableID(tableID, rewriteRule)
- if newTableID == 0 {
- return 0, false, errors.Errorf("newTableID is 0, tableID: %d", tableID)
- }
- return newTableID, false, nil
- })
- return errors.Trace(err)
- /* TODO: we can use range filter to skip restoring the index kv using accelerated indexing feature
- filter := rtree.NewRangeTree()
- err = recorder.Iterate(func(tableID int64, indexID int64, info *ingestrec.IngestIndexInfo) error {
- // range after table ID rewritten
- startKey := tablecodec.EncodeTableIndexPrefix(tableID, indexID)
- endKey := tablecodec.EncodeTableIndexPrefix(tableID, indexID+1)
- rg := rtree.Range{
- StartKey: codec.EncodeBytes([]byte{}, startKey),
- EndKey: codec.EncodeBytes([]byte{}, endKey),
- }
- filter.InsertRange(rg)
- return nil
- })
- return errors.Trace(err)
- */
-}
-
-// MockClient create a fake client used to test.
-func MockClient(dbs map[string]*metautil.Database) *Client {
- return &Client{databases: dbs}
-}
-
-func CheckKeyspaceBREnable(ctx context.Context, pdClient pd.Client) error {
- return version.CheckClusterVersion(ctx, pdClient, version.CheckVersionForKeyspaceBR)
-}
-
-func CheckNewCollationEnable(
- backupNewCollationEnable string,
- g glue.Glue,
- storage kv.Storage,
- CheckRequirements bool,
-) (bool, error) {
- se, err := g.CreateSession(storage)
- if err != nil {
- return false, errors.Trace(err)
- }
-
- newCollationEnable, err := se.GetGlobalVariable(utils.GetTidbNewCollationEnabled())
- if err != nil {
- return false, errors.Trace(err)
- }
- // collate.newCollationEnabled is set to 1 when the collate package is initialized,
- // so we need to modify this value according to the config of the cluster
- // before using the collate package.
- enabled := newCollationEnable == "True"
- // modify collate.newCollationEnabled according to the config of the cluster
- collate.SetNewCollationEnabledForTest(enabled)
- log.Info(fmt.Sprintf("set %s", utils.TidbNewCollationEnabled), zap.Bool("new_collation_enabled", enabled))
-
- if backupNewCollationEnable == "" {
- if CheckRequirements {
- return enabled, errors.Annotatef(berrors.ErrUnknown,
- "the value '%s' not found in backupmeta. "+
- "you can use \"SELECT VARIABLE_VALUE FROM mysql.tidb WHERE VARIABLE_NAME='%s';\" to manually check the config. "+
- "if you ensure the value '%s' in backup cluster is as same as restore cluster, use --check-requirements=false to skip this check",
- utils.TidbNewCollationEnabled, utils.TidbNewCollationEnabled, utils.TidbNewCollationEnabled)
- }
- log.Warn(fmt.Sprintf("the config '%s' is not in backupmeta", utils.TidbNewCollationEnabled))
- return enabled, nil
- }
-
- if !strings.EqualFold(backupNewCollationEnable, newCollationEnable) {
- return enabled, errors.Annotatef(berrors.ErrUnknown,
- "the config '%s' not match, upstream:%v, downstream: %v",
- utils.TidbNewCollationEnabled, backupNewCollationEnable, newCollationEnable)
- }
-
- return enabled, nil
-}
-
-type waitTiFlashBackoffer struct {
- Attempts int
- BaseBackoff time.Duration
-}
-
-// NextBackoff returns a duration to wait before retrying again
-func (b *waitTiFlashBackoffer) NextBackoff(error) time.Duration {
- bo := b.BaseBackoff
- b.Attempts--
- if b.Attempts == 0 {
- return 0
- }
- b.BaseBackoff *= 2
- if b.BaseBackoff > 32*time.Second {
- b.BaseBackoff = 32 * time.Second
- }
- return bo
-}
-
-// Attempt returns the remain attempt times
-func (b *waitTiFlashBackoffer) Attempt() int {
- return b.Attempts
-}
diff --git a/br/pkg/restore/data/BUILD.bazel b/br/pkg/restore/data/BUILD.bazel
index 76e082ee4c91a..9ed1383a56e99 100644
--- a/br/pkg/restore/data/BUILD.bazel
+++ b/br/pkg/restore/data/BUILD.bazel
@@ -46,7 +46,7 @@ go_test(
deps = [
":data",
"//br/pkg/conn",
- "//br/pkg/gluetidb",
+ "//br/pkg/gluetidb/mock",
"//br/pkg/pdutil",
"@com_github_pingcap_kvproto//pkg/metapb",
"@com_github_pingcap_kvproto//pkg/recoverdatapb",
diff --git a/br/pkg/restore/data/data_test.go b/br/pkg/restore/data/data_test.go
index 7c3a5a3242d21..516d625429ca1 100644
--- a/br/pkg/restore/data/data_test.go
+++ b/br/pkg/restore/data/data_test.go
@@ -8,7 +8,7 @@ import (
"github.com/pingcap/kvproto/pkg/metapb"
recovpb "github.com/pingcap/kvproto/pkg/recoverdatapb"
"github.com/pingcap/tidb/br/pkg/conn"
- "github.com/pingcap/tidb/br/pkg/gluetidb"
+ gluemock "github.com/pingcap/tidb/br/pkg/gluetidb/mock"
"github.com/pingcap/tidb/br/pkg/pdutil"
"github.com/pingcap/tidb/br/pkg/restore/data"
"github.com/stretchr/testify/require"
@@ -89,7 +89,7 @@ func createStores() []*metapb.Store {
func createDataSuite(t *testing.T) *testData {
tikvClient, _, pdClient, err := testutils.NewMockTiKV("", nil)
require.NoError(t, err)
- mockGlue := &gluetidb.MockGlue{}
+ mockGlue := &gluemock.MockGlue{}
ctx, cancel := context.WithCancel(context.Background())
mockMgr := &conn.Mgr{PdController: &pdutil.PdController{}}
mockMgr.SetPDClient(pdClient)
diff --git a/br/pkg/restore/db_test.go b/br/pkg/restore/db_test.go
deleted file mode 100644
index 800058e63b367..0000000000000
--- a/br/pkg/restore/db_test.go
+++ /dev/null
@@ -1,429 +0,0 @@
-// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
-
-package restore_test
-
-import (
- "context"
- "encoding/json"
- "math"
- "strconv"
- "testing"
-
- "github.com/golang/protobuf/proto"
- backuppb "github.com/pingcap/kvproto/pkg/brpb"
- "github.com/pingcap/kvproto/pkg/encryptionpb"
- "github.com/pingcap/tidb/br/pkg/backup"
- "github.com/pingcap/tidb/br/pkg/gluetidb"
- "github.com/pingcap/tidb/br/pkg/metautil"
- "github.com/pingcap/tidb/br/pkg/mock"
- "github.com/pingcap/tidb/br/pkg/restore"
- "github.com/pingcap/tidb/br/pkg/storage"
- "github.com/pingcap/tidb/pkg/infoschema"
- "github.com/pingcap/tidb/pkg/meta/autoid"
- "github.com/pingcap/tidb/pkg/parser/model"
- "github.com/pingcap/tidb/pkg/parser/mysql"
- "github.com/pingcap/tidb/pkg/parser/types"
- "github.com/pingcap/tidb/pkg/session"
- "github.com/pingcap/tidb/pkg/testkit"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
- "github.com/tikv/client-go/v2/oracle"
-)
-
-type testRestoreSchemaSuite struct {
- mock *mock.Cluster
- mockGlue *gluetidb.MockGlue
- storage storage.ExternalStorage
-}
-
-func createRestoreSchemaSuite(t *testing.T) *testRestoreSchemaSuite {
- var err error
- s := new(testRestoreSchemaSuite)
- s.mockGlue = &gluetidb.MockGlue{}
- s.mock, err = mock.NewCluster()
- require.NoError(t, err)
- base := t.TempDir()
- s.storage, err = storage.NewLocalStorage(base)
- require.NoError(t, err)
- require.NoError(t, s.mock.Start())
- t.Cleanup(func() {
- s.mock.Stop()
- })
- return s
-}
-
-func TestRestoreAutoIncID(t *testing.T) {
- s := createRestoreSchemaSuite(t)
- tk := testkit.NewTestKit(t, s.mock.Storage)
- tk.MustExec("use test")
- tk.MustExec("set @@sql_mode=''")
- tk.MustExec("drop table if exists `\"t\"`;")
- // Test SQL Mode
- tk.MustExec("create table `\"t\"` (" +
- "a int not null," +
- "time timestamp not null default '0000-00-00 00:00:00');",
- )
- tk.MustExec("insert into `\"t\"` values (10, '0000-00-00 00:00:00');")
- // Query the current AutoIncID
- autoIncID, err := strconv.ParseUint(tk.MustQuery("admin show `\"t\"` next_row_id").Rows()[0][3].(string), 10, 64)
- require.NoErrorf(t, err, "Error query auto inc id: %s", err)
- // Get schemas of db and table
- info, err := s.mock.Domain.GetSnapshotInfoSchema(math.MaxUint64)
- require.NoErrorf(t, err, "Error get snapshot info schema: %s", err)
- dbInfo, exists := info.SchemaByName(model.NewCIStr("test"))
- require.Truef(t, exists, "Error get db info")
- tableInfo, err := info.TableByName(model.NewCIStr("test"), model.NewCIStr("\"t\""))
- require.NoErrorf(t, err, "Error get table info: %s", err)
- table := metautil.Table{
- Info: tableInfo.Meta(),
- DB: dbInfo,
- }
- // Get the next AutoIncID
- idAlloc := autoid.NewAllocator(s.mock.Domain, dbInfo.ID, table.Info.ID, false, autoid.RowIDAllocType)
- globalAutoID, err := idAlloc.NextGlobalAutoID()
- require.NoErrorf(t, err, "Error allocate next auto id")
- require.Equal(t, uint64(globalAutoID), autoIncID)
- // Alter AutoIncID to the next AutoIncID + 100
- table.Info.AutoIncID = globalAutoID + 100
- db, _, err := restore.NewDB(gluetidb.New(), s.mock.Storage, "STRICT")
- require.NoErrorf(t, err, "Error create DB")
- tk.MustExec("drop database if exists test;")
- // Test empty collate value
- table.DB.Charset = "utf8mb4"
- table.DB.Collate = ""
- err = db.CreateDatabase(context.Background(), table.DB)
- require.NoErrorf(t, err, "Error create empty collate db: %s %s", err, s.mock.DSN)
- tk.MustExec("drop database if exists test;")
- // Test empty charset value
- table.DB.Charset = ""
- table.DB.Collate = "utf8mb4_bin"
- err = db.CreateDatabase(context.Background(), table.DB)
- require.NoErrorf(t, err, "Error create empty charset db: %s %s", err, s.mock.DSN)
- uniqueMap := make(map[restore.UniqueTableName]bool)
- err = db.CreateTable(context.Background(), &table, uniqueMap, false, nil)
- require.NoErrorf(t, err, "Error create table: %s %s", err, s.mock.DSN)
-
- tk.MustExec("use test")
- autoIncID, err = strconv.ParseUint(tk.MustQuery("admin show `\"t\"` next_row_id").Rows()[0][3].(string), 10, 64)
- require.NoErrorf(t, err, "Error query auto inc id: %s", err)
- // Check if AutoIncID is altered successfully.
- require.Equal(t, uint64(globalAutoID+100), autoIncID)
-
- // try again, failed due to table exists.
- table.Info.AutoIncID = globalAutoID + 200
- err = db.CreateTable(context.Background(), &table, uniqueMap, false, nil)
- require.NoError(t, err)
- // Check if AutoIncID is not altered.
- autoIncID, err = strconv.ParseUint(tk.MustQuery("admin show `\"t\"` next_row_id").Rows()[0][3].(string), 10, 64)
- require.NoErrorf(t, err, "Error query auto inc id: %s", err)
- require.Equal(t, uint64(globalAutoID+100), autoIncID)
-
- // try again, success because we use alter sql in unique map.
- table.Info.AutoIncID = globalAutoID + 300
- uniqueMap[restore.UniqueTableName{"test", "\"t\""}] = true
- err = db.CreateTable(context.Background(), &table, uniqueMap, false, nil)
- require.NoError(t, err)
- // Check if AutoIncID is altered to globalAutoID + 300.
- autoIncID, err = strconv.ParseUint(tk.MustQuery("admin show `\"t\"` next_row_id").Rows()[0][3].(string), 10, 64)
- require.NoErrorf(t, err, "Error query auto inc id: %s", err)
- require.Equal(t, uint64(globalAutoID+300), autoIncID)
-}
-
-func TestCreateTablesInDb(t *testing.T) {
- s := createRestoreSchemaSuite(t)
- info, err := s.mock.Domain.GetSnapshotInfoSchema(math.MaxUint64)
- require.NoErrorf(t, err, "Error get snapshot info schema: %s", err)
-
- dbSchema, isExist := info.SchemaByName(model.NewCIStr("test"))
- require.True(t, isExist)
-
- tables := make([]*metautil.Table, 4)
- intField := types.NewFieldType(mysql.TypeLong)
- intField.SetCharset("binary")
- ddlJobMap := make(map[restore.UniqueTableName]bool)
- for i := len(tables) - 1; i >= 0; i-- {
- tables[i] = &metautil.Table{
- DB: dbSchema,
- Info: &model.TableInfo{
- ID: int64(i),
- Name: model.NewCIStr("test" + strconv.Itoa(i)),
- Columns: []*model.ColumnInfo{{
- ID: 1,
- Name: model.NewCIStr("id"),
- FieldType: *intField,
- State: model.StatePublic,
- }},
- Charset: "utf8mb4",
- Collate: "utf8mb4_bin",
- },
- }
- ddlJobMap[restore.UniqueTableName{dbSchema.Name.String(), tables[i].Info.Name.String()}] = false
- }
- db, _, err := restore.NewDB(gluetidb.New(), s.mock.Storage, "STRICT")
- require.NoError(t, err)
-
- err = db.CreateTables(context.Background(), tables, ddlJobMap, false, nil)
- require.NoError(t, err)
-}
-
-func TestFilterDDLJobs(t *testing.T) {
- s := createRestoreSchemaSuite(t)
- tk := testkit.NewTestKit(t, s.mock.Storage)
- tk.MustExec("CREATE DATABASE IF NOT EXISTS test_db;")
- tk.MustExec("CREATE TABLE IF NOT EXISTS test_db.test_table (c1 INT);")
- lastTS, err := s.mock.GetOracle().GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope})
- require.NoErrorf(t, err, "Error get last ts: %s", err)
- tk.MustExec("RENAME TABLE test_db.test_table to test_db.test_table1;")
- tk.MustExec("DROP TABLE test_db.test_table1;")
- tk.MustExec("DROP DATABASE test_db;")
- tk.MustExec("CREATE DATABASE test_db;")
- tk.MustExec("USE test_db;")
- tk.MustExec("CREATE TABLE test_table1 (c2 CHAR(255));")
- tk.MustExec("RENAME TABLE test_table1 to test_table;")
- tk.MustExec("TRUNCATE TABLE test_table;")
-
- ts, err := s.mock.GetOracle().GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope})
- require.NoErrorf(t, err, "Error get ts: %s", err)
-
- cipher := backuppb.CipherInfo{
- CipherType: encryptionpb.EncryptionMethod_PLAINTEXT,
- }
-
- metaWriter := metautil.NewMetaWriter(s.storage, metautil.MetaFileSize, false, "", &cipher)
- ctx := context.Background()
- metaWriter.StartWriteMetasAsync(ctx, metautil.AppendDDL)
- s.mockGlue.SetSession(tk.Session())
- err = backup.WriteBackupDDLJobs(metaWriter, s.mockGlue, s.mock.Storage, lastTS, ts, false)
- require.NoErrorf(t, err, "Error get ddl jobs: %s", err)
- err = metaWriter.FinishWriteMetas(ctx, metautil.AppendDDL)
- require.NoErrorf(t, err, "Flush failed", err)
- err = metaWriter.FlushBackupMeta(ctx)
- require.NoErrorf(t, err, "Finially flush backupmeta failed", err)
- infoSchema, err := s.mock.Domain.GetSnapshotInfoSchema(ts)
- require.NoErrorf(t, err, "Error get snapshot info schema: %s", err)
- dbInfo, ok := infoSchema.SchemaByName(model.NewCIStr("test_db"))
- require.Truef(t, ok, "DB info not exist")
- tableInfo, err := infoSchema.TableByName(model.NewCIStr("test_db"), model.NewCIStr("test_table"))
- require.NoErrorf(t, err, "Error get table info: %s", err)
- tables := []*metautil.Table{{
- DB: dbInfo,
- Info: tableInfo.Meta(),
- }}
- metaBytes, err := s.storage.ReadFile(ctx, metautil.MetaFile)
- require.NoError(t, err)
- mockMeta := &backuppb.BackupMeta{}
- err = proto.Unmarshal(metaBytes, mockMeta)
- require.NoError(t, err)
- // check the schema version
- require.Equal(t, int32(metautil.MetaV1), mockMeta.Version)
- metaReader := metautil.NewMetaReader(mockMeta, s.storage, &cipher)
- allDDLJobsBytes, err := metaReader.ReadDDLs(ctx)
- require.NoError(t, err)
- var allDDLJobs []*model.Job
- err = json.Unmarshal(allDDLJobsBytes, &allDDLJobs)
- require.NoError(t, err)
-
- ddlJobs := restore.FilterDDLJobs(allDDLJobs, tables)
- for _, job := range ddlJobs {
- t.Logf("get ddl job: %s", job.Query)
- }
- require.Equal(t, 7, len(ddlJobs))
-}
-
-func TestFilterDDLJobsV2(t *testing.T) {
- s := createRestoreSchemaSuite(t)
- tk := testkit.NewTestKit(t, s.mock.Storage)
- tk.MustExec("CREATE DATABASE IF NOT EXISTS test_db;")
- tk.MustExec("CREATE TABLE IF NOT EXISTS test_db.test_table (c1 INT);")
- lastTS, err := s.mock.GetOracle().GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope})
- require.NoErrorf(t, err, "Error get last ts: %s", err)
- tk.MustExec("RENAME TABLE test_db.test_table to test_db.test_table1;")
- tk.MustExec("DROP TABLE test_db.test_table1;")
- tk.MustExec("DROP DATABASE test_db;")
- tk.MustExec("CREATE DATABASE test_db;")
- tk.MustExec("USE test_db;")
- tk.MustExec("CREATE TABLE test_table1 (c2 CHAR(255));")
- tk.MustExec("RENAME TABLE test_table1 to test_table;")
- tk.MustExec("TRUNCATE TABLE test_table;")
-
- ts, err := s.mock.GetOracle().GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope})
- require.NoErrorf(t, err, "Error get ts: %s", err)
-
- cipher := backuppb.CipherInfo{
- CipherType: encryptionpb.EncryptionMethod_PLAINTEXT,
- }
-
- metaWriter := metautil.NewMetaWriter(s.storage, metautil.MetaFileSize, true, "", &cipher)
- ctx := context.Background()
- metaWriter.StartWriteMetasAsync(ctx, metautil.AppendDDL)
- s.mockGlue.SetSession(tk.Session())
- err = backup.WriteBackupDDLJobs(metaWriter, s.mockGlue, s.mock.Storage, lastTS, ts, false)
- require.NoErrorf(t, err, "Error get ddl jobs: %s", err)
- err = metaWriter.FinishWriteMetas(ctx, metautil.AppendDDL)
- require.NoErrorf(t, err, "Flush failed", err)
- err = metaWriter.FlushBackupMeta(ctx)
- require.NoErrorf(t, err, "Flush BackupMeta failed", err)
-
- infoSchema, err := s.mock.Domain.GetSnapshotInfoSchema(ts)
- require.NoErrorf(t, err, "Error get snapshot info schema: %s", err)
- dbInfo, ok := infoSchema.SchemaByName(model.NewCIStr("test_db"))
- require.Truef(t, ok, "DB info not exist")
- tableInfo, err := infoSchema.TableByName(model.NewCIStr("test_db"), model.NewCIStr("test_table"))
- require.NoErrorf(t, err, "Error get table info: %s", err)
- tables := []*metautil.Table{{
- DB: dbInfo,
- Info: tableInfo.Meta(),
- }}
- metaBytes, err := s.storage.ReadFile(ctx, metautil.MetaFile)
- require.NoError(t, err)
- mockMeta := &backuppb.BackupMeta{}
- err = proto.Unmarshal(metaBytes, mockMeta)
- require.NoError(t, err)
- // check the schema version
- require.Equal(t, int32(metautil.MetaV2), mockMeta.Version)
- metaReader := metautil.NewMetaReader(mockMeta, s.storage, &cipher)
- allDDLJobsBytes, err := metaReader.ReadDDLs(ctx)
- require.NoError(t, err)
- var allDDLJobs []*model.Job
- err = json.Unmarshal(allDDLJobsBytes, &allDDLJobs)
- require.NoError(t, err)
-
- ddlJobs := restore.FilterDDLJobs(allDDLJobs, tables)
- for _, job := range ddlJobs {
- t.Logf("get ddl job: %s", job.Query)
- }
- require.Equal(t, 7, len(ddlJobs))
-}
-
-func TestDB_ExecDDL(t *testing.T) {
- s := createRestoreSchemaSuite(t)
-
- ctx := context.Background()
- ddlJobs := []*model.Job{
- {
- Type: model.ActionAddIndex,
- Query: "CREATE DATABASE IF NOT EXISTS test_db;",
- BinlogInfo: &model.HistoryInfo{},
- },
- {
- Type: model.ActionAddIndex,
- Query: "",
- BinlogInfo: &model.HistoryInfo{},
- },
- }
-
- db, _, err := restore.NewDB(gluetidb.New(), s.mock.Storage, "STRICT")
- require.NoError(t, err)
-
- for _, ddlJob := range ddlJobs {
- err = db.ExecDDL(ctx, ddlJob)
- assert.NoError(t, err)
- }
-}
-
-func TestFilterDDLJobByRules(t *testing.T) {
- ddlJobs := []*model.Job{
- {
- Type: model.ActionSetTiFlashReplica,
- },
- {
- Type: model.ActionAddPrimaryKey,
- },
- {
- Type: model.ActionUpdateTiFlashReplicaStatus,
- },
- {
- Type: model.ActionCreateTable,
- },
- {
- Type: model.ActionLockTable,
- },
- {
- Type: model.ActionAddIndex,
- },
- {
- Type: model.ActionUnlockTable,
- },
- {
- Type: model.ActionCreateSchema,
- },
- {
- Type: model.ActionModifyColumn,
- },
- }
-
- expectedDDLTypes := []model.ActionType{
- model.ActionAddPrimaryKey,
- model.ActionCreateTable,
- model.ActionAddIndex,
- model.ActionCreateSchema,
- model.ActionModifyColumn,
- }
-
- ddlJobs = restore.FilterDDLJobByRules(ddlJobs, restore.DDLJobBlockListRule)
-
- require.Equal(t, len(expectedDDLTypes), len(ddlJobs))
- for i, ddlJob := range ddlJobs {
- assert.Equal(t, expectedDDLTypes[i], ddlJob.Type)
- }
-}
-
-func TestGetExistedUserDBs(t *testing.T) {
- m, err := mock.NewCluster()
- require.Nil(t, err)
- defer m.Stop()
- dom := m.Domain
-
- dbs := restore.GetExistedUserDBs(dom)
- require.Equal(t, 0, len(dbs))
-
- builder, err := infoschema.NewBuilder(dom, nil, nil).InitWithDBInfos(
- []*model.DBInfo{
- {Name: model.NewCIStr("mysql")},
- {Name: model.NewCIStr("test")},
- },
- nil, nil, 1)
- require.Nil(t, err)
- dom.MockInfoCacheAndLoadInfoSchema(builder.Build(math.MaxUint64))
- dbs = restore.GetExistedUserDBs(dom)
- require.Equal(t, 0, len(dbs))
-
- builder, err = infoschema.NewBuilder(dom, nil, nil).InitWithDBInfos(
- []*model.DBInfo{
- {Name: model.NewCIStr("mysql")},
- {Name: model.NewCIStr("test")},
- {Name: model.NewCIStr("d1")},
- },
- nil, nil, 1)
- require.Nil(t, err)
- dom.MockInfoCacheAndLoadInfoSchema(builder.Build(math.MaxUint64))
- dbs = restore.GetExistedUserDBs(dom)
- require.Equal(t, 1, len(dbs))
-
- builder, err = infoschema.NewBuilder(dom, nil, nil).InitWithDBInfos(
- []*model.DBInfo{
- {Name: model.NewCIStr("mysql")},
- {Name: model.NewCIStr("d1")},
- {
- Name: model.NewCIStr("test"),
- Tables: []*model.TableInfo{{ID: 1, Name: model.NewCIStr("t1"), State: model.StatePublic}},
- State: model.StatePublic,
- },
- },
- nil, nil, 1)
- require.Nil(t, err)
- dom.MockInfoCacheAndLoadInfoSchema(builder.Build(math.MaxUint64))
- dbs = restore.GetExistedUserDBs(dom)
- require.Equal(t, 2, len(dbs))
-}
-
-// NOTICE: Once there is a new system table, BR needs to ensure that it is correctly classified:
-//
-// - IF it is an unrecoverable table, please add the table name into `unRecoverableTable`.
-// - IF it is an system privilege table, please add the table name into `sysPrivilegeTableMap`.
-// - IF it is an statistics table, please add the table name into `statsTables`.
-//
-// The above variables are in the file br/pkg/restore/systable_restore.go
-func TestMonitorTheSystemTableIncremental(t *testing.T) {
- require.Equal(t, int64(196), session.CurrentBootstrapVersion)
-}
diff --git a/br/pkg/restore/file_importer/import.go b/br/pkg/restore/file_importer/import.go
deleted file mode 100644
index 30320ee22e5b4..0000000000000
--- a/br/pkg/restore/file_importer/import.go
+++ /dev/null
@@ -1,1583 +0,0 @@
-// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
-
-package file_importer
-
-import (
- "bytes"
- "context"
- "crypto/tls"
- "fmt"
- "math/rand"
- "strings"
- "sync"
- "sync/atomic"
- "time"
-
- "github.com/google/uuid"
- "github.com/pingcap/errors"
- "github.com/pingcap/failpoint"
- backuppb "github.com/pingcap/kvproto/pkg/brpb"
- "github.com/pingcap/kvproto/pkg/import_sstpb"
- "github.com/pingcap/kvproto/pkg/kvrpcpb"
- "github.com/pingcap/kvproto/pkg/metapb"
- "github.com/pingcap/log"
- "github.com/pingcap/tidb/br/pkg/conn"
- "github.com/pingcap/tidb/br/pkg/conn/util"
- berrors "github.com/pingcap/tidb/br/pkg/errors"
- "github.com/pingcap/tidb/br/pkg/logutil"
- logrestore "github.com/pingcap/tidb/br/pkg/restore/log_restore"
- "github.com/pingcap/tidb/br/pkg/restore/split"
- restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
- "github.com/pingcap/tidb/br/pkg/stream"
- "github.com/pingcap/tidb/br/pkg/summary"
- "github.com/pingcap/tidb/br/pkg/utils"
- "github.com/pingcap/tidb/pkg/kv"
- "github.com/pingcap/tidb/pkg/util/codec"
- kvutil "github.com/tikv/client-go/v2/util"
- pd "github.com/tikv/pd/client"
- "go.uber.org/multierr"
- "go.uber.org/zap"
- "golang.org/x/exp/maps"
- "golang.org/x/sync/errgroup"
- "google.golang.org/grpc"
- "google.golang.org/grpc/backoff"
- "google.golang.org/grpc/codes"
- "google.golang.org/grpc/credentials"
- "google.golang.org/grpc/credentials/insecure"
- "google.golang.org/grpc/keepalive"
- "google.golang.org/grpc/status"
-)
-
-type KvMode int
-
-const (
- TiDB KvMode = iota
- Raw
- Txn
-)
-
-const (
- gRPCBackOffMaxDelay = 3 * time.Second
- // Todo: make it configable
- gRPCTimeOut = 25 * time.Minute
-)
-
-// RewriteMode is a mode flag that tells the TiKV how to handle the rewrite rules.
-type RewriteMode int
-
-const (
- // RewriteModeLegacy means no rewrite rule is applied.
- RewriteModeLegacy RewriteMode = iota
-
- // RewriteModeKeyspace means the rewrite rule could be applied to keyspace.
- RewriteModeKeyspace
-)
-
-// ImporterClient is used to import a file to TiKV.
-type ImporterClient interface {
- ClearFiles(
- ctx context.Context,
- storeID uint64,
- req *import_sstpb.ClearRequest,
- ) (*import_sstpb.ClearResponse, error)
-
- ApplyKVFile(
- ctx context.Context,
- storeID uint64,
- req *import_sstpb.ApplyRequest,
- ) (*import_sstpb.ApplyResponse, error)
-
- DownloadSST(
- ctx context.Context,
- storeID uint64,
- req *import_sstpb.DownloadRequest,
- ) (*import_sstpb.DownloadResponse, error)
-
- IngestSST(
- ctx context.Context,
- storeID uint64,
- req *import_sstpb.IngestRequest,
- ) (*import_sstpb.IngestResponse, error)
- MultiIngest(
- ctx context.Context,
- storeID uint64,
- req *import_sstpb.MultiIngestRequest,
- ) (*import_sstpb.IngestResponse, error)
-
- SetDownloadSpeedLimit(
- ctx context.Context,
- storeID uint64,
- req *import_sstpb.SetDownloadSpeedLimitRequest,
- ) (*import_sstpb.SetDownloadSpeedLimitResponse, error)
-
- GetImportClient(
- ctx context.Context,
- storeID uint64,
- ) (import_sstpb.ImportSSTClient, error)
-
- CloseGrpcClient() error
-
- SupportMultiIngest(ctx context.Context, stores []uint64) (bool, error)
-}
-
-type importClient struct {
- metaClient split.SplitClient
- mu sync.Mutex
- // Notice: In order to avoid leak for BRIE via SQL, it needs to close grpc client connection before br task exits.
- // So it caches the grpc connection instead of import_sstpb.ImportSSTClient.
- // used for any request except the ingest reqeust
- conns map[uint64]*grpc.ClientConn
- // used for ingest request
- ingestConns map[uint64]*grpc.ClientConn
-
- tlsConf *tls.Config
- keepaliveConf keepalive.ClientParameters
-}
-
-// NewImportClient returns a new ImporterClient.
-func NewImportClient(metaClient split.SplitClient, tlsConf *tls.Config, keepaliveConf keepalive.ClientParameters) ImporterClient {
- return &importClient{
- metaClient: metaClient,
- conns: make(map[uint64]*grpc.ClientConn),
- ingestConns: make(map[uint64]*grpc.ClientConn),
- tlsConf: tlsConf,
- keepaliveConf: keepaliveConf,
- }
-}
-
-func (ic *importClient) ClearFiles(
- ctx context.Context,
- storeID uint64,
- req *import_sstpb.ClearRequest,
-) (*import_sstpb.ClearResponse, error) {
- client, err := ic.GetImportClient(ctx, storeID)
- if err != nil {
- return nil, errors.Trace(err)
- }
- return client.ClearFiles(ctx, req)
-}
-
-func (ic *importClient) ApplyKVFile(
- ctx context.Context,
- storeID uint64,
- req *import_sstpb.ApplyRequest,
-) (*import_sstpb.ApplyResponse, error) {
- client, err := ic.GetImportClient(ctx, storeID)
- if err != nil {
- return nil, errors.Trace(err)
- }
- return client.Apply(ctx, req)
-}
-
-func (ic *importClient) DownloadSST(
- ctx context.Context,
- storeID uint64,
- req *import_sstpb.DownloadRequest,
-) (*import_sstpb.DownloadResponse, error) {
- client, err := ic.GetImportClient(ctx, storeID)
- if err != nil {
- return nil, errors.Trace(err)
- }
- return client.Download(ctx, req)
-}
-
-func (ic *importClient) SetDownloadSpeedLimit(
- ctx context.Context,
- storeID uint64,
- req *import_sstpb.SetDownloadSpeedLimitRequest,
-) (*import_sstpb.SetDownloadSpeedLimitResponse, error) {
- client, err := ic.GetImportClient(ctx, storeID)
- if err != nil {
- return nil, errors.Trace(err)
- }
- return client.SetDownloadSpeedLimit(ctx, req)
-}
-
-func (ic *importClient) IngestSST(
- ctx context.Context,
- storeID uint64,
- req *import_sstpb.IngestRequest,
-) (*import_sstpb.IngestResponse, error) {
- client, err := ic.GetIngestClient(ctx, storeID)
- if err != nil {
- return nil, errors.Trace(err)
- }
- return client.Ingest(ctx, req)
-}
-
-func (ic *importClient) MultiIngest(
- ctx context.Context,
- storeID uint64,
- req *import_sstpb.MultiIngestRequest,
-) (*import_sstpb.IngestResponse, error) {
- client, err := ic.GetIngestClient(ctx, storeID)
- if err != nil {
- return nil, errors.Trace(err)
- }
- return client.MultiIngest(ctx, req)
-}
-
-func (ic *importClient) createGrpcConn(
- ctx context.Context,
- storeID uint64,
-) (*grpc.ClientConn, error) {
- store, err := ic.metaClient.GetStore(ctx, storeID)
- if err != nil {
- return nil, errors.Trace(err)
- }
- opt := grpc.WithTransportCredentials(insecure.NewCredentials())
- if ic.tlsConf != nil {
- opt = grpc.WithTransportCredentials(credentials.NewTLS(ic.tlsConf))
- }
- addr := store.GetPeerAddress()
- if addr == "" {
- addr = store.GetAddress()
- }
- bfConf := backoff.DefaultConfig
- bfConf.MaxDelay = gRPCBackOffMaxDelay
- conn, err := grpc.DialContext(
- ctx,
- addr,
- opt,
- grpc.WithBlock(),
- grpc.FailOnNonTempDialError(true),
- grpc.WithConnectParams(grpc.ConnectParams{Backoff: bfConf}),
- grpc.WithKeepaliveParams(ic.keepaliveConf),
- )
- return conn, errors.Trace(err)
-}
-
-func (ic *importClient) cachedConnectionFrom(
- ctx context.Context,
- storeID uint64,
- caches map[uint64]*grpc.ClientConn,
-) (import_sstpb.ImportSSTClient, error) {
- ic.mu.Lock()
- defer ic.mu.Unlock()
- conn, ok := caches[storeID]
- if ok {
- return import_sstpb.NewImportSSTClient(conn), nil
- }
- conn, err := ic.createGrpcConn(ctx, storeID)
- if err != nil {
- return nil, errors.Trace(err)
- }
- caches[storeID] = conn
- return import_sstpb.NewImportSSTClient(conn), nil
-}
-
-func (ic *importClient) GetImportClient(
- ctx context.Context,
- storeID uint64,
-) (import_sstpb.ImportSSTClient, error) {
- return ic.cachedConnectionFrom(ctx, storeID, ic.conns)
-}
-
-func (ic *importClient) GetIngestClient(
- ctx context.Context,
- storeID uint64,
-) (import_sstpb.ImportSSTClient, error) {
- return ic.cachedConnectionFrom(ctx, storeID, ic.ingestConns)
-}
-
-func (ic *importClient) CloseGrpcClient() error {
- ic.mu.Lock()
- defer ic.mu.Unlock()
- for id, conn := range ic.conns {
- if err := conn.Close(); err != nil {
- return errors.Trace(err)
- }
- delete(ic.conns, id)
- }
- for id, conn := range ic.ingestConns {
- if err := conn.Close(); err != nil {
- return errors.Trace(err)
- }
- delete(ic.ingestConns, id)
- }
- return nil
-}
-
-func (ic *importClient) SupportMultiIngest(ctx context.Context, stores []uint64) (bool, error) {
- for _, storeID := range stores {
- _, err := ic.MultiIngest(ctx, storeID, &import_sstpb.MultiIngestRequest{})
- if err != nil {
- if s, ok := status.FromError(err); ok {
- if s.Code() == codes.Unimplemented {
- return false, nil
- }
- }
- return false, errors.Trace(err)
- }
- }
- return true, nil
-}
-
-type storeTokenChannelMap struct {
- sync.RWMutex
- tokens map[uint64]chan struct{}
-}
-
-func (s *storeTokenChannelMap) acquireTokenCh(storeID uint64, bufferSize uint) chan struct{} {
- s.RLock()
- tokenCh, ok := s.tokens[storeID]
- // handle the case that the store is new-scaled in the cluster
- if !ok {
- s.RUnlock()
- s.Lock()
- // Notice: worker channel can't replaced, because it is still used after unlock.
- if tokenCh, ok = s.tokens[storeID]; !ok {
- tokenCh = utils.BuildWorkerTokenChannel(bufferSize)
- s.tokens[storeID] = tokenCh
- }
- s.Unlock()
- } else {
- s.RUnlock()
- }
- return tokenCh
-}
-
-func (s *storeTokenChannelMap) ShouldBlock() bool {
- s.RLock()
- defer s.RUnlock()
- if len(s.tokens) == 0 {
- // never block if there is no store worker pool
- return false
- }
- for _, pool := range s.tokens {
- if len(pool) > 0 {
- // At least one store worker pool has available worker
- return false
- }
- }
- return true
-}
-
-func newStoreTokenChannelMap(stores []*metapb.Store, bufferSize uint) *storeTokenChannelMap {
- storeTokenChannelMap := &storeTokenChannelMap{
- sync.RWMutex{},
- make(map[uint64]chan struct{}),
- }
- if bufferSize == 0 {
- return storeTokenChannelMap
- }
- for _, store := range stores {
- ch := utils.BuildWorkerTokenChannel(bufferSize)
- storeTokenChannelMap.tokens[store.Id] = ch
- }
- return storeTokenChannelMap
-}
-
-// FileImporter used to import a file to TiKV.
-type FileImporter struct {
- metaClient split.SplitClient
- importClient ImporterClient
- backend *backuppb.StorageBackend
-
- downloadTokensMap *storeTokenChannelMap
- ingestTokensMap *storeTokenChannelMap
-
- concurrencyPerStore uint
- useTokenBucket bool
-
- kvMode KvMode
- rawStartKey []byte
- rawEndKey []byte
- SupportMultiIngest bool
- rewriteMode RewriteMode
-
- cacheKey string
- cond *sync.Cond
-}
-
-// NewFileImporter returns a new file importClient.
-func NewFileImporter(
- metaClient split.SplitClient,
- importClient ImporterClient,
- backend *backuppb.StorageBackend,
- isRawKvMode bool,
- isTxnKvMode bool,
- stores []*metapb.Store,
- rewriteMode RewriteMode,
- concurrencyPerStore uint,
- useTokenBucket bool,
-) FileImporter {
- kvMode := TiDB
- if isRawKvMode {
- kvMode = Raw
- }
- if isTxnKvMode {
- kvMode = Txn
- }
-
- downloadTokensMap := newStoreTokenChannelMap(stores, 0)
- ingestTokensMap := newStoreTokenChannelMap(stores, 0)
-
- if useTokenBucket {
- downloadTokensMap = newStoreTokenChannelMap(stores, concurrencyPerStore)
- ingestTokensMap = newStoreTokenChannelMap(stores, concurrencyPerStore)
- }
- return FileImporter{
- metaClient: metaClient,
- backend: backend,
- importClient: importClient,
- downloadTokensMap: downloadTokensMap,
- ingestTokensMap: ingestTokensMap,
- kvMode: kvMode,
- rewriteMode: rewriteMode,
- cacheKey: fmt.Sprintf("BR-%s-%d", time.Now().Format("20060102150405"), rand.Int63()),
- concurrencyPerStore: concurrencyPerStore,
- useTokenBucket: useTokenBucket,
- cond: sync.NewCond(new(sync.Mutex)),
- }
-}
-
-func (importer *FileImporter) WaitUntilUnblock() {
- importer.cond.L.Lock()
- for importer.ShouldBlock() {
- // wait for download worker notified
- importer.cond.Wait()
- }
- importer.cond.L.Unlock()
-}
-
-func (importer *FileImporter) ShouldBlock() bool {
- if importer != nil && importer.useTokenBucket {
- return importer.downloadTokensMap.ShouldBlock() || importer.ingestTokensMap.ShouldBlock()
- }
- return false
-}
-
-func (importer *FileImporter) releaseToken(tokenCh chan struct{}) {
- tokenCh <- struct{}{}
- // finish the task, notify the main goroutine to continue
- importer.cond.L.Lock()
- importer.cond.Signal()
- importer.cond.L.Unlock()
-}
-
-func (importer *FileImporter) Close() error {
- if importer != nil && importer.importClient != nil {
- return importer.importClient.CloseGrpcClient()
- }
- return nil
-}
-
-// CheckMultiIngestSupport checks whether all stores support multi-ingest
-func (importer *FileImporter) CheckMultiIngestSupport(ctx context.Context, pdClient pd.Client) error {
- allStores, err := util.GetAllTiKVStores(ctx, pdClient, util.SkipTiFlash)
- if err != nil {
- return errors.Trace(err)
- }
- storeIDs := make([]uint64, 0, len(allStores))
- for _, s := range allStores {
- if s.State != metapb.StoreState_Up {
- continue
- }
- storeIDs = append(storeIDs, s.Id)
- }
-
- support, err := importer.importClient.SupportMultiIngest(ctx, storeIDs)
- if err != nil {
- return errors.Trace(err)
- }
- importer.SupportMultiIngest = support
- log.L().Info("multi ingest support", zap.Bool("support", support))
- return nil
-}
-
-// SetRawRange sets the range to be restored in raw kv mode.
-func (importer *FileImporter) SetRawRange(startKey, endKey []byte) error {
- if importer.kvMode != Raw {
- return errors.Annotate(berrors.ErrRestoreModeMismatch, "file importer is not in raw kv mode")
- }
- importer.rawStartKey = startKey
- importer.rawEndKey = endKey
- return nil
-}
-
-func getKeyRangeByMode(mode KvMode) func(f *backuppb.File, rules *restoreutils.RewriteRules) ([]byte, []byte, error) {
- switch mode {
- case Raw:
- return func(f *backuppb.File, rules *restoreutils.RewriteRules) ([]byte, []byte, error) {
- return f.GetStartKey(), f.GetEndKey(), nil
- }
- case Txn:
- return func(f *backuppb.File, rules *restoreutils.RewriteRules) ([]byte, []byte, error) {
- start, end := f.GetStartKey(), f.GetEndKey()
- if len(start) != 0 {
- start = codec.EncodeBytes([]byte{}, f.GetStartKey())
- }
- if len(end) != 0 {
- end = codec.EncodeBytes([]byte{}, f.GetEndKey())
- }
- return start, end, nil
- }
- default:
- return func(f *backuppb.File, rules *restoreutils.RewriteRules) ([]byte, []byte, error) {
- return restoreutils.GetRewriteRawKeys(f, rules)
- }
- }
-}
-
-var GetKeyRangeByModeForTest = getKeyRangeByMode
-
-// getKeyRangeForFiles gets the maximum range on files.
-func (importer *FileImporter) getKeyRangeForFiles(
- files []*backuppb.File,
- rewriteRules *restoreutils.RewriteRules,
-) ([]byte, []byte, error) {
- var (
- startKey, endKey []byte
- start, end []byte
- err error
- )
- getRangeFn := getKeyRangeByMode(importer.kvMode)
- for _, f := range files {
- start, end, err = getRangeFn(f, rewriteRules)
- if err != nil {
- return nil, nil, errors.Trace(err)
- }
- if len(startKey) == 0 || bytes.Compare(start, startKey) < 0 {
- startKey = start
- }
- if len(endKey) == 0 || bytes.Compare(endKey, end) < 0 {
- endKey = end
- }
- }
-
- log.Debug("rewrite file keys", logutil.Files(files),
- logutil.Key("startKey", startKey), logutil.Key("endKey", endKey))
- return startKey, endKey, nil
-}
-
-// Import tries to import a file.
-func (importer *FileImporter) ImportKVFileForRegion(
- ctx context.Context,
- files []*logrestore.LogDataFileInfo,
- rule *restoreutils.RewriteRules,
- shiftStartTS uint64,
- startTS uint64,
- restoreTS uint64,
- info *split.RegionInfo,
- supportBatch bool,
-) RPCResult {
- // Try to download file.
- result := importer.downloadAndApplyKVFile(ctx, files, rule, info, shiftStartTS, startTS, restoreTS, supportBatch)
- if !result.OK() {
- errDownload := result.Err
- for _, e := range multierr.Errors(errDownload) {
- switch errors.Cause(e) { // nolint:errorlint
- case berrors.ErrKVRewriteRuleNotFound, berrors.ErrKVRangeIsEmpty:
- // Skip this region
- logutil.CL(ctx).Warn("download file skipped",
- logutil.Region(info.Region),
- logutil.ShortError(e))
- return RPCResultOK()
- }
- }
- logutil.CL(ctx).Warn("download and apply file failed",
- logutil.ShortError(&result))
- return result
- }
- summary.CollectInt("RegionInvolved", 1)
- return RPCResultOK()
-}
-
-func (importer *FileImporter) ClearFiles(ctx context.Context, pdClient pd.Client, prefix string) error {
- allStores, err := conn.GetAllTiKVStoresWithRetry(ctx, pdClient, util.SkipTiFlash)
- if err != nil {
- return errors.Trace(err)
- }
- for _, s := range allStores {
- if s.State != metapb.StoreState_Up {
- continue
- }
- req := &import_sstpb.ClearRequest{
- Prefix: prefix,
- }
- _, err = importer.importClient.ClearFiles(ctx, s.GetId(), req)
- if err != nil {
- log.Warn("cleanup kv files failed", zap.Uint64("store", s.GetId()), zap.Error(err))
- }
- }
- return nil
-}
-
-func FilterFilesByRegion(
- files []*logrestore.LogDataFileInfo,
- ranges []kv.KeyRange,
- r *split.RegionInfo,
-) ([]*logrestore.LogDataFileInfo, error) {
- if len(files) != len(ranges) {
- return nil, errors.Annotatef(berrors.ErrInvalidArgument,
- "count of files no equals count of ranges, file-count:%v, ranges-count:%v",
- len(files), len(ranges))
- }
-
- output := make([]*logrestore.LogDataFileInfo, 0, len(files))
- if r != nil && r.Region != nil {
- for i, f := range files {
- if bytes.Compare(r.Region.StartKey, ranges[i].EndKey) <= 0 &&
- (len(r.Region.EndKey) == 0 || bytes.Compare(r.Region.EndKey, ranges[i].StartKey) >= 0) {
- output = append(output, f)
- }
- }
- } else {
- output = files
- }
-
- return output, nil
-}
-
-// ImportKVFiles restores the kv events.
-func (importer *FileImporter) ImportKVFiles(
- ctx context.Context,
- files []*logrestore.LogDataFileInfo,
- rule *restoreutils.RewriteRules,
- shiftStartTS uint64,
- startTS uint64,
- restoreTS uint64,
- supportBatch bool,
-) error {
- var (
- startKey []byte
- endKey []byte
- ranges = make([]kv.KeyRange, len(files))
- err error
- )
-
- if !supportBatch && len(files) > 1 {
- return errors.Annotatef(berrors.ErrInvalidArgument,
- "do not support batch apply but files count:%v > 1", len(files))
- }
- log.Debug("import kv files", zap.Int("batch file count", len(files)))
-
- for i, f := range files {
- ranges[i].StartKey, ranges[i].EndKey, err = restoreutils.GetRewriteEncodedKeys(f, rule)
- if err != nil {
- return errors.Trace(err)
- }
-
- if len(startKey) == 0 || bytes.Compare(ranges[i].StartKey, startKey) < 0 {
- startKey = ranges[i].StartKey
- }
- if len(endKey) == 0 || bytes.Compare(ranges[i].EndKey, endKey) > 0 {
- endKey = ranges[i].EndKey
- }
- }
-
- log.Debug("rewrite file keys",
- logutil.Key("startKey", startKey), logutil.Key("endKey", endKey))
-
- // This RetryState will retry 45 time, about 10 min.
- rs := utils.InitialRetryState(45, 100*time.Millisecond, 15*time.Second)
- ctl := OverRegionsInRange(startKey, endKey, importer.metaClient, &rs)
- err = ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) RPCResult {
- subfiles, errFilter := FilterFilesByRegion(files, ranges, r)
- if errFilter != nil {
- return RPCResultFromError(errFilter)
- }
- if len(subfiles) == 0 {
- return RPCResultOK()
- }
- return importer.ImportKVFileForRegion(ctx, subfiles, rule, shiftStartTS, startTS, restoreTS, r, supportBatch)
- })
- return errors.Trace(err)
-}
-
-// ImportSSTFiles tries to import a file.
-// All rules must contain encoded keys.
-func (importer *FileImporter) ImportSSTFiles(
- ctx context.Context,
- files []*backuppb.File,
- rewriteRules *restoreutils.RewriteRules,
- cipher *backuppb.CipherInfo,
- apiVersion kvrpcpb.APIVersion,
-) error {
- start := time.Now()
- log.Debug("import file", logutil.Files(files))
-
- // Rewrite the start key and end key of file to scan regions
- startKey, endKey, err := importer.getKeyRangeForFiles(files, rewriteRules)
- if err != nil {
- return errors.Trace(err)
- }
-
- downloadFn := importer.download
- if importer.useTokenBucket {
- downloadFn = importer.downloadV2
- }
-
- err = utils.WithRetry(ctx, func() error {
- // Scan regions covered by the file range
- regionInfos, errScanRegion := split.PaginateScanRegion(
- ctx, importer.metaClient, startKey, endKey, split.ScanRegionPaginationLimit)
- if errScanRegion != nil {
- return errors.Trace(errScanRegion)
- }
-
- log.Debug("scan regions", logutil.Files(files), zap.Int("count", len(regionInfos)))
- // Try to download and ingest the file in every region
- regionLoop:
- for _, regionInfo := range regionInfos {
- info := regionInfo
- // Try to download file.
- downloadMetas, errDownload := downloadFn(ctx, info, files, rewriteRules, cipher, apiVersion)
- if errDownload != nil {
- for _, e := range multierr.Errors(errDownload) {
- switch errors.Cause(e) { // nolint:errorlint
- case berrors.ErrKVRewriteRuleNotFound, berrors.ErrKVRangeIsEmpty:
- // Skip this region
- log.Warn("download file skipped",
- logutil.Files(files),
- logutil.Region(info.Region),
- logutil.Key("startKey", startKey),
- logutil.Key("endKey", endKey),
- logutil.Key("file-simple-start", files[0].StartKey),
- logutil.Key("file-simple-end", files[0].EndKey),
- logutil.ShortError(e))
- continue regionLoop
- }
- }
- log.Warn("download file failed, retry later",
- logutil.Files(files),
- logutil.Region(info.Region),
- logutil.Key("startKey", startKey),
- logutil.Key("endKey", endKey),
- logutil.ShortError(errDownload))
- return errors.Trace(errDownload)
- }
- log.Debug("download file done",
- zap.String("file-sample", files[0].Name), zap.Stringer("take", time.Since(start)),
- logutil.Key("start", files[0].StartKey), logutil.Key("end", files[0].EndKey))
- start = time.Now()
- if errIngest := importer.ingest(ctx, files, info, downloadMetas); errIngest != nil {
- log.Warn("ingest file failed, retry later",
- logutil.Files(files),
- logutil.SSTMetas(downloadMetas),
- logutil.Region(info.Region),
- zap.Error(errIngest))
- return errors.Trace(errIngest)
- }
- log.Debug("ingest file done", zap.String("file-sample", files[0].Name), zap.Stringer("take", time.Since(start)))
- }
-
- for _, f := range files {
- summary.CollectSuccessUnit(summary.TotalKV, 1, f.TotalKvs)
- summary.CollectSuccessUnit(summary.TotalBytes, 1, f.TotalBytes)
- }
- return nil
- }, utils.NewImportSSTBackoffer())
- if err != nil {
- log.Error("import sst file failed after retry, stop the whole progress", logutil.Files(files), zap.Error(err))
- return errors.Trace(err)
- }
- return nil
-}
-
-func (importer *FileImporter) SetDownloadSpeedLimit(ctx context.Context, storeID, rateLimit uint64) error {
- req := &import_sstpb.SetDownloadSpeedLimitRequest{
- SpeedLimit: rateLimit,
- }
- _, err := importer.importClient.SetDownloadSpeedLimit(ctx, storeID, req)
- return errors.Trace(err)
-}
-
-func (importer *FileImporter) download(
- ctx context.Context,
- regionInfo *split.RegionInfo,
- files []*backuppb.File,
- rewriteRules *restoreutils.RewriteRules,
- cipher *backuppb.CipherInfo,
- apiVersion kvrpcpb.APIVersion,
-) ([]*import_sstpb.SSTMeta, error) {
- var (
- downloadMetas = make([]*import_sstpb.SSTMeta, 0, len(files))
- remainFiles = files
- )
- errDownload := utils.WithRetry(ctx, func() error {
- var e error
- for i, f := range remainFiles {
- var downloadMeta *import_sstpb.SSTMeta
- // we treat Txn kv file as Raw kv file. because we don't have table id to decode
- if importer.kvMode == Raw || importer.kvMode == Txn {
- downloadMeta, e = importer.downloadRawKVSST(ctx, regionInfo, f, cipher, apiVersion)
- } else {
- downloadMeta, e = importer.downloadSST(ctx, regionInfo, f, rewriteRules, cipher, apiVersion)
- }
-
- failpoint.Inject("restore-storage-error", func(val failpoint.Value) {
- msg := val.(string)
- log.Debug("failpoint restore-storage-error injected.", zap.String("msg", msg))
- e = errors.Annotate(e, msg)
- })
- failpoint.Inject("restore-gRPC-error", func(_ failpoint.Value) {
- log.Warn("the connection to TiKV has been cut by a neko, meow :3")
- e = status.Error(codes.Unavailable, "the connection to TiKV has been cut by a neko, meow :3")
- })
- if isDecryptSstErr(e) {
- log.Info("fail to decrypt when download sst, try again with no-crypt", logutil.File(f))
- if importer.kvMode == Raw || importer.kvMode == Txn {
- downloadMeta, e = importer.downloadRawKVSST(ctx, regionInfo, f, nil, apiVersion)
- } else {
- downloadMeta, e = importer.downloadSST(ctx, regionInfo, f, rewriteRules, nil, apiVersion)
- }
- }
-
- if e != nil {
- remainFiles = remainFiles[i:]
- return errors.Trace(e)
- }
-
- downloadMetas = append(downloadMetas, downloadMeta)
- }
-
- return nil
- }, utils.NewDownloadSSTBackoffer())
-
- return downloadMetas, errDownload
-}
-
-// GetSSTMetaFromFile compares the keys in file, region and rewrite rules, then returns a sst conn.
-// The range of the returned sst meta is [regionRule.NewKeyPrefix, append(regionRule.NewKeyPrefix, 0xff)].
-func GetSSTMetaFromFile(
- id []byte,
- file *backuppb.File,
- region *metapb.Region,
- regionRule *import_sstpb.RewriteRule,
- rewriteMode RewriteMode,
-) (meta *import_sstpb.SSTMeta, err error) {
- r := *region
- // If the rewrite mode is for keyspace, then the region bound should be decoded.
- if rewriteMode == RewriteModeKeyspace {
- if len(region.GetStartKey()) > 0 {
- _, r.StartKey, err = codec.DecodeBytes(region.GetStartKey(), nil)
- if err != nil {
- return
- }
- }
- if len(region.GetEndKey()) > 0 {
- _, r.EndKey, err = codec.DecodeBytes(region.GetEndKey(), nil)
- if err != nil {
- return
- }
- }
- }
-
- // Get the column family of the file by the file name.
- var cfName string
- if strings.Contains(file.GetName(), restoreutils.DefaultCFName) {
- cfName = restoreutils.DefaultCFName
- } else if strings.Contains(file.GetName(), restoreutils.WriteCFName) {
- cfName = restoreutils.WriteCFName
- }
- // Find the overlapped part between the file and the region.
- // Here we rewrites the keys to compare with the keys of the region.
- rangeStart := regionRule.GetNewKeyPrefix()
- // rangeStart = max(rangeStart, region.StartKey)
- if bytes.Compare(rangeStart, r.GetStartKey()) < 0 {
- rangeStart = r.GetStartKey()
- }
-
- // Append 10 * 0xff to make sure rangeEnd cover all file key
- // If choose to regionRule.NewKeyPrefix + 1, it may cause WrongPrefix here
- // https://github.com/tikv/tikv/blob/970a9bf2a9ea782a455ae579ad237aaf6cb1daec/
- // components/sst_importer/src/sst_importer.rs#L221
- suffix := []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
- rangeEnd := append(append([]byte{}, regionRule.GetNewKeyPrefix()...), suffix...)
- // rangeEnd = min(rangeEnd, region.EndKey)
- if len(r.GetEndKey()) > 0 && bytes.Compare(rangeEnd, r.GetEndKey()) > 0 {
- rangeEnd = r.GetEndKey()
- }
-
- if bytes.Compare(rangeStart, rangeEnd) > 0 {
- log.Panic("range start exceed range end",
- logutil.File(file),
- logutil.Key("startKey", rangeStart),
- logutil.Key("endKey", rangeEnd))
- }
-
- log.Debug("get sstMeta",
- logutil.Region(region),
- logutil.File(file),
- logutil.Key("startKey", rangeStart),
- logutil.Key("endKey", rangeEnd))
-
- return &import_sstpb.SSTMeta{
- Uuid: id,
- CfName: cfName,
- Range: &import_sstpb.Range{
- Start: rangeStart,
- End: rangeEnd,
- },
- Length: file.GetSize_(),
- RegionId: region.GetId(),
- RegionEpoch: region.GetRegionEpoch(),
- CipherIv: file.GetCipherIv(),
- }, nil
-}
-
-func (importer *FileImporter) downloadSST(
- ctx context.Context,
- regionInfo *split.RegionInfo,
- file *backuppb.File,
- rewriteRules *restoreutils.RewriteRules,
- cipher *backuppb.CipherInfo,
- apiVersion kvrpcpb.APIVersion,
-) (*import_sstpb.SSTMeta, error) {
- uid := uuid.New()
- id := uid[:]
- // Get the rewrite rule for the file.
- fileRule := restoreutils.FindMatchedRewriteRule(file, rewriteRules)
- if fileRule == nil {
- return nil, errors.Trace(berrors.ErrKVRewriteRuleNotFound)
- }
-
- // For the legacy version of TiKV, we need to encode the key prefix, since in the legacy
- // version, the TiKV will rewrite the key with the encoded prefix without decoding the keys in
- // the SST file. For the new version of TiKV that support keyspace rewrite, we don't need to
- // encode the key prefix. The TiKV will decode the keys in the SST file and rewrite the keys
- // with the plain prefix and encode the keys before writing to SST.
-
- // for the keyspace rewrite mode
- rule := *fileRule
- // for the legacy rewrite mode
- if importer.rewriteMode == RewriteModeLegacy {
- rule.OldKeyPrefix = restoreutils.EncodeKeyPrefix(fileRule.GetOldKeyPrefix())
- rule.NewKeyPrefix = restoreutils.EncodeKeyPrefix(fileRule.GetNewKeyPrefix())
- }
-
- sstMeta, err := GetSSTMetaFromFile(id, file, regionInfo.Region, &rule, importer.rewriteMode)
- if err != nil {
- return nil, err
- }
-
- req := &import_sstpb.DownloadRequest{
- Sst: *sstMeta,
- StorageBackend: importer.backend,
- Name: file.GetName(),
- RewriteRule: rule,
- CipherInfo: cipher,
- StorageCacheId: importer.cacheKey,
- // For the older version of TiDB, the request type will be default to `import_sstpb.RequestType_Legacy`
- RequestType: import_sstpb.DownloadRequestType_Keyspace,
- Context: &kvrpcpb.Context{
- ResourceControlContext: &kvrpcpb.ResourceControlContext{
- ResourceGroupName: "", // TODO,
- },
- RequestSource: kvutil.BuildRequestSource(true, kv.InternalTxnBR, kvutil.ExplicitTypeBR),
- },
- }
- log.Debug("download SST",
- logutil.SSTMeta(sstMeta),
- logutil.File(file),
- logutil.Region(regionInfo.Region),
- logutil.Leader(regionInfo.Leader),
- )
-
- var atomicResp atomic.Pointer[import_sstpb.DownloadResponse]
- eg, ectx := errgroup.WithContext(ctx)
- for _, p := range regionInfo.Region.GetPeers() {
- peer := p
- eg.Go(func() error {
- resp, err := importer.importClient.DownloadSST(ectx, peer.GetStoreId(), req)
- if err != nil {
- return errors.Trace(err)
- }
- if resp.GetError() != nil {
- return errors.Annotate(berrors.ErrKVDownloadFailed, resp.GetError().GetMessage())
- }
- if resp.GetIsEmpty() {
- return errors.Trace(berrors.ErrKVRangeIsEmpty)
- }
-
- log.Debug("download from peer",
- logutil.Region(regionInfo.Region),
- logutil.Peer(peer),
- logutil.Key("resp-range-start", resp.Range.Start),
- logutil.Key("resp-range-end", resp.Range.End),
- zap.Bool("resp-isempty", resp.IsEmpty),
- zap.Uint32("resp-crc32", resp.Crc32),
- )
- atomicResp.Store(resp)
- return nil
- })
- }
- if err := eg.Wait(); err != nil {
- return nil, err
- }
-
- downloadResp := atomicResp.Load()
- sstMeta.Range.Start = restoreutils.TruncateTS(downloadResp.Range.GetStart())
- sstMeta.Range.End = restoreutils.TruncateTS(downloadResp.Range.GetEnd())
- sstMeta.ApiVersion = apiVersion
- return sstMeta, nil
-}
-
-func (importer *FileImporter) downloadRawKVSST(
- ctx context.Context,
- regionInfo *split.RegionInfo,
- file *backuppb.File,
- cipher *backuppb.CipherInfo,
- apiVersion kvrpcpb.APIVersion,
-) (*import_sstpb.SSTMeta, error) {
- uid := uuid.New()
- id := uid[:]
- // Empty rule
- var rule import_sstpb.RewriteRule
- sstMeta, err := GetSSTMetaFromFile(id, file, regionInfo.Region, &rule, RewriteModeLegacy)
- if err != nil {
- return nil, err
- }
-
- // Cut the SST file's range to fit in the restoring range.
- if bytes.Compare(importer.rawStartKey, sstMeta.Range.GetStart()) > 0 {
- sstMeta.Range.Start = importer.rawStartKey
- }
- if len(importer.rawEndKey) > 0 &&
- (len(sstMeta.Range.GetEnd()) == 0 || bytes.Compare(importer.rawEndKey, sstMeta.Range.GetEnd()) <= 0) {
- sstMeta.Range.End = importer.rawEndKey
- sstMeta.EndKeyExclusive = true
- }
- if bytes.Compare(sstMeta.Range.GetStart(), sstMeta.Range.GetEnd()) > 0 {
- return nil, errors.Trace(berrors.ErrKVRangeIsEmpty)
- }
-
- req := &import_sstpb.DownloadRequest{
- Sst: *sstMeta,
- StorageBackend: importer.backend,
- Name: file.GetName(),
- RewriteRule: rule,
- IsRawKv: true,
- CipherInfo: cipher,
- StorageCacheId: importer.cacheKey,
- Context: &kvrpcpb.Context{
- ResourceControlContext: &kvrpcpb.ResourceControlContext{
- ResourceGroupName: "", // TODO,
- },
- RequestSource: kvutil.BuildRequestSource(true, kv.InternalTxnBR, kvutil.ExplicitTypeBR),
- },
- }
- log.Debug("download SST", logutil.SSTMeta(sstMeta), logutil.Region(regionInfo.Region))
-
- var atomicResp atomic.Pointer[import_sstpb.DownloadResponse]
- eg, ectx := errgroup.WithContext(ctx)
- for _, p := range regionInfo.Region.GetPeers() {
- peer := p
- eg.Go(func() error {
- resp, err := importer.importClient.DownloadSST(ectx, peer.GetStoreId(), req)
- if err != nil {
- return errors.Trace(err)
- }
- if resp.GetError() != nil {
- return errors.Annotate(berrors.ErrKVDownloadFailed, resp.GetError().GetMessage())
- }
- if resp.GetIsEmpty() {
- return errors.Trace(berrors.ErrKVRangeIsEmpty)
- }
-
- atomicResp.Store(resp)
- return nil
- })
- }
-
- if err := eg.Wait(); err != nil {
- return nil, err
- }
-
- downloadResp := atomicResp.Load()
- sstMeta.Range.Start = downloadResp.Range.GetStart()
- sstMeta.Range.End = downloadResp.Range.GetEnd()
- sstMeta.ApiVersion = apiVersion
- return sstMeta, nil
-}
-
-// a new way to download ssts files
-// 1. download write + default sst files at peer level.
-// 2. control the download concurrency per store.
-func (importer *FileImporter) downloadV2(
- ctx context.Context,
- regionInfo *split.RegionInfo,
- files []*backuppb.File,
- rewriteRules *restoreutils.RewriteRules,
- cipher *backuppb.CipherInfo,
- apiVersion kvrpcpb.APIVersion,
-) ([]*import_sstpb.SSTMeta, error) {
- var (
- downloadMetas = make([]*import_sstpb.SSTMeta, 0, len(files))
- )
- errDownload := utils.WithRetry(ctx, func() error {
- var e error
- // we treat Txn kv file as Raw kv file. because we don't have table id to decode
- if importer.kvMode == Raw || importer.kvMode == Txn {
- downloadMetas, e = importer.downloadRawKVSSTV2(ctx, regionInfo, files, cipher, apiVersion)
- } else {
- downloadMetas, e = importer.downloadSSTV2(ctx, regionInfo, files, rewriteRules, cipher, apiVersion)
- }
-
- failpoint.Inject("restore-storage-error", func(val failpoint.Value) {
- msg := val.(string)
- log.Debug("failpoint restore-storage-error injected.", zap.String("msg", msg))
- e = errors.Annotate(e, msg)
- })
- failpoint.Inject("restore-gRPC-error", func(_ failpoint.Value) {
- log.Warn("the connection to TiKV has been cut by a neko, meow :3")
- e = status.Error(codes.Unavailable, "the connection to TiKV has been cut by a neko, meow :3")
- })
- if isDecryptSstErr(e) {
- log.Info("fail to decrypt when download sst, try again with no-crypt", logutil.Files(files))
- if importer.kvMode == Raw || importer.kvMode == Txn {
- downloadMetas, e = importer.downloadRawKVSSTV2(ctx, regionInfo, files, nil, apiVersion)
- } else {
- downloadMetas, e = importer.downloadSSTV2(ctx, regionInfo, files, rewriteRules, nil, apiVersion)
- }
- }
- if e != nil {
- return errors.Trace(e)
- }
-
- return nil
- }, utils.NewDownloadSSTBackoffer())
-
- return downloadMetas, errDownload
-}
-
-func (importer *FileImporter) buildDownloadRequest(
- file *backuppb.File,
- rewriteRules *restoreutils.RewriteRules,
- regionInfo *split.RegionInfo,
- cipher *backuppb.CipherInfo,
-) (*import_sstpb.DownloadRequest, import_sstpb.SSTMeta, error) {
- uid := uuid.New()
- id := uid[:]
- // Get the rewrite rule for the file.
- fileRule := restoreutils.FindMatchedRewriteRule(file, rewriteRules)
- if fileRule == nil {
- return nil, import_sstpb.SSTMeta{}, errors.Trace(berrors.ErrKVRewriteRuleNotFound)
- }
-
- // For the legacy version of TiKV, we need to encode the key prefix, since in the legacy
- // version, the TiKV will rewrite the key with the encoded prefix without decoding the keys in
- // the SST file. For the new version of TiKV that support keyspace rewrite, we don't need to
- // encode the key prefix. The TiKV will decode the keys in the SST file and rewrite the keys
- // with the plain prefix and encode the keys before writing to SST.
-
- // for the keyspace rewrite mode
- rule := *fileRule
- // for the legacy rewrite mode
- if importer.rewriteMode == RewriteModeLegacy {
- rule.OldKeyPrefix = restoreutils.EncodeKeyPrefix(fileRule.GetOldKeyPrefix())
- rule.NewKeyPrefix = restoreutils.EncodeKeyPrefix(fileRule.GetNewKeyPrefix())
- }
-
- sstMeta, err := GetSSTMetaFromFile(id, file, regionInfo.Region, &rule, importer.rewriteMode)
- if err != nil {
- return nil, import_sstpb.SSTMeta{}, err
- }
-
- req := &import_sstpb.DownloadRequest{
- Sst: *sstMeta,
- StorageBackend: importer.backend,
- Name: file.GetName(),
- RewriteRule: rule,
- CipherInfo: cipher,
- StorageCacheId: importer.cacheKey,
- // For the older version of TiDB, the request type will be default to `import_sstpb.RequestType_Legacy`
- RequestType: import_sstpb.DownloadRequestType_Keyspace,
- Context: &kvrpcpb.Context{
- ResourceControlContext: &kvrpcpb.ResourceControlContext{
- ResourceGroupName: "", // TODO,
- },
- RequestSource: kvutil.BuildRequestSource(true, kv.InternalTxnBR, kvutil.ExplicitTypeBR),
- },
- }
- return req, *sstMeta, nil
-}
-
-func (importer *FileImporter) downloadSSTV2(
- ctx context.Context,
- regionInfo *split.RegionInfo,
- files []*backuppb.File,
- rewriteRules *restoreutils.RewriteRules,
- cipher *backuppb.CipherInfo,
- apiVersion kvrpcpb.APIVersion,
-) ([]*import_sstpb.SSTMeta, error) {
- var mu sync.Mutex
- downloadMetasMap := make(map[string]import_sstpb.SSTMeta)
- resultMetasMap := make(map[string]*import_sstpb.SSTMeta)
- downloadReqsMap := make(map[string]*import_sstpb.DownloadRequest)
- for _, file := range files {
- req, sstMeta, err := importer.buildDownloadRequest(file, rewriteRules, regionInfo, cipher)
- if err != nil {
- return nil, errors.Trace(err)
- }
- sstMeta.ApiVersion = apiVersion
- downloadMetasMap[file.Name] = sstMeta
- downloadReqsMap[file.Name] = req
- }
-
- eg, ectx := errgroup.WithContext(ctx)
- for _, p := range regionInfo.Region.GetPeers() {
- peer := p
- eg.Go(func() error {
- if importer.useTokenBucket {
- tokenCh := importer.downloadTokensMap.acquireTokenCh(peer.GetStoreId(), importer.concurrencyPerStore)
- select {
- case <-ectx.Done():
- return ectx.Err()
- case <-tokenCh:
- }
- defer func() {
- importer.releaseToken(tokenCh)
- }()
- }
- for _, file := range files {
- req, ok := downloadReqsMap[file.Name]
- if !ok {
- return errors.New("not found file key for download request")
- }
- var err error
- var resp *import_sstpb.DownloadResponse
- resp, err = utils.WithRetryV2(ectx, utils.NewDownloadSSTBackoffer(), func(ctx context.Context) (*import_sstpb.DownloadResponse, error) {
- dctx, cancel := context.WithTimeout(ctx, gRPCTimeOut)
- defer cancel()
- return importer.importClient.DownloadSST(dctx, peer.GetStoreId(), req)
- })
- if err != nil {
- return errors.Trace(err)
- }
- if resp.GetError() != nil {
- return errors.Annotate(berrors.ErrKVDownloadFailed, resp.GetError().GetMessage())
- }
- if resp.GetIsEmpty() {
- return errors.Trace(berrors.ErrKVRangeIsEmpty)
- }
-
- mu.Lock()
- sstMeta, ok := downloadMetasMap[file.Name]
- if !ok {
- mu.Unlock()
- return errors.Errorf("not found file %s for download sstMeta", file.Name)
- }
- sstMeta.Range = &import_sstpb.Range{
- Start: restoreutils.TruncateTS(resp.Range.GetStart()),
- End: restoreutils.TruncateTS(resp.Range.GetEnd()),
- }
- resultMetasMap[file.Name] = &sstMeta
- mu.Unlock()
-
- log.Debug("download from peer",
- logutil.Region(regionInfo.Region),
- logutil.File(file),
- logutil.Peer(peer),
- logutil.Key("resp-range-start", resp.Range.Start),
- logutil.Key("resp-range-end", resp.Range.End),
- zap.Bool("resp-isempty", resp.IsEmpty),
- zap.Uint32("resp-crc32", resp.Crc32),
- zap.Int("len files", len(files)),
- )
- }
- return nil
- })
- }
- if err := eg.Wait(); err != nil {
- return nil, err
- }
- return maps.Values(resultMetasMap), nil
-}
-
-func (importer *FileImporter) downloadRawKVSSTV2(
- ctx context.Context,
- regionInfo *split.RegionInfo,
- files []*backuppb.File,
- cipher *backuppb.CipherInfo,
- apiVersion kvrpcpb.APIVersion,
-) ([]*import_sstpb.SSTMeta, error) {
- downloadMetas := make([]*import_sstpb.SSTMeta, 0, len(files))
- for _, file := range files {
- uid := uuid.New()
- id := uid[:]
- // Empty rule
- var rule import_sstpb.RewriteRule
- sstMeta, err := GetSSTMetaFromFile(id, file, regionInfo.Region, &rule, RewriteModeLegacy)
- if err != nil {
- return nil, err
- }
-
- // Cut the SST file's range to fit in the restoring range.
- if bytes.Compare(importer.rawStartKey, sstMeta.Range.GetStart()) > 0 {
- sstMeta.Range.Start = importer.rawStartKey
- }
- if len(importer.rawEndKey) > 0 &&
- (len(sstMeta.Range.GetEnd()) == 0 || bytes.Compare(importer.rawEndKey, sstMeta.Range.GetEnd()) <= 0) {
- sstMeta.Range.End = importer.rawEndKey
- sstMeta.EndKeyExclusive = true
- }
- if bytes.Compare(sstMeta.Range.GetStart(), sstMeta.Range.GetEnd()) > 0 {
- return nil, errors.Trace(berrors.ErrKVRangeIsEmpty)
- }
-
- req := &import_sstpb.DownloadRequest{
- Sst: *sstMeta,
- StorageBackend: importer.backend,
- Name: file.GetName(),
- RewriteRule: rule,
- IsRawKv: true,
- CipherInfo: cipher,
- StorageCacheId: importer.cacheKey,
- }
- log.Debug("download SST", logutil.SSTMeta(sstMeta), logutil.Region(regionInfo.Region))
-
- var atomicResp atomic.Pointer[import_sstpb.DownloadResponse]
- eg, ectx := errgroup.WithContext(ctx)
- for _, p := range regionInfo.Region.GetPeers() {
- peer := p
- eg.Go(func() error {
- resp, err := importer.importClient.DownloadSST(ectx, peer.GetStoreId(), req)
- if err != nil {
- return errors.Trace(err)
- }
- if resp.GetError() != nil {
- return errors.Annotate(berrors.ErrKVDownloadFailed, resp.GetError().GetMessage())
- }
- if resp.GetIsEmpty() {
- return errors.Trace(berrors.ErrKVRangeIsEmpty)
- }
-
- atomicResp.Store(resp)
- return nil
- })
- }
-
- if err := eg.Wait(); err != nil {
- return nil, err
- }
-
- downloadResp := atomicResp.Load()
- sstMeta.Range.Start = downloadResp.Range.GetStart()
- sstMeta.Range.End = downloadResp.Range.GetEnd()
- sstMeta.ApiVersion = apiVersion
- downloadMetas = append(downloadMetas, sstMeta)
- }
- return downloadMetas, nil
-}
-
-func (importer *FileImporter) ingest(
- ctx context.Context,
- files []*backuppb.File,
- info *split.RegionInfo,
- downloadMetas []*import_sstpb.SSTMeta,
-) error {
- if importer.useTokenBucket {
- tokenCh := importer.ingestTokensMap.acquireTokenCh(info.Leader.GetStoreId(), importer.concurrencyPerStore)
- select {
- case <-ctx.Done():
- return ctx.Err()
- case <-tokenCh:
- }
- defer func() {
- importer.releaseToken(tokenCh)
- }()
- }
- for {
- ingestResp, errIngest := importer.ingestSSTs(ctx, downloadMetas, info)
- if errIngest != nil {
- return errors.Trace(errIngest)
- }
-
- errPb := ingestResp.GetError()
- switch {
- case errPb == nil:
- return nil
- case errPb.NotLeader != nil:
- // If error is `NotLeader`, update the region info and retry
- var newInfo *split.RegionInfo
- if newLeader := errPb.GetNotLeader().GetLeader(); newLeader != nil {
- newInfo = &split.RegionInfo{
- Leader: newLeader,
- Region: info.Region,
- }
- } else {
- for {
- // Slow path, get region from PD
- newInfo, errIngest = importer.metaClient.GetRegion(
- ctx, info.Region.GetStartKey())
- if errIngest != nil {
- return errors.Trace(errIngest)
- }
- if newInfo != nil {
- break
- }
- // do not get region info, wait a second and GetRegion() again.
- log.Warn("ingest get region by key return nil", logutil.Region(info.Region),
- logutil.Files(files),
- logutil.SSTMetas(downloadMetas),
- )
- time.Sleep(time.Second)
- }
- }
-
- if !split.CheckRegionEpoch(newInfo, info) {
- return errors.Trace(berrors.ErrKVEpochNotMatch)
- }
- log.Debug("ingest sst returns not leader error, retry it",
- logutil.Files(files),
- logutil.SSTMetas(downloadMetas),
- logutil.Region(info.Region),
- zap.Stringer("newLeader", newInfo.Leader))
- info = newInfo
- case errPb.EpochNotMatch != nil:
- // TODO handle epoch not match error
- // 1. retry download if needed
- // 2. retry ingest
- return errors.Trace(berrors.ErrKVEpochNotMatch)
- case errPb.KeyNotInRegion != nil:
- return errors.Trace(berrors.ErrKVKeyNotInRegion)
- default:
- // Other errors like `ServerIsBusy`, `RegionNotFound`, etc. should be retryable
- return errors.Annotatef(berrors.ErrKVIngestFailed, "ingest error %s", errPb)
- }
- }
-}
-
-func (importer *FileImporter) ingestSSTs(
- ctx context.Context,
- sstMetas []*import_sstpb.SSTMeta,
- regionInfo *split.RegionInfo,
-) (*import_sstpb.IngestResponse, error) {
- leader := regionInfo.Leader
- if leader == nil {
- return nil, errors.Annotatef(berrors.ErrPDLeaderNotFound,
- "region id %d has no leader", regionInfo.Region.Id)
- }
- reqCtx := &kvrpcpb.Context{
- RegionId: regionInfo.Region.GetId(),
- RegionEpoch: regionInfo.Region.GetRegionEpoch(),
- Peer: leader,
- ResourceControlContext: &kvrpcpb.ResourceControlContext{
- ResourceGroupName: "", // TODO,
- },
- RequestSource: kvutil.BuildRequestSource(true, kv.InternalTxnBR, kvutil.ExplicitTypeBR),
- }
-
- if !importer.SupportMultiIngest {
- // TODO: not sure we need this check
- if len(sstMetas) != 1 {
- panic("do not support batch ingest")
- }
- req := &import_sstpb.IngestRequest{
- Context: reqCtx,
- Sst: sstMetas[0],
- }
- log.Debug("ingest SST", logutil.SSTMeta(sstMetas[0]), logutil.Leader(leader))
- resp, err := importer.importClient.IngestSST(ctx, leader.GetStoreId(), req)
- return resp, errors.Trace(err)
- }
-
- req := &import_sstpb.MultiIngestRequest{
- Context: reqCtx,
- Ssts: sstMetas,
- }
- log.Debug("ingest SSTs", logutil.SSTMetas(sstMetas), logutil.Leader(leader))
- resp, err := importer.importClient.MultiIngest(ctx, leader.GetStoreId(), req)
- return resp, errors.Trace(err)
-}
-
-func (importer *FileImporter) downloadAndApplyKVFile(
- ctx context.Context,
- files []*logrestore.LogDataFileInfo,
- rules *restoreutils.RewriteRules,
- regionInfo *split.RegionInfo,
- shiftStartTS uint64,
- startTS uint64,
- restoreTS uint64,
- supportBatch bool,
-) RPCResult {
- leader := regionInfo.Leader
- if leader == nil {
- return RPCResultFromError(errors.Annotatef(berrors.ErrPDLeaderNotFound,
- "region id %d has no leader", regionInfo.Region.Id))
- }
-
- metas := make([]*import_sstpb.KVMeta, 0, len(files))
- rewriteRules := make([]*import_sstpb.RewriteRule, 0, len(files))
-
- for _, file := range files {
- // Get the rewrite rule for the file.
- fileRule := restoreutils.FindMatchedRewriteRule(file, rules)
- if fileRule == nil {
- return RPCResultFromError(errors.Annotatef(berrors.ErrKVRewriteRuleNotFound,
- "rewrite rule for file %+v not find (in %+v)", file, rules))
- }
- rule := import_sstpb.RewriteRule{
- OldKeyPrefix: restoreutils.EncodeKeyPrefix(fileRule.GetOldKeyPrefix()),
- NewKeyPrefix: restoreutils.EncodeKeyPrefix(fileRule.GetNewKeyPrefix()),
- }
-
- meta := &import_sstpb.KVMeta{
- Name: file.Path,
- Cf: file.Cf,
- RangeOffset: file.RangeOffset,
- Length: file.Length,
- RangeLength: file.RangeLength,
- IsDelete: file.Type == backuppb.FileType_Delete,
- StartTs: func() uint64 {
- if file.Cf == stream.DefaultCF {
- return shiftStartTS
- }
- return startTS
- }(),
- RestoreTs: restoreTS,
- StartKey: regionInfo.Region.GetStartKey(),
- EndKey: regionInfo.Region.GetEndKey(),
- Sha256: file.GetSha256(),
- CompressionType: file.CompressionType,
- }
-
- metas = append(metas, meta)
- rewriteRules = append(rewriteRules, &rule)
- }
-
- reqCtx := &kvrpcpb.Context{
- RegionId: regionInfo.Region.GetId(),
- RegionEpoch: regionInfo.Region.GetRegionEpoch(),
- Peer: leader,
- }
-
- var req *import_sstpb.ApplyRequest
- if supportBatch {
- req = &import_sstpb.ApplyRequest{
- Metas: metas,
- StorageBackend: importer.backend,
- RewriteRules: rewriteRules,
- Context: reqCtx,
- StorageCacheId: importer.cacheKey,
- }
- } else {
- req = &import_sstpb.ApplyRequest{
- Meta: metas[0],
- StorageBackend: importer.backend,
- RewriteRule: *rewriteRules[0],
- Context: reqCtx,
- StorageCacheId: importer.cacheKey,
- }
- }
-
- log.Debug("apply kv file", logutil.Leader(leader))
- resp, err := importer.importClient.ApplyKVFile(ctx, leader.GetStoreId(), req)
- if err != nil {
- return RPCResultFromError(errors.Trace(err))
- }
- if resp.GetError() != nil {
- logutil.CL(ctx).Warn("import meet error", zap.Stringer("error", resp.GetError()))
- return RPCResultFromPBError(resp.GetError())
- }
- return RPCResultOK()
-}
-
-func isDecryptSstErr(err error) bool {
- return err != nil &&
- strings.Contains(err.Error(), "Engine Engine") &&
- strings.Contains(err.Error(), "Corruption: Bad table magic number")
-}
diff --git a/br/pkg/restore/import_mode_switcher.go b/br/pkg/restore/import_mode_switcher.go
new file mode 100644
index 0000000000000..0ae69f4a6a0af
--- /dev/null
+++ b/br/pkg/restore/import_mode_switcher.go
@@ -0,0 +1,195 @@
+// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
+
+package restore
+
+import (
+ "context"
+ "crypto/tls"
+ "time"
+
+ _ "github.com/go-sql-driver/mysql" // mysql driver
+ "github.com/pingcap/errors"
+ "github.com/pingcap/kvproto/pkg/import_sstpb"
+ "github.com/pingcap/log"
+ "github.com/pingcap/tidb/br/pkg/conn"
+ "github.com/pingcap/tidb/br/pkg/conn/util"
+ "github.com/pingcap/tidb/br/pkg/pdutil"
+ tidbutil "github.com/pingcap/tidb/pkg/util"
+ pd "github.com/tikv/pd/client"
+ "go.uber.org/zap"
+ "golang.org/x/sync/errgroup"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/backoff"
+ "google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/credentials/insecure"
+)
+
+type ImportModeSwitcher struct {
+ pdClient pd.Client
+
+ switchModeInterval time.Duration
+ tlsConf *tls.Config
+
+ switchCh chan struct{}
+}
+
+func NewImportModeSwitcher(
+ pdClient pd.Client,
+ switchModeInterval time.Duration,
+ tlsConf *tls.Config,
+) *ImportModeSwitcher {
+ return &ImportModeSwitcher{
+ pdClient: pdClient,
+ switchModeInterval: switchModeInterval,
+ tlsConf: tlsConf,
+ switchCh: make(chan struct{}),
+ }
+}
+
+// switchToNormalMode switch tikv cluster to normal mode.
+func (switcher *ImportModeSwitcher) switchToNormalMode(ctx context.Context) error {
+ close(switcher.switchCh)
+ return switcher.switchTiKVMode(ctx, import_sstpb.SwitchMode_Normal)
+}
+
+func (switcher *ImportModeSwitcher) switchTiKVMode(
+ ctx context.Context,
+ mode import_sstpb.SwitchMode,
+) error {
+ stores, err := util.GetAllTiKVStores(ctx, switcher.pdClient, util.SkipTiFlash)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ bfConf := backoff.DefaultConfig
+ bfConf.MaxDelay = time.Second * 3
+
+ workerPool := tidbutil.NewWorkerPool(uint(len(stores)), "switch import mode")
+ eg, ectx := errgroup.WithContext(ctx)
+ for _, store := range stores {
+ if err := ectx.Err(); err != nil {
+ return errors.Trace(err)
+ }
+
+ finalStore := store
+ workerPool.ApplyOnErrorGroup(eg,
+ func() error {
+ opt := grpc.WithTransportCredentials(insecure.NewCredentials())
+ if switcher.tlsConf != nil {
+ opt = grpc.WithTransportCredentials(credentials.NewTLS(switcher.tlsConf))
+ }
+ gctx, cancel := context.WithTimeout(ectx, time.Second*5)
+ connection, err := grpc.DialContext(
+ gctx,
+ finalStore.GetAddress(),
+ opt,
+ grpc.WithBlock(),
+ grpc.FailOnNonTempDialError(true),
+ grpc.WithConnectParams(grpc.ConnectParams{Backoff: bfConf}),
+ // we don't need to set keepalive timeout here, because the connection lives
+ // at most 5s. (shorter than minimal value for keepalive time!)
+ )
+ cancel()
+ if err != nil {
+ return errors.Trace(err)
+ }
+ client := import_sstpb.NewImportSSTClient(connection)
+ _, err = client.SwitchMode(ctx, &import_sstpb.SwitchModeRequest{
+ Mode: mode,
+ })
+ if err != nil {
+ return errors.Trace(err)
+ }
+ err = connection.Close()
+ if err != nil {
+ log.Error("close grpc connection failed in switch mode", zap.Error(err))
+ }
+ return nil
+ })
+ }
+
+ if err = eg.Wait(); err != nil {
+ return errors.Trace(err)
+ }
+ return nil
+}
+
+// switchToImportMode switch tikv cluster to import mode.
+func (switcher *ImportModeSwitcher) switchToImportMode(
+ ctx context.Context,
+) {
+ // tikv automatically switch to normal mode in every 10 minutes
+ // so we need ping tikv in less than 10 minute
+ go func() {
+ tick := time.NewTicker(switcher.switchModeInterval)
+ defer tick.Stop()
+
+ // [important!] switch tikv mode into import at the beginning
+ log.Info("switch to import mode at beginning")
+ err := switcher.switchTiKVMode(ctx, import_sstpb.SwitchMode_Import)
+ if err != nil {
+ log.Warn("switch to import mode failed", zap.Error(err))
+ }
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-tick.C:
+ log.Info("switch to import mode")
+ err := switcher.switchTiKVMode(ctx, import_sstpb.SwitchMode_Import)
+ if err != nil {
+ log.Warn("switch to import mode failed", zap.Error(err))
+ }
+ case <-switcher.switchCh:
+ log.Info("stop automatic switch to import mode")
+ return
+ }
+ }
+ }()
+}
+
+// RestorePreWork executes some prepare work before restore.
+// TODO make this function returns a restore post work.
+func RestorePreWork(
+ ctx context.Context,
+ mgr *conn.Mgr,
+ switcher *ImportModeSwitcher,
+ isOnline bool,
+ switchToImport bool,
+) (pdutil.UndoFunc, *pdutil.ClusterConfig, error) {
+ if isOnline {
+ return pdutil.Nop, nil, nil
+ }
+
+ if switchToImport {
+ // Switch TiKV cluster to import mode (adjust rocksdb configuration).
+ switcher.switchToImportMode(ctx)
+ }
+
+ return mgr.RemoveSchedulersWithConfig(ctx)
+}
+
+// RestorePostWork executes some post work after restore.
+// TODO: aggregate all lifetime manage methods into batcher's context manager field.
+func RestorePostWork(
+ ctx context.Context,
+ switcher *ImportModeSwitcher,
+ restoreSchedulers pdutil.UndoFunc,
+ isOnline bool,
+) {
+ if isOnline {
+ return
+ }
+
+ if ctx.Err() != nil {
+ log.Warn("context canceled, try shutdown")
+ ctx = context.Background()
+ }
+
+ if err := switcher.switchToNormalMode(ctx); err != nil {
+ log.Warn("fail to switch to normal mode", zap.Error(err))
+ }
+ if err := restoreSchedulers(ctx); err != nil {
+ log.Warn("failed to restore PD schedulers", zap.Error(err))
+ }
+}
diff --git a/br/pkg/restore/internal/import_client/BUILD.bazel b/br/pkg/restore/internal/import_client/BUILD.bazel
new file mode 100644
index 0000000000000..572bf61357d04
--- /dev/null
+++ b/br/pkg/restore/internal/import_client/BUILD.bazel
@@ -0,0 +1,20 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "import_client",
+ srcs = ["import_client.go"],
+ importpath = "github.com/pingcap/tidb/br/pkg/restore/internal/import_client",
+ visibility = ["//br/pkg/restore:__subpackages__"],
+ deps = [
+ "//br/pkg/restore/split",
+ "@com_github_pingcap_errors//:errors",
+ "@com_github_pingcap_kvproto//pkg/import_sstpb",
+ "@org_golang_google_grpc//:grpc",
+ "@org_golang_google_grpc//backoff",
+ "@org_golang_google_grpc//codes",
+ "@org_golang_google_grpc//credentials",
+ "@org_golang_google_grpc//credentials/insecure",
+ "@org_golang_google_grpc//keepalive",
+ "@org_golang_google_grpc//status",
+ ],
+)
diff --git a/br/pkg/restore/internal/import_client/import_client.go b/br/pkg/restore/internal/import_client/import_client.go
new file mode 100644
index 0000000000000..ee41058c9b063
--- /dev/null
+++ b/br/pkg/restore/internal/import_client/import_client.go
@@ -0,0 +1,260 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package importclient
+
+import (
+ "context"
+ "crypto/tls"
+ "sync"
+ "time"
+
+ "github.com/pingcap/errors"
+ "github.com/pingcap/kvproto/pkg/import_sstpb"
+ "github.com/pingcap/tidb/br/pkg/restore/split"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/backoff"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/credentials/insecure"
+ "google.golang.org/grpc/keepalive"
+ "google.golang.org/grpc/status"
+)
+
+const (
+ gRPCBackOffMaxDelay = 3 * time.Second
+)
+
+// ImporterClient is used to import a file to TiKV.
+type ImporterClient interface {
+ ClearFiles(
+ ctx context.Context,
+ storeID uint64,
+ req *import_sstpb.ClearRequest,
+ ) (*import_sstpb.ClearResponse, error)
+
+ ApplyKVFile(
+ ctx context.Context,
+ storeID uint64,
+ req *import_sstpb.ApplyRequest,
+ ) (*import_sstpb.ApplyResponse, error)
+
+ DownloadSST(
+ ctx context.Context,
+ storeID uint64,
+ req *import_sstpb.DownloadRequest,
+ ) (*import_sstpb.DownloadResponse, error)
+
+ MultiIngest(
+ ctx context.Context,
+ storeID uint64,
+ req *import_sstpb.MultiIngestRequest,
+ ) (*import_sstpb.IngestResponse, error)
+
+ SetDownloadSpeedLimit(
+ ctx context.Context,
+ storeID uint64,
+ req *import_sstpb.SetDownloadSpeedLimitRequest,
+ ) (*import_sstpb.SetDownloadSpeedLimitResponse, error)
+
+ GetImportClient(
+ ctx context.Context,
+ storeID uint64,
+ ) (import_sstpb.ImportSSTClient, error)
+
+ CloseGrpcClient() error
+
+ CheckMultiIngestSupport(ctx context.Context, stores []uint64) error
+}
+
+type importClient struct {
+ metaClient split.SplitClient
+ mu sync.Mutex
+ // Notice: In order to avoid leak for BRIE via SQL, it needs to close grpc client connection before br task exits.
+ // So it caches the grpc connection instead of import_sstpb.ImportSSTClient.
+ // used for any request except the ingest reqeust
+ conns map[uint64]*grpc.ClientConn
+ // used for ingest request
+ ingestConns map[uint64]*grpc.ClientConn
+
+ tlsConf *tls.Config
+ keepaliveConf keepalive.ClientParameters
+}
+
+// NewImportClient returns a new importerClient.
+func NewImportClient(metaClient split.SplitClient, tlsConf *tls.Config, keepaliveConf keepalive.ClientParameters) ImporterClient {
+ return &importClient{
+ metaClient: metaClient,
+ conns: make(map[uint64]*grpc.ClientConn),
+ ingestConns: make(map[uint64]*grpc.ClientConn),
+ tlsConf: tlsConf,
+ keepaliveConf: keepaliveConf,
+ }
+}
+
+func (ic *importClient) ClearFiles(
+ ctx context.Context,
+ storeID uint64,
+ req *import_sstpb.ClearRequest,
+) (*import_sstpb.ClearResponse, error) {
+ client, err := ic.GetImportClient(ctx, storeID)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ return client.ClearFiles(ctx, req)
+}
+
+func (ic *importClient) ApplyKVFile(
+ ctx context.Context,
+ storeID uint64,
+ req *import_sstpb.ApplyRequest,
+) (*import_sstpb.ApplyResponse, error) {
+ client, err := ic.GetImportClient(ctx, storeID)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ return client.Apply(ctx, req)
+}
+
+func (ic *importClient) DownloadSST(
+ ctx context.Context,
+ storeID uint64,
+ req *import_sstpb.DownloadRequest,
+) (*import_sstpb.DownloadResponse, error) {
+ client, err := ic.GetImportClient(ctx, storeID)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ return client.Download(ctx, req)
+}
+
+func (ic *importClient) SetDownloadSpeedLimit(
+ ctx context.Context,
+ storeID uint64,
+ req *import_sstpb.SetDownloadSpeedLimitRequest,
+) (*import_sstpb.SetDownloadSpeedLimitResponse, error) {
+ client, err := ic.GetImportClient(ctx, storeID)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ return client.SetDownloadSpeedLimit(ctx, req)
+}
+
+func (ic *importClient) MultiIngest(
+ ctx context.Context,
+ storeID uint64,
+ req *import_sstpb.MultiIngestRequest,
+) (*import_sstpb.IngestResponse, error) {
+ client, err := ic.GetIngestClient(ctx, storeID)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ return client.MultiIngest(ctx, req)
+}
+
+func (ic *importClient) createGrpcConn(
+ ctx context.Context,
+ storeID uint64,
+) (*grpc.ClientConn, error) {
+ store, err := ic.metaClient.GetStore(ctx, storeID)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ opt := grpc.WithTransportCredentials(insecure.NewCredentials())
+ if ic.tlsConf != nil {
+ opt = grpc.WithTransportCredentials(credentials.NewTLS(ic.tlsConf))
+ }
+ addr := store.GetPeerAddress()
+ if addr == "" {
+ addr = store.GetAddress()
+ }
+ bfConf := backoff.DefaultConfig
+ bfConf.MaxDelay = gRPCBackOffMaxDelay
+ conn, err := grpc.DialContext(
+ ctx,
+ addr,
+ opt,
+ grpc.WithBlock(),
+ grpc.FailOnNonTempDialError(true),
+ grpc.WithConnectParams(grpc.ConnectParams{Backoff: bfConf}),
+ grpc.WithKeepaliveParams(ic.keepaliveConf),
+ )
+ return conn, errors.Trace(err)
+}
+
+func (ic *importClient) cachedConnectionFrom(
+ ctx context.Context,
+ storeID uint64,
+ caches map[uint64]*grpc.ClientConn,
+) (import_sstpb.ImportSSTClient, error) {
+ ic.mu.Lock()
+ defer ic.mu.Unlock()
+ conn, ok := caches[storeID]
+ if ok {
+ return import_sstpb.NewImportSSTClient(conn), nil
+ }
+ conn, err := ic.createGrpcConn(ctx, storeID)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ caches[storeID] = conn
+ return import_sstpb.NewImportSSTClient(conn), nil
+}
+
+func (ic *importClient) GetImportClient(
+ ctx context.Context,
+ storeID uint64,
+) (import_sstpb.ImportSSTClient, error) {
+ return ic.cachedConnectionFrom(ctx, storeID, ic.conns)
+}
+
+func (ic *importClient) GetIngestClient(
+ ctx context.Context,
+ storeID uint64,
+) (import_sstpb.ImportSSTClient, error) {
+ return ic.cachedConnectionFrom(ctx, storeID, ic.ingestConns)
+}
+
+func (ic *importClient) CloseGrpcClient() error {
+ ic.mu.Lock()
+ defer ic.mu.Unlock()
+ for id, conn := range ic.conns {
+ if err := conn.Close(); err != nil {
+ return errors.Trace(err)
+ }
+ delete(ic.conns, id)
+ }
+ for id, conn := range ic.ingestConns {
+ if err := conn.Close(); err != nil {
+ return errors.Trace(err)
+ }
+ delete(ic.ingestConns, id)
+ }
+ return nil
+}
+
+func (ic *importClient) CheckMultiIngestSupport(ctx context.Context, stores []uint64) error {
+ for _, storeID := range stores {
+ _, err := ic.MultiIngest(ctx, storeID, &import_sstpb.MultiIngestRequest{})
+ if err != nil {
+ if s, ok := status.FromError(err); ok {
+ if s.Code() == codes.Unimplemented {
+ return errors.Errorf("tikv node doesn't support multi ingest. (store id %d)", storeID)
+ }
+ }
+ return errors.Annotatef(err, "failed to check multi ingest support. (store id %d)", storeID)
+ }
+ }
+ return nil
+}
diff --git a/br/pkg/restore/internal/log_split/BUILD.bazel b/br/pkg/restore/internal/log_split/BUILD.bazel
new file mode 100644
index 0000000000000..7b3cc6bdd626f
--- /dev/null
+++ b/br/pkg/restore/internal/log_split/BUILD.bazel
@@ -0,0 +1,51 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+ name = "log_split",
+ srcs = [
+ "split.go",
+ "sum_sorted.go",
+ ],
+ importpath = "github.com/pingcap/tidb/br/pkg/restore/internal/log_split",
+ visibility = ["//br/pkg/restore:__subpackages__"],
+ deps = [
+ "//br/pkg/logutil",
+ "//br/pkg/restore/internal/utils",
+ "//br/pkg/restore/split",
+ "//br/pkg/restore/utils",
+ "//br/pkg/rtree",
+ "//br/pkg/utils",
+ "//pkg/kv",
+ "//pkg/tablecodec",
+ "//pkg/util",
+ "//pkg/util/codec",
+ "@com_github_google_btree//:btree",
+ "@com_github_pingcap_errors//:errors",
+ "@com_github_pingcap_kvproto//pkg/brpb",
+ "@com_github_pingcap_log//:log",
+ "@org_golang_x_sync//errgroup",
+ "@org_uber_go_zap//:zap",
+ ],
+)
+
+go_test(
+ name = "log_split_test",
+ timeout = "short",
+ srcs = [
+ "split_test.go",
+ "sum_sorted_test.go",
+ ],
+ flaky = True,
+ shard_count = 3,
+ deps = [
+ ":log_split",
+ "//br/pkg/restore/internal/utils",
+ "//br/pkg/restore/split",
+ "//br/pkg/restore/utils",
+ "//br/pkg/utiltest",
+ "//pkg/tablecodec",
+ "//pkg/util/codec",
+ "@com_github_pingcap_kvproto//pkg/import_sstpb",
+ "@com_github_stretchr_testify//require",
+ ],
+)
diff --git a/br/pkg/restore/log_restore/split.go b/br/pkg/restore/internal/log_split/split.go
similarity index 77%
rename from br/pkg/restore/log_restore/split.go
rename to br/pkg/restore/internal/log_split/split.go
index 4e0cdb3d8c60b..021135c1e71b5 100644
--- a/br/pkg/restore/log_restore/split.go
+++ b/br/pkg/restore/internal/log_split/split.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package logrestore
+package logsplit
import (
"bytes"
@@ -24,10 +24,10 @@ import (
"github.com/pingcap/errors"
backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/log"
+ "github.com/pingcap/tidb/br/pkg/restore/internal/utils"
"github.com/pingcap/tidb/br/pkg/restore/split"
- "github.com/pingcap/tidb/br/pkg/restore/utils"
+ restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
"github.com/pingcap/tidb/br/pkg/rtree"
- "github.com/pingcap/tidb/br/pkg/utils/iter"
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/codec"
@@ -38,25 +38,25 @@ import (
type rewriteSplitter struct {
rewriteKey []byte
tableID int64
- rule *utils.RewriteRules
- splitter *split.SplitHelper
+ rule *restoreutils.RewriteRules
+ splitter *SplitHelper
}
type splitHelperIterator struct {
tableSplitters []*rewriteSplitter
}
-func (iter *splitHelperIterator) Traverse(fn func(v split.Valued, endKey []byte, rule *utils.RewriteRules) bool) {
+func (iter *splitHelperIterator) Traverse(fn func(v Valued, endKey []byte, rule *restoreutils.RewriteRules) bool) {
for _, entry := range iter.tableSplitters {
endKey := codec.EncodeBytes([]byte{}, tablecodec.EncodeTablePrefix(entry.tableID+1))
rule := entry.rule
- entry.splitter.Traverse(func(v split.Valued) bool {
+ entry.splitter.Traverse(func(v Valued) bool {
return fn(v, endKey, rule)
})
}
}
-func NewSplitHelperIteratorForTest(helper *split.SplitHelper, tableID int64, rule *utils.RewriteRules) *splitHelperIterator {
+func NewSplitHelperIteratorForTest(helper *SplitHelper, tableID int64, rule *restoreutils.RewriteRules) *splitHelperIterator {
return &splitHelperIterator{
tableSplitters: []*rewriteSplitter{
{
@@ -69,8 +69,8 @@ func NewSplitHelperIteratorForTest(helper *split.SplitHelper, tableID int64, rul
}
type LogSplitHelper struct {
- tableSplitter map[int64]*split.SplitHelper
- rules map[int64]*utils.RewriteRules
+ tableSplitter map[int64]*SplitHelper
+ rules map[int64]*restoreutils.RewriteRules
client split.SplitClient
pool *util.WorkerPool
eg *errgroup.Group
@@ -80,9 +80,9 @@ type LogSplitHelper struct {
splitThreSholdKeys int64
}
-func NewLogSplitHelper(rules map[int64]*utils.RewriteRules, client split.SplitClient, splitSize uint64, splitKeys int64) *LogSplitHelper {
+func NewLogSplitHelper(rules map[int64]*restoreutils.RewriteRules, client split.SplitClient, splitSize uint64, splitKeys int64) *LogSplitHelper {
return &LogSplitHelper{
- tableSplitter: make(map[int64]*split.SplitHelper),
+ tableSplitter: make(map[int64]*SplitHelper),
rules: rules,
client: client,
pool: util.NewWorkerPool(128, "split region"),
@@ -102,7 +102,7 @@ func (helper *LogSplitHelper) iterator() *splitHelperIterator {
log.Info("skip splitting due to no table id matched", zap.Int64("tableID", tableID))
continue
}
- newTableID := utils.GetRewriteTableID(tableID, rewriteRule)
+ newTableID := restoreutils.GetRewriteTableID(tableID, rewriteRule)
if newTableID == 0 {
log.Warn("failed to get the rewrite table id", zap.Int64("tableID", tableID))
continue
@@ -135,23 +135,23 @@ func (helper *LogSplitHelper) Merge(file *backuppb.DataFileInfo) {
}
splitHelper, exist := helper.tableSplitter[file.TableId]
if !exist {
- splitHelper = split.NewSplitHelper()
+ splitHelper = NewSplitHelper()
helper.tableSplitter[file.TableId] = splitHelper
}
- splitHelper.Merge(split.Valued{
- Key: split.Span{
+ splitHelper.Merge(Valued{
+ Key: Span{
StartKey: file.StartKey,
EndKey: file.EndKey,
},
- Value: split.Value{
+ Value: Value{
Size: file.Length,
Number: file.NumberOfEntries,
},
})
}
-type splitFunc = func(context.Context, *utils.RegionSplitter, uint64, int64, *split.RegionInfo, []split.Valued) error
+type splitFunc = func(context.Context, *utils.RegionSplitter, uint64, int64, *split.RegionInfo, []Valued) error
func (helper *LogSplitHelper) splitRegionByPoints(
ctx context.Context,
@@ -159,7 +159,7 @@ func (helper *LogSplitHelper) splitRegionByPoints(
initialLength uint64,
initialNumber int64,
region *split.RegionInfo,
- valueds []split.Valued,
+ valueds []Valued,
) error {
var (
splitPoints [][]byte = make([][]byte, 0)
@@ -231,7 +231,7 @@ func SplitPoint(
// region span +------------------------------------+
// +initial length+ +end valued+
// regionValueds is the ranges array overlapped with `regionInfo`
- regionValueds []split.Valued = nil
+ regionValueds []Valued = nil
// regionInfo is the region to be split
regionInfo *split.RegionInfo = nil
// intialLength is the length of the part of the first range overlapped with the region
@@ -244,7 +244,7 @@ func SplitPoint(
regionOverCount uint64 = 0
)
- iter.Traverse(func(v split.Valued, endKey []byte, rule *utils.RewriteRules) bool {
+ iter.Traverse(func(v Valued, endKey []byte, rule *restoreutils.RewriteRules) bool {
if v.Value.Number == 0 || v.Value.Size == 0 {
return true
}
@@ -253,7 +253,7 @@ func SplitPoint(
vEndKey []byte
)
// use `vStartKey` and `vEndKey` to compare with region's key
- vStartKey, vEndKey, err = utils.GetRewriteEncodedKeys(v, rule)
+ vStartKey, vEndKey, err = restoreutils.GetRewriteEncodedKeys(v, rule)
if err != nil {
return false
}
@@ -299,19 +299,19 @@ func SplitPoint(
if len(regionValueds) > 0 && regionInfo != region {
// add a part of the range as the end part
if bytes.Compare(vStartKey, regionInfo.Region.EndKey) < 0 {
- regionValueds = append(regionValueds, split.NewValued(vStartKey, regionInfo.Region.EndKey, split.Value{Size: endLength, Number: endNumber}))
+ regionValueds = append(regionValueds, NewValued(vStartKey, regionInfo.Region.EndKey, Value{Size: endLength, Number: endNumber}))
}
// try to split the region
err = splitF(ctx, regionSplitter, initialLength, initialNumber, regionInfo, regionValueds)
if err != nil {
return false
}
- regionValueds = make([]split.Valued, 0)
+ regionValueds = make([]Valued, 0)
}
if regionOverCount == 1 {
// the region completely contains the range
- regionValueds = append(regionValueds, split.Valued{
- Key: split.Span{
+ regionValueds = append(regionValueds, Valued{
+ Key: Span{
StartKey: vStartKey,
EndKey: vEndKey,
},
@@ -391,52 +391,3 @@ func (helper *LogSplitHelper) Split(ctx context.Context) error {
return nil
}
-
-type LogFilesIterWithSplitHelper struct {
- iter LogIter
- helper *LogSplitHelper
- buffer []*LogDataFileInfo
- next int
-}
-
-const SplitFilesBufferSize = 4096
-
-func NewLogFilesIterWithSplitHelper(iter LogIter, rules map[int64]*utils.RewriteRules, client split.SplitClient, splitSize uint64, splitKeys int64) LogIter {
- return &LogFilesIterWithSplitHelper{
- iter: iter,
- helper: NewLogSplitHelper(rules, client, splitSize, splitKeys),
- buffer: nil,
- next: 0,
- }
-}
-
-func (splitIter *LogFilesIterWithSplitHelper) TryNext(ctx context.Context) iter.IterResult[*LogDataFileInfo] {
- if splitIter.next >= len(splitIter.buffer) {
- splitIter.buffer = make([]*LogDataFileInfo, 0, SplitFilesBufferSize)
- for r := splitIter.iter.TryNext(ctx); !r.Finished; r = splitIter.iter.TryNext(ctx) {
- if r.Err != nil {
- return r
- }
- f := r.Item
- splitIter.helper.Merge(f.DataFileInfo)
- splitIter.buffer = append(splitIter.buffer, f)
- if len(splitIter.buffer) >= SplitFilesBufferSize {
- break
- }
- }
- splitIter.next = 0
- if len(splitIter.buffer) == 0 {
- return iter.Done[*LogDataFileInfo]()
- }
- log.Info("start to split the regions")
- startTime := time.Now()
- if err := splitIter.helper.Split(ctx); err != nil {
- return iter.Throw[*LogDataFileInfo](errors.Trace(err))
- }
- log.Info("end to split the regions", zap.Duration("takes", time.Since(startTime)))
- }
-
- res := iter.Emit(splitIter.buffer[splitIter.next])
- splitIter.next += 1
- return res
-}
diff --git a/br/pkg/restore/log_restore/split_test.go b/br/pkg/restore/internal/log_split/split_test.go
similarity index 55%
rename from br/pkg/restore/log_restore/split_test.go
rename to br/pkg/restore/internal/log_split/split_test.go
index 27af9db2975e1..99e30b78b22b8 100644
--- a/br/pkg/restore/log_restore/split_test.go
+++ b/br/pkg/restore/internal/log_split/split_test.go
@@ -12,64 +12,24 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package logrestore
+package logsplit_test
import (
- "bytes"
"context"
"fmt"
"testing"
- backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/kvproto/pkg/import_sstpb"
- "github.com/pingcap/kvproto/pkg/metapb"
+ logsplit "github.com/pingcap/tidb/br/pkg/restore/internal/log_split"
+ "github.com/pingcap/tidb/br/pkg/restore/internal/utils"
"github.com/pingcap/tidb/br/pkg/restore/split"
- "github.com/pingcap/tidb/br/pkg/restore/utils"
- "github.com/pingcap/tidb/br/pkg/utils/iter"
+ restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
+ "github.com/pingcap/tidb/br/pkg/utiltest"
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/util/codec"
"github.com/stretchr/testify/require"
)
-type fakeSplitClient struct {
- split.SplitClient
- regions []*split.RegionInfo
-}
-
-func newFakeSplitClient() *fakeSplitClient {
- return &fakeSplitClient{
- regions: make([]*split.RegionInfo, 0),
- }
-}
-
-func (f *fakeSplitClient) AppendRegion(startKey, endKey []byte) {
- f.regions = append(f.regions, &split.RegionInfo{
- Region: &metapb.Region{
- StartKey: startKey,
- EndKey: endKey,
- },
- })
-}
-
-func (f *fakeSplitClient) ScanRegions(ctx context.Context, startKey, endKey []byte, limit int) ([]*split.RegionInfo, error) {
- result := make([]*split.RegionInfo, 0)
- count := 0
- for _, rng := range f.regions {
- if bytes.Compare(rng.Region.StartKey, endKey) <= 0 && bytes.Compare(rng.Region.EndKey, startKey) > 0 {
- result = append(result, rng)
- count++
- }
- if count >= limit {
- break
- }
- }
- return result, nil
-}
-
-func (f *fakeSplitClient) WaitRegionsScattered(context.Context, []*split.RegionInfo) (int, error) {
- return 0, nil
-}
-
func keyWithTablePrefix(tableID int64, key string) []byte {
rawKey := append(tablecodec.GenTableRecordPrefix(tableID), []byte(key)...)
return codec.EncodeBytes([]byte{}, rawKey)
@@ -79,7 +39,7 @@ func TestSplitPoint(t *testing.T) {
ctx := context.Background()
var oldTableID int64 = 50
var tableID int64 = 100
- rewriteRules := &utils.RewriteRules{
+ rewriteRules := &restoreutils.RewriteRules{
Data: []*import_sstpb.RewriteRule{
{
OldKeyPrefix: tablecodec.EncodeTablePrefix(oldTableID),
@@ -92,18 +52,18 @@ func TestSplitPoint(t *testing.T) {
// +---+ +---+ +---------+
// +-------------+----------+---------+
// region: a f h j
- splitHelper := split.NewSplitHelper()
- splitHelper.Merge(split.Valued{Key: split.Span{StartKey: keyWithTablePrefix(oldTableID, "b"), EndKey: keyWithTablePrefix(oldTableID, "c")}, Value: split.Value{Size: 100, Number: 100}})
- splitHelper.Merge(split.Valued{Key: split.Span{StartKey: keyWithTablePrefix(oldTableID, "d"), EndKey: keyWithTablePrefix(oldTableID, "e")}, Value: split.Value{Size: 200, Number: 200}})
- splitHelper.Merge(split.Valued{Key: split.Span{StartKey: keyWithTablePrefix(oldTableID, "g"), EndKey: keyWithTablePrefix(oldTableID, "i")}, Value: split.Value{Size: 300, Number: 300}})
- client := newFakeSplitClient()
+ splitHelper := logsplit.NewSplitHelper()
+ splitHelper.Merge(logsplit.Valued{Key: logsplit.Span{StartKey: keyWithTablePrefix(oldTableID, "b"), EndKey: keyWithTablePrefix(oldTableID, "c")}, Value: logsplit.Value{Size: 100, Number: 100}})
+ splitHelper.Merge(logsplit.Valued{Key: logsplit.Span{StartKey: keyWithTablePrefix(oldTableID, "d"), EndKey: keyWithTablePrefix(oldTableID, "e")}, Value: logsplit.Value{Size: 200, Number: 200}})
+ splitHelper.Merge(logsplit.Valued{Key: logsplit.Span{StartKey: keyWithTablePrefix(oldTableID, "g"), EndKey: keyWithTablePrefix(oldTableID, "i")}, Value: logsplit.Value{Size: 300, Number: 300}})
+ client := utiltest.NewFakeSplitClient()
client.AppendRegion(keyWithTablePrefix(tableID, "a"), keyWithTablePrefix(tableID, "f"))
client.AppendRegion(keyWithTablePrefix(tableID, "f"), keyWithTablePrefix(tableID, "h"))
client.AppendRegion(keyWithTablePrefix(tableID, "h"), keyWithTablePrefix(tableID, "j"))
client.AppendRegion(keyWithTablePrefix(tableID, "j"), keyWithTablePrefix(tableID+1, "a"))
- iter := NewSplitHelperIteratorForTest(splitHelper, tableID, rewriteRules)
- err := SplitPoint(ctx, iter, client, func(ctx context.Context, rs *utils.RegionSplitter, u uint64, o int64, ri *split.RegionInfo, v []split.Valued) error {
+ iter := logsplit.NewSplitHelperIteratorForTest(splitHelper, tableID, rewriteRules)
+ err := logsplit.SplitPoint(ctx, iter, client, func(ctx context.Context, rs *utils.RegionSplitter, u uint64, o int64, ri *split.RegionInfo, v []logsplit.Valued) error {
require.Equal(t, u, uint64(0))
require.Equal(t, o, int64(0))
require.Equal(t, ri.Region.StartKey, keyWithTablePrefix(tableID, "a"))
@@ -129,7 +89,7 @@ func TestSplitPoint2(t *testing.T) {
ctx := context.Background()
var oldTableID int64 = 50
var tableID int64 = 100
- rewriteRules := &utils.RewriteRules{
+ rewriteRules := &restoreutils.RewriteRules{
Data: []*import_sstpb.RewriteRule{
{
OldKeyPrefix: tablecodec.EncodeTablePrefix(oldTableID),
@@ -142,13 +102,13 @@ func TestSplitPoint2(t *testing.T) {
// +---+ +---+ +-----------------+ +----+ +--------+
// +---------------+--+.....+----+------------+---------+
// region: a g >128 h m o
- splitHelper := split.NewSplitHelper()
- splitHelper.Merge(split.Valued{Key: split.Span{StartKey: keyWithTablePrefix(oldTableID, "b"), EndKey: keyWithTablePrefix(oldTableID, "c")}, Value: split.Value{Size: 100, Number: 100}})
- splitHelper.Merge(split.Valued{Key: split.Span{StartKey: keyWithTablePrefix(oldTableID, "d"), EndKey: keyWithTablePrefix(oldTableID, "e")}, Value: split.Value{Size: 200, Number: 200}})
- splitHelper.Merge(split.Valued{Key: split.Span{StartKey: keyWithTablePrefix(oldTableID, "f"), EndKey: keyWithTablePrefix(oldTableID, "i")}, Value: split.Value{Size: 300, Number: 300}})
- splitHelper.Merge(split.Valued{Key: split.Span{StartKey: keyWithTablePrefix(oldTableID, "j"), EndKey: keyWithTablePrefix(oldTableID, "k")}, Value: split.Value{Size: 200, Number: 200}})
- splitHelper.Merge(split.Valued{Key: split.Span{StartKey: keyWithTablePrefix(oldTableID, "l"), EndKey: keyWithTablePrefix(oldTableID, "n")}, Value: split.Value{Size: 200, Number: 200}})
- client := newFakeSplitClient()
+ splitHelper := logsplit.NewSplitHelper()
+ splitHelper.Merge(logsplit.Valued{Key: logsplit.Span{StartKey: keyWithTablePrefix(oldTableID, "b"), EndKey: keyWithTablePrefix(oldTableID, "c")}, Value: logsplit.Value{Size: 100, Number: 100}})
+ splitHelper.Merge(logsplit.Valued{Key: logsplit.Span{StartKey: keyWithTablePrefix(oldTableID, "d"), EndKey: keyWithTablePrefix(oldTableID, "e")}, Value: logsplit.Value{Size: 200, Number: 200}})
+ splitHelper.Merge(logsplit.Valued{Key: logsplit.Span{StartKey: keyWithTablePrefix(oldTableID, "f"), EndKey: keyWithTablePrefix(oldTableID, "i")}, Value: logsplit.Value{Size: 300, Number: 300}})
+ splitHelper.Merge(logsplit.Valued{Key: logsplit.Span{StartKey: keyWithTablePrefix(oldTableID, "j"), EndKey: keyWithTablePrefix(oldTableID, "k")}, Value: logsplit.Value{Size: 200, Number: 200}})
+ splitHelper.Merge(logsplit.Valued{Key: logsplit.Span{StartKey: keyWithTablePrefix(oldTableID, "l"), EndKey: keyWithTablePrefix(oldTableID, "n")}, Value: logsplit.Value{Size: 200, Number: 200}})
+ client := utiltest.NewFakeSplitClient()
client.AppendRegion(keyWithTablePrefix(tableID, "a"), keyWithTablePrefix(tableID, "g"))
client.AppendRegion(keyWithTablePrefix(tableID, "g"), keyWithTablePrefix(tableID, getCharFromNumber("g", 0)))
for i := 0; i < 256; i++ {
@@ -160,8 +120,8 @@ func TestSplitPoint2(t *testing.T) {
client.AppendRegion(keyWithTablePrefix(tableID, "o"), keyWithTablePrefix(tableID+1, "a"))
firstSplit := true
- iter := NewSplitHelperIteratorForTest(splitHelper, tableID, rewriteRules)
- err := SplitPoint(ctx, iter, client, func(ctx context.Context, rs *utils.RegionSplitter, u uint64, o int64, ri *split.RegionInfo, v []split.Valued) error {
+ iter := logsplit.NewSplitHelperIteratorForTest(splitHelper, tableID, rewriteRules)
+ err := logsplit.SplitPoint(ctx, iter, client, func(ctx context.Context, rs *utils.RegionSplitter, u uint64, o int64, ri *split.RegionInfo, v []logsplit.Valued) error {
if firstSplit {
require.Equal(t, u, uint64(0))
require.Equal(t, o, int64(0))
@@ -194,46 +154,3 @@ func TestSplitPoint2(t *testing.T) {
})
require.NoError(t, err)
}
-
-type mockLogIter struct {
- next int
-}
-
-func (m *mockLogIter) TryNext(ctx context.Context) iter.IterResult[*LogDataFileInfo] {
- if m.next > 10000 {
- return iter.Done[*LogDataFileInfo]()
- }
- m.next += 1
- return iter.Emit(&LogDataFileInfo{
- DataFileInfo: &backuppb.DataFileInfo{
- StartKey: []byte(fmt.Sprintf("a%d", m.next)),
- EndKey: []byte("b"),
- Length: 1024, // 1 KB
- },
- })
-}
-
-func TestLogFilesIterWithSplitHelper(t *testing.T) {
- var tableID int64 = 76
- var oldTableID int64 = 80
- rewriteRules := &utils.RewriteRules{
- Data: []*import_sstpb.RewriteRule{
- {
- OldKeyPrefix: tablecodec.EncodeTablePrefix(oldTableID),
- NewKeyPrefix: tablecodec.EncodeTablePrefix(tableID),
- },
- },
- }
- rewriteRulesMap := map[int64]*utils.RewriteRules{
- oldTableID: rewriteRules,
- }
- mockIter := &mockLogIter{}
- ctx := context.Background()
- logIter := NewLogFilesIterWithSplitHelper(mockIter, rewriteRulesMap, newFakeSplitClient(), 144*1024*1024, 1440000)
- next := 0
- for r := logIter.TryNext(ctx); !r.Finished; r = logIter.TryNext(ctx) {
- require.NoError(t, r.Err)
- next += 1
- require.Equal(t, []byte(fmt.Sprintf("a%d", next)), r.Item.StartKey)
- }
-}
diff --git a/br/pkg/restore/split/sum_sorted.go b/br/pkg/restore/internal/log_split/sum_sorted.go
similarity index 99%
rename from br/pkg/restore/split/sum_sorted.go
rename to br/pkg/restore/internal/log_split/sum_sorted.go
index 1ab51588ba6ca..fb5d3d8f9a0a2 100644
--- a/br/pkg/restore/split/sum_sorted.go
+++ b/br/pkg/restore/internal/log_split/sum_sorted.go
@@ -1,5 +1,5 @@
// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0.
-package split
+package logsplit
import (
"bytes"
diff --git a/br/pkg/restore/split/sum_sorted_test.go b/br/pkg/restore/internal/log_split/sum_sorted_test.go
similarity index 77%
rename from br/pkg/restore/split/sum_sorted_test.go
rename to br/pkg/restore/internal/log_split/sum_sorted_test.go
index 3a3b3db6d90eb..1810d31029ed7 100644
--- a/br/pkg/restore/split/sum_sorted_test.go
+++ b/br/pkg/restore/internal/log_split/sum_sorted_test.go
@@ -1,16 +1,16 @@
// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0.
-package split_test
+package logsplit_test
import (
"testing"
- "github.com/pingcap/tidb/br/pkg/restore/split"
+ logsplit "github.com/pingcap/tidb/br/pkg/restore/internal/log_split"
"github.com/stretchr/testify/require"
)
-func v(s, e string, val split.Value) split.Valued {
- return split.Valued{
- Key: split.Span{
+func v(s, e string, val logsplit.Value) logsplit.Valued {
+ return logsplit.Valued{
+ Key: logsplit.Span{
StartKey: []byte(s),
EndKey: []byte(e),
},
@@ -18,8 +18,8 @@ func v(s, e string, val split.Value) split.Valued {
}
}
-func mb(b uint64) split.Value {
- return split.Value{
+func mb(b uint64) logsplit.Value {
+ return logsplit.Value{
Size: b * 1024 * 1024,
Number: int64(b),
}
@@ -27,11 +27,11 @@ func mb(b uint64) split.Value {
func TestSumSorted(t *testing.T) {
cases := []struct {
- values []split.Valued
+ values []logsplit.Valued
result []uint64
}{
{
- values: []split.Valued{
+ values: []logsplit.Valued{
v("a", "f", mb(100)),
v("a", "c", mb(200)),
v("d", "g", mb(100)),
@@ -39,7 +39,7 @@ func TestSumSorted(t *testing.T) {
result: []uint64{0, 250, 25, 75, 50, 0},
},
{
- values: []split.Valued{
+ values: []logsplit.Valued{
v("a", "f", mb(100)),
v("a", "c", mb(200)),
v("d", "f", mb(100)),
@@ -47,7 +47,7 @@ func TestSumSorted(t *testing.T) {
result: []uint64{0, 250, 25, 125, 0},
},
{
- values: []split.Valued{
+ values: []logsplit.Valued{
v("a", "f", mb(100)),
v("a", "c", mb(200)),
v("c", "f", mb(100)),
@@ -55,7 +55,7 @@ func TestSumSorted(t *testing.T) {
result: []uint64{0, 250, 150, 0},
},
{
- values: []split.Valued{
+ values: []logsplit.Valued{
v("a", "f", mb(100)),
v("a", "c", mb(200)),
v("c", "f", mb(100)),
@@ -64,7 +64,7 @@ func TestSumSorted(t *testing.T) {
result: []uint64{0, 250, 50, 150, 50, 0},
},
{
- values: []split.Valued{
+ values: []logsplit.Valued{
v("a", "f", mb(100)),
v("a", "c", mb(200)),
v("c", "f", mb(100)),
@@ -74,7 +74,7 @@ func TestSumSorted(t *testing.T) {
result: []uint64{0, 250, 25, 75, 200, 50, 0},
},
{
- values: []split.Valued{
+ values: []logsplit.Valued{
v("a", "f", mb(100)),
v("a", "c", mb(200)),
v("c", "f", mb(100)),
@@ -84,7 +84,7 @@ func TestSumSorted(t *testing.T) {
result: []uint64{0, 250, 25, 75, 200, 100, 0},
},
{
- values: []split.Valued{
+ values: []logsplit.Valued{
v("a", "f", mb(100)),
v("a", "c", mb(200)),
v("c", "f", mb(100)),
@@ -94,7 +94,7 @@ func TestSumSorted(t *testing.T) {
result: []uint64{0, 250, 25, 75, 200, 75, 25, 0},
},
{
- values: []split.Valued{
+ values: []logsplit.Valued{
v("a", "f", mb(100)),
v("a", "c", mb(200)),
v("c", "f", mb(100)),
@@ -104,7 +104,7 @@ func TestSumSorted(t *testing.T) {
result: []uint64{0, 250, 25, 75, 200, 75, 25, 0},
},
{
- values: []split.Valued{
+ values: []logsplit.Valued{
v("a", "f", mb(100)),
v("a", "c", mb(200)),
v("c", "f", mb(100)),
@@ -114,7 +114,7 @@ func TestSumSorted(t *testing.T) {
result: []uint64{0, 250, 100, 200, 75, 25, 0},
},
{
- values: []split.Valued{
+ values: []logsplit.Valued{
v("a", "f", mb(100)),
v("a", "c", mb(200)),
v("c", "f", mb(100)),
@@ -126,13 +126,13 @@ func TestSumSorted(t *testing.T) {
}
for _, ca := range cases {
- full := split.NewSplitHelper()
+ full := logsplit.NewSplitHelper()
for _, v := range ca.values {
full.Merge(v)
}
i := 0
- full.Traverse(func(v split.Valued) bool {
+ full.Traverse(func(v logsplit.Valued) bool {
require.Equal(t, mb(ca.result[i]), v.Value)
i++
return true
diff --git a/br/pkg/restore/internal/prealloc_db/BUILD.bazel b/br/pkg/restore/internal/prealloc_db/BUILD.bazel
new file mode 100644
index 0000000000000..c5205aea23179
--- /dev/null
+++ b/br/pkg/restore/internal/prealloc_db/BUILD.bazel
@@ -0,0 +1,45 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+ name = "prealloc_db",
+ srcs = ["db.go"],
+ importpath = "github.com/pingcap/tidb/br/pkg/restore/internal/prealloc_db",
+ visibility = ["//br/pkg/restore:__subpackages__"],
+ deps = [
+ "//br/pkg/glue",
+ "//br/pkg/metautil",
+ "//br/pkg/restore",
+ "//br/pkg/restore/internal/prealloc_table_id",
+ "//br/pkg/utils",
+ "//pkg/ddl",
+ "//pkg/kv",
+ "//pkg/parser/model",
+ "//pkg/parser/mysql",
+ "//pkg/sessionctx/variable",
+ "@com_github_pingcap_errors//:errors",
+ "@com_github_pingcap_log//:log",
+ "@org_uber_go_zap//:zap",
+ ],
+)
+
+go_test(
+ name = "prealloc_db_test",
+ timeout = "short",
+ srcs = ["db_test.go"],
+ flaky = True,
+ shard_count = 4,
+ deps = [
+ ":prealloc_db",
+ "//br/pkg/gluetidb",
+ "//br/pkg/metautil",
+ "//br/pkg/restore",
+ "//br/pkg/utiltest",
+ "//pkg/meta/autoid",
+ "//pkg/parser/model",
+ "//pkg/parser/mysql",
+ "//pkg/parser/types",
+ "//pkg/testkit",
+ "@com_github_stretchr_testify//assert",
+ "@com_github_stretchr_testify//require",
+ ],
+)
diff --git a/br/pkg/restore/db.go b/br/pkg/restore/internal/prealloc_db/db.go
similarity index 70%
rename from br/pkg/restore/db.go
rename to br/pkg/restore/internal/prealloc_db/db.go
index f09f420ceaa19..7c9a661d8b895 100644
--- a/br/pkg/restore/db.go
+++ b/br/pkg/restore/internal/prealloc_db/db.go
@@ -1,27 +1,24 @@
// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
-package restore
+package preallocdb
import (
- "cmp"
"context"
"fmt"
- "slices"
"sync"
"github.com/pingcap/errors"
"github.com/pingcap/log"
"github.com/pingcap/tidb/br/pkg/glue"
"github.com/pingcap/tidb/br/pkg/metautil"
- prealloctableid "github.com/pingcap/tidb/br/pkg/restore/prealloc_table_id"
+ "github.com/pingcap/tidb/br/pkg/restore"
+ prealloctableid "github.com/pingcap/tidb/br/pkg/restore/internal/prealloc_table_id"
"github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/pkg/ddl"
- "github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
- tidbutil "github.com/pingcap/tidb/pkg/util"
"go.uber.org/zap"
)
@@ -31,20 +28,6 @@ type DB struct {
preallocedIDs *prealloctableid.PreallocIDs
}
-type UniqueTableName struct {
- DB string
- Table string
-}
-
-type DDLJobFilterRule func(ddlJob *model.Job) bool
-
-var incrementalRestoreActionBlockList = map[model.ActionType]struct{}{
- model.ActionSetTiFlashReplica: {},
- model.ActionUpdateTiFlashReplicaStatus: {},
- model.ActionLockTable: {},
- model.ActionUnlockTable: {},
-}
-
// NewDB returns a new DB.
func NewDB(g glue.Glue, store kv.Storage, policyMode string) (*DB, bool, error) {
se, err := g.CreateSession(store)
@@ -81,7 +64,11 @@ func NewDB(g glue.Glue, store kv.Storage, policyMode string) (*DB, bool, error)
}, supportPolicy, nil
}
-func (db *DB) registerPreallocatedIDs(ids *prealloctableid.PreallocIDs) {
+func (db *DB) Session() glue.Session {
+ return db.se
+}
+
+func (db *DB) RegisterPreallocatedIDs(ids *prealloctableid.PreallocIDs) {
db.preallocedIDs = ids
}
@@ -170,7 +157,21 @@ func (db *DB) CreatePlacementPolicy(ctx context.Context, policy *model.PolicyInf
}
// CreateDatabase executes a CREATE DATABASE SQL.
-func (db *DB) CreateDatabase(ctx context.Context, schema *model.DBInfo) error {
+func (db *DB) CreateDatabase(ctx context.Context, schema *model.DBInfo, supportPolicy bool, policyMap *sync.Map) error {
+ log.Info("create database", zap.Stringer("name", schema.Name))
+
+ if !supportPolicy {
+ log.Info("set placementPolicyRef to nil when target tidb not support policy",
+ zap.Stringer("database", schema.Name))
+ schema.PlacementPolicyRef = nil
+ }
+
+ if schema.PlacementPolicyRef != nil {
+ if err := db.ensurePlacementPolicy(ctx, schema.PlacementPolicyRef.Name, policyMap); err != nil {
+ return errors.Trace(err)
+ }
+ }
+
err := db.se.CreateDatabase(ctx, schema)
if err != nil {
log.Error("create database failed", zap.Stringer("db", schema.Name), zap.Error(err))
@@ -235,7 +236,7 @@ func (db *DB) restoreSequence(ctx context.Context, table *metautil.Table) error
return errors.Trace(err)
}
-func (db *DB) CreateTablePostRestore(ctx context.Context, table *metautil.Table, toBeCorrectedTables map[UniqueTableName]bool) error {
+func (db *DB) CreateTablePostRestore(ctx context.Context, table *metautil.Table, toBeCorrectedTables map[restore.UniqueTableName]bool) error {
var restoreMetaSQL string
var err error
switch {
@@ -247,7 +248,7 @@ func (db *DB) CreateTablePostRestore(ctx context.Context, table *metautil.Table,
return errors.Trace(err)
}
// only table exists in restored cluster during incremental restoration should do alter after creation.
- case toBeCorrectedTables[UniqueTableName{table.DB.Name.String(), table.Info.Name.String()}]:
+ case toBeCorrectedTables[restore.UniqueTableName{DB: table.DB.Name.String(), Table: table.Info.Name.String()}]:
if utils.NeedAutoID(table.Info) {
restoreMetaSQL = fmt.Sprintf(
"alter table %s.%s auto_increment = %d;",
@@ -294,7 +295,7 @@ func (db *DB) tableIDAllocFilter() ddl.AllocTableIDIf {
// CreateTables execute a internal CREATE TABLES.
func (db *DB) CreateTables(ctx context.Context, tables []*metautil.Table,
- ddlTables map[UniqueTableName]bool, supportPolicy bool, policyMap *sync.Map) error {
+ ddlTables map[restore.UniqueTableName]bool, supportPolicy bool, policyMap *sync.Map) error {
if batchSession, ok := db.se.(glue.BatchCreateTableSession); ok {
m := map[string][]*model.TableInfo{}
for _, table := range tables {
@@ -329,7 +330,7 @@ func (db *DB) CreateTables(ctx context.Context, tables []*metautil.Table,
// CreateTable executes a CREATE TABLE SQL.
func (db *DB) CreateTable(ctx context.Context, table *metautil.Table,
- ddlTables map[UniqueTableName]bool, supportPolicy bool, policyMap *sync.Map) error {
+ ddlTables map[restore.UniqueTableName]bool, supportPolicy bool, policyMap *sync.Map) error {
if !supportPolicy {
log.Info("set placementPolicyRef to nil when target tidb not support policy",
zap.Stringer("table", table.Info.Name), zap.Stringer("db", table.DB.Name))
@@ -400,115 +401,3 @@ func (db *DB) ensureTablePlacementPolicies(ctx context.Context, tableInfo *model
return nil
}
-
-// FilterDDLJobs filters ddl jobs.
-func FilterDDLJobs(allDDLJobs []*model.Job, tables []*metautil.Table) (ddlJobs []*model.Job) {
- // Sort the ddl jobs by schema version in descending order.
- slices.SortFunc(allDDLJobs, func(i, j *model.Job) int {
- return cmp.Compare(j.BinlogInfo.SchemaVersion, i.BinlogInfo.SchemaVersion)
- })
- dbs := getDatabases(tables)
- for _, db := range dbs {
- // These maps is for solving some corner case.
- // e.g. let "t=2" indicates that the id of database "t" is 2, if the ddl execution sequence is:
- // rename "a" to "b"(a=1) -> drop "b"(b=1) -> create "b"(b=2) -> rename "b" to "a"(a=2)
- // Which we cannot find the "create" DDL by name and id directly.
- // To cover †his case, we must find all names and ids the database/table ever had.
- dbIDs := make(map[int64]bool)
- dbIDs[db.ID] = true
- dbNames := make(map[string]bool)
- dbNames[db.Name.String()] = true
- for _, job := range allDDLJobs {
- if job.BinlogInfo.DBInfo != nil {
- if dbIDs[job.SchemaID] || dbNames[job.BinlogInfo.DBInfo.Name.String()] {
- ddlJobs = append(ddlJobs, job)
- // The the jobs executed with the old id, like the step 2 in the example above.
- dbIDs[job.SchemaID] = true
- // For the jobs executed after rename, like the step 3 in the example above.
- dbNames[job.BinlogInfo.DBInfo.Name.String()] = true
- }
- }
- }
- }
-
- for _, table := range tables {
- tableIDs := make(map[int64]bool)
- tableIDs[table.Info.ID] = true
- tableNames := make(map[UniqueTableName]bool)
- name := UniqueTableName{table.DB.Name.String(), table.Info.Name.String()}
- tableNames[name] = true
- for _, job := range allDDLJobs {
- if job.BinlogInfo.TableInfo != nil {
- name = UniqueTableName{job.SchemaName, job.BinlogInfo.TableInfo.Name.String()}
- if tableIDs[job.TableID] || tableNames[name] {
- ddlJobs = append(ddlJobs, job)
- tableIDs[job.TableID] = true
- // For truncate table, the id may be changed
- tableIDs[job.BinlogInfo.TableInfo.ID] = true
- tableNames[name] = true
- }
- }
- }
- }
- return ddlJobs
-}
-
-// FilterDDLJobByRules if one of rules returns true, the job in srcDDLJobs will be filtered.
-func FilterDDLJobByRules(srcDDLJobs []*model.Job, rules ...DDLJobFilterRule) (dstDDLJobs []*model.Job) {
- dstDDLJobs = make([]*model.Job, 0, len(srcDDLJobs))
- for _, ddlJob := range srcDDLJobs {
- passed := true
- for _, rule := range rules {
- if rule(ddlJob) {
- passed = false
- break
- }
- }
-
- if passed {
- dstDDLJobs = append(dstDDLJobs, ddlJob)
- }
- }
-
- return
-}
-
-// DDLJobBlockListRule rule for filter ddl job with type in block list.
-func DDLJobBlockListRule(ddlJob *model.Job) bool {
- return checkIsInActions(ddlJob.Type, incrementalRestoreActionBlockList)
-}
-
-// GetExistedUserDBs get dbs created or modified by users
-func GetExistedUserDBs(dom *domain.Domain) []*model.DBInfo {
- databases := dom.InfoSchema().AllSchemas()
- existedDatabases := make([]*model.DBInfo, 0, 16)
- for _, db := range databases {
- dbName := db.Name.L
- if tidbutil.IsMemOrSysDB(dbName) {
- continue
- } else if dbName == "test" && len(db.Tables) == 0 {
- // tidb create test db on fresh cluster
- // if it's empty we don't take it as user db
- continue
- }
- existedDatabases = append(existedDatabases, db)
- }
-
- return existedDatabases
-}
-
-func getDatabases(tables []*metautil.Table) (dbs []*model.DBInfo) {
- dbIDs := make(map[int64]bool)
- for _, table := range tables {
- if !dbIDs[table.DB.ID] {
- dbs = append(dbs, table.DB)
- dbIDs[table.DB.ID] = true
- }
- }
- return
-}
-
-func checkIsInActions(action model.ActionType, actions map[model.ActionType]struct{}) bool {
- _, ok := actions[action]
- return ok
-}
diff --git a/br/pkg/restore/internal/prealloc_db/db_test.go b/br/pkg/restore/internal/prealloc_db/db_test.go
new file mode 100644
index 0000000000000..7b2f11dd21ef7
--- /dev/null
+++ b/br/pkg/restore/internal/prealloc_db/db_test.go
@@ -0,0 +1,263 @@
+// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
+
+package preallocdb_test
+
+import (
+ "context"
+ "math"
+ "strconv"
+ "testing"
+
+ "github.com/pingcap/tidb/br/pkg/gluetidb"
+ "github.com/pingcap/tidb/br/pkg/metautil"
+ "github.com/pingcap/tidb/br/pkg/restore"
+ preallocdb "github.com/pingcap/tidb/br/pkg/restore/internal/prealloc_db"
+ "github.com/pingcap/tidb/br/pkg/utiltest"
+ "github.com/pingcap/tidb/pkg/meta/autoid"
+ "github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/pingcap/tidb/pkg/parser/mysql"
+ "github.com/pingcap/tidb/pkg/parser/types"
+ "github.com/pingcap/tidb/pkg/testkit"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestRestoreAutoIncID(t *testing.T) {
+ s := utiltest.CreateRestoreSchemaSuite(t)
+ tk := testkit.NewTestKit(t, s.Mock.Storage)
+ tk.MustExec("use test")
+ tk.MustExec("set @@sql_mode=''")
+ tk.MustExec("drop table if exists `\"t\"`;")
+ // Test SQL Mode
+ tk.MustExec("create table `\"t\"` (" +
+ "a int not null," +
+ "time timestamp not null default '0000-00-00 00:00:00');",
+ )
+ tk.MustExec("insert into `\"t\"` values (10, '0000-00-00 00:00:00');")
+ // Query the current AutoIncID
+ autoIncID, err := strconv.ParseUint(tk.MustQuery("admin show `\"t\"` next_row_id").Rows()[0][3].(string), 10, 64)
+ require.NoErrorf(t, err, "Error query auto inc id: %s", err)
+ // Get schemas of db and table
+ info, err := s.Mock.Domain.GetSnapshotInfoSchema(math.MaxUint64)
+ require.NoErrorf(t, err, "Error get snapshot info schema: %s", err)
+ dbInfo, exists := info.SchemaByName(model.NewCIStr("test"))
+ require.Truef(t, exists, "Error get db info")
+ tableInfo, err := info.TableByName(model.NewCIStr("test"), model.NewCIStr("\"t\""))
+ require.NoErrorf(t, err, "Error get table info: %s", err)
+ table := metautil.Table{
+ Info: tableInfo.Meta(),
+ DB: dbInfo,
+ }
+ // Get the next AutoIncID
+ idAlloc := autoid.NewAllocator(s.Mock.Domain, dbInfo.ID, table.Info.ID, false, autoid.RowIDAllocType)
+ globalAutoID, err := idAlloc.NextGlobalAutoID()
+ require.NoErrorf(t, err, "Error allocate next auto id")
+ require.Equal(t, uint64(globalAutoID), autoIncID)
+ // Alter AutoIncID to the next AutoIncID + 100
+ table.Info.AutoIncID = globalAutoID + 100
+ db, _, err := preallocdb.NewDB(gluetidb.New(), s.Mock.Storage, "STRICT")
+ require.NoErrorf(t, err, "Error create DB")
+ tk.MustExec("drop database if exists test;")
+ // Test empty collate value
+ table.DB.Charset = "utf8mb4"
+ table.DB.Collate = ""
+ err = db.CreateDatabase(context.Background(), table.DB, false, nil)
+ require.NoErrorf(t, err, "Error create empty collate db: %s %s", err, s.Mock.DSN)
+ tk.MustExec("drop database if exists test;")
+ // Test empty charset value
+ table.DB.Charset = ""
+ table.DB.Collate = "utf8mb4_bin"
+ err = db.CreateDatabase(context.Background(), table.DB, false, nil)
+ require.NoErrorf(t, err, "Error create empty charset db: %s %s", err, s.Mock.DSN)
+ uniqueMap := make(map[restore.UniqueTableName]bool)
+ err = db.CreateTable(context.Background(), &table, uniqueMap, false, nil)
+ require.NoErrorf(t, err, "Error create table: %s %s", err, s.Mock.DSN)
+
+ tk.MustExec("use test")
+ autoIncID, err = strconv.ParseUint(tk.MustQuery("admin show `\"t\"` next_row_id").Rows()[0][3].(string), 10, 64)
+ require.NoErrorf(t, err, "Error query auto inc id: %s", err)
+ // Check if AutoIncID is altered successfully.
+ require.Equal(t, uint64(globalAutoID+100), autoIncID)
+
+ // try again, failed due to table exists.
+ table.Info.AutoIncID = globalAutoID + 200
+ err = db.CreateTable(context.Background(), &table, uniqueMap, false, nil)
+ require.NoError(t, err)
+ // Check if AutoIncID is not altered.
+ autoIncID, err = strconv.ParseUint(tk.MustQuery("admin show `\"t\"` next_row_id").Rows()[0][3].(string), 10, 64)
+ require.NoErrorf(t, err, "Error query auto inc id: %s", err)
+ require.Equal(t, uint64(globalAutoID+100), autoIncID)
+
+ // try again, success because we use alter sql in unique map.
+ table.Info.AutoIncID = globalAutoID + 300
+ uniqueMap[restore.UniqueTableName{DB: "test", Table: "\"t\""}] = true
+ err = db.CreateTable(context.Background(), &table, uniqueMap, false, nil)
+ require.NoError(t, err)
+ // Check if AutoIncID is altered to globalAutoID + 300.
+ autoIncID, err = strconv.ParseUint(tk.MustQuery("admin show `\"t\"` next_row_id").Rows()[0][3].(string), 10, 64)
+ require.NoErrorf(t, err, "Error query auto inc id: %s", err)
+ require.Equal(t, uint64(globalAutoID+300), autoIncID)
+}
+
+func TestCreateTablesInDb(t *testing.T) {
+ s := utiltest.CreateRestoreSchemaSuite(t)
+ info, err := s.Mock.Domain.GetSnapshotInfoSchema(math.MaxUint64)
+ require.NoErrorf(t, err, "Error get snapshot info schema: %s", err)
+
+ dbSchema, isExist := info.SchemaByName(model.NewCIStr("test"))
+ require.True(t, isExist)
+
+ tables := make([]*metautil.Table, 4)
+ intField := types.NewFieldType(mysql.TypeLong)
+ intField.SetCharset("binary")
+ ddlJobMap := make(map[restore.UniqueTableName]bool)
+ for i := len(tables) - 1; i >= 0; i-- {
+ tables[i] = &metautil.Table{
+ DB: dbSchema,
+ Info: &model.TableInfo{
+ ID: int64(i),
+ Name: model.NewCIStr("test" + strconv.Itoa(i)),
+ Columns: []*model.ColumnInfo{{
+ ID: 1,
+ Name: model.NewCIStr("id"),
+ FieldType: *intField,
+ State: model.StatePublic,
+ }},
+ Charset: "utf8mb4",
+ Collate: "utf8mb4_bin",
+ },
+ }
+ ddlJobMap[restore.UniqueTableName{DB: dbSchema.Name.String(), Table: tables[i].Info.Name.String()}] = false
+ }
+ db, _, err := preallocdb.NewDB(gluetidb.New(), s.Mock.Storage, "STRICT")
+ require.NoError(t, err)
+
+ err = db.CreateTables(context.Background(), tables, ddlJobMap, false, nil)
+ require.NoError(t, err)
+}
+
+func TestDB_ExecDDL(t *testing.T) {
+ s := utiltest.CreateRestoreSchemaSuite(t)
+
+ ctx := context.Background()
+ ddlJobs := []*model.Job{
+ {
+ Type: model.ActionAddIndex,
+ Query: "CREATE DATABASE IF NOT EXISTS test_db;",
+ BinlogInfo: &model.HistoryInfo{},
+ },
+ {
+ Type: model.ActionAddIndex,
+ Query: "",
+ BinlogInfo: &model.HistoryInfo{},
+ },
+ }
+
+ db, _, err := preallocdb.NewDB(gluetidb.New(), s.Mock.Storage, "STRICT")
+ require.NoError(t, err)
+
+ for _, ddlJob := range ddlJobs {
+ err = db.ExecDDL(ctx, ddlJob)
+ assert.NoError(t, err)
+ }
+}
+
+func TestCreateTableConsistent(t *testing.T) {
+ ctx := context.Background()
+ s := utiltest.CreateRestoreSchemaSuite(t)
+ tk := testkit.NewTestKit(t, s.Mock.Storage)
+ tk.MustExec("use test")
+ tk.MustExec("set @@sql_mode=''")
+
+ db, supportPolicy, err := preallocdb.NewDB(gluetidb.New(), s.Mock.Storage, "STRICT")
+ require.NoError(t, err)
+ require.True(t, supportPolicy)
+ defer db.Close()
+
+ getTableInfo := func(name string) (*model.DBInfo, *model.TableInfo) {
+ info, err := s.Mock.Domain.GetSnapshotInfoSchema(math.MaxUint64)
+ require.NoError(t, err)
+ dbInfo, exists := info.SchemaByName(model.NewCIStr("test"))
+ require.True(t, exists)
+ tableInfo, err := info.TableByName(model.NewCIStr("test"), model.NewCIStr(name))
+ require.NoError(t, err)
+ return dbInfo, tableInfo.Meta()
+ }
+ tk.MustExec("create sequence test.s increment by 1 minvalue = 10;")
+ dbInfo, seqInfo := getTableInfo("s")
+ tk.MustExec("drop sequence test.s;")
+
+ newSeqInfo := seqInfo.Clone()
+ newSeqInfo.ID += 100
+ newTables := []*metautil.Table{
+ {
+ DB: dbInfo.Clone(),
+ Info: newSeqInfo,
+ },
+ }
+ err = db.CreateTables(ctx, newTables, nil, false, nil)
+ require.NoError(t, err)
+ r11 := tk.MustQuery("select nextval(s)").Rows()
+ r12 := tk.MustQuery("show create table test.s").Rows()
+
+ tk.MustExec("drop sequence test.s;")
+
+ newSeqInfo = seqInfo.Clone()
+ newSeqInfo.ID += 100
+ newTable := &metautil.Table{DB: dbInfo.Clone(), Info: newSeqInfo}
+ err = db.CreateTable(ctx, newTable, nil, false, nil)
+ require.NoError(t, err)
+ r21 := tk.MustQuery("select nextval(s)").Rows()
+ r22 := tk.MustQuery("show create table test.s").Rows()
+
+ require.Equal(t, r11, r21)
+ require.Equal(t, r12, r22)
+
+ tk.MustExec("drop sequence test.s;")
+ tk.MustExec("create table t (a int);")
+ tk.MustExec("create view v as select * from t;")
+
+ _, tblInfo := getTableInfo("t")
+ _, viewInfo := getTableInfo("v")
+ tk.MustExec("drop table t;")
+ tk.MustExec("drop view v;")
+
+ newTblInfo := tblInfo.Clone()
+ newTblInfo.ID += 100
+ newViewInfo := viewInfo.Clone()
+ newViewInfo.ID += 100
+ newTables = []*metautil.Table{
+ {
+ DB: dbInfo.Clone(),
+ Info: newTblInfo,
+ },
+ {
+ DB: dbInfo.Clone(),
+ Info: newViewInfo,
+ },
+ }
+ err = db.CreateTables(ctx, newTables, nil, false, nil)
+ require.NoError(t, err)
+ r11 = tk.MustQuery("show create table t;").Rows()
+ r12 = tk.MustQuery("show create view v;").Rows()
+
+ tk.MustExec("drop table t;")
+ tk.MustExec("drop view v;")
+
+ newTblInfo = tblInfo.Clone()
+ newTblInfo.ID += 200
+ newTable = &metautil.Table{DB: dbInfo.Clone(), Info: newTblInfo}
+ err = db.CreateTable(ctx, newTable, nil, false, nil)
+ require.NoError(t, err)
+ newViewInfo = viewInfo.Clone()
+ newViewInfo.ID += 200
+ newTable = &metautil.Table{DB: dbInfo.Clone(), Info: newViewInfo}
+ err = db.CreateTable(ctx, newTable, nil, false, nil)
+ require.NoError(t, err)
+
+ r21 = tk.MustQuery("show create table t;").Rows()
+ r22 = tk.MustQuery("show create view v;").Rows()
+
+ require.Equal(t, r11, r21)
+ require.Equal(t, r12, r22)
+}
diff --git a/br/pkg/restore/prealloc_table_id/BUILD.bazel b/br/pkg/restore/internal/prealloc_table_id/BUILD.bazel
similarity index 77%
rename from br/pkg/restore/prealloc_table_id/BUILD.bazel
rename to br/pkg/restore/internal/prealloc_table_id/BUILD.bazel
index 9496362873763..c19b294e2bf16 100644
--- a/br/pkg/restore/prealloc_table_id/BUILD.bazel
+++ b/br/pkg/restore/internal/prealloc_table_id/BUILD.bazel
@@ -3,8 +3,8 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "prealloc_table_id",
srcs = ["alloc.go"],
- importpath = "github.com/pingcap/tidb/br/pkg/restore/prealloc_table_id",
- visibility = ["//visibility:public"],
+ importpath = "github.com/pingcap/tidb/br/pkg/restore/internal/prealloc_table_id",
+ visibility = ["//br/pkg/restore:__subpackages__"],
deps = [
"//br/pkg/metautil",
"//pkg/parser/model",
@@ -16,7 +16,6 @@ go_test(
timeout = "short",
srcs = ["alloc_test.go"],
flaky = True,
- race = "on",
deps = [
":prealloc_table_id",
"//br/pkg/metautil",
diff --git a/br/pkg/restore/prealloc_table_id/alloc.go b/br/pkg/restore/internal/prealloc_table_id/alloc.go
similarity index 100%
rename from br/pkg/restore/prealloc_table_id/alloc.go
rename to br/pkg/restore/internal/prealloc_table_id/alloc.go
diff --git a/br/pkg/restore/prealloc_table_id/alloc_test.go b/br/pkg/restore/internal/prealloc_table_id/alloc_test.go
similarity index 97%
rename from br/pkg/restore/prealloc_table_id/alloc_test.go
rename to br/pkg/restore/internal/prealloc_table_id/alloc_test.go
index 7fc05f943c04a..ae526c8fd6676 100644
--- a/br/pkg/restore/prealloc_table_id/alloc_test.go
+++ b/br/pkg/restore/internal/prealloc_table_id/alloc_test.go
@@ -7,7 +7,7 @@ import (
"testing"
"github.com/pingcap/tidb/br/pkg/metautil"
- prealloctableid "github.com/pingcap/tidb/br/pkg/restore/prealloc_table_id"
+ prealloctableid "github.com/pingcap/tidb/br/pkg/restore/internal/prealloc_table_id"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/stretchr/testify/require"
)
diff --git a/br/pkg/restore/rawkv/BUILD.bazel b/br/pkg/restore/internal/rawkv/BUILD.bazel
similarity index 91%
rename from br/pkg/restore/rawkv/BUILD.bazel
rename to br/pkg/restore/internal/rawkv/BUILD.bazel
index cac9142aac686..f6d5f8835fb94 100644
--- a/br/pkg/restore/rawkv/BUILD.bazel
+++ b/br/pkg/restore/internal/rawkv/BUILD.bazel
@@ -3,7 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "rawkv",
srcs = ["rawkv_client.go"],
- importpath = "github.com/pingcap/tidb/br/pkg/restore/rawkv",
+ importpath = "github.com/pingcap/tidb/br/pkg/restore/internal/rawkv",
visibility = ["//visibility:public"],
deps = [
"//br/pkg/restore/utils",
diff --git a/br/pkg/restore/rawkv/rawkv_client.go b/br/pkg/restore/internal/rawkv/rawkv_client.go
similarity index 100%
rename from br/pkg/restore/rawkv/rawkv_client.go
rename to br/pkg/restore/internal/rawkv/rawkv_client.go
diff --git a/br/pkg/restore/rawkv/rawkv_client_test.go b/br/pkg/restore/internal/rawkv/rawkv_client_test.go
similarity index 98%
rename from br/pkg/restore/rawkv/rawkv_client_test.go
rename to br/pkg/restore/internal/rawkv/rawkv_client_test.go
index 79b12c68292c1..ae17ca29ac033 100644
--- a/br/pkg/restore/rawkv/rawkv_client_test.go
+++ b/br/pkg/restore/internal/rawkv/rawkv_client_test.go
@@ -10,7 +10,7 @@ import (
"github.com/pingcap/errors"
berrors "github.com/pingcap/tidb/br/pkg/errors"
- rawclient "github.com/pingcap/tidb/br/pkg/restore/rawkv"
+ rawclient "github.com/pingcap/tidb/br/pkg/restore/internal/rawkv"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/util/codec"
"github.com/stretchr/testify/require"
diff --git a/br/pkg/restore/internal/utils/BUILD.bazel b/br/pkg/restore/internal/utils/BUILD.bazel
new file mode 100644
index 0000000000000..607722774f9d0
--- /dev/null
+++ b/br/pkg/restore/internal/utils/BUILD.bazel
@@ -0,0 +1,36 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+ name = "utils",
+ srcs = ["split.go"],
+ importpath = "github.com/pingcap/tidb/br/pkg/restore/internal/utils",
+ visibility = ["//br/pkg/restore:__subpackages__"],
+ deps = [
+ "//br/pkg/errors",
+ "//br/pkg/logutil",
+ "//br/pkg/restore/split",
+ "//br/pkg/rtree",
+ "@com_github_opentracing_opentracing_go//:opentracing-go",
+ "@com_github_pingcap_errors//:errors",
+ "@com_github_pingcap_log//:log",
+ "@org_uber_go_zap//:zap",
+ ],
+)
+
+go_test(
+ name = "utils_test",
+ timeout = "short",
+ srcs = ["split_test.go"],
+ flaky = True,
+ shard_count = 4,
+ deps = [
+ ":utils",
+ "//br/pkg/restore/split",
+ "//br/pkg/restore/utils",
+ "//br/pkg/rtree",
+ "//pkg/tablecodec",
+ "//pkg/util/codec",
+ "@com_github_pingcap_kvproto//pkg/import_sstpb",
+ "@com_github_stretchr_testify//require",
+ ],
+)
diff --git a/br/pkg/restore/utils/split.go b/br/pkg/restore/internal/utils/split.go
similarity index 100%
rename from br/pkg/restore/utils/split.go
rename to br/pkg/restore/internal/utils/split.go
diff --git a/br/pkg/restore/utils/split_test.go b/br/pkg/restore/internal/utils/split_test.go
similarity index 57%
rename from br/pkg/restore/utils/split_test.go
rename to br/pkg/restore/internal/utils/split_test.go
index 3d6e4c44c5b26..a1a45e5513313 100644
--- a/br/pkg/restore/utils/split_test.go
+++ b/br/pkg/restore/internal/utils/split_test.go
@@ -7,9 +7,11 @@ import (
"testing"
"github.com/pingcap/kvproto/pkg/import_sstpb"
+ "github.com/pingcap/tidb/br/pkg/restore/internal/utils"
"github.com/pingcap/tidb/br/pkg/restore/split"
- "github.com/pingcap/tidb/br/pkg/restore/utils"
+ restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
"github.com/pingcap/tidb/br/pkg/rtree"
+ "github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/util/codec"
"github.com/stretchr/testify/require"
)
@@ -48,7 +50,7 @@ func TestSplitAndScatter(t *testing.T) {
ranges := initRanges()
rules := initRewriteRules()
for i, rg := range ranges {
- tmp, err := utils.RewriteRange(&rg, rules)
+ tmp, err := restoreutils.RewriteRange(&rg, rules)
require.NoError(t, err)
ranges[i] = *tmp
}
@@ -121,7 +123,7 @@ func initRanges() []rtree.Range {
return ranges[:]
}
-func initRewriteRules() *utils.RewriteRules {
+func initRewriteRules() *restoreutils.RewriteRules {
var rules [2]*import_sstpb.RewriteRule
rules[0] = &import_sstpb.RewriteRule{
OldKeyPrefix: []byte("aa"),
@@ -131,7 +133,85 @@ func initRewriteRules() *utils.RewriteRules {
OldKeyPrefix: []byte("cc"),
NewKeyPrefix: []byte("bb"),
}
- return &utils.RewriteRules{
+ return &restoreutils.RewriteRules{
Data: rules[:],
}
}
+
+func TestSortRange(t *testing.T) {
+ dataRules := []*import_sstpb.RewriteRule{
+ {OldKeyPrefix: tablecodec.GenTableRecordPrefix(1), NewKeyPrefix: tablecodec.GenTableRecordPrefix(4)},
+ {OldKeyPrefix: tablecodec.GenTableRecordPrefix(2), NewKeyPrefix: tablecodec.GenTableRecordPrefix(5)},
+ }
+ rewriteRules := &restoreutils.RewriteRules{
+ Data: dataRules,
+ }
+ ranges1 := []rtree.Range{
+ {
+ StartKey: append(tablecodec.GenTableRecordPrefix(1), []byte("aaa")...),
+ EndKey: append(tablecodec.GenTableRecordPrefix(1), []byte("bbb")...), Files: nil,
+ },
+ }
+ for i, rg := range ranges1 {
+ tmp, _ := restoreutils.RewriteRange(&rg, rewriteRules)
+ ranges1[i] = *tmp
+ }
+ rs1, err := utils.SortRanges(ranges1)
+ require.NoErrorf(t, err, "sort range1 failed: %v", err)
+ rangeEquals(t, rs1, []rtree.Range{
+ {
+ StartKey: append(tablecodec.GenTableRecordPrefix(4), []byte("aaa")...),
+ EndKey: append(tablecodec.GenTableRecordPrefix(4), []byte("bbb")...), Files: nil,
+ },
+ })
+
+ ranges2 := []rtree.Range{
+ {
+ StartKey: append(tablecodec.GenTableRecordPrefix(1), []byte("aaa")...),
+ EndKey: append(tablecodec.GenTableRecordPrefix(2), []byte("bbb")...), Files: nil,
+ },
+ }
+ for _, rg := range ranges2 {
+ _, err := restoreutils.RewriteRange(&rg, rewriteRules)
+ require.Error(t, err)
+ require.Regexp(t, "table id mismatch.*", err.Error())
+ }
+
+ ranges3 := []rtree.Range{
+ {StartKey: []byte("aaa"), EndKey: []byte("aae")},
+ {StartKey: []byte("aae"), EndKey: []byte("aaz")},
+ {StartKey: []byte("ccd"), EndKey: []byte("ccf")},
+ {StartKey: []byte("ccf"), EndKey: []byte("ccj")},
+ }
+ rewriteRules1 := &restoreutils.RewriteRules{
+ Data: []*import_sstpb.RewriteRule{
+ {
+ OldKeyPrefix: []byte("aa"),
+ NewKeyPrefix: []byte("xx"),
+ }, {
+ OldKeyPrefix: []byte("cc"),
+ NewKeyPrefix: []byte("bb"),
+ },
+ },
+ }
+ for i, rg := range ranges3 {
+ tmp, _ := restoreutils.RewriteRange(&rg, rewriteRules1)
+ ranges3[i] = *tmp
+ }
+ rs3, err := utils.SortRanges(ranges3)
+ require.NoErrorf(t, err, "sort range1 failed: %v", err)
+ rangeEquals(t, rs3, []rtree.Range{
+ {StartKey: []byte("bbd"), EndKey: []byte("bbf"), Files: nil},
+ {StartKey: []byte("bbf"), EndKey: []byte("bbj"), Files: nil},
+ {StartKey: []byte("xxa"), EndKey: []byte("xxe"), Files: nil},
+ {StartKey: []byte("xxe"), EndKey: []byte("xxz"), Files: nil},
+ })
+}
+
+func rangeEquals(t *testing.T, obtained, expected []rtree.Range) {
+ require.Equal(t, len(expected), len(obtained))
+ for i := range obtained {
+ require.Equal(t, expected[i].StartKey, obtained[i].StartKey)
+ require.Equal(t, expected[i].EndKey, obtained[i].EndKey)
+ }
+}
diff --git a/br/pkg/restore/file_importer/BUILD.bazel b/br/pkg/restore/log_client/BUILD.bazel
similarity index 54%
rename from br/pkg/restore/file_importer/BUILD.bazel
rename to br/pkg/restore/log_client/BUILD.bazel
index a9d6cc246a1f8..25431dd3e6d4a 100644
--- a/br/pkg/restore/file_importer/BUILD.bazel
+++ b/br/pkg/restore/log_client/BUILD.bazel
@@ -1,27 +1,50 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
- name = "file_importer",
+ name = "log_client",
srcs = [
+ "client.go",
"import.go",
"import_retry.go",
+ "log_file_manager.go",
+ "log_file_map.go",
],
- importpath = "github.com/pingcap/tidb/br/pkg/restore/file_importer",
+ importpath = "github.com/pingcap/tidb/br/pkg/restore/log_client",
visibility = ["//visibility:public"],
deps = [
+ "//br/pkg/checkpoint",
+ "//br/pkg/checksum",
"//br/pkg/conn",
"//br/pkg/conn/util",
"//br/pkg/errors",
+ "//br/pkg/glue",
"//br/pkg/logutil",
- "//br/pkg/restore/log_restore",
+ "//br/pkg/metautil",
+ "//br/pkg/restore",
+ "//br/pkg/restore/ingestrec",
+ "//br/pkg/restore/internal/import_client",
+ "//br/pkg/restore/internal/log_split",
+ "//br/pkg/restore/internal/rawkv",
"//br/pkg/restore/split",
+ "//br/pkg/restore/tiflashrec",
"//br/pkg/restore/utils",
+ "//br/pkg/storage",
"//br/pkg/stream",
"//br/pkg/summary",
"//br/pkg/utils",
+ "//br/pkg/utils/iter",
+ "//br/pkg/version",
+ "//pkg/ddl/util",
+ "//pkg/domain",
"//pkg/kv",
+ "//pkg/meta",
+ "//pkg/parser/model",
+ "//pkg/util",
"//pkg/util/codec",
- "@com_github_google_uuid//:uuid",
+ "//pkg/util/redact",
+ "//pkg/util/table-filter",
+ "@com_github_fatih_color//:color",
+ "@com_github_opentracing_opentracing_go//:opentracing-go",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_kvproto//pkg/brpb",
@@ -30,39 +53,53 @@ go_library(
"@com_github_pingcap_kvproto//pkg/kvrpcpb",
"@com_github_pingcap_kvproto//pkg/metapb",
"@com_github_pingcap_log//:log",
+ "@com_github_tikv_client_go_v2//config",
"@com_github_tikv_client_go_v2//kv",
"@com_github_tikv_client_go_v2//util",
"@com_github_tikv_pd_client//:client",
- "@org_golang_google_grpc//:grpc",
- "@org_golang_google_grpc//backoff",
+ "@com_github_tikv_pd_client//http",
"@org_golang_google_grpc//codes",
- "@org_golang_google_grpc//credentials",
- "@org_golang_google_grpc//credentials/insecure",
"@org_golang_google_grpc//keepalive",
"@org_golang_google_grpc//status",
- "@org_golang_x_exp//maps",
"@org_golang_x_sync//errgroup",
"@org_uber_go_multierr//:multierr",
"@org_uber_go_zap//:zap",
+ "@org_uber_go_zap//zapcore",
],
)
go_test(
- name = "file_importer_test",
+ name = "log_client_test",
timeout = "short",
srcs = [
+ "client_test.go",
+ "export_test.go",
"import_retry_test.go",
"import_test.go",
+ "log_file_manager_test.go",
+ "log_file_map_test.go",
+ "main_test.go",
],
+ embed = [":log_client"],
flaky = True,
- shard_count = 11,
+ shard_count = 36,
deps = [
- ":file_importer",
+ "//br/pkg/errors",
+ "//br/pkg/gluetidb",
+ "//br/pkg/mock",
"//br/pkg/restore/split",
"//br/pkg/restore/utils",
+ "//br/pkg/storage",
+ "//br/pkg/stream",
"//br/pkg/utils",
+ "//br/pkg/utils/iter",
+ "//br/pkg/utiltest",
+ "//pkg/kv",
"//pkg/store/pdtypes",
+ "//pkg/tablecodec",
+ "//pkg/testkit/testsetup",
"//pkg/util/codec",
+ "//pkg/util/table-filter",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_kvproto//pkg/brpb",
@@ -70,8 +107,13 @@ go_test(
"@com_github_pingcap_kvproto//pkg/import_sstpb",
"@com_github_pingcap_kvproto//pkg/metapb",
"@com_github_pingcap_kvproto//pkg/pdpb",
+ "@com_github_pingcap_log//:log",
"@com_github_stretchr_testify//require",
"@org_golang_google_grpc//codes",
+ "@org_golang_google_grpc//keepalive",
"@org_golang_google_grpc//status",
+ "@org_uber_go_goleak//:goleak",
+ "@org_uber_go_zap//:zap",
+ "@org_uber_go_zap//zapcore",
],
)
diff --git a/br/pkg/restore/log_client/client.go b/br/pkg/restore/log_client/client.go
new file mode 100644
index 0000000000000..c7c9f2a8ce9bb
--- /dev/null
+++ b/br/pkg/restore/log_client/client.go
@@ -0,0 +1,1657 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package logclient
+
+import (
+ "cmp"
+ "context"
+ "crypto/tls"
+ "fmt"
+ "math"
+ "slices"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/fatih/color"
+ "github.com/opentracing/opentracing-go"
+ "github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
+ backuppb "github.com/pingcap/kvproto/pkg/brpb"
+ "github.com/pingcap/log"
+ "github.com/pingcap/tidb/br/pkg/checkpoint"
+ "github.com/pingcap/tidb/br/pkg/checksum"
+ "github.com/pingcap/tidb/br/pkg/conn"
+ "github.com/pingcap/tidb/br/pkg/conn/util"
+ "github.com/pingcap/tidb/br/pkg/glue"
+ "github.com/pingcap/tidb/br/pkg/logutil"
+ "github.com/pingcap/tidb/br/pkg/metautil"
+ "github.com/pingcap/tidb/br/pkg/restore"
+ "github.com/pingcap/tidb/br/pkg/restore/ingestrec"
+ importclient "github.com/pingcap/tidb/br/pkg/restore/internal/import_client"
+ logsplit "github.com/pingcap/tidb/br/pkg/restore/internal/log_split"
+ "github.com/pingcap/tidb/br/pkg/restore/internal/rawkv"
+ "github.com/pingcap/tidb/br/pkg/restore/split"
+ "github.com/pingcap/tidb/br/pkg/restore/tiflashrec"
+ restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
+ "github.com/pingcap/tidb/br/pkg/storage"
+ "github.com/pingcap/tidb/br/pkg/stream"
+ "github.com/pingcap/tidb/br/pkg/summary"
+ "github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/br/pkg/utils/iter"
+ "github.com/pingcap/tidb/br/pkg/version"
+ ddlutil "github.com/pingcap/tidb/pkg/ddl/util"
+ "github.com/pingcap/tidb/pkg/domain"
+ "github.com/pingcap/tidb/pkg/kv"
+ "github.com/pingcap/tidb/pkg/meta"
+ "github.com/pingcap/tidb/pkg/parser/model"
+ tidbutil "github.com/pingcap/tidb/pkg/util"
+ filter "github.com/pingcap/tidb/pkg/util/table-filter"
+ "github.com/tikv/client-go/v2/config"
+ kvutil "github.com/tikv/client-go/v2/util"
+ pd "github.com/tikv/pd/client"
+ pdhttp "github.com/tikv/pd/client/http"
+ "go.uber.org/zap"
+ "go.uber.org/zap/zapcore"
+ "golang.org/x/sync/errgroup"
+ "google.golang.org/grpc/keepalive"
+)
+
+const MetaKVBatchSize = 64 * 1024 * 1024
+const maxSplitKeysOnce = 10240
+
+// rawKVBatchCount specifies the count of entries that the rawkv client puts into TiKV.
+const rawKVBatchCount = 64
+
+type LogClient struct {
+ cipher *backuppb.CipherInfo
+ pdClient pd.Client
+ pdHTTPClient pdhttp.Client
+ clusterID uint64
+ dom *domain.Domain
+ tlsConf *tls.Config
+ keepaliveConf keepalive.ClientParameters
+
+ rawKVClient *rawkv.RawKVBatchClient
+ storage storage.ExternalStorage
+
+ se glue.Session
+
+ // currentTS is used for rewrite meta kv when restore stream.
+ // Can not use `restoreTS` directly, because schema created in `full backup` maybe is new than `restoreTS`.
+ currentTS uint64
+
+ *LogFileManager
+
+ workerPool *tidbutil.WorkerPool
+ fileImporter *LogFileImporter
+
+ // the query to insert rows into table `gc_delete_range`, lack of ts.
+ deleteRangeQuery []*stream.PreDelRangeQuery
+ deleteRangeQueryCh chan *stream.PreDelRangeQuery
+ deleteRangeQueryWaitGroup sync.WaitGroup
+
+ // checkpoint information for log restore
+ useCheckpoint bool
+}
+
+// NewRestoreClient returns a new RestoreClient.
+func NewRestoreClient(
+ pdClient pd.Client,
+ pdHTTPCli pdhttp.Client,
+ tlsConf *tls.Config,
+ keepaliveConf keepalive.ClientParameters,
+) *LogClient {
+ return &LogClient{
+ pdClient: pdClient,
+ pdHTTPClient: pdHTTPCli,
+ tlsConf: tlsConf,
+ keepaliveConf: keepaliveConf,
+ deleteRangeQuery: make([]*stream.PreDelRangeQuery, 0),
+ deleteRangeQueryCh: make(chan *stream.PreDelRangeQuery, 10),
+ }
+}
+
+// Close a client.
+func (rc *LogClient) Close() {
+ // close the connection, and it must be succeed when in SQL mode.
+ if rc.se != nil {
+ rc.se.Close()
+ }
+
+ if rc.rawKVClient != nil {
+ rc.rawKVClient.Close()
+ }
+
+ if err := rc.fileImporter.Close(); err != nil {
+ log.Warn("failed to close file improter")
+ }
+
+ log.Info("Restore client closed")
+}
+
+func (rc *LogClient) SetRawKVBatchClient(
+ ctx context.Context,
+ pdAddrs []string,
+ security config.Security,
+) error {
+ rawkvClient, err := rawkv.NewRawkvClient(ctx, pdAddrs, security)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ rc.rawKVClient = rawkv.NewRawKVBatchClient(rawkvClient, rawKVBatchCount)
+ return nil
+}
+
+func (rc *LogClient) SetCrypter(crypter *backuppb.CipherInfo) {
+ rc.cipher = crypter
+}
+
+func (rc *LogClient) SetConcurrency(c uint) {
+ log.Info("download worker pool", zap.Uint("size", c))
+ rc.workerPool = tidbutil.NewWorkerPool(c, "file")
+}
+
+func (rc *LogClient) SetStorage(ctx context.Context, backend *backuppb.StorageBackend, opts *storage.ExternalStorageOptions) error {
+ var err error
+ rc.storage, err = storage.New(ctx, backend, opts)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ return nil
+}
+
+func (rc *LogClient) SetCurrentTS(ts uint64) {
+ rc.currentTS = ts
+}
+
+// GetClusterID gets the cluster id from down-stream cluster.
+func (rc *LogClient) GetClusterID(ctx context.Context) uint64 {
+ if rc.clusterID <= 0 {
+ rc.clusterID = rc.pdClient.GetClusterID(ctx)
+ }
+ return rc.clusterID
+}
+
+func (rc *LogClient) GetDomain() *domain.Domain {
+ return rc.dom
+}
+
+func (rc *LogClient) CleanUpKVFiles(
+ ctx context.Context,
+) error {
+ // Current we only have v1 prefix.
+ // In the future, we can add more operation for this interface.
+ return rc.fileImporter.ClearFiles(ctx, rc.pdClient, "v1")
+}
+
+func (rc *LogClient) StartCheckpointRunnerForLogRestore(ctx context.Context, taskName string) (*checkpoint.CheckpointRunner[checkpoint.LogRestoreKeyType, checkpoint.LogRestoreValueType], error) {
+ runner, err := checkpoint.StartCheckpointRunnerForLogRestore(ctx, rc.storage, rc.cipher, taskName)
+ return runner, errors.Trace(err)
+}
+
+// Init create db connection and domain for storage.
+func (rc *LogClient) Init(g glue.Glue, store kv.Storage) error {
+ var err error
+ rc.se, err = g.CreateSession(store)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ // Set SQL mode to None for avoiding SQL compatibility problem
+ err = rc.se.Execute(context.Background(), "set @@sql_mode=''")
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ rc.dom, err = g.GetDomain(store)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ return nil
+}
+
+func (rc *LogClient) InitClients(ctx context.Context, backend *backuppb.StorageBackend) {
+ stores, err := conn.GetAllTiKVStoresWithRetry(ctx, rc.pdClient, util.SkipTiFlash)
+ if err != nil {
+ log.Fatal("failed to get stores", zap.Error(err))
+ }
+
+ metaClient := split.NewClient(rc.pdClient, rc.pdHTTPClient, rc.tlsConf, maxSplitKeysOnce, len(stores)+1)
+ importCli := importclient.NewImportClient(metaClient, rc.tlsConf, rc.keepaliveConf)
+ rc.fileImporter = NewLogFileImporter(metaClient, importCli, backend)
+}
+
+func (rc *LogClient) InitCheckpointMetadataForLogRestore(ctx context.Context, taskName string, gcRatio string) (string, error) {
+ rc.useCheckpoint = true
+
+ // it shows that the user has modified gc-ratio, if `gcRatio` doesn't equal to "1.1".
+ // update the `gcRatio` for checkpoint metadata.
+ if gcRatio == utils.DefaultGcRatioVal {
+ // if the checkpoint metadata exists in the external storage, the restore is not
+ // for the first time.
+ exists, err := checkpoint.ExistsRestoreCheckpoint(ctx, rc.storage, taskName)
+ if err != nil {
+ return "", errors.Trace(err)
+ }
+
+ if exists {
+ // load the checkpoint since this is not the first time to restore
+ meta, err := checkpoint.LoadCheckpointMetadataForRestore(ctx, rc.storage, taskName)
+ if err != nil {
+ return "", errors.Trace(err)
+ }
+
+ log.Info("reuse gc ratio from checkpoint metadata", zap.String("gc-ratio", gcRatio))
+ return meta.GcRatio, nil
+ }
+ }
+
+ // initialize the checkpoint metadata since it is the first time to restore.
+ log.Info("save gc ratio into checkpoint metadata", zap.String("gc-ratio", gcRatio))
+ if err := checkpoint.SaveCheckpointMetadataForRestore(ctx, rc.storage, &checkpoint.CheckpointMetadataForRestore{
+ GcRatio: gcRatio,
+ }, taskName); err != nil {
+ return gcRatio, errors.Trace(err)
+ }
+
+ return gcRatio, nil
+}
+
+func (rc *LogClient) InstallLogFileManager(ctx context.Context, startTS, restoreTS uint64, metadataDownloadBatchSize uint) error {
+ init := LogFileManagerInit{
+ StartTS: startTS,
+ RestoreTS: restoreTS,
+ Storage: rc.storage,
+
+ MetadataDownloadBatchSize: metadataDownloadBatchSize,
+ }
+ var err error
+ rc.LogFileManager, err = CreateLogFileManager(ctx, init)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+type FilesInRegion struct {
+ defaultSize uint64
+ defaultKVCount int64
+ writeSize uint64
+ writeKVCount int64
+
+ defaultFiles []*LogDataFileInfo
+ writeFiles []*LogDataFileInfo
+ deleteFiles []*LogDataFileInfo
+}
+
+type FilesInTable struct {
+ regionMapFiles map[int64]*FilesInRegion
+}
+
+func ApplyKVFilesWithBatchMethod(
+ ctx context.Context,
+ logIter LogIter,
+ batchCount int,
+ batchSize uint64,
+ applyFunc func(files []*LogDataFileInfo, kvCount int64, size uint64),
+ applyWg *sync.WaitGroup,
+) error {
+ var (
+ tableMapFiles = make(map[int64]*FilesInTable)
+ tmpFiles = make([]*LogDataFileInfo, 0, batchCount)
+ tmpSize uint64 = 0
+ tmpKVCount int64 = 0
+ )
+ for r := logIter.TryNext(ctx); !r.Finished; r = logIter.TryNext(ctx) {
+ if r.Err != nil {
+ return r.Err
+ }
+
+ f := r.Item
+ if f.GetType() == backuppb.FileType_Put && f.GetLength() >= batchSize {
+ applyFunc([]*LogDataFileInfo{f}, f.GetNumberOfEntries(), f.GetLength())
+ continue
+ }
+
+ fit, exist := tableMapFiles[f.TableId]
+ if !exist {
+ fit = &FilesInTable{
+ regionMapFiles: make(map[int64]*FilesInRegion),
+ }
+ tableMapFiles[f.TableId] = fit
+ }
+ fs, exist := fit.regionMapFiles[f.RegionId]
+ if !exist {
+ fs = &FilesInRegion{}
+ fit.regionMapFiles[f.RegionId] = fs
+ }
+
+ if f.GetType() == backuppb.FileType_Delete {
+ if fs.defaultFiles == nil {
+ fs.deleteFiles = make([]*LogDataFileInfo, 0)
+ }
+ fs.deleteFiles = append(fs.deleteFiles, f)
+ } else {
+ if f.GetCf() == stream.DefaultCF {
+ if fs.defaultFiles == nil {
+ fs.defaultFiles = make([]*LogDataFileInfo, 0, batchCount)
+ }
+ fs.defaultFiles = append(fs.defaultFiles, f)
+ fs.defaultSize += f.Length
+ fs.defaultKVCount += f.GetNumberOfEntries()
+ if len(fs.defaultFiles) >= batchCount || fs.defaultSize >= batchSize {
+ applyFunc(fs.defaultFiles, fs.defaultKVCount, fs.defaultSize)
+ fs.defaultFiles = nil
+ fs.defaultSize = 0
+ fs.defaultKVCount = 0
+ }
+ } else {
+ if fs.writeFiles == nil {
+ fs.writeFiles = make([]*LogDataFileInfo, 0, batchCount)
+ }
+ fs.writeFiles = append(fs.writeFiles, f)
+ fs.writeSize += f.GetLength()
+ fs.writeKVCount += f.GetNumberOfEntries()
+ if len(fs.writeFiles) >= batchCount || fs.writeSize >= batchSize {
+ applyFunc(fs.writeFiles, fs.writeKVCount, fs.writeSize)
+ fs.writeFiles = nil
+ fs.writeSize = 0
+ fs.writeKVCount = 0
+ }
+ }
+ }
+ }
+
+ for _, fwt := range tableMapFiles {
+ for _, fs := range fwt.regionMapFiles {
+ if len(fs.defaultFiles) > 0 {
+ applyFunc(fs.defaultFiles, fs.defaultKVCount, fs.defaultSize)
+ }
+ if len(fs.writeFiles) > 0 {
+ applyFunc(fs.writeFiles, fs.writeKVCount, fs.writeSize)
+ }
+ }
+ }
+
+ applyWg.Wait()
+ for _, fwt := range tableMapFiles {
+ for _, fs := range fwt.regionMapFiles {
+ for _, d := range fs.deleteFiles {
+ tmpFiles = append(tmpFiles, d)
+ tmpSize += d.GetLength()
+ tmpKVCount += d.GetNumberOfEntries()
+
+ if len(tmpFiles) >= batchCount || tmpSize >= batchSize {
+ applyFunc(tmpFiles, tmpKVCount, tmpSize)
+ tmpFiles = make([]*LogDataFileInfo, 0, batchCount)
+ tmpSize = 0
+ tmpKVCount = 0
+ }
+ }
+ if len(tmpFiles) > 0 {
+ applyFunc(tmpFiles, tmpKVCount, tmpSize)
+ tmpFiles = make([]*LogDataFileInfo, 0, batchCount)
+ tmpSize = 0
+ tmpKVCount = 0
+ }
+ }
+ }
+
+ return nil
+}
+
+func ApplyKVFilesWithSingelMethod(
+ ctx context.Context,
+ files LogIter,
+ applyFunc func(file []*LogDataFileInfo, kvCount int64, size uint64),
+ applyWg *sync.WaitGroup,
+) error {
+ deleteKVFiles := make([]*LogDataFileInfo, 0)
+
+ for r := files.TryNext(ctx); !r.Finished; r = files.TryNext(ctx) {
+ if r.Err != nil {
+ return r.Err
+ }
+
+ f := r.Item
+ if f.GetType() == backuppb.FileType_Delete {
+ deleteKVFiles = append(deleteKVFiles, f)
+ continue
+ }
+ applyFunc([]*LogDataFileInfo{f}, f.GetNumberOfEntries(), f.GetLength())
+ }
+
+ applyWg.Wait()
+ log.Info("restore delete files", zap.Int("count", len(deleteKVFiles)))
+ for _, file := range deleteKVFiles {
+ f := file
+ applyFunc([]*LogDataFileInfo{f}, f.GetNumberOfEntries(), f.GetLength())
+ }
+
+ return nil
+}
+
+func (rc *LogClient) RestoreKVFiles(
+ ctx context.Context,
+ rules map[int64]*restoreutils.RewriteRules,
+ idrules map[int64]int64,
+ logIter LogIter,
+ runner *checkpoint.CheckpointRunner[checkpoint.LogRestoreKeyType, checkpoint.LogRestoreValueType],
+ pitrBatchCount uint32,
+ pitrBatchSize uint32,
+ updateStats func(kvCount uint64, size uint64),
+ onProgress func(cnt int64),
+) error {
+ var (
+ err error
+ fileCount = 0
+ start = time.Now()
+ supportBatch = version.CheckPITRSupportBatchKVFiles()
+ skipFile = 0
+ )
+ defer func() {
+ if err == nil {
+ elapsed := time.Since(start)
+ log.Info("Restore KV files", zap.Duration("take", elapsed))
+ summary.CollectSuccessUnit("files", fileCount, elapsed)
+ }
+ }()
+
+ if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
+ span1 := span.Tracer().StartSpan("Client.RestoreKVFiles", opentracing.ChildOf(span.Context()))
+ defer span1.Finish()
+ ctx = opentracing.ContextWithSpan(ctx, span1)
+ }
+
+ var applyWg sync.WaitGroup
+ eg, ectx := errgroup.WithContext(ctx)
+ applyFunc := func(files []*LogDataFileInfo, kvCount int64, size uint64) {
+ if len(files) == 0 {
+ return
+ }
+ // get rewrite rule from table id.
+ // because the tableID of files is the same.
+ rule, ok := rules[files[0].TableId]
+ if !ok {
+ // TODO handle new created table
+ // For this version we do not handle new created table after full backup.
+ // in next version we will perform rewrite and restore meta key to restore new created tables.
+ // so we can simply skip the file that doesn't have the rule here.
+ onProgress(int64(len(files)))
+ summary.CollectInt("FileSkip", len(files))
+ log.Debug("skip file due to table id not matched", zap.Int64("table-id", files[0].TableId))
+ skipFile += len(files)
+ } else {
+ applyWg.Add(1)
+ downstreamId := idrules[files[0].TableId]
+ rc.workerPool.ApplyOnErrorGroup(eg, func() (err error) {
+ fileStart := time.Now()
+ defer applyWg.Done()
+ defer func() {
+ onProgress(int64(len(files)))
+ updateStats(uint64(kvCount), size)
+ summary.CollectInt("File", len(files))
+
+ if err == nil {
+ filenames := make([]string, 0, len(files))
+ if runner == nil {
+ for _, f := range files {
+ filenames = append(filenames, f.Path+", ")
+ }
+ } else {
+ for _, f := range files {
+ filenames = append(filenames, f.Path+", ")
+ if e := checkpoint.AppendRangeForLogRestore(ectx, runner, f.MetaDataGroupName, downstreamId, f.OffsetInMetaGroup, f.OffsetInMergedGroup); e != nil {
+ err = errors.Annotate(e, "failed to append checkpoint data")
+ break
+ }
+ }
+ }
+ log.Info("import files done", zap.Int("batch-count", len(files)), zap.Uint64("batch-size", size),
+ zap.Duration("take", time.Since(fileStart)), zap.Strings("files", filenames))
+ }
+ }()
+
+ return rc.fileImporter.ImportKVFiles(ectx, files, rule, rc.shiftStartTS, rc.startTS, rc.restoreTS, supportBatch)
+ })
+ }
+ }
+
+ rc.workerPool.ApplyOnErrorGroup(eg, func() error {
+ if supportBatch {
+ err = ApplyKVFilesWithBatchMethod(ectx, logIter, int(pitrBatchCount), uint64(pitrBatchSize), applyFunc, &applyWg)
+ } else {
+ err = ApplyKVFilesWithSingelMethod(ectx, logIter, applyFunc, &applyWg)
+ }
+ return errors.Trace(err)
+ })
+
+ if err = eg.Wait(); err != nil {
+ summary.CollectFailureUnit("file", err)
+ log.Error("restore files failed", zap.Error(err))
+ }
+
+ log.Info("total skip files due to table id not matched", zap.Int("count", skipFile))
+ if skipFile > 0 {
+ log.Debug("table id in full backup storage", zap.Any("tables", rules))
+ }
+
+ return errors.Trace(err)
+}
+
+func (rc *LogClient) initSchemasMap(
+ ctx context.Context,
+ clusterID uint64,
+ restoreTS uint64,
+) ([]*backuppb.PitrDBMap, error) {
+ filename := metautil.PitrIDMapsFilename(clusterID, restoreTS)
+ exist, err := rc.storage.FileExists(ctx, filename)
+ if err != nil {
+ return nil, errors.Annotatef(err, "failed to check filename:%s ", filename)
+ } else if !exist {
+ log.Info("pitr id maps isn't existed", zap.String("file", filename))
+ return nil, nil
+ }
+
+ metaData, err := rc.storage.ReadFile(ctx, filename)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ backupMeta := &backuppb.BackupMeta{}
+ if err = backupMeta.Unmarshal(metaData); err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ return backupMeta.GetDbMaps(), nil
+}
+
+func initFullBackupTables(
+ ctx context.Context,
+ s storage.ExternalStorage,
+ tableFilter filter.Filter,
+) (map[int64]*metautil.Table, error) {
+ metaData, err := s.ReadFile(ctx, metautil.MetaFile)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ backupMeta := &backuppb.BackupMeta{}
+ if err = backupMeta.Unmarshal(metaData); err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ // read full backup databases to get map[table]table.Info
+ reader := metautil.NewMetaReader(backupMeta, s, nil)
+
+ databases, err := metautil.LoadBackupTables(ctx, reader, false)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+
+ tables := make(map[int64]*metautil.Table)
+ for _, db := range databases {
+ dbName := db.Info.Name.O
+ if name, ok := utils.GetSysDBName(db.Info.Name); utils.IsSysDB(name) && ok {
+ dbName = name
+ }
+
+ if !tableFilter.MatchSchema(dbName) {
+ continue
+ }
+
+ for _, table := range db.Tables {
+ // check this db is empty.
+ if table.Info == nil {
+ tables[db.Info.ID] = table
+ continue
+ }
+ if !tableFilter.MatchTable(dbName, table.Info.Name.O) {
+ continue
+ }
+ tables[table.Info.ID] = table
+ }
+ }
+
+ return tables, nil
+}
+
+type FullBackupStorageConfig struct {
+ Backend *backuppb.StorageBackend
+ Opts *storage.ExternalStorageOptions
+}
+
+type InitSchemaConfig struct {
+ // required
+ IsNewTask bool
+ HasFullRestore bool
+ TableFilter filter.Filter
+
+ // optional
+ TiFlashRecorder *tiflashrec.TiFlashRecorder
+ FullBackupStorage *FullBackupStorageConfig
+}
+
+// InitSchemasReplaceForDDL gets schemas information Mapping from old schemas to new schemas.
+// It is used to rewrite meta kv-event.
+func (rc *LogClient) InitSchemasReplaceForDDL(
+ ctx context.Context,
+ cfg *InitSchemaConfig,
+) (*stream.SchemasReplace, error) {
+ var (
+ err error
+ dbMaps []*backuppb.PitrDBMap
+ // the id map doesn't need to construct only when it is not the first execution
+ needConstructIdMap bool
+
+ dbReplaces = make(map[stream.UpstreamID]*stream.DBReplace)
+ )
+
+ // not new task, load schemas map from external storage
+ if !cfg.IsNewTask {
+ log.Info("try to load pitr id maps")
+ needConstructIdMap = false
+ dbMaps, err = rc.initSchemasMap(ctx, rc.GetClusterID(ctx), rc.restoreTS)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ }
+
+ // a new task, but without full snapshot restore, tries to load
+ // schemas map whose `restore-ts`` is the task's `start-ts`.
+ if len(dbMaps) <= 0 && !cfg.HasFullRestore {
+ log.Info("try to load pitr id maps of the previous task", zap.Uint64("start-ts", rc.startTS))
+ needConstructIdMap = true
+ dbMaps, err = rc.initSchemasMap(ctx, rc.GetClusterID(ctx), rc.startTS)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ }
+
+ if len(dbMaps) <= 0 {
+ log.Info("no id maps, build the table replaces from cluster and full backup schemas")
+ needConstructIdMap = true
+ s, err := storage.New(ctx, cfg.FullBackupStorage.Backend, cfg.FullBackupStorage.Opts)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ fullBackupTables, err := initFullBackupTables(ctx, s, cfg.TableFilter)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ for _, t := range fullBackupTables {
+ dbName, _ := utils.GetSysDBCIStrName(t.DB.Name)
+ newDBInfo, exist := rc.dom.InfoSchema().SchemaByName(dbName)
+ if !exist {
+ log.Info("db not existed", zap.String("dbname", dbName.String()))
+ continue
+ }
+
+ dbReplace, exist := dbReplaces[t.DB.ID]
+ if !exist {
+ dbReplace = stream.NewDBReplace(t.DB.Name.O, newDBInfo.ID)
+ dbReplaces[t.DB.ID] = dbReplace
+ }
+
+ if t.Info == nil {
+ // If the db is empty, skip it.
+ continue
+ }
+ newTableInfo, err := restore.GetTableSchema(rc.GetDomain(), dbName, t.Info.Name)
+ if err != nil {
+ log.Info("table not existed", zap.String("tablename", dbName.String()+"."+t.Info.Name.String()))
+ continue
+ }
+
+ dbReplace.TableMap[t.Info.ID] = &stream.TableReplace{
+ Name: newTableInfo.Name.O,
+ TableID: newTableInfo.ID,
+ PartitionMap: restoreutils.GetPartitionIDMap(newTableInfo, t.Info),
+ IndexMap: restoreutils.GetIndexIDMap(newTableInfo, t.Info),
+ }
+ }
+ } else {
+ dbReplaces = stream.FromSchemaMaps(dbMaps)
+ }
+
+ for oldDBID, dbReplace := range dbReplaces {
+ log.Info("replace info", func() []zapcore.Field {
+ fields := make([]zapcore.Field, 0, (len(dbReplace.TableMap)+1)*3)
+ fields = append(fields,
+ zap.String("dbName", dbReplace.Name),
+ zap.Int64("oldID", oldDBID),
+ zap.Int64("newID", dbReplace.DbID))
+ for oldTableID, tableReplace := range dbReplace.TableMap {
+ fields = append(fields,
+ zap.String("table", tableReplace.Name),
+ zap.Int64("oldID", oldTableID),
+ zap.Int64("newID", tableReplace.TableID))
+ }
+ return fields
+ }()...)
+ }
+
+ rp := stream.NewSchemasReplace(
+ dbReplaces, needConstructIdMap, cfg.TiFlashRecorder, rc.currentTS, cfg.TableFilter, rc.GenGlobalID, rc.GenGlobalIDs,
+ rc.RecordDeleteRange)
+ return rp, nil
+}
+
+func SortMetaKVFiles(files []*backuppb.DataFileInfo) []*backuppb.DataFileInfo {
+ slices.SortFunc(files, func(i, j *backuppb.DataFileInfo) int {
+ if c := cmp.Compare(i.GetMinTs(), j.GetMinTs()); c != 0 {
+ return c
+ }
+ if c := cmp.Compare(i.GetMaxTs(), j.GetMaxTs()); c != 0 {
+ return c
+ }
+ return cmp.Compare(i.GetResolvedTs(), j.GetResolvedTs())
+ })
+ return files
+}
+
+// RestoreMetaKVFiles tries to restore files about meta kv-event from stream-backup.
+func (rc *LogClient) RestoreMetaKVFiles(
+ ctx context.Context,
+ files []*backuppb.DataFileInfo,
+ schemasReplace *stream.SchemasReplace,
+ updateStats func(kvCount uint64, size uint64),
+ progressInc func(),
+) error {
+ filesInWriteCF := make([]*backuppb.DataFileInfo, 0, len(files))
+ filesInDefaultCF := make([]*backuppb.DataFileInfo, 0, len(files))
+
+ // The k-v events in default CF should be restored firstly. The reason is that:
+ // The error of transactions of meta could happen if restore write CF events successfully,
+ // but failed to restore default CF events.
+ for _, f := range files {
+ if f.Cf == stream.WriteCF {
+ filesInWriteCF = append(filesInWriteCF, f)
+ continue
+ }
+ if f.Type == backuppb.FileType_Delete {
+ // this should happen abnormally.
+ // only do some preventive checks here.
+ log.Warn("detected delete file of meta key, skip it", zap.Any("file", f))
+ continue
+ }
+ if f.Cf == stream.DefaultCF {
+ filesInDefaultCF = append(filesInDefaultCF, f)
+ }
+ }
+ filesInDefaultCF = SortMetaKVFiles(filesInDefaultCF)
+ filesInWriteCF = SortMetaKVFiles(filesInWriteCF)
+
+ failpoint.Inject("failed-before-id-maps-saved", func(_ failpoint.Value) {
+ failpoint.Return(errors.New("failpoint: failed before id maps saved"))
+ })
+
+ log.Info("start to restore meta files",
+ zap.Int("total files", len(files)),
+ zap.Int("default files", len(filesInDefaultCF)),
+ zap.Int("write files", len(filesInWriteCF)))
+
+ if schemasReplace.NeedConstructIdMap() {
+ // Preconstruct the map and save it into external storage.
+ if err := rc.PreConstructAndSaveIDMap(
+ ctx,
+ filesInWriteCF,
+ filesInDefaultCF,
+ schemasReplace,
+ ); err != nil {
+ return errors.Trace(err)
+ }
+ }
+ failpoint.Inject("failed-after-id-maps-saved", func(_ failpoint.Value) {
+ failpoint.Return(errors.New("failpoint: failed after id maps saved"))
+ })
+
+ // run the rewrite and restore meta-kv into TiKV cluster.
+ if err := RestoreMetaKVFilesWithBatchMethod(
+ ctx,
+ filesInDefaultCF,
+ filesInWriteCF,
+ schemasReplace,
+ updateStats,
+ progressInc,
+ rc.RestoreBatchMetaKVFiles,
+ ); err != nil {
+ return errors.Trace(err)
+ }
+
+ // Update global schema version and report all of TiDBs.
+ if err := rc.UpdateSchemaVersion(ctx); err != nil {
+ return errors.Trace(err)
+ }
+ return nil
+}
+
+// PreConstructAndSaveIDMap constructs id mapping and save it.
+func (rc *LogClient) PreConstructAndSaveIDMap(
+ ctx context.Context,
+ fsInWriteCF, fsInDefaultCF []*backuppb.DataFileInfo,
+ sr *stream.SchemasReplace,
+) error {
+ sr.SetPreConstructMapStatus()
+
+ if err := rc.constructIDMap(ctx, fsInWriteCF, sr); err != nil {
+ return errors.Trace(err)
+ }
+ if err := rc.constructIDMap(ctx, fsInDefaultCF, sr); err != nil {
+ return errors.Trace(err)
+ }
+
+ if err := rc.SaveIDMap(ctx, sr); err != nil {
+ return errors.Trace(err)
+ }
+ return nil
+}
+
+func (rc *LogClient) constructIDMap(
+ ctx context.Context,
+ fs []*backuppb.DataFileInfo,
+ sr *stream.SchemasReplace,
+) error {
+ for _, f := range fs {
+ entries, _, err := rc.ReadAllEntries(ctx, f, math.MaxUint64)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ for _, entry := range entries {
+ if _, err := sr.RewriteKvEntry(&entry.E, f.GetCf()); err != nil {
+ return errors.Trace(err)
+ }
+ }
+ }
+ return nil
+}
+
+func RestoreMetaKVFilesWithBatchMethod(
+ ctx context.Context,
+ defaultFiles []*backuppb.DataFileInfo,
+ writeFiles []*backuppb.DataFileInfo,
+ schemasReplace *stream.SchemasReplace,
+ updateStats func(kvCount uint64, size uint64),
+ progressInc func(),
+ restoreBatch func(
+ ctx context.Context,
+ files []*backuppb.DataFileInfo,
+ schemasReplace *stream.SchemasReplace,
+ kvEntries []*KvEntryWithTS,
+ filterTS uint64,
+ updateStats func(kvCount uint64, size uint64),
+ progressInc func(),
+ cf string,
+ ) ([]*KvEntryWithTS, error),
+) error {
+ // the average size of each KV is 2560 Bytes
+ // kvEntries is kvs left by the previous batch
+ const kvSize = 2560
+ var (
+ rangeMin uint64
+ rangeMax uint64
+ err error
+
+ batchSize uint64 = 0
+ defaultIdx int = 0
+ writeIdx int = 0
+
+ defaultKvEntries = make([]*KvEntryWithTS, 0)
+ writeKvEntries = make([]*KvEntryWithTS, 0)
+ )
+ // Set restoreKV to SchemaReplace.
+ schemasReplace.SetRestoreKVStatus()
+
+ for i, f := range defaultFiles {
+ if i == 0 {
+ rangeMax = f.MaxTs
+ rangeMin = f.MinTs
+ batchSize = f.Length
+ } else {
+ if f.MinTs <= rangeMax && batchSize+f.Length <= MetaKVBatchSize {
+ rangeMin = min(rangeMin, f.MinTs)
+ rangeMax = max(rangeMax, f.MaxTs)
+ batchSize += f.Length
+ } else {
+ // Either f.MinTS > rangeMax or f.MinTs is the filterTs we need.
+ // So it is ok to pass f.MinTs as filterTs.
+ defaultKvEntries, err = restoreBatch(ctx, defaultFiles[defaultIdx:i], schemasReplace, defaultKvEntries, f.MinTs, updateStats, progressInc, stream.DefaultCF)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ defaultIdx = i
+ rangeMin = f.MinTs
+ rangeMax = f.MaxTs
+ // the initial batch size is the size of left kvs and the current file length.
+ batchSize = uint64(len(defaultKvEntries)*kvSize) + f.Length
+
+ // restore writeCF kv to f.MinTs
+ var toWriteIdx int
+ for toWriteIdx = writeIdx; toWriteIdx < len(writeFiles); toWriteIdx++ {
+ if writeFiles[toWriteIdx].MinTs >= f.MinTs {
+ break
+ }
+ }
+ writeKvEntries, err = restoreBatch(ctx, writeFiles[writeIdx:toWriteIdx], schemasReplace, writeKvEntries, f.MinTs, updateStats, progressInc, stream.WriteCF)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ writeIdx = toWriteIdx
+ }
+ }
+ }
+
+ // restore the left meta kv files and entries
+ // Notice: restoreBatch needs to realize the parameter `files` and `kvEntries` might be empty
+ // Assert: defaultIdx <= len(defaultFiles) && writeIdx <= len(writeFiles)
+ _, err = restoreBatch(ctx, defaultFiles[defaultIdx:], schemasReplace, defaultKvEntries, math.MaxUint64, updateStats, progressInc, stream.DefaultCF)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ _, err = restoreBatch(ctx, writeFiles[writeIdx:], schemasReplace, writeKvEntries, math.MaxUint64, updateStats, progressInc, stream.WriteCF)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ return nil
+}
+
+func (rc *LogClient) RestoreBatchMetaKVFiles(
+ ctx context.Context,
+ files []*backuppb.DataFileInfo,
+ schemasReplace *stream.SchemasReplace,
+ kvEntries []*KvEntryWithTS,
+ filterTS uint64,
+ updateStats func(kvCount uint64, size uint64),
+ progressInc func(),
+ cf string,
+) ([]*KvEntryWithTS, error) {
+ nextKvEntries := make([]*KvEntryWithTS, 0)
+ curKvEntries := make([]*KvEntryWithTS, 0)
+ if len(files) == 0 && len(kvEntries) == 0 {
+ return nextKvEntries, nil
+ }
+
+ // filter the kv from kvEntries again.
+ for _, kv := range kvEntries {
+ if kv.Ts < filterTS {
+ curKvEntries = append(curKvEntries, kv)
+ } else {
+ nextKvEntries = append(nextKvEntries, kv)
+ }
+ }
+
+ // read all of entries from files.
+ for _, f := range files {
+ es, nextEs, err := rc.ReadAllEntries(ctx, f, filterTS)
+ if err != nil {
+ return nextKvEntries, errors.Trace(err)
+ }
+
+ curKvEntries = append(curKvEntries, es...)
+ nextKvEntries = append(nextKvEntries, nextEs...)
+ }
+
+ // sort these entries.
+ slices.SortFunc(curKvEntries, func(i, j *KvEntryWithTS) int {
+ return cmp.Compare(i.Ts, j.Ts)
+ })
+
+ // restore these entries with rawPut() method.
+ kvCount, size, err := rc.restoreMetaKvEntries(ctx, schemasReplace, curKvEntries, cf)
+ if err != nil {
+ return nextKvEntries, errors.Trace(err)
+ }
+
+ if schemasReplace.IsRestoreKVStatus() {
+ updateStats(kvCount, size)
+ for i := 0; i < len(files); i++ {
+ progressInc()
+ }
+ }
+ return nextKvEntries, nil
+}
+
+func (rc *LogClient) restoreMetaKvEntries(
+ ctx context.Context,
+ sr *stream.SchemasReplace,
+ entries []*KvEntryWithTS,
+ columnFamily string,
+) (uint64, uint64, error) {
+ var (
+ kvCount uint64
+ size uint64
+ )
+
+ rc.rawKVClient.SetColumnFamily(columnFamily)
+
+ for _, entry := range entries {
+ log.Debug("before rewrte entry", zap.Uint64("key-ts", entry.Ts), zap.Int("key-len", len(entry.E.Key)),
+ zap.Int("value-len", len(entry.E.Value)), zap.ByteString("key", entry.E.Key))
+
+ newEntry, err := sr.RewriteKvEntry(&entry.E, columnFamily)
+ if err != nil {
+ log.Error("rewrite txn entry failed", zap.Int("klen", len(entry.E.Key)),
+ logutil.Key("txn-key", entry.E.Key))
+ return 0, 0, errors.Trace(err)
+ } else if newEntry == nil {
+ continue
+ }
+ log.Debug("after rewrite entry", zap.Int("new-key-len", len(newEntry.Key)),
+ zap.Int("new-value-len", len(entry.E.Value)), zap.ByteString("new-key", newEntry.Key))
+
+ failpoint.Inject("failed-to-restore-metakv", func(_ failpoint.Value) {
+ failpoint.Return(0, 0, errors.Errorf("failpoint: failed to restore metakv"))
+ })
+ if err := rc.rawKVClient.Put(ctx, newEntry.Key, newEntry.Value, entry.Ts); err != nil {
+ return 0, 0, errors.Trace(err)
+ }
+ // for failpoint, we need to flush the cache in rawKVClient every time
+ failpoint.Inject("do-not-put-metakv-in-batch", func(_ failpoint.Value) {
+ if err := rc.rawKVClient.PutRest(ctx); err != nil {
+ failpoint.Return(0, 0, errors.Trace(err))
+ }
+ })
+ kvCount++
+ size += uint64(len(newEntry.Key) + len(newEntry.Value))
+ }
+
+ return kvCount, size, rc.rawKVClient.PutRest(ctx)
+}
+
+// GenGlobalID generates a global id by transaction way.
+func (rc *LogClient) GenGlobalID(ctx context.Context) (int64, error) {
+ var id int64
+ storage := rc.GetDomain().Store()
+
+ ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnBR)
+ err := kv.RunInNewTxn(
+ ctx,
+ storage,
+ true,
+ func(ctx context.Context, txn kv.Transaction) error {
+ var e error
+ t := meta.NewMeta(txn)
+ id, e = t.GenGlobalID()
+ return e
+ })
+
+ return id, err
+}
+
+// GenGlobalIDs generates several global ids by transaction way.
+func (rc *LogClient) GenGlobalIDs(ctx context.Context, n int) ([]int64, error) {
+ ids := make([]int64, 0)
+ storage := rc.GetDomain().Store()
+
+ ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnBR)
+ err := kv.RunInNewTxn(
+ ctx,
+ storage,
+ true,
+ func(ctx context.Context, txn kv.Transaction) error {
+ var e error
+ t := meta.NewMeta(txn)
+ ids, e = t.GenGlobalIDs(n)
+ return e
+ })
+
+ return ids, err
+}
+
+// UpdateSchemaVersion updates schema version by transaction way.
+func (rc *LogClient) UpdateSchemaVersion(ctx context.Context) error {
+ storage := rc.GetDomain().Store()
+ var schemaVersion int64
+
+ ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnBR)
+ if err := kv.RunInNewTxn(
+ ctx,
+ storage,
+ true,
+ func(ctx context.Context, txn kv.Transaction) error {
+ t := meta.NewMeta(txn)
+ var e error
+ // To trigger full-reload instead of diff-reload, we need to increase the schema version
+ // by at least `domain.LoadSchemaDiffVersionGapThreshold`.
+ schemaVersion, e = t.GenSchemaVersions(64 + domain.LoadSchemaDiffVersionGapThreshold)
+ if e != nil {
+ return e
+ }
+ // add the diff key so that the domain won't retry to reload the schemas with `schemaVersion` frequently.
+ return t.SetSchemaDiff(&model.SchemaDiff{
+ Version: schemaVersion,
+ Type: model.ActionNone,
+ SchemaID: -1,
+ TableID: -1,
+ RegenerateSchemaMap: true,
+ })
+ },
+ ); err != nil {
+ return errors.Trace(err)
+ }
+
+ log.Info("update global schema version", zap.Int64("global-schema-version", schemaVersion))
+
+ ver := strconv.FormatInt(schemaVersion, 10)
+ if err := ddlutil.PutKVToEtcd(
+ ctx,
+ rc.GetDomain().GetEtcdClient(),
+ math.MaxInt,
+ ddlutil.DDLGlobalSchemaVersion,
+ ver,
+ ); err != nil {
+ return errors.Annotatef(err, "failed to put global schema verson %v to etcd", ver)
+ }
+
+ return nil
+}
+
+func (rc *LogClient) WrapLogFilesIterWithSplitHelper(logIter LogIter, rules map[int64]*restoreutils.RewriteRules, g glue.Glue, store kv.Storage) (LogIter, error) {
+ se, err := g.CreateSession(store)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ execCtx := se.GetSessionCtx().GetRestrictedSQLExecutor()
+ splitSize, splitKeys := utils.GetRegionSplitInfo(execCtx)
+ log.Info("get split threshold from tikv config", zap.Uint64("split-size", splitSize), zap.Int64("split-keys", splitKeys))
+ client := split.NewClient(rc.pdClient, rc.pdHTTPClient, rc.tlsConf, maxSplitKeysOnce, 3)
+ return NewLogFilesIterWithSplitHelper(logIter, rules, client, splitSize, splitKeys), nil
+}
+
+func (rc *LogClient) generateKvFilesSkipMap(ctx context.Context, downstreamIdset map[int64]struct{}, taskName string) (*LogFilesSkipMap, error) {
+ skipMap := NewLogFilesSkipMap()
+ t, err := checkpoint.WalkCheckpointFileForRestore(ctx, rc.storage, rc.cipher, taskName, func(groupKey checkpoint.LogRestoreKeyType, off checkpoint.LogRestoreValueMarshaled) {
+ for tableID, foffs := range off.Foffs {
+ // filter out the checkpoint data of dropped table
+ if _, exists := downstreamIdset[tableID]; exists {
+ for _, foff := range foffs {
+ skipMap.Insert(groupKey, off.Goff, foff)
+ }
+ }
+ }
+ })
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ summary.AdjustStartTimeToEarlierTime(t)
+ return skipMap, nil
+}
+
+func (rc *LogClient) WrapLogFilesIterWithCheckpoint(
+ ctx context.Context,
+ logIter LogIter,
+ downstreamIdset map[int64]struct{},
+ taskName string,
+ updateStats func(kvCount, size uint64),
+ onProgress func(),
+) (LogIter, error) {
+ skipMap, err := rc.generateKvFilesSkipMap(ctx, downstreamIdset, taskName)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ return iter.FilterOut(logIter, func(d *LogDataFileInfo) bool {
+ if skipMap.NeedSkip(d.MetaDataGroupName, d.OffsetInMetaGroup, d.OffsetInMergedGroup) {
+ onProgress()
+ updateStats(uint64(d.NumberOfEntries), d.Length)
+ return true
+ }
+ return false
+ }), nil
+}
+
+const (
+ alterTableDropIndexSQL = "ALTER TABLE %n.%n DROP INDEX %n"
+ alterTableAddIndexFormat = "ALTER TABLE %%n.%%n ADD INDEX %%n(%s)"
+ alterTableAddUniqueIndexFormat = "ALTER TABLE %%n.%%n ADD UNIQUE KEY %%n(%s)"
+ alterTableAddPrimaryFormat = "ALTER TABLE %%n.%%n ADD PRIMARY KEY (%s) NONCLUSTERED"
+)
+
+func (rc *LogClient) generateRepairIngestIndexSQLs(
+ ctx context.Context,
+ ingestRecorder *ingestrec.IngestRecorder,
+ allSchema []*model.DBInfo,
+ taskName string,
+) ([]checkpoint.CheckpointIngestIndexRepairSQL, bool, error) {
+ var sqls []checkpoint.CheckpointIngestIndexRepairSQL
+ if rc.useCheckpoint {
+ exists, err := checkpoint.ExistsCheckpointIngestIndexRepairSQLs(ctx, rc.storage, taskName)
+ if err != nil {
+ return sqls, false, errors.Trace(err)
+ }
+ if exists {
+ checkpointSQLs, err := checkpoint.LoadCheckpointIngestIndexRepairSQLs(ctx, rc.storage, taskName)
+ if err != nil {
+ return sqls, false, errors.Trace(err)
+ }
+ sqls = checkpointSQLs.SQLs
+ log.Info("load ingest index repair sqls from checkpoint", zap.String("category", "ingest"), zap.Reflect("sqls", sqls))
+ return sqls, true, nil
+ }
+ }
+
+ ingestRecorder.UpdateIndexInfo(allSchema)
+ if err := ingestRecorder.Iterate(func(_, indexID int64, info *ingestrec.IngestIndexInfo) error {
+ var (
+ addSQL strings.Builder
+ addArgs []any = make([]any, 0, 5+len(info.ColumnArgs))
+ )
+ if info.IsPrimary {
+ addSQL.WriteString(fmt.Sprintf(alterTableAddPrimaryFormat, info.ColumnList))
+ addArgs = append(addArgs, info.SchemaName.O, info.TableName.O)
+ addArgs = append(addArgs, info.ColumnArgs...)
+ } else if info.IndexInfo.Unique {
+ addSQL.WriteString(fmt.Sprintf(alterTableAddUniqueIndexFormat, info.ColumnList))
+ addArgs = append(addArgs, info.SchemaName.O, info.TableName.O, info.IndexInfo.Name.O)
+ addArgs = append(addArgs, info.ColumnArgs...)
+ } else {
+ addSQL.WriteString(fmt.Sprintf(alterTableAddIndexFormat, info.ColumnList))
+ addArgs = append(addArgs, info.SchemaName.O, info.TableName.O, info.IndexInfo.Name.O)
+ addArgs = append(addArgs, info.ColumnArgs...)
+ }
+ // USING BTREE/HASH/RTREE
+ indexTypeStr := info.IndexInfo.Tp.String()
+ if len(indexTypeStr) > 0 {
+ addSQL.WriteString(" USING ")
+ addSQL.WriteString(indexTypeStr)
+ }
+
+ // COMMENT [...]
+ if len(info.IndexInfo.Comment) > 0 {
+ addSQL.WriteString(" COMMENT %?")
+ addArgs = append(addArgs, info.IndexInfo.Comment)
+ }
+
+ if info.IndexInfo.Invisible {
+ addSQL.WriteString(" INVISIBLE")
+ } else {
+ addSQL.WriteString(" VISIBLE")
+ }
+
+ sqls = append(sqls, checkpoint.CheckpointIngestIndexRepairSQL{
+ IndexID: indexID,
+ SchemaName: info.SchemaName,
+ TableName: info.TableName,
+ IndexName: info.IndexInfo.Name.O,
+ AddSQL: addSQL.String(),
+ AddArgs: addArgs,
+ })
+
+ return nil
+ }); err != nil {
+ return sqls, false, errors.Trace(err)
+ }
+
+ if rc.useCheckpoint && len(sqls) > 0 {
+ if err := checkpoint.SaveCheckpointIngestIndexRepairSQLs(ctx, rc.storage, &checkpoint.CheckpointIngestIndexRepairSQLs{
+ SQLs: sqls,
+ }, taskName); err != nil {
+ return sqls, false, errors.Trace(err)
+ }
+ }
+ return sqls, false, nil
+}
+
+// RepairIngestIndex drops the indexes from IngestRecorder and re-add them.
+func (rc *LogClient) RepairIngestIndex(ctx context.Context, ingestRecorder *ingestrec.IngestRecorder, g glue.Glue, taskName string) error {
+ info := rc.dom.InfoSchema()
+
+ sqls, fromCheckpoint, err := rc.generateRepairIngestIndexSQLs(ctx, ingestRecorder, info.AllSchemas(), taskName)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ console := glue.GetConsole(g)
+NEXTSQL:
+ for _, sql := range sqls {
+ progressTitle := fmt.Sprintf("repair ingest index %s for table %s.%s", sql.IndexName, sql.SchemaName, sql.TableName)
+
+ tableInfo, err := info.TableByName(sql.SchemaName, sql.TableName)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ oldIndexIDFound := false
+ if fromCheckpoint {
+ for _, idx := range tableInfo.Indices() {
+ indexInfo := idx.Meta()
+ if indexInfo.ID == sql.IndexID {
+ // the original index id is not dropped
+ oldIndexIDFound = true
+ break
+ }
+ // what if index's state is not public?
+ if indexInfo.Name.O == sql.IndexName {
+ // find the same name index, but not the same index id,
+ // which means the repaired index id is created
+ if _, err := fmt.Fprintf(console.Out(), "%s ... %s\n", progressTitle, color.HiGreenString("SKIPPED DUE TO CHECKPOINT MODE")); err != nil {
+ return errors.Trace(err)
+ }
+ continue NEXTSQL
+ }
+ }
+ }
+
+ if err := func(sql checkpoint.CheckpointIngestIndexRepairSQL) error {
+ w := console.StartProgressBar(progressTitle, glue.OnlyOneTask)
+ defer w.Close()
+
+ // TODO: When the TiDB supports the DROP and CREATE the same name index in one SQL,
+ // the checkpoint for ingest recorder can be removed and directly use the SQL:
+ // ALTER TABLE db.tbl DROP INDEX `i_1`, ADD IDNEX `i_1` ...
+ //
+ // This SQL is compatible with checkpoint: If one ingest index has been recreated by
+ // the SQL, the index's id would be another one. In the next retry execution, BR can
+ // not find the ingest index's dropped id so that BR regards it as a dropped index by
+ // restored metakv and then skips repairing it.
+
+ // only when first execution or old index id is not dropped
+ if !fromCheckpoint || oldIndexIDFound {
+ if err := rc.se.ExecuteInternal(ctx, alterTableDropIndexSQL, sql.SchemaName.O, sql.TableName.O, sql.IndexName); err != nil {
+ return errors.Trace(err)
+ }
+ }
+ failpoint.Inject("failed-before-create-ingest-index", func(v failpoint.Value) {
+ if v != nil && v.(bool) {
+ failpoint.Return(errors.New("failed before create ingest index"))
+ }
+ })
+ // create the repaired index when first execution or not found it
+ if err := rc.se.ExecuteInternal(ctx, sql.AddSQL, sql.AddArgs...); err != nil {
+ return errors.Trace(err)
+ }
+ w.Inc()
+ if err := w.Wait(ctx); err != nil {
+ return errors.Trace(err)
+ }
+ return nil
+ }(sql); err != nil {
+ return errors.Trace(err)
+ }
+ }
+
+ return nil
+}
+
+func (rc *LogClient) RecordDeleteRange(sql *stream.PreDelRangeQuery) {
+ rc.deleteRangeQueryCh <- sql
+}
+
+// use channel to save the delete-range query to make it thread-safety.
+func (rc *LogClient) RunGCRowsLoader(ctx context.Context) {
+ rc.deleteRangeQueryWaitGroup.Add(1)
+
+ go func() {
+ defer rc.deleteRangeQueryWaitGroup.Done()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case query, ok := <-rc.deleteRangeQueryCh:
+ if !ok {
+ return
+ }
+ rc.deleteRangeQuery = append(rc.deleteRangeQuery, query)
+ }
+ }
+ }()
+}
+
+// InsertGCRows insert the querys into table `gc_delete_range`
+func (rc *LogClient) InsertGCRows(ctx context.Context) error {
+ close(rc.deleteRangeQueryCh)
+ rc.deleteRangeQueryWaitGroup.Wait()
+ ts, err := restore.GetTSWithRetry(ctx, rc.pdClient)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ jobIDMap := make(map[int64]int64)
+ for _, query := range rc.deleteRangeQuery {
+ paramsList := make([]any, 0, len(query.ParamsList)*5)
+ for _, params := range query.ParamsList {
+ newJobID, exists := jobIDMap[params.JobID]
+ if !exists {
+ newJobID, err = rc.GenGlobalID(ctx)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ jobIDMap[params.JobID] = newJobID
+ }
+ log.Info("insert into the delete range",
+ zap.Int64("jobID", newJobID),
+ zap.Int64("elemID", params.ElemID),
+ zap.String("startKey", params.StartKey),
+ zap.String("endKey", params.EndKey),
+ zap.Uint64("ts", ts))
+ // (job_id, elem_id, start_key, end_key, ts)
+ paramsList = append(paramsList, newJobID, params.ElemID, params.StartKey, params.EndKey, ts)
+ }
+ if len(paramsList) > 0 {
+ // trim the ',' behind the query.Sql if exists
+ // that's when the rewrite rule of the last table id is not exist
+ sql := strings.TrimSuffix(query.Sql, ",")
+ if err := rc.se.ExecuteInternal(ctx, sql, paramsList...); err != nil {
+ return errors.Trace(err)
+ }
+ }
+ }
+ return nil
+}
+
+// only for unit test
+func (rc *LogClient) GetGCRows() []*stream.PreDelRangeQuery {
+ close(rc.deleteRangeQueryCh)
+ rc.deleteRangeQueryWaitGroup.Wait()
+ return rc.deleteRangeQuery
+}
+
+// SaveIDMap saves the id mapping information.
+func (rc *LogClient) SaveIDMap(
+ ctx context.Context,
+ sr *stream.SchemasReplace,
+) error {
+ idMaps := sr.TidySchemaMaps()
+ clusterID := rc.GetClusterID(ctx)
+ metaFileName := metautil.PitrIDMapsFilename(clusterID, rc.restoreTS)
+ metaWriter := metautil.NewMetaWriter(rc.storage, metautil.MetaFileSize, false, metaFileName, nil)
+ metaWriter.Update(func(m *backuppb.BackupMeta) {
+ // save log startTS to backupmeta file
+ m.ClusterId = clusterID
+ m.DbMaps = idMaps
+ })
+
+ if err := metaWriter.FlushBackupMeta(ctx); err != nil {
+ return errors.Trace(err)
+ }
+ if rc.useCheckpoint {
+ var items map[int64]model.TiFlashReplicaInfo
+ if sr.TiflashRecorder != nil {
+ items = sr.TiflashRecorder.GetItems()
+ }
+ log.Info("save checkpoint task info with InLogRestoreAndIdMapPersist status")
+ if err := checkpoint.SaveCheckpointTaskInfoForLogRestore(ctx, rc.storage, &checkpoint.CheckpointTaskInfoForLogRestore{
+ Progress: checkpoint.InLogRestoreAndIdMapPersist,
+ StartTS: rc.startTS,
+ RestoreTS: rc.restoreTS,
+ RewriteTS: rc.currentTS,
+ TiFlashItems: items,
+ }, rc.GetClusterID(ctx)); err != nil {
+ return errors.Trace(err)
+ }
+ }
+ return nil
+}
+
+// called by failpoint, only used for test
+// it would print the checksum result into the log, and
+// the auto-test script records them to compare another
+// cluster's checksum.
+func (rc *LogClient) FailpointDoChecksumForLogRestore(
+ ctx context.Context,
+ kvClient kv.Client,
+ pdClient pd.Client,
+ idrules map[int64]int64,
+ rewriteRules map[int64]*restoreutils.RewriteRules,
+) (finalErr error) {
+ startTS, err := restore.GetTSWithRetry(ctx, rc.pdClient)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ // set gc safepoint for checksum
+ sp := utils.BRServiceSafePoint{
+ BackupTS: startTS,
+ TTL: utils.DefaultBRGCSafePointTTL,
+ ID: utils.MakeSafePointID(),
+ }
+ cctx, gcSafePointKeeperCancel := context.WithCancel(ctx)
+ defer func() {
+ log.Info("start to remove gc-safepoint keeper")
+ // close the gc safe point keeper at first
+ gcSafePointKeeperCancel()
+ // set the ttl to 0 to remove the gc-safe-point
+ sp.TTL = 0
+ if err := utils.UpdateServiceSafePoint(ctx, pdClient, sp); err != nil {
+ log.Warn("failed to update service safe point, backup may fail if gc triggered",
+ zap.Error(err),
+ )
+ }
+ log.Info("finish removing gc-safepoint keeper")
+ }()
+ err = utils.StartServiceSafePointKeeper(cctx, pdClient, sp)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ eg, ectx := errgroup.WithContext(ctx)
+ pool := tidbutil.NewWorkerPool(4, "checksum for log restore")
+ infoSchema := rc.GetDomain().InfoSchema()
+ // downstream id -> upstream id
+ reidRules := make(map[int64]int64)
+ for upstreamID, downstreamID := range idrules {
+ reidRules[downstreamID] = upstreamID
+ }
+ for upstreamID, downstreamID := range idrules {
+ newTable, ok := infoSchema.TableByID(downstreamID)
+ if !ok {
+ // a dropped table
+ continue
+ }
+ rewriteRule, ok := rewriteRules[upstreamID]
+ if !ok {
+ continue
+ }
+ newTableInfo := newTable.Meta()
+ var definitions []model.PartitionDefinition
+ if newTableInfo.Partition != nil {
+ for _, def := range newTableInfo.Partition.Definitions {
+ upid, ok := reidRules[def.ID]
+ if !ok {
+ log.Panic("no rewrite rule for parition table id", zap.Int64("id", def.ID))
+ }
+ definitions = append(definitions, model.PartitionDefinition{
+ ID: upid,
+ })
+ }
+ }
+ oldPartition := &model.PartitionInfo{
+ Definitions: definitions,
+ }
+ oldTable := &metautil.Table{
+ Info: &model.TableInfo{
+ ID: upstreamID,
+ Indices: newTableInfo.Indices,
+ Partition: oldPartition,
+ },
+ }
+ pool.ApplyOnErrorGroup(eg, func() error {
+ exe, err := checksum.NewExecutorBuilder(newTableInfo, startTS).
+ SetOldTable(oldTable).
+ SetConcurrency(4).
+ SetOldKeyspace(rewriteRule.OldKeyspace).
+ SetNewKeyspace(rewriteRule.NewKeyspace).
+ SetExplicitRequestSourceType(kvutil.ExplicitTypeBR).
+ Build()
+ if err != nil {
+ return errors.Trace(err)
+ }
+ checksumResp, err := exe.Execute(ectx, kvClient, func() {})
+ if err != nil {
+ return errors.Trace(err)
+ }
+ // print to log so that the test script can get final checksum
+ log.Info("failpoint checksum completed",
+ zap.String("table-name", newTableInfo.Name.O),
+ zap.Int64("upstream-id", oldTable.Info.ID),
+ zap.Uint64("checksum", checksumResp.Checksum),
+ zap.Uint64("total-kvs", checksumResp.TotalKvs),
+ zap.Uint64("total-bytes", checksumResp.TotalBytes),
+ )
+ return nil
+ })
+ }
+
+ return eg.Wait()
+}
+
+type LogFilesIterWithSplitHelper struct {
+ iter LogIter
+ helper *logsplit.LogSplitHelper
+ buffer []*LogDataFileInfo
+ next int
+}
+
+const SplitFilesBufferSize = 4096
+
+func NewLogFilesIterWithSplitHelper(iter LogIter, rules map[int64]*restoreutils.RewriteRules, client split.SplitClient, splitSize uint64, splitKeys int64) LogIter {
+ return &LogFilesIterWithSplitHelper{
+ iter: iter,
+ helper: logsplit.NewLogSplitHelper(rules, client, splitSize, splitKeys),
+ buffer: nil,
+ next: 0,
+ }
+}
+
+func (splitIter *LogFilesIterWithSplitHelper) TryNext(ctx context.Context) iter.IterResult[*LogDataFileInfo] {
+ if splitIter.next >= len(splitIter.buffer) {
+ splitIter.buffer = make([]*LogDataFileInfo, 0, SplitFilesBufferSize)
+ for r := splitIter.iter.TryNext(ctx); !r.Finished; r = splitIter.iter.TryNext(ctx) {
+ if r.Err != nil {
+ return r
+ }
+ f := r.Item
+ splitIter.helper.Merge(f.DataFileInfo)
+ splitIter.buffer = append(splitIter.buffer, f)
+ if len(splitIter.buffer) >= SplitFilesBufferSize {
+ break
+ }
+ }
+ splitIter.next = 0
+ if len(splitIter.buffer) == 0 {
+ return iter.Done[*LogDataFileInfo]()
+ }
+ log.Info("start to split the regions")
+ startTime := time.Now()
+ if err := splitIter.helper.Split(ctx); err != nil {
+ return iter.Throw[*LogDataFileInfo](errors.Trace(err))
+ }
+ log.Info("end to split the regions", zap.Duration("takes", time.Since(startTime)))
+ }
+
+ res := iter.Emit(splitIter.buffer[splitIter.next])
+ splitIter.next += 1
+ return res
+}
diff --git a/br/pkg/restore/client_test.go b/br/pkg/restore/log_client/client_test.go
similarity index 50%
rename from br/pkg/restore/client_test.go
rename to br/pkg/restore/log_client/client_test.go
index b7497c23e466f..9b3b487e36de1 100644
--- a/br/pkg/restore/client_test.go
+++ b/br/pkg/restore/log_client/client_test.go
@@ -1,627 +1,44 @@
-// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
-
-package restore_test
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package logclient_test
import (
"context"
"fmt"
"math"
- "slices"
- "sort"
- "strconv"
"sync"
"testing"
"time"
- "github.com/pingcap/errors"
- "github.com/pingcap/failpoint"
backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/kvproto/pkg/import_sstpb"
- "github.com/pingcap/kvproto/pkg/metapb"
- berrors "github.com/pingcap/tidb/br/pkg/errors"
"github.com/pingcap/tidb/br/pkg/gluetidb"
- "github.com/pingcap/tidb/br/pkg/metautil"
"github.com/pingcap/tidb/br/pkg/mock"
- "github.com/pingcap/tidb/br/pkg/restore"
- fileimporter "github.com/pingcap/tidb/br/pkg/restore/file_importer"
- logrestore "github.com/pingcap/tidb/br/pkg/restore/log_restore"
- "github.com/pingcap/tidb/br/pkg/restore/tiflashrec"
+ logclient "github.com/pingcap/tidb/br/pkg/restore/log_client"
+ "github.com/pingcap/tidb/br/pkg/restore/utils"
"github.com/pingcap/tidb/br/pkg/stream"
- "github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/br/pkg/utils/iter"
- "github.com/pingcap/tidb/pkg/parser/model"
- "github.com/pingcap/tidb/pkg/parser/mysql"
- "github.com/pingcap/tidb/pkg/parser/types"
+ "github.com/pingcap/tidb/br/pkg/utiltest"
"github.com/pingcap/tidb/pkg/tablecodec"
filter "github.com/pingcap/tidb/pkg/util/table-filter"
"github.com/stretchr/testify/require"
- pd "github.com/tikv/pd/client"
"google.golang.org/grpc/keepalive"
)
var mc *mock.Cluster
-var defaultKeepaliveCfg = keepalive.ClientParameters{
- Time: 3 * time.Second,
- Timeout: 10 * time.Second,
-}
-
-func TestCreateTables(t *testing.T) {
- m := mc
- g := gluetidb.New()
- client := restore.NewRestoreClient(m.PDClient, m.PDHTTPCli, nil, defaultKeepaliveCfg)
- err := client.Init(g, m.Storage)
- require.NoError(t, err)
-
- info, err := m.Domain.GetSnapshotInfoSchema(math.MaxUint64)
- require.NoError(t, err)
- dbSchema, isExist := info.SchemaByName(model.NewCIStr("test"))
- require.True(t, isExist)
-
- client.SetBatchDdlSize(1)
- tables := make([]*metautil.Table, 4)
- intField := types.NewFieldType(mysql.TypeLong)
- intField.SetCharset("binary")
- for i := len(tables) - 1; i >= 0; i-- {
- tables[i] = &metautil.Table{
- DB: dbSchema,
- Info: &model.TableInfo{
- ID: int64(i),
- Name: model.NewCIStr("test" + strconv.Itoa(i)),
- Columns: []*model.ColumnInfo{{
- ID: 1,
- Name: model.NewCIStr("id"),
- FieldType: *intField,
- State: model.StatePublic,
- }},
- Charset: "utf8mb4",
- Collate: "utf8mb4_bin",
- },
- }
- }
- rules, newTables, err := client.CreateTables(m.Domain, tables, 0)
- require.NoError(t, err)
- // make sure tables and newTables have same order
- for i, tbl := range tables {
- require.Equal(t, tbl.Info.Name, newTables[i].Name)
- }
- for _, nt := range newTables {
- require.Regexp(t, "test[0-3]", nt.Name.String())
- }
- oldTableIDExist := make(map[int64]bool)
- newTableIDExist := make(map[int64]bool)
- for _, tr := range rules.Data {
- oldTableID := tablecodec.DecodeTableID(tr.GetOldKeyPrefix())
- require.False(t, oldTableIDExist[oldTableID], "table rule duplicate old table id")
- oldTableIDExist[oldTableID] = true
-
- newTableID := tablecodec.DecodeTableID(tr.GetNewKeyPrefix())
- require.False(t, newTableIDExist[newTableID], "table rule duplicate new table id")
- newTableIDExist[newTableID] = true
- }
-
- for i := 0; i < len(tables); i++ {
- require.True(t, oldTableIDExist[int64(i)], "table rule does not exist")
- }
-}
-
-func TestIsOnline(t *testing.T) {
- m := mc
- g := gluetidb.New()
- client := restore.NewRestoreClient(m.PDClient, m.PDHTTPCli, nil, defaultKeepaliveCfg)
- err := client.Init(g, m.Storage)
- require.NoError(t, err)
-
- require.False(t, client.IsOnline())
- client.EnableOnline()
- require.True(t, client.IsOnline())
-}
-
-func getStartedMockedCluster(t *testing.T) *mock.Cluster {
- t.Helper()
- cluster, err := mock.NewCluster()
- require.NoError(t, err)
- err = cluster.Start()
- require.NoError(t, err)
- return cluster
-}
-
-func TestNeedCheckTargetClusterFresh(t *testing.T) {
- // cannot use shared `mc`, other parallel case may change it.
- cluster := getStartedMockedCluster(t)
- defer cluster.Stop()
-
- g := gluetidb.New()
- client := restore.NewRestoreClient(cluster.PDClient, cluster.PDHTTPCli, nil, defaultKeepaliveCfg)
- err := client.Init(g, cluster.Storage)
- require.NoError(t, err)
-
- // not set filter and first run with checkpoint
- require.True(t, client.NeedCheckFreshCluster(false, true))
-
- // skip check when has checkpoint
- require.False(t, client.NeedCheckFreshCluster(false, false))
-
- // skip check when set --filter
- require.False(t, client.NeedCheckFreshCluster(true, false))
-
- // skip check when has set --filter and has checkpoint
- require.False(t, client.NeedCheckFreshCluster(true, true))
-
- require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/br/pkg/restore/mock-incr-backup-data", "return(false)"))
- defer func() {
- require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/br/pkg/restore/mock-incr-backup-data"))
- }()
- // skip check when increment backup
- require.False(t, client.NeedCheckFreshCluster(false, true))
-}
-
-func TestCheckTargetClusterFresh(t *testing.T) {
- // cannot use shared `mc`, other parallel case may change it.
- cluster := getStartedMockedCluster(t)
- defer cluster.Stop()
-
- g := gluetidb.New()
- client := restore.NewRestoreClient(cluster.PDClient, cluster.PDHTTPCli, nil, defaultKeepaliveCfg)
- err := client.Init(g, cluster.Storage)
- require.NoError(t, err)
-
- ctx := context.Background()
- require.NoError(t, client.CheckTargetClusterFresh(ctx))
-
- require.NoError(t, client.CreateDatabases(ctx, []*metautil.Database{{Info: &model.DBInfo{Name: model.NewCIStr("user_db")}}}))
- require.True(t, berrors.ErrRestoreNotFreshCluster.Equal(client.CheckTargetClusterFresh(ctx)))
-}
-
-func TestCheckTargetClusterFreshWithTable(t *testing.T) {
- // cannot use shared `mc`, other parallel case may change it.
- cluster := getStartedMockedCluster(t)
- defer cluster.Stop()
-
- g := gluetidb.New()
- client := restore.NewRestoreClient(cluster.PDClient, cluster.PDHTTPCli, nil, defaultKeepaliveCfg)
- err := client.Init(g, cluster.Storage)
- require.NoError(t, err)
-
- ctx := context.Background()
- info, err := cluster.Domain.GetSnapshotInfoSchema(math.MaxUint64)
- require.NoError(t, err)
- dbSchema, isExist := info.SchemaByName(model.NewCIStr("test"))
- require.True(t, isExist)
- intField := types.NewFieldType(mysql.TypeLong)
- intField.SetCharset("binary")
- table := &metautil.Table{
- DB: dbSchema,
- Info: &model.TableInfo{
- ID: int64(1),
- Name: model.NewCIStr("t"),
- Columns: []*model.ColumnInfo{{
- ID: 1,
- Name: model.NewCIStr("id"),
- FieldType: *intField,
- State: model.StatePublic,
- }},
- Charset: "utf8mb4",
- Collate: "utf8mb4_bin",
- },
- }
- _, _, err = client.CreateTables(cluster.Domain, []*metautil.Table{table}, 0)
- require.NoError(t, err)
-
- require.True(t, berrors.ErrRestoreNotFreshCluster.Equal(client.CheckTargetClusterFresh(ctx)))
-}
-
-func TestCheckSysTableCompatibility(t *testing.T) {
- cluster := mc
- g := gluetidb.New()
- client := restore.NewRestoreClient(cluster.PDClient, cluster.PDHTTPCli, nil, defaultKeepaliveCfg)
- err := client.Init(g, cluster.Storage)
- require.NoError(t, err)
-
- info, err := cluster.Domain.GetSnapshotInfoSchema(math.MaxUint64)
- require.NoError(t, err)
- dbSchema, isExist := info.SchemaByName(model.NewCIStr(mysql.SystemDB))
- require.True(t, isExist)
- tmpSysDB := dbSchema.Clone()
- tmpSysDB.Name = utils.TemporaryDBName(mysql.SystemDB)
- sysDB := model.NewCIStr(mysql.SystemDB)
- userTI, err := client.GetTableSchema(cluster.Domain, sysDB, model.NewCIStr("user"))
- require.NoError(t, err)
-
- // user table in cluster have more columns(success)
- mockedUserTI := userTI.Clone()
- userTI.Columns = append(userTI.Columns, &model.ColumnInfo{Name: model.NewCIStr("new-name")})
- err = client.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{
- DB: tmpSysDB,
- Info: mockedUserTI,
- }})
- require.NoError(t, err)
- userTI.Columns = userTI.Columns[:len(userTI.Columns)-1]
-
- // user table in cluster have less columns(failed)
- mockedUserTI = userTI.Clone()
- mockedUserTI.Columns = append(mockedUserTI.Columns, &model.ColumnInfo{Name: model.NewCIStr("new-name")})
- err = client.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{
- DB: tmpSysDB,
- Info: mockedUserTI,
- }})
- require.True(t, berrors.ErrRestoreIncompatibleSys.Equal(err))
-
- // column order mismatch(success)
- mockedUserTI = userTI.Clone()
- mockedUserTI.Columns[4], mockedUserTI.Columns[5] = mockedUserTI.Columns[5], mockedUserTI.Columns[4]
- err = client.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{
- DB: tmpSysDB,
- Info: mockedUserTI,
- }})
- require.NoError(t, err)
-
- // incompatible column type
- mockedUserTI = userTI.Clone()
- mockedUserTI.Columns[0].FieldType.SetFlen(2000) // Columns[0] is `Host` char(255)
- err = client.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{
- DB: tmpSysDB,
- Info: mockedUserTI,
- }})
- require.True(t, berrors.ErrRestoreIncompatibleSys.Equal(err))
-
- // compatible
- mockedUserTI = userTI.Clone()
- err = client.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{
- DB: tmpSysDB,
- Info: mockedUserTI,
- }})
- require.NoError(t, err)
-
- // use the mysql.db table to test for column count mismatch.
- dbTI, err := client.GetTableSchema(cluster.Domain, sysDB, model.NewCIStr("db"))
- require.NoError(t, err)
-
- // other system tables in cluster have more columns(failed)
- mockedDBTI := dbTI.Clone()
- dbTI.Columns = append(dbTI.Columns, &model.ColumnInfo{Name: model.NewCIStr("new-name")})
- err = client.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{
- DB: tmpSysDB,
- Info: mockedDBTI,
- }})
- require.True(t, berrors.ErrRestoreIncompatibleSys.Equal(err))
-}
-
-func TestInitFullClusterRestore(t *testing.T) {
- cluster := mc
- g := gluetidb.New()
- client := restore.NewRestoreClient(cluster.PDClient, cluster.PDHTTPCli, nil, defaultKeepaliveCfg)
- err := client.Init(g, cluster.Storage)
- require.NoError(t, err)
-
- // explicit filter
- client.InitFullClusterRestore(true)
- require.False(t, client.IsFullClusterRestore())
-
- client.InitFullClusterRestore(false)
- require.True(t, client.IsFullClusterRestore())
- // set it to false again
- client.InitFullClusterRestore(true)
- require.False(t, client.IsFullClusterRestore())
-
- require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/br/pkg/restore/mock-incr-backup-data", "return(true)"))
- defer func() {
- require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/br/pkg/restore/mock-incr-backup-data"))
- }()
- client.InitFullClusterRestore(false)
- require.False(t, client.IsFullClusterRestore())
-}
-
-func TestPreCheckTableClusterIndex(t *testing.T) {
- m := mc
- g := gluetidb.New()
- client := restore.NewRestoreClient(m.PDClient, m.PDHTTPCli, nil, defaultKeepaliveCfg)
- err := client.Init(g, m.Storage)
- require.NoError(t, err)
-
- info, err := m.Domain.GetSnapshotInfoSchema(math.MaxUint64)
- require.NoError(t, err)
- dbSchema, isExist := info.SchemaByName(model.NewCIStr("test"))
- require.True(t, isExist)
-
- tables := make([]*metautil.Table, 4)
- intField := types.NewFieldType(mysql.TypeLong)
- intField.SetCharset("binary")
- for i := len(tables) - 1; i >= 0; i-- {
- tables[i] = &metautil.Table{
- DB: dbSchema,
- Info: &model.TableInfo{
- ID: int64(i),
- Name: model.NewCIStr("test" + strconv.Itoa(i)),
- Columns: []*model.ColumnInfo{{
- ID: 1,
- Name: model.NewCIStr("id"),
- FieldType: *intField,
- State: model.StatePublic,
- }},
- Charset: "utf8mb4",
- Collate: "utf8mb4_bin",
- },
- }
- }
- _, _, err = client.CreateTables(m.Domain, tables, 0)
- require.NoError(t, err)
-
- // exist different tables
- tables[1].Info.IsCommonHandle = true
- err = client.PreCheckTableClusterIndex(tables, nil, m.Domain)
- require.Error(t, err)
- require.Regexp(t, `.*@@tidb_enable_clustered_index should be ON \(backup table = true, created table = false\).*`, err.Error())
-
- // exist different DDLs
- jobs := []*model.Job{{
- ID: 5,
- Type: model.ActionCreateTable,
- SchemaName: "test",
- Query: "",
- BinlogInfo: &model.HistoryInfo{
- TableInfo: &model.TableInfo{
- Name: model.NewCIStr("test1"),
- IsCommonHandle: true,
- },
- },
- }}
- err = client.PreCheckTableClusterIndex(nil, jobs, m.Domain)
- require.Error(t, err)
- require.Regexp(t, `.*@@tidb_enable_clustered_index should be ON \(backup table = true, created table = false\).*`, err.Error())
-
- // should pass pre-check cluster index
- tables[1].Info.IsCommonHandle = false
- jobs[0].BinlogInfo.TableInfo.IsCommonHandle = false
- require.Nil(t, client.PreCheckTableClusterIndex(tables, jobs, m.Domain))
-}
-
-type fakePDClient struct {
- pd.Client
- stores []*metapb.Store
-
- notLeader bool
- retryTimes *int
-}
-
-func (fpdc fakePDClient) GetAllStores(context.Context, ...pd.GetStoreOption) ([]*metapb.Store, error) {
- return append([]*metapb.Store{}, fpdc.stores...), nil
-}
-
-func (fpdc fakePDClient) GetTS(ctx context.Context) (int64, int64, error) {
- (*fpdc.retryTimes)++
- if *fpdc.retryTimes >= 3 { // the mock PD leader switched successfully
- fpdc.notLeader = false
- }
-
- if fpdc.notLeader {
- return 0, 0, errors.Errorf("rpc error: code = Unknown desc = [PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster")
- }
- return 1, 1, nil
-}
-
-func TestGetTSWithRetry(t *testing.T) {
- t.Run("PD leader is healthy:", func(t *testing.T) {
- retryTimes := -1000
- pDClient := fakePDClient{notLeader: false, retryTimes: &retryTimes}
- client := restore.NewRestoreClient(pDClient, nil, nil, defaultKeepaliveCfg)
- _, err := client.GetTSWithRetry(context.Background())
- require.NoError(t, err)
- })
-
- t.Run("PD leader failure:", func(t *testing.T) {
- require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/br/pkg/utils/set-attempt-to-one", "1*return(true)"))
- defer func() {
- require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/br/pkg/utils/set-attempt-to-one"))
- }()
- retryTimes := -1000
- pDClient := fakePDClient{notLeader: true, retryTimes: &retryTimes}
- client := restore.NewRestoreClient(pDClient, nil, nil, defaultKeepaliveCfg)
- _, err := client.GetTSWithRetry(context.Background())
- require.Error(t, err)
- })
-
- t.Run("PD leader switch successfully", func(t *testing.T) {
- retryTimes := 0
- pDClient := fakePDClient{notLeader: true, retryTimes: &retryTimes}
- client := restore.NewRestoreClient(pDClient, nil, nil, defaultKeepaliveCfg)
- _, err := client.GetTSWithRetry(context.Background())
- require.NoError(t, err)
- })
-}
-
-func TestPreCheckTableTiFlashReplicas(t *testing.T) {
- m := mc
- mockStores := []*metapb.Store{
- {
- Id: 1,
- Labels: []*metapb.StoreLabel{
- {
- Key: "engine",
- Value: "tiflash",
- },
- },
- },
- {
- Id: 2,
- Labels: []*metapb.StoreLabel{
- {
- Key: "engine",
- Value: "tiflash",
- },
- },
- },
- }
-
- g := gluetidb.New()
- client := restore.NewRestoreClient(fakePDClient{
- stores: mockStores,
- }, nil, nil, defaultKeepaliveCfg)
- err := client.Init(g, m.Storage)
- require.NoError(t, err)
-
- tables := make([]*metautil.Table, 4)
- for i := 0; i < len(tables); i++ {
- tiflashReplica := &model.TiFlashReplicaInfo{
- Count: uint64(i),
- }
- if i == 0 {
- tiflashReplica = nil
- }
-
- tables[i] = &metautil.Table{
- DB: &model.DBInfo{Name: model.NewCIStr("test")},
- Info: &model.TableInfo{
- ID: int64(i),
- Name: model.NewCIStr("test" + strconv.Itoa(i)),
- TiFlashReplica: tiflashReplica,
- },
- }
- }
- ctx := context.Background()
- require.Nil(t, client.PreCheckTableTiFlashReplica(ctx, tables, nil))
-
- for i := 0; i < len(tables); i++ {
- if i == 0 || i > 2 {
- require.Nil(t, tables[i].Info.TiFlashReplica)
- } else {
- require.NotNil(t, tables[i].Info.TiFlashReplica)
- obtainCount := int(tables[i].Info.TiFlashReplica.Count)
- require.Equal(t, i, obtainCount)
- }
- }
-
- require.Nil(t, client.PreCheckTableTiFlashReplica(ctx, tables, tiflashrec.New()))
- for i := 0; i < len(tables); i++ {
- require.Nil(t, tables[i].Info.TiFlashReplica)
- }
-}
-
-// Mock ImporterClient interface
-type FakeImporterClient struct {
- fileimporter.ImporterClient
-}
-
-// Record the stores that have communicated
-type RecordStores struct {
- mu sync.Mutex
- stores []uint64
-}
-
-func NewRecordStores() RecordStores {
- return RecordStores{stores: make([]uint64, 0)}
-}
-
-func (r *RecordStores) put(id uint64) {
- r.mu.Lock()
- defer r.mu.Unlock()
- r.stores = append(r.stores, id)
-}
-
-func (r *RecordStores) sort() {
- r.mu.Lock()
- defer r.mu.Unlock()
- slices.Sort(r.stores)
-}
-
-func (r *RecordStores) len() int {
- r.mu.Lock()
- defer r.mu.Unlock()
- return len(r.stores)
-}
-
-func (r *RecordStores) get(i int) uint64 {
- r.mu.Lock()
- defer r.mu.Unlock()
- return r.stores[i]
-}
-
-func (r *RecordStores) toString() string {
- r.mu.Lock()
- defer r.mu.Unlock()
- return fmt.Sprintf("%v", r.stores)
-}
-
-var recordStores RecordStores
-
-const (
- SET_SPEED_LIMIT_ERROR = 999999
- WORKING_TIME = 100
-)
-
-func (fakeImportCli FakeImporterClient) SetDownloadSpeedLimit(
- ctx context.Context,
- storeID uint64,
- req *import_sstpb.SetDownloadSpeedLimitRequest,
-) (*import_sstpb.SetDownloadSpeedLimitResponse, error) {
- if storeID == SET_SPEED_LIMIT_ERROR {
- return nil, fmt.Errorf("storeID:%v ERROR", storeID)
- }
-
- time.Sleep(WORKING_TIME * time.Millisecond) // simulate doing 100 ms work
- recordStores.put(storeID)
- return nil, nil
-}
-
-func TestSetSpeedLimit(t *testing.T) {
- mockStores := []*metapb.Store{
- {Id: 1},
- {Id: 2},
- {Id: 3},
- {Id: 4},
- {Id: 5},
- {Id: 6},
- {Id: 7},
- {Id: 8},
- {Id: 9},
- {Id: 10},
- }
-
- // 1. The cost of concurrent communication is expected to be less than the cost of serial communication.
- client := restore.NewRestoreClient(fakePDClient{
- stores: mockStores,
- }, nil, nil, defaultKeepaliveCfg)
- ctx := context.Background()
-
- recordStores = NewRecordStores()
- start := time.Now()
- err := restore.MockCallSetSpeedLimit(ctx, FakeImporterClient{}, client, 10)
- cost := time.Since(start)
- require.NoError(t, err)
-
- recordStores.sort()
- t.Logf("Total Cost: %v\n", cost)
- t.Logf("Has Communicated: %v\n", recordStores.toString())
-
- serialCost := len(mockStores) * WORKING_TIME
- require.Less(t, cost, time.Duration(serialCost)*time.Millisecond)
- require.Equal(t, len(mockStores), recordStores.len())
- for i := 0; i < recordStores.len(); i++ {
- require.Equal(t, mockStores[i].Id, recordStores.get(i))
- }
-
- // 2. Expect the number of communicated stores to be less than the length of the mockStore
- // Because subsequent unstarted communications are aborted when an error is encountered.
- recordStores = NewRecordStores()
- mockStores[5].Id = SET_SPEED_LIMIT_ERROR // setting a fault store
- client = restore.NewRestoreClient(fakePDClient{
- stores: mockStores,
- }, nil, nil, defaultKeepaliveCfg)
-
- // Concurrency needs to be less than the number of stores
- err = restore.MockCallSetSpeedLimit(ctx, FakeImporterClient{}, client, 2)
- require.Error(t, err)
- t.Log(err)
-
- recordStores.sort()
- sort.Slice(mockStores, func(i, j int) bool { return mockStores[i].Id < mockStores[j].Id })
- t.Logf("Has Communicated: %v\n", recordStores.toString())
- require.Less(t, recordStores.len(), len(mockStores))
- for i := 0; i < recordStores.len(); i++ {
- require.Equal(t, mockStores[i].Id, recordStores.get(i))
- }
-}
-
var deleteRangeQueryList = []*stream.PreDelRangeQuery{
{
Sql: "INSERT IGNORE INTO mysql.gc_delete_range VALUES (%?, %?, %?, %?, %?), (%?, %?, %?, %?, %?)",
@@ -659,34 +76,12 @@ var deleteRangeQueryList = []*stream.PreDelRangeQuery{
},
}
-func TestDeleteRangeQuery(t *testing.T) {
+func TestDeleteRangeQueryExec(t *testing.T) {
ctx := context.Background()
m := mc
- mockStores := []*metapb.Store{
- {
- Id: 1,
- Labels: []*metapb.StoreLabel{
- {
- Key: "engine",
- Value: "tiflash",
- },
- },
- },
- {
- Id: 2,
- Labels: []*metapb.StoreLabel{
- {
- Key: "engine",
- Value: "tiflash",
- },
- },
- },
- }
-
g := gluetidb.New()
- client := restore.NewRestoreClient(fakePDClient{
- stores: mockStores,
- }, nil, nil, defaultKeepaliveCfg)
+ client := logclient.NewRestoreClient(
+ utiltest.NewFakePDClient(nil, false, nil), nil, nil, keepalive.ClientParameters{})
err := client.Init(g, m.Storage)
require.NoError(t, err)
@@ -695,48 +90,17 @@ func TestDeleteRangeQuery(t *testing.T) {
for _, query := range deleteRangeQueryList {
client.RecordDeleteRange(query)
}
- querys := client.GetGCRows()
- require.Equal(t, len(querys), len(deleteRangeQueryList))
- for i, query := range querys {
- expected_query := deleteRangeQueryList[i]
- require.Equal(t, expected_query.Sql, query.Sql)
- require.Equal(t, len(expected_query.ParamsList), len(query.ParamsList))
- for j := range expected_query.ParamsList {
- require.Equal(t, expected_query.ParamsList[j], query.ParamsList[j])
- }
- }
+
+ require.NoError(t, client.InsertGCRows(ctx))
}
-func TestDeleteRangeQueryExec(t *testing.T) {
+func TestDeleteRangeQuery(t *testing.T) {
ctx := context.Background()
m := mc
- mockStores := []*metapb.Store{
- {
- Id: 1,
- Labels: []*metapb.StoreLabel{
- {
- Key: "engine",
- Value: "tiflash",
- },
- },
- },
- {
- Id: 2,
- Labels: []*metapb.StoreLabel{
- {
- Key: "engine",
- Value: "tiflash",
- },
- },
- },
- }
g := gluetidb.New()
- retryCnt := 1
- client := restore.NewRestoreClient(fakePDClient{
- stores: mockStores,
- retryTimes: &retryCnt,
- }, nil, nil, defaultKeepaliveCfg)
+ client := logclient.NewRestoreClient(
+ utiltest.NewFakePDClient(nil, false, nil), nil, nil, keepalive.ClientParameters{})
err := client.Init(g, m.Storage)
require.NoError(t, err)
@@ -745,8 +109,16 @@ func TestDeleteRangeQueryExec(t *testing.T) {
for _, query := range deleteRangeQueryList {
client.RecordDeleteRange(query)
}
-
- require.NoError(t, client.InsertGCRows(ctx))
+ querys := client.GetGCRows()
+ require.Equal(t, len(querys), len(deleteRangeQueryList))
+ for i, query := range querys {
+ expected_query := deleteRangeQueryList[i]
+ require.Equal(t, expected_query.Sql, query.Sql)
+ require.Equal(t, len(expected_query.ParamsList), len(query.ParamsList))
+ for j := range expected_query.ParamsList {
+ require.Equal(t, expected_query.ParamsList[j], query.ParamsList[j])
+ }
+ }
}
func MockEmptySchemasReplace() *stream.SchemasReplace {
@@ -755,7 +127,7 @@ func MockEmptySchemasReplace() *stream.SchemasReplace {
dbMap,
true,
nil,
- 9527,
+ 1,
filter.All(),
nil,
nil,
@@ -764,10 +136,10 @@ func MockEmptySchemasReplace() *stream.SchemasReplace {
}
func TestRestoreBatchMetaKVFiles(t *testing.T) {
- client := restore.MockClient(nil)
+ client := logclient.NewRestoreClient(nil, nil, nil, keepalive.ClientParameters{})
files := []*backuppb.DataFileInfo{}
// test empty files and entries
- next, err := client.RestoreBatchMetaKVFiles(context.Background(), files[0:], nil, make([]*logrestore.KvEntryWithTS, 0), math.MaxUint64, nil, nil, "")
+ next, err := client.RestoreBatchMetaKVFiles(context.Background(), files[0:], nil, make([]*logclient.KvEntryWithTS, 0), math.MaxUint64, nil, nil, "")
require.NoError(t, err)
require.Equal(t, 0, len(next))
}
@@ -777,9 +149,8 @@ func TestRestoreMetaKVFilesWithBatchMethod1(t *testing.T) {
files_write := []*backuppb.DataFileInfo{}
batchCount := 0
- client := restore.MockClient(nil)
sr := MockEmptySchemasReplace()
- err := client.RestoreMetaKVFilesWithBatchMethod(
+ err := logclient.RestoreMetaKVFilesWithBatchMethod(
context.Background(),
files_default,
files_write,
@@ -790,12 +161,12 @@ func TestRestoreMetaKVFilesWithBatchMethod1(t *testing.T) {
ctx context.Context,
files []*backuppb.DataFileInfo,
schemasReplace *stream.SchemasReplace,
- entries []*logrestore.KvEntryWithTS,
+ entries []*logclient.KvEntryWithTS,
filterTS uint64,
updateStats func(kvCount uint64, size uint64),
progressInc func(),
cf string,
- ) ([]*logrestore.KvEntryWithTS, error) {
+ ) ([]*logclient.KvEntryWithTS, error) {
require.Equal(t, 0, len(entries))
require.Equal(t, 0, len(files))
batchCount++
@@ -817,9 +188,8 @@ func TestRestoreMetaKVFilesWithBatchMethod2_default_empty(t *testing.T) {
}
batchCount := 0
- client := restore.MockClient(nil)
sr := MockEmptySchemasReplace()
- err := client.RestoreMetaKVFilesWithBatchMethod(
+ err := logclient.RestoreMetaKVFilesWithBatchMethod(
context.Background(),
files_default,
files_write,
@@ -830,12 +200,12 @@ func TestRestoreMetaKVFilesWithBatchMethod2_default_empty(t *testing.T) {
ctx context.Context,
files []*backuppb.DataFileInfo,
schemasReplace *stream.SchemasReplace,
- entries []*logrestore.KvEntryWithTS,
+ entries []*logclient.KvEntryWithTS,
filterTS uint64,
updateStats func(kvCount uint64, size uint64),
progressInc func(),
cf string,
- ) ([]*logrestore.KvEntryWithTS, error) {
+ ) ([]*logclient.KvEntryWithTS, error) {
if len(entries) == 0 && len(files) == 0 {
require.Equal(t, stream.DefaultCF, cf)
batchCount++
@@ -864,9 +234,8 @@ func TestRestoreMetaKVFilesWithBatchMethod2_write_empty_1(t *testing.T) {
files_write := []*backuppb.DataFileInfo{}
batchCount := 0
- client := restore.MockClient(nil)
sr := MockEmptySchemasReplace()
- err := client.RestoreMetaKVFilesWithBatchMethod(
+ err := logclient.RestoreMetaKVFilesWithBatchMethod(
context.Background(),
files_default,
files_write,
@@ -877,12 +246,12 @@ func TestRestoreMetaKVFilesWithBatchMethod2_write_empty_1(t *testing.T) {
ctx context.Context,
files []*backuppb.DataFileInfo,
schemasReplace *stream.SchemasReplace,
- entries []*logrestore.KvEntryWithTS,
+ entries []*logclient.KvEntryWithTS,
filterTS uint64,
updateStats func(kvCount uint64, size uint64),
progressInc func(),
cf string,
- ) ([]*logrestore.KvEntryWithTS, error) {
+ ) ([]*logclient.KvEntryWithTS, error) {
if len(entries) == 0 && len(files) == 0 {
require.Equal(t, stream.WriteCF, cf)
batchCount++
@@ -906,22 +275,21 @@ func TestRestoreMetaKVFilesWithBatchMethod2_write_empty_2(t *testing.T) {
Path: "f1",
MinTs: 100,
MaxTs: 120,
- Length: restore.MetaKVBatchSize - 1000,
+ Length: logclient.MetaKVBatchSize - 1000,
},
{
Path: "f2",
MinTs: 110,
MaxTs: 1100,
- Length: restore.MetaKVBatchSize,
+ Length: logclient.MetaKVBatchSize,
},
}
files_write := []*backuppb.DataFileInfo{}
emptyCount := 0
batchCount := 0
- client := restore.MockClient(nil)
sr := MockEmptySchemasReplace()
- err := client.RestoreMetaKVFilesWithBatchMethod(
+ err := logclient.RestoreMetaKVFilesWithBatchMethod(
context.Background(),
files_default,
files_write,
@@ -932,12 +300,12 @@ func TestRestoreMetaKVFilesWithBatchMethod2_write_empty_2(t *testing.T) {
ctx context.Context,
files []*backuppb.DataFileInfo,
schemasReplace *stream.SchemasReplace,
- entries []*logrestore.KvEntryWithTS,
+ entries []*logclient.KvEntryWithTS,
filterTS uint64,
updateStats func(kvCount uint64, size uint64),
progressInc func(),
cf string,
- ) ([]*logrestore.KvEntryWithTS, error) {
+ ) ([]*logclient.KvEntryWithTS, error) {
if len(entries) == 0 && len(files) == 0 {
// write - write
require.Equal(t, stream.WriteCF, cf)
@@ -973,22 +341,21 @@ func TestRestoreMetaKVFilesWithBatchMethod_with_entries(t *testing.T) {
Path: "f1",
MinTs: 100,
MaxTs: 120,
- Length: restore.MetaKVBatchSize - 1000,
+ Length: logclient.MetaKVBatchSize - 1000,
},
{
Path: "f2",
MinTs: 110,
MaxTs: 1100,
- Length: restore.MetaKVBatchSize,
+ Length: logclient.MetaKVBatchSize,
},
}
files_write := []*backuppb.DataFileInfo{}
emptyCount := 0
batchCount := 0
- client := restore.MockClient(nil)
sr := MockEmptySchemasReplace()
- err := client.RestoreMetaKVFilesWithBatchMethod(
+ err := logclient.RestoreMetaKVFilesWithBatchMethod(
context.Background(),
files_default,
files_write,
@@ -999,12 +366,12 @@ func TestRestoreMetaKVFilesWithBatchMethod_with_entries(t *testing.T) {
ctx context.Context,
files []*backuppb.DataFileInfo,
schemasReplace *stream.SchemasReplace,
- entries []*logrestore.KvEntryWithTS,
+ entries []*logclient.KvEntryWithTS,
filterTS uint64,
updateStats func(kvCount uint64, size uint64),
progressInc func(),
cf string,
- ) ([]*logrestore.KvEntryWithTS, error) {
+ ) ([]*logclient.KvEntryWithTS, error) {
if len(entries) == 0 && len(files) == 0 {
// write - write
require.Equal(t, stream.WriteCF, cf)
@@ -1094,9 +461,8 @@ func TestRestoreMetaKVFilesWithBatchMethod3(t *testing.T) {
result := make(map[int][]*backuppb.DataFileInfo)
resultKV := make(map[int]int)
- client := restore.MockClient(nil)
sr := MockEmptySchemasReplace()
- err := client.RestoreMetaKVFilesWithBatchMethod(
+ err := logclient.RestoreMetaKVFilesWithBatchMethod(
context.Background(),
defaultFiles,
writeFiles,
@@ -1107,17 +473,17 @@ func TestRestoreMetaKVFilesWithBatchMethod3(t *testing.T) {
ctx context.Context,
fs []*backuppb.DataFileInfo,
schemasReplace *stream.SchemasReplace,
- entries []*logrestore.KvEntryWithTS,
+ entries []*logclient.KvEntryWithTS,
filterTS uint64,
updateStats func(kvCount uint64, size uint64),
progressInc func(),
cf string,
- ) ([]*logrestore.KvEntryWithTS, error) {
+ ) ([]*logclient.KvEntryWithTS, error) {
result[batchCount] = fs
t.Log(filterTS)
resultKV[batchCount] = len(entries)
batchCount++
- return make([]*logrestore.KvEntryWithTS, batchCount), nil
+ return make([]*logclient.KvEntryWithTS, batchCount), nil
},
)
require.Nil(t, err)
@@ -1181,9 +547,8 @@ func TestRestoreMetaKVFilesWithBatchMethod4(t *testing.T) {
batchCount := 0
result := make(map[int][]*backuppb.DataFileInfo)
- client := restore.MockClient(nil)
sr := MockEmptySchemasReplace()
- err := client.RestoreMetaKVFilesWithBatchMethod(
+ err := logclient.RestoreMetaKVFilesWithBatchMethod(
context.Background(),
defaultFiles,
writeFiles,
@@ -1194,12 +559,12 @@ func TestRestoreMetaKVFilesWithBatchMethod4(t *testing.T) {
ctx context.Context,
fs []*backuppb.DataFileInfo,
schemasReplace *stream.SchemasReplace,
- entries []*logrestore.KvEntryWithTS,
+ entries []*logclient.KvEntryWithTS,
filterTS uint64,
updateStats func(kvCount uint64, size uint64),
progressInc func(),
cf string,
- ) ([]*logrestore.KvEntryWithTS, error) {
+ ) ([]*logclient.KvEntryWithTS, error) {
result[batchCount] = fs
batchCount++
return nil, nil
@@ -1262,9 +627,8 @@ func TestRestoreMetaKVFilesWithBatchMethod5(t *testing.T) {
batchCount := 0
result := make(map[int][]*backuppb.DataFileInfo)
- client := restore.MockClient(nil)
sr := MockEmptySchemasReplace()
- err := client.RestoreMetaKVFilesWithBatchMethod(
+ err := logclient.RestoreMetaKVFilesWithBatchMethod(
context.Background(),
defaultFiles,
writeFiles,
@@ -1275,12 +639,12 @@ func TestRestoreMetaKVFilesWithBatchMethod5(t *testing.T) {
ctx context.Context,
fs []*backuppb.DataFileInfo,
schemasReplace *stream.SchemasReplace,
- entries []*logrestore.KvEntryWithTS,
+ entries []*logclient.KvEntryWithTS,
filterTS uint64,
updateStats func(kvCount uint64, size uint64),
progressInc func(),
cf string,
- ) ([]*logrestore.KvEntryWithTS, error) {
+ ) ([]*logclient.KvEntryWithTS, error) {
result[batchCount] = fs
batchCount++
return nil, nil
@@ -1306,7 +670,7 @@ func TestRestoreMetaKVFilesWithBatchMethod6(t *testing.T) {
Path: "f2",
MinTs: 100,
MaxTs: 120,
- Length: restore.MetaKVBatchSize - 100,
+ Length: logclient.MetaKVBatchSize - 100,
},
{
Path: "f3",
@@ -1360,9 +724,8 @@ func TestRestoreMetaKVFilesWithBatchMethod6(t *testing.T) {
result := make(map[int][]*backuppb.DataFileInfo)
resultKV := make(map[int]int)
- client := restore.MockClient(nil)
sr := MockEmptySchemasReplace()
- err := client.RestoreMetaKVFilesWithBatchMethod(
+ err := logclient.RestoreMetaKVFilesWithBatchMethod(
context.Background(),
defaultFiles,
writeFiles,
@@ -1373,17 +736,17 @@ func TestRestoreMetaKVFilesWithBatchMethod6(t *testing.T) {
ctx context.Context,
fs []*backuppb.DataFileInfo,
schemasReplace *stream.SchemasReplace,
- entries []*logrestore.KvEntryWithTS,
+ entries []*logclient.KvEntryWithTS,
filterTS uint64,
updateStats func(kvCount uint64, size uint64),
progressInc func(),
cf string,
- ) ([]*logrestore.KvEntryWithTS, error) {
+ ) ([]*logclient.KvEntryWithTS, error) {
result[batchCount] = fs
t.Log(filterTS)
resultKV[batchCount] = len(entries)
batchCount++
- return make([]*logrestore.KvEntryWithTS, batchCount), nil
+ return make([]*logclient.KvEntryWithTS, batchCount), nil
},
)
require.Nil(t, err)
@@ -1436,7 +799,7 @@ func TestSortMetaKVFiles(t *testing.T) {
},
}
- files = restore.SortMetaKVFiles(files)
+ files = logclient.SortMetaKVFiles(files)
require.Equal(t, len(files), 5)
require.Equal(t, files[0].Path, "f1")
require.Equal(t, files[1].Path, "f2")
@@ -1445,9 +808,9 @@ func TestSortMetaKVFiles(t *testing.T) {
require.Equal(t, files[4].Path, "f5")
}
-func toLogDataFileInfoIter(logIter iter.TryNextor[*backuppb.DataFileInfo]) logrestore.LogIter {
- return iter.Map(logIter, func(d *backuppb.DataFileInfo) *logrestore.LogDataFileInfo {
- return &logrestore.LogDataFileInfo{
+func toLogDataFileInfoIter(logIter iter.TryNextor[*backuppb.DataFileInfo]) logclient.LogIter {
+ return iter.Map(logIter, func(d *backuppb.DataFileInfo) *logclient.LogDataFileInfo {
+ return &logclient.LogDataFileInfo{
DataFileInfo: d,
}
})
@@ -1483,7 +846,7 @@ func TestApplyKVFilesWithSingelMethod(t *testing.T) {
}
var applyWg sync.WaitGroup
applyFunc := func(
- files []*logrestore.LogDataFileInfo,
+ files []*logclient.LogDataFileInfo,
kvCount int64,
size uint64,
) {
@@ -1494,7 +857,7 @@ func TestApplyKVFilesWithSingelMethod(t *testing.T) {
}
}
- restore.ApplyKVFilesWithSingelMethod(
+ logclient.ApplyKVFilesWithSingelMethod(
context.TODO(),
toLogDataFileInfoIter(iter.FromSlice(ds)),
applyFunc,
@@ -1555,7 +918,7 @@ func TestApplyKVFilesWithBatchMethod1(t *testing.T) {
}
var applyWg sync.WaitGroup
applyFunc := func(
- files []*logrestore.LogDataFileInfo,
+ files []*logclient.LogDataFileInfo,
kvCount int64,
size uint64,
) {
@@ -1568,7 +931,7 @@ func TestApplyKVFilesWithBatchMethod1(t *testing.T) {
logs = append(logs, log)
}
- restore.ApplyKVFilesWithBatchMethod(
+ logclient.ApplyKVFilesWithBatchMethod(
context.TODO(),
toLogDataFileInfoIter(iter.FromSlice(ds)),
batchCount,
@@ -1645,7 +1008,7 @@ func TestApplyKVFilesWithBatchMethod2(t *testing.T) {
}
var applyWg sync.WaitGroup
applyFunc := func(
- files []*logrestore.LogDataFileInfo,
+ files []*logclient.LogDataFileInfo,
kvCount int64,
size uint64,
) {
@@ -1658,7 +1021,7 @@ func TestApplyKVFilesWithBatchMethod2(t *testing.T) {
logs = append(logs, log)
}
- restore.ApplyKVFilesWithBatchMethod(
+ logclient.ApplyKVFilesWithBatchMethod(
context.TODO(),
toLogDataFileInfoIter(iter.FromSlice(ds)),
batchCount,
@@ -1729,7 +1092,7 @@ func TestApplyKVFilesWithBatchMethod3(t *testing.T) {
}
var applyWg sync.WaitGroup
applyFunc := func(
- files []*logrestore.LogDataFileInfo,
+ files []*logclient.LogDataFileInfo,
kvCount int64,
size uint64,
) {
@@ -1742,7 +1105,7 @@ func TestApplyKVFilesWithBatchMethod3(t *testing.T) {
logs = append(logs, log)
}
- restore.ApplyKVFilesWithBatchMethod(
+ logclient.ApplyKVFilesWithBatchMethod(
context.TODO(),
toLogDataFileInfoIter(iter.FromSlice(ds)),
batchCount,
@@ -1811,7 +1174,7 @@ func TestApplyKVFilesWithBatchMethod4(t *testing.T) {
}
var applyWg sync.WaitGroup
applyFunc := func(
- files []*logrestore.LogDataFileInfo,
+ files []*logclient.LogDataFileInfo,
kvCount int64,
size uint64,
) {
@@ -1824,7 +1187,7 @@ func TestApplyKVFilesWithBatchMethod4(t *testing.T) {
logs = append(logs, log)
}
- restore.ApplyKVFilesWithBatchMethod(
+ logclient.ApplyKVFilesWithBatchMethod(
context.TODO(),
toLogDataFileInfoIter(iter.FromSlice(ds)),
batchCount,
@@ -1889,7 +1252,7 @@ func TestApplyKVFilesWithBatchMethod5(t *testing.T) {
}
var applyWg sync.WaitGroup
applyFunc := func(
- files []*logrestore.LogDataFileInfo,
+ files []*logclient.LogDataFileInfo,
kvCount int64,
size uint64,
) {
@@ -1908,7 +1271,7 @@ func TestApplyKVFilesWithBatchMethod5(t *testing.T) {
}()
}
- restore.ApplyKVFilesWithBatchMethod(
+ logclient.ApplyKVFilesWithBatchMethod(
context.TODO(),
toLogDataFileInfoIter(iter.FromSlice(ds)),
2,
@@ -1921,7 +1284,7 @@ func TestApplyKVFilesWithBatchMethod5(t *testing.T) {
require.Equal(t, backuppb.FileType_Delete, types[len(types)-1])
types = make([]backuppb.FileType, 0)
- restore.ApplyKVFilesWithSingelMethod(
+ logclient.ApplyKVFilesWithSingelMethod(
context.TODO(),
toLogDataFileInfoIter(iter.FromSlice(ds)),
applyFunc,
@@ -1932,80 +1295,45 @@ func TestApplyKVFilesWithBatchMethod5(t *testing.T) {
require.Equal(t, backuppb.FileType_Delete, types[len(types)-1])
}
-func TestCheckNewCollationEnable(t *testing.T) {
- caseList := []struct {
- backupMeta *backuppb.BackupMeta
- newCollationEnableInCluster string
- CheckRequirements bool
- isErr bool
- }{
- {
- backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "True"},
- newCollationEnableInCluster: "True",
- CheckRequirements: true,
- isErr: false,
- },
- {
- backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "True"},
- newCollationEnableInCluster: "False",
- CheckRequirements: true,
- isErr: true,
- },
- {
- backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "False"},
- newCollationEnableInCluster: "True",
- CheckRequirements: true,
- isErr: true,
- },
- {
- backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "False"},
- newCollationEnableInCluster: "false",
- CheckRequirements: true,
- isErr: false,
- },
- {
- backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "False"},
- newCollationEnableInCluster: "True",
- CheckRequirements: false,
- isErr: true,
- },
- {
- backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "True"},
- newCollationEnableInCluster: "False",
- CheckRequirements: false,
- isErr: true,
- },
- {
- backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: ""},
- newCollationEnableInCluster: "True",
- CheckRequirements: false,
- isErr: false,
- },
- {
- backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: ""},
- newCollationEnableInCluster: "True",
- CheckRequirements: true,
- isErr: true,
+type mockLogIter struct {
+ next int
+}
+
+func (m *mockLogIter) TryNext(ctx context.Context) iter.IterResult[*logclient.LogDataFileInfo] {
+ if m.next > 10000 {
+ return iter.Done[*logclient.LogDataFileInfo]()
+ }
+ m.next += 1
+ return iter.Emit(&logclient.LogDataFileInfo{
+ DataFileInfo: &backuppb.DataFileInfo{
+ StartKey: []byte(fmt.Sprintf("a%d", m.next)),
+ EndKey: []byte("b"),
+ Length: 1024, // 1 KB
},
- {
- backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: ""},
- newCollationEnableInCluster: "False",
- CheckRequirements: false,
- isErr: false,
+ })
+}
+
+func TestLogFilesIterWithSplitHelper(t *testing.T) {
+ var tableID int64 = 76
+ var oldTableID int64 = 80
+ rewriteRules := &utils.RewriteRules{
+ Data: []*import_sstpb.RewriteRule{
+ {
+ OldKeyPrefix: tablecodec.EncodeTablePrefix(oldTableID),
+ NewKeyPrefix: tablecodec.EncodeTablePrefix(tableID),
+ },
},
}
-
- for i, ca := range caseList {
- g := &gluetidb.MockGlue{
- GlobalVars: map[string]string{"new_collation_enabled": ca.newCollationEnableInCluster},
- }
- enabled, err := restore.CheckNewCollationEnable(ca.backupMeta.GetNewCollationsEnabled(), g, nil, ca.CheckRequirements)
- t.Logf("[%d] Got Error: %v\n", i, err)
- if ca.isErr {
- require.Error(t, err)
- } else {
- require.NoError(t, err)
- }
- require.Equal(t, ca.newCollationEnableInCluster == "True", enabled)
+ rewriteRulesMap := map[int64]*utils.RewriteRules{
+ oldTableID: rewriteRules,
+ }
+ mockIter := &mockLogIter{}
+ ctx := context.Background()
+ logIter := logclient.NewLogFilesIterWithSplitHelper(mockIter, rewriteRulesMap, utiltest.NewFakeSplitClient(), 144*1024*1024, 1440000)
+ next := 0
+ for r := logIter.TryNext(ctx); !r.Finished; r = logIter.TryNext(ctx) {
+ require.NoError(t, r.Err)
+ next += 1
+ require.Equal(t, []byte(fmt.Sprintf("a%d", next)), r.Item.StartKey)
}
}
diff --git a/br/pkg/restore/log_restore/export_test.go b/br/pkg/restore/log_client/export_test.go
similarity index 94%
rename from br/pkg/restore/log_restore/export_test.go
rename to br/pkg/restore/log_client/export_test.go
index 492230146c21b..c28d46c066798 100644
--- a/br/pkg/restore/log_restore/export_test.go
+++ b/br/pkg/restore/log_client/export_test.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package logrestore
+package logclient
import (
"context"
@@ -21,6 +21,8 @@ import (
"github.com/pingcap/tidb/br/pkg/utils/iter"
)
+var FilterFilesByRegion = filterFilesByRegion
+
// readStreamMetaByTS is used for streaming task. collect all meta file by TS, it is for test usage.
func (rc *LogFileManager) ReadStreamMeta(ctx context.Context) ([]Meta, error) {
metas, err := rc.streamingMeta(ctx)
diff --git a/br/pkg/restore/log_client/import.go b/br/pkg/restore/log_client/import.go
new file mode 100644
index 0000000000000..41abe7fe5c426
--- /dev/null
+++ b/br/pkg/restore/log_client/import.go
@@ -0,0 +1,300 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package logclient
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "math/rand"
+ "time"
+
+ "github.com/pingcap/errors"
+ backuppb "github.com/pingcap/kvproto/pkg/brpb"
+ "github.com/pingcap/kvproto/pkg/import_sstpb"
+ "github.com/pingcap/kvproto/pkg/kvrpcpb"
+ "github.com/pingcap/kvproto/pkg/metapb"
+ "github.com/pingcap/log"
+ "github.com/pingcap/tidb/br/pkg/conn"
+ "github.com/pingcap/tidb/br/pkg/conn/util"
+ berrors "github.com/pingcap/tidb/br/pkg/errors"
+ "github.com/pingcap/tidb/br/pkg/logutil"
+ importclient "github.com/pingcap/tidb/br/pkg/restore/internal/import_client"
+ "github.com/pingcap/tidb/br/pkg/restore/split"
+ restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
+ "github.com/pingcap/tidb/br/pkg/stream"
+ "github.com/pingcap/tidb/br/pkg/summary"
+ "github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/pkg/kv"
+ pd "github.com/tikv/pd/client"
+ "go.uber.org/multierr"
+ "go.uber.org/zap"
+)
+
+type LogFileImporter struct {
+ metaClient split.SplitClient
+ importClient importclient.ImporterClient
+ backend *backuppb.StorageBackend
+
+ cacheKey string
+}
+
+// NewFileImporter returns a new file importClient.
+func NewLogFileImporter(
+ metaClient split.SplitClient,
+ importClient importclient.ImporterClient,
+ backend *backuppb.StorageBackend,
+) *LogFileImporter {
+ return &LogFileImporter{
+ metaClient: metaClient,
+ backend: backend,
+ importClient: importClient,
+ cacheKey: fmt.Sprintf("BR-%s-%d", time.Now().Format("20060102150405"), rand.Int63()),
+ }
+}
+
+func (importer *LogFileImporter) Close() error {
+ if importer != nil && importer.importClient != nil {
+ return importer.importClient.CloseGrpcClient()
+ }
+ return nil
+}
+
+func (importer *LogFileImporter) ClearFiles(ctx context.Context, pdClient pd.Client, prefix string) error {
+ allStores, err := conn.GetAllTiKVStoresWithRetry(ctx, pdClient, util.SkipTiFlash)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ for _, s := range allStores {
+ if s.State != metapb.StoreState_Up {
+ continue
+ }
+ req := &import_sstpb.ClearRequest{
+ Prefix: prefix,
+ }
+ _, err = importer.importClient.ClearFiles(ctx, s.GetId(), req)
+ if err != nil {
+ log.Warn("cleanup kv files failed", zap.Uint64("store", s.GetId()), zap.Error(err))
+ }
+ }
+ return nil
+}
+
+// ImportKVFiles restores the kv events.
+func (importer *LogFileImporter) ImportKVFiles(
+ ctx context.Context,
+ files []*LogDataFileInfo,
+ rule *restoreutils.RewriteRules,
+ shiftStartTS uint64,
+ startTS uint64,
+ restoreTS uint64,
+ supportBatch bool,
+) error {
+ var (
+ startKey []byte
+ endKey []byte
+ ranges = make([]kv.KeyRange, len(files))
+ err error
+ )
+
+ if !supportBatch && len(files) > 1 {
+ return errors.Annotatef(berrors.ErrInvalidArgument,
+ "do not support batch apply but files count:%v > 1", len(files))
+ }
+ log.Debug("import kv files", zap.Int("batch file count", len(files)))
+
+ for i, f := range files {
+ ranges[i].StartKey, ranges[i].EndKey, err = restoreutils.GetRewriteEncodedKeys(f, rule)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ if len(startKey) == 0 || bytes.Compare(ranges[i].StartKey, startKey) < 0 {
+ startKey = ranges[i].StartKey
+ }
+ if len(endKey) == 0 || bytes.Compare(ranges[i].EndKey, endKey) > 0 {
+ endKey = ranges[i].EndKey
+ }
+ }
+
+ log.Debug("rewrite file keys",
+ logutil.Key("startKey", startKey), logutil.Key("endKey", endKey))
+
+ // This RetryState will retry 45 time, about 10 min.
+ rs := utils.InitialRetryState(45, 100*time.Millisecond, 15*time.Second)
+ ctl := OverRegionsInRange(startKey, endKey, importer.metaClient, &rs)
+ err = ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) RPCResult {
+ subfiles, errFilter := filterFilesByRegion(files, ranges, r)
+ if errFilter != nil {
+ return RPCResultFromError(errFilter)
+ }
+ if len(subfiles) == 0 {
+ return RPCResultOK()
+ }
+ return importer.importKVFileForRegion(ctx, subfiles, rule, shiftStartTS, startTS, restoreTS, r, supportBatch)
+ })
+ return errors.Trace(err)
+}
+
+func filterFilesByRegion(
+ files []*LogDataFileInfo,
+ ranges []kv.KeyRange,
+ r *split.RegionInfo,
+) ([]*LogDataFileInfo, error) {
+ if len(files) != len(ranges) {
+ return nil, errors.Annotatef(berrors.ErrInvalidArgument,
+ "count of files no equals count of ranges, file-count:%v, ranges-count:%v",
+ len(files), len(ranges))
+ }
+
+ output := make([]*LogDataFileInfo, 0, len(files))
+ if r != nil && r.Region != nil {
+ for i, f := range files {
+ if bytes.Compare(r.Region.StartKey, ranges[i].EndKey) <= 0 &&
+ (len(r.Region.EndKey) == 0 || bytes.Compare(r.Region.EndKey, ranges[i].StartKey) >= 0) {
+ output = append(output, f)
+ }
+ }
+ } else {
+ output = files
+ }
+
+ return output, nil
+}
+
+// Import tries to import a file.
+func (importer *LogFileImporter) importKVFileForRegion(
+ ctx context.Context,
+ files []*LogDataFileInfo,
+ rule *restoreutils.RewriteRules,
+ shiftStartTS uint64,
+ startTS uint64,
+ restoreTS uint64,
+ info *split.RegionInfo,
+ supportBatch bool,
+) RPCResult {
+ // Try to download file.
+ result := importer.downloadAndApplyKVFile(ctx, files, rule, info, shiftStartTS, startTS, restoreTS, supportBatch)
+ if !result.OK() {
+ errDownload := result.Err
+ for _, e := range multierr.Errors(errDownload) {
+ switch errors.Cause(e) { // nolint:errorlint
+ case berrors.ErrKVRewriteRuleNotFound, berrors.ErrKVRangeIsEmpty:
+ // Skip this region
+ logutil.CL(ctx).Warn("download file skipped",
+ logutil.Region(info.Region),
+ logutil.ShortError(e))
+ return RPCResultOK()
+ }
+ }
+ logutil.CL(ctx).Warn("download and apply file failed",
+ logutil.ShortError(&result))
+ return result
+ }
+ summary.CollectInt("RegionInvolved", 1)
+ return RPCResultOK()
+}
+
+func (importer *LogFileImporter) downloadAndApplyKVFile(
+ ctx context.Context,
+ files []*LogDataFileInfo,
+ rules *restoreutils.RewriteRules,
+ regionInfo *split.RegionInfo,
+ shiftStartTS uint64,
+ startTS uint64,
+ restoreTS uint64,
+ supportBatch bool,
+) RPCResult {
+ leader := regionInfo.Leader
+ if leader == nil {
+ return RPCResultFromError(errors.Annotatef(berrors.ErrPDLeaderNotFound,
+ "region id %d has no leader", regionInfo.Region.Id))
+ }
+
+ metas := make([]*import_sstpb.KVMeta, 0, len(files))
+ rewriteRules := make([]*import_sstpb.RewriteRule, 0, len(files))
+
+ for _, file := range files {
+ // Get the rewrite rule for the file.
+ fileRule := restoreutils.FindMatchedRewriteRule(file, rules)
+ if fileRule == nil {
+ return RPCResultFromError(errors.Annotatef(berrors.ErrKVRewriteRuleNotFound,
+ "rewrite rule for file %+v not find (in %+v)", file, rules))
+ }
+ rule := import_sstpb.RewriteRule{
+ OldKeyPrefix: restoreutils.EncodeKeyPrefix(fileRule.GetOldKeyPrefix()),
+ NewKeyPrefix: restoreutils.EncodeKeyPrefix(fileRule.GetNewKeyPrefix()),
+ }
+
+ meta := &import_sstpb.KVMeta{
+ Name: file.Path,
+ Cf: file.Cf,
+ RangeOffset: file.RangeOffset,
+ Length: file.Length,
+ RangeLength: file.RangeLength,
+ IsDelete: file.Type == backuppb.FileType_Delete,
+ StartTs: func() uint64 {
+ if file.Cf == stream.DefaultCF {
+ return shiftStartTS
+ }
+ return startTS
+ }(),
+ RestoreTs: restoreTS,
+ StartKey: regionInfo.Region.GetStartKey(),
+ EndKey: regionInfo.Region.GetEndKey(),
+ Sha256: file.GetSha256(),
+ CompressionType: file.CompressionType,
+ }
+
+ metas = append(metas, meta)
+ rewriteRules = append(rewriteRules, &rule)
+ }
+
+ reqCtx := &kvrpcpb.Context{
+ RegionId: regionInfo.Region.GetId(),
+ RegionEpoch: regionInfo.Region.GetRegionEpoch(),
+ Peer: leader,
+ }
+
+ var req *import_sstpb.ApplyRequest
+ if supportBatch {
+ req = &import_sstpb.ApplyRequest{
+ Metas: metas,
+ StorageBackend: importer.backend,
+ RewriteRules: rewriteRules,
+ Context: reqCtx,
+ StorageCacheId: importer.cacheKey,
+ }
+ } else {
+ req = &import_sstpb.ApplyRequest{
+ Meta: metas[0],
+ StorageBackend: importer.backend,
+ RewriteRule: *rewriteRules[0],
+ Context: reqCtx,
+ StorageCacheId: importer.cacheKey,
+ }
+ }
+
+ log.Debug("apply kv file", logutil.Leader(leader))
+ resp, err := importer.importClient.ApplyKVFile(ctx, leader.GetStoreId(), req)
+ if err != nil {
+ return RPCResultFromError(errors.Trace(err))
+ }
+ if resp.GetError() != nil {
+ logutil.CL(ctx).Warn("import meet error", zap.Stringer("error", resp.GetError()))
+ return RPCResultFromPBError(resp.GetError())
+ }
+ return RPCResultOK()
+}
diff --git a/br/pkg/restore/file_importer/import_retry.go b/br/pkg/restore/log_client/import_retry.go
similarity index 98%
rename from br/pkg/restore/file_importer/import_retry.go
rename to br/pkg/restore/log_client/import_retry.go
index b394ba372de05..93f454d6252e5 100644
--- a/br/pkg/restore/file_importer/import_retry.go
+++ b/br/pkg/restore/log_client/import_retry.go
@@ -1,6 +1,6 @@
// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0.
-package file_importer
+package logclient
import (
"context"
@@ -53,7 +53,7 @@ func OverRegionsInRange(start, end []byte, metaClient split.SplitClient, retrySt
}
}
-func (o *OverRegionsInRangeController) onError(ctx context.Context, result RPCResult, region *split.RegionInfo) {
+func (o *OverRegionsInRangeController) onError(_ context.Context, result RPCResult, region *split.RegionInfo) {
o.errors = multierr.Append(o.errors, errors.Annotatef(&result, "execute over region %v failed", region.Region))
// TODO: Maybe handle some of region errors like `epoch not match`?
}
diff --git a/br/pkg/restore/file_importer/import_retry_test.go b/br/pkg/restore/log_client/import_retry_test.go
similarity index 85%
rename from br/pkg/restore/file_importer/import_retry_test.go
rename to br/pkg/restore/log_client/import_retry_test.go
index 1dfa55fb1f773..fd4fc32c18317 100644
--- a/br/pkg/restore/file_importer/import_retry_test.go
+++ b/br/pkg/restore/log_client/import_retry_test.go
@@ -1,6 +1,6 @@
// Copyright 2021 PingCAP, Inc. Licensed under Apache-2.0.
-package file_importer_test
+package logclient_test
import (
"bytes"
@@ -19,7 +19,7 @@ import (
"github.com/pingcap/kvproto/pkg/import_sstpb"
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/kvproto/pkg/pdpb"
- fileimporter "github.com/pingcap/tidb/br/pkg/restore/file_importer"
+ logclient "github.com/pingcap/tidb/br/pkg/restore/log_client"
"github.com/pingcap/tidb/br/pkg/restore/split"
"github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/pkg/store/pdtypes"
@@ -228,35 +228,35 @@ func TestScanSuccess(t *testing.T) {
ctx := context.Background()
// make exclusive to inclusive.
- ctl := fileimporter.OverRegionsInRange([]byte("aa"), []byte("aay"), cli, &rs)
+ ctl := logclient.OverRegionsInRange([]byte("aa"), []byte("aay"), cli, &rs)
collectedRegions := []*split.RegionInfo{}
- ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) fileimporter.RPCResult {
+ ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) logclient.RPCResult {
collectedRegions = append(collectedRegions, r)
- return fileimporter.RPCResultOK()
+ return logclient.RPCResultOK()
})
assertRegions(t, collectedRegions, "", "aay", "bba")
- ctl = fileimporter.OverRegionsInRange([]byte("aaz"), []byte("bb"), cli, &rs)
+ ctl = logclient.OverRegionsInRange([]byte("aaz"), []byte("bb"), cli, &rs)
collectedRegions = []*split.RegionInfo{}
- ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) fileimporter.RPCResult {
+ ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) logclient.RPCResult {
collectedRegions = append(collectedRegions, r)
- return fileimporter.RPCResultOK()
+ return logclient.RPCResultOK()
})
assertRegions(t, collectedRegions, "aay", "bba", "bbh", "cca")
- ctl = fileimporter.OverRegionsInRange([]byte("aa"), []byte("cc"), cli, &rs)
+ ctl = logclient.OverRegionsInRange([]byte("aa"), []byte("cc"), cli, &rs)
collectedRegions = []*split.RegionInfo{}
- ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) fileimporter.RPCResult {
+ ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) logclient.RPCResult {
collectedRegions = append(collectedRegions, r)
- return fileimporter.RPCResultOK()
+ return logclient.RPCResultOK()
})
assertRegions(t, collectedRegions, "", "aay", "bba", "bbh", "cca", "")
- ctl = fileimporter.OverRegionsInRange([]byte("aa"), []byte(""), cli, &rs)
+ ctl = logclient.OverRegionsInRange([]byte("aa"), []byte(""), cli, &rs)
collectedRegions = []*split.RegionInfo{}
- ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) fileimporter.RPCResult {
+ ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) logclient.RPCResult {
collectedRegions = append(collectedRegions, r)
- return fileimporter.RPCResultOK()
+ return logclient.RPCResultOK()
})
assertRegions(t, collectedRegions, "", "aay", "bba", "bbh", "cca", "")
}
@@ -265,7 +265,7 @@ func TestNotLeader(t *testing.T) {
// region: [, aay), [aay, bba), [bba, bbh), [bbh, cca), [cca, )
cli := initTestClient(false)
rs := utils.InitialRetryState(1, 0, 0)
- ctl := fileimporter.OverRegionsInRange([]byte(""), []byte(""), cli, &rs)
+ ctl := logclient.OverRegionsInRange([]byte(""), []byte(""), cli, &rs)
ctx := context.Background()
notLeader := errorpb.Error{
@@ -279,17 +279,17 @@ func TestNotLeader(t *testing.T) {
meetRegions := []*split.RegionInfo{}
// record all regions we meet with id == 2.
idEqualsTo2Regions := []*split.RegionInfo{}
- err := ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) fileimporter.RPCResult {
+ err := ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) logclient.RPCResult {
if r.Region.Id == 2 {
idEqualsTo2Regions = append(idEqualsTo2Regions, r)
}
if r.Region.Id == 2 && (r.Leader == nil || r.Leader.Id != 42) {
- return fileimporter.RPCResult{
+ return logclient.RPCResult{
StoreError: ¬Leader,
}
}
meetRegions = append(meetRegions, r)
- return fileimporter.RPCResultOK()
+ return logclient.RPCResultOK()
})
require.NoError(t, err)
@@ -305,7 +305,7 @@ func TestServerIsBusy(t *testing.T) {
// region: [, aay), [aay, bba), [bba, bbh), [bbh, cca), [cca, )
cli := initTestClient(false)
rs := utils.InitialRetryState(2, 0, 0)
- ctl := fileimporter.OverRegionsInRange([]byte(""), []byte(""), cli, &rs)
+ ctl := logclient.OverRegionsInRange([]byte(""), []byte(""), cli, &rs)
ctx := context.Background()
serverIsBusy := errorpb.Error{
@@ -319,16 +319,16 @@ func TestServerIsBusy(t *testing.T) {
// record all regions we meet with id == 2.
idEqualsTo2Regions := []*split.RegionInfo{}
theFirstRun := true
- err := ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) fileimporter.RPCResult {
+ err := ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) logclient.RPCResult {
if theFirstRun && r.Region.Id == 2 {
idEqualsTo2Regions = append(idEqualsTo2Regions, r)
theFirstRun = false
- return fileimporter.RPCResult{
+ return logclient.RPCResult{
StoreError: &serverIsBusy,
}
}
meetRegions = append(meetRegions, r)
- return fileimporter.RPCResultOK()
+ return logclient.RPCResultOK()
})
require.NoError(t, err)
@@ -338,15 +338,15 @@ func TestServerIsBusy(t *testing.T) {
}
func TestServerIsBusyWithMemoryIsLimited(t *testing.T) {
- _ = failpoint.Enable("github.com/pingcap/tidb/br/pkg/restore/hint-memory-is-limited", "return(true)")
+ _ = failpoint.Enable("github.com/pingcap/tidb/br/pkg/restore/log_client/hint-memory-is-limited", "return(true)")
defer func() {
- _ = failpoint.Disable("github.com/pingcap/tidb/br/pkg/restore/hint-memory-is-limited")
+ _ = failpoint.Disable("github.com/pingcap/tidb/br/pkg/restore/log_client/hint-memory-is-limited")
}()
// region: [, aay), [aay, bba), [bba, bbh), [bbh, cca), [cca, )
cli := initTestClient(false)
rs := utils.InitialRetryState(2, 0, 0)
- ctl := fileimporter.OverRegionsInRange([]byte(""), []byte(""), cli, &rs)
+ ctl := logclient.OverRegionsInRange([]byte(""), []byte(""), cli, &rs)
ctx := context.Background()
serverIsBusy := errorpb.Error{
@@ -360,16 +360,16 @@ func TestServerIsBusyWithMemoryIsLimited(t *testing.T) {
// record all regions we meet with id == 2.
idEqualsTo2Regions := []*split.RegionInfo{}
theFirstRun := true
- err := ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) fileimporter.RPCResult {
+ err := ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) logclient.RPCResult {
if theFirstRun && r.Region.Id == 2 {
idEqualsTo2Regions = append(idEqualsTo2Regions, r)
theFirstRun = false
- return fileimporter.RPCResult{
+ return logclient.RPCResult{
StoreError: &serverIsBusy,
}
}
meetRegions = append(meetRegions, r)
- return fileimporter.RPCResultOK()
+ return logclient.RPCResultOK()
})
require.NoError(t, err)
@@ -398,7 +398,7 @@ func TestEpochNotMatch(t *testing.T) {
// region: [, aay), [aay, bba), [bba, bbh), [bbh, cca), [cca, )
cli := initTestClient(false)
rs := utils.InitialRetryState(2, 0, 0)
- ctl := fileimporter.OverRegionsInRange([]byte(""), []byte(""), cli, &rs)
+ ctl := logclient.OverRegionsInRange([]byte(""), []byte(""), cli, &rs)
ctx := context.Background()
printPDRegion("cli", cli.regionsInfo.Regions)
@@ -432,18 +432,18 @@ func TestEpochNotMatch(t *testing.T) {
firstRunRegions := []*split.RegionInfo{}
secondRunRegions := []*split.RegionInfo{}
isSecondRun := false
- err = ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) fileimporter.RPCResult {
+ err = ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) logclient.RPCResult {
if !isSecondRun && r.Region.Id == left.Region.Id {
mergeRegion()
isSecondRun = true
- return fileimporter.RPCResultFromPBError(epochNotMatch)
+ return logclient.RPCResultFromPBError(epochNotMatch)
}
if isSecondRun {
secondRunRegions = append(secondRunRegions, r)
} else {
firstRunRegions = append(firstRunRegions, r)
}
- return fileimporter.RPCResultOK()
+ return logclient.RPCResultOK()
})
printRegion("first", firstRunRegions)
printRegion("second", secondRunRegions)
@@ -457,7 +457,7 @@ func TestRegionSplit(t *testing.T) {
// region: [, aay), [aay, bba), [bba, bbh), [bbh, cca), [cca, )
cli := initTestClient(false)
rs := utils.InitialRetryState(2, 0, 0)
- ctl := fileimporter.OverRegionsInRange([]byte(""), []byte(""), cli, &rs)
+ ctl := logclient.OverRegionsInRange([]byte(""), []byte(""), cli, &rs)
ctx := context.Background()
printPDRegion("cli", cli.regionsInfo.Regions)
@@ -510,18 +510,18 @@ func TestRegionSplit(t *testing.T) {
firstRunRegions := []*split.RegionInfo{}
secondRunRegions := []*split.RegionInfo{}
isSecondRun := false
- err = ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) fileimporter.RPCResult {
+ err = ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) logclient.RPCResult {
if !isSecondRun && r.Region.Id == target.Region.Id {
splitRegion()
isSecondRun = true
- return fileimporter.RPCResultFromPBError(epochNotMatch)
+ return logclient.RPCResultFromPBError(epochNotMatch)
}
if isSecondRun {
secondRunRegions = append(secondRunRegions, r)
} else {
firstRunRegions = append(firstRunRegions, r)
}
- return fileimporter.RPCResultOK()
+ return logclient.RPCResultOK()
})
printRegion("first", firstRunRegions)
printRegion("second", secondRunRegions)
@@ -535,7 +535,7 @@ func TestRetryBackoff(t *testing.T) {
// region: [, aay), [aay, bba), [bba, bbh), [bbh, cca), [cca, )
cli := initTestClient(false)
rs := utils.InitialRetryState(2, time.Millisecond, 10*time.Millisecond)
- ctl := fileimporter.OverRegionsInRange([]byte(""), []byte(""), cli, &rs)
+ ctl := logclient.OverRegionsInRange([]byte(""), []byte(""), cli, &rs)
ctx := context.Background()
printPDRegion("cli", cli.regionsInfo.Regions)
@@ -552,12 +552,12 @@ func TestRetryBackoff(t *testing.T) {
},
}}
isSecondRun := false
- err = ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) fileimporter.RPCResult {
+ err = ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) logclient.RPCResult {
if !isSecondRun && r.Region.Id == left.Region.Id {
isSecondRun = true
- return fileimporter.RPCResultFromPBError(epochNotLeader)
+ return logclient.RPCResultFromPBError(epochNotLeader)
}
- return fileimporter.RPCResultOK()
+ return logclient.RPCResultOK()
})
printPDRegion("cli", cli.regionsInfo.Regions)
require.Equal(t, 1, rs.Attempt())
@@ -567,10 +567,10 @@ func TestRetryBackoff(t *testing.T) {
}
func TestWrappedError(t *testing.T) {
- result := fileimporter.RPCResultFromError(errors.Trace(status.Error(codes.Unavailable, "the server is slacking. ><=·>")))
- require.Equal(t, result.StrategyForRetry(), fileimporter.StrategyFromThisRegion)
- result = fileimporter.RPCResultFromError(errors.Trace(status.Error(codes.Unknown, "the server said something hard to understand")))
- require.Equal(t, result.StrategyForRetry(), fileimporter.StrategyGiveUp)
+ result := logclient.RPCResultFromError(errors.Trace(status.Error(codes.Unavailable, "the server is slacking. ><=·>")))
+ require.Equal(t, result.StrategyForRetry(), logclient.StrategyFromThisRegion)
+ result = logclient.RPCResultFromError(errors.Trace(status.Error(codes.Unknown, "the server said something hard to understand")))
+ require.Equal(t, result.StrategyForRetry(), logclient.StrategyGiveUp)
}
func envInt(name string, def int) int {
@@ -586,15 +586,15 @@ func TestPaginateScanLeader(t *testing.T) {
// region: [, aay), [aay, bba), [bba, bbh), [bbh, cca), [cca, )
cli := initTestClient(false)
rs := utils.InitialRetryState(2, time.Millisecond, 10*time.Millisecond)
- ctl := fileimporter.OverRegionsInRange([]byte("aa"), []byte("aaz"), cli, &rs)
+ ctl := logclient.OverRegionsInRange([]byte("aa"), []byte("aaz"), cli, &rs)
ctx := context.Background()
cli.InjectErr = true
cli.InjectTimes = int32(envInt("PAGINATE_SCAN_LEADER_FAILURE_COUNT", 2))
collectedRegions := []*split.RegionInfo{}
- ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) fileimporter.RPCResult {
+ ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) logclient.RPCResult {
collectedRegions = append(collectedRegions, r)
- return fileimporter.RPCResultOK()
+ return logclient.RPCResultOK()
})
assertRegions(t, collectedRegions, "", "aay", "bba")
}
diff --git a/br/pkg/restore/log_restore/import_test.go b/br/pkg/restore/log_client/import_test.go
similarity index 81%
rename from br/pkg/restore/log_restore/import_test.go
rename to br/pkg/restore/log_client/import_test.go
index 786766037538d..04445730cdfa0 100644
--- a/br/pkg/restore/log_restore/import_test.go
+++ b/br/pkg/restore/log_client/import_test.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package logrestore_test
+package logclient_test
import (
"context"
@@ -21,8 +21,7 @@ import (
backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/kvproto/pkg/metapb"
berrors "github.com/pingcap/tidb/br/pkg/errors"
- fileimporter "github.com/pingcap/tidb/br/pkg/restore/file_importer"
- logrestore "github.com/pingcap/tidb/br/pkg/restore/log_restore"
+ logclient "github.com/pingcap/tidb/br/pkg/restore/log_client"
"github.com/pingcap/tidb/br/pkg/restore/split"
"github.com/pingcap/tidb/pkg/kv"
"github.com/stretchr/testify/require"
@@ -30,7 +29,7 @@ import (
func TestImportKVFiles(t *testing.T) {
var (
- importer = fileimporter.FileImporter{}
+ importer = logclient.LogFileImporter{}
ctx = context.Background()
shiftStartTS uint64 = 100
startTS uint64 = 200
@@ -39,7 +38,7 @@ func TestImportKVFiles(t *testing.T) {
err := importer.ImportKVFiles(
ctx,
- []*logrestore.LogDataFileInfo{
+ []*logclient.LogDataFileInfo{
{
DataFileInfo: &backuppb.DataFileInfo{
Path: "log3",
@@ -61,7 +60,7 @@ func TestImportKVFiles(t *testing.T) {
}
func TestFilterFilesByRegion(t *testing.T) {
- files := []*logrestore.LogDataFileInfo{
+ files := []*logclient.LogDataFileInfo{
{
DataFileInfo: &backuppb.DataFileInfo{
Path: "log3",
@@ -85,7 +84,7 @@ func TestFilterFilesByRegion(t *testing.T) {
testCases := []struct {
r split.RegionInfo
- subfiles []*logrestore.LogDataFileInfo
+ subfiles []*logclient.LogDataFileInfo
err error
}{
{
@@ -95,7 +94,7 @@ func TestFilterFilesByRegion(t *testing.T) {
EndKey: []byte("1110"),
},
},
- subfiles: []*logrestore.LogDataFileInfo{},
+ subfiles: []*logclient.LogDataFileInfo{},
err: nil,
},
{
@@ -105,7 +104,7 @@ func TestFilterFilesByRegion(t *testing.T) {
EndKey: []byte("1111"),
},
},
- subfiles: []*logrestore.LogDataFileInfo{
+ subfiles: []*logclient.LogDataFileInfo{
files[0],
},
err: nil,
@@ -117,7 +116,7 @@ func TestFilterFilesByRegion(t *testing.T) {
EndKey: []byte("2222"),
},
},
- subfiles: []*logrestore.LogDataFileInfo{
+ subfiles: []*logclient.LogDataFileInfo{
files[0],
},
err: nil,
@@ -129,7 +128,7 @@ func TestFilterFilesByRegion(t *testing.T) {
EndKey: []byte("3332"),
},
},
- subfiles: []*logrestore.LogDataFileInfo{
+ subfiles: []*logclient.LogDataFileInfo{
files[0],
},
err: nil,
@@ -141,7 +140,7 @@ func TestFilterFilesByRegion(t *testing.T) {
EndKey: []byte("3332"),
},
},
- subfiles: []*logrestore.LogDataFileInfo{},
+ subfiles: []*logclient.LogDataFileInfo{},
err: nil,
},
{
@@ -151,7 +150,7 @@ func TestFilterFilesByRegion(t *testing.T) {
EndKey: []byte("3333"),
},
},
- subfiles: []*logrestore.LogDataFileInfo{
+ subfiles: []*logclient.LogDataFileInfo{
files[1],
},
err: nil,
@@ -163,7 +162,7 @@ func TestFilterFilesByRegion(t *testing.T) {
EndKey: []byte("5555"),
},
},
- subfiles: []*logrestore.LogDataFileInfo{
+ subfiles: []*logclient.LogDataFileInfo{
files[1],
},
err: nil,
@@ -175,7 +174,7 @@ func TestFilterFilesByRegion(t *testing.T) {
EndKey: nil,
},
},
- subfiles: []*logrestore.LogDataFileInfo{
+ subfiles: []*logclient.LogDataFileInfo{
files[1],
},
err: nil,
@@ -193,7 +192,7 @@ func TestFilterFilesByRegion(t *testing.T) {
}
for _, c := range testCases {
- subfile, err := fileimporter.FilterFilesByRegion(files, ranges, &c.r)
+ subfile, err := logclient.FilterFilesByRegion(files, ranges, &c.r)
require.Equal(t, err, c.err)
require.Equal(t, subfile, c.subfiles)
}
diff --git a/br/pkg/restore/log_restore/log_client.go b/br/pkg/restore/log_client/log_file_manager.go
similarity index 94%
rename from br/pkg/restore/log_restore/log_client.go
rename to br/pkg/restore/log_client/log_file_manager.go
index c64a24c00a68f..b5a30f44d806c 100644
--- a/br/pkg/restore/log_restore/log_client.go
+++ b/br/pkg/restore/log_client/log_file_manager.go
@@ -1,6 +1,6 @@
// Copyright 2022 PingCAP, Inc. Licensed under Apache-2.0.
-package logrestore
+package logclient
import (
"bytes"
@@ -50,14 +50,14 @@ type Log = *backuppb.DataFileInfo
type LogFileManager struct {
// startTS and restoreTS are used for kv file restore.
// TiKV will filter the key space that don't belong to [startTS, restoreTS].
- StartTS uint64
- RestoreTS uint64
+ startTS uint64
+ restoreTS uint64
// If the commitTS of txn-entry belong to [startTS, restoreTS],
// the startTS of txn-entry may be smaller than startTS.
// We need maintain and restore more entries in default cf
// (the startTS in these entries belong to [shiftStartTS, startTS]).
- ShiftStartTS uint64
+ shiftStartTS uint64
storage storage.ExternalStorage
helper *stream.MetadataHelper
@@ -83,8 +83,8 @@ type DDLMetaGroup struct {
// Generally the config cannot be changed during its lifetime.
func CreateLogFileManager(ctx context.Context, init LogFileManagerInit) (*LogFileManager, error) {
fm := &LogFileManager{
- StartTS: init.StartTS,
- RestoreTS: init.RestoreTS,
+ startTS: init.StartTS,
+ restoreTS: init.RestoreTS,
storage: init.Storage,
helper: stream.NewMetadataHelper(),
@@ -98,7 +98,7 @@ func CreateLogFileManager(ctx context.Context, init LogFileManagerInit) (*LogFil
}
func (rc *LogFileManager) ShiftTS() uint64 {
- return rc.ShiftStartTS
+ return rc.shiftStartTS
}
func (rc *LogFileManager) loadShiftTS(ctx context.Context) error {
@@ -115,7 +115,7 @@ func (rc *LogFileManager) loadShiftTS(ctx context.Context) error {
log.Info("read meta from storage and parse", zap.String("path", path), zap.Uint64("min-ts", m.MinTs),
zap.Uint64("max-ts", m.MaxTs), zap.Int32("meta-version", int32(m.MetaVersion)))
- ts, ok := stream.UpdateShiftTS(m, rc.StartTS, rc.RestoreTS)
+ ts, ok := stream.UpdateShiftTS(m, rc.startTS, rc.restoreTS)
shiftTS.Lock()
if ok && (!shiftTS.exists || shiftTS.value > ts) {
shiftTS.value = ts
@@ -129,15 +129,15 @@ func (rc *LogFileManager) loadShiftTS(ctx context.Context) error {
return err
}
if !shiftTS.exists {
- rc.ShiftStartTS = rc.StartTS
+ rc.shiftStartTS = rc.startTS
return nil
}
- rc.ShiftStartTS = shiftTS.value
+ rc.shiftStartTS = shiftTS.value
return nil
}
func (rc *LogFileManager) streamingMeta(ctx context.Context) (MetaIter, error) {
- return rc.streamingMetaByTS(ctx, rc.RestoreTS)
+ return rc.streamingMetaByTS(ctx, rc.restoreTS)
}
func (rc *LogFileManager) streamingMetaByTS(ctx context.Context, restoreTS uint64) (MetaIter, error) {
@@ -146,7 +146,7 @@ func (rc *LogFileManager) streamingMetaByTS(ctx context.Context, restoreTS uint6
return nil, err
}
filtered := iter.FilterOut(it, func(metadata *backuppb.Metadata) bool {
- return restoreTS < metadata.MinTs || metadata.MaxTs < rc.ShiftStartTS
+ return restoreTS < metadata.MinTs || metadata.MaxTs < rc.shiftStartTS
})
return filtered, nil
}
@@ -213,9 +213,9 @@ func (rc *LogFileManager) FilterDataFiles(ms MetaIter) LogIter {
// ShouldFilterOut checks whether a file should be filtered out via the current client.
func (rc *LogFileManager) ShouldFilterOut(d *backuppb.DataFileInfo) bool {
- return d.MinTs > rc.RestoreTS ||
- (d.Cf == stream.WriteCF && d.MaxTs < rc.StartTS) ||
- (d.Cf == stream.DefaultCF && d.MaxTs < rc.ShiftStartTS)
+ return d.MinTs > rc.restoreTS ||
+ (d.Cf == stream.WriteCF && d.MaxTs < rc.startTS) ||
+ (d.Cf == stream.DefaultCF && d.MaxTs < rc.shiftStartTS)
}
func (rc *LogFileManager) collectDDLFilesAndPrepareCache(
@@ -347,11 +347,11 @@ func (rc *LogFileManager) ReadAllEntries(
// The commitTs in write CF need be limited on [startTs, restoreTs].
// We can restore more key-value in default CF.
- if ts > rc.RestoreTS {
+ if ts > rc.restoreTS {
continue
- } else if file.Cf == stream.WriteCF && ts < rc.StartTS {
+ } else if file.Cf == stream.WriteCF && ts < rc.startTS {
continue
- } else if file.Cf == stream.DefaultCF && ts < rc.ShiftStartTS {
+ } else if file.Cf == stream.DefaultCF && ts < rc.shiftStartTS {
continue
}
diff --git a/br/pkg/restore/log_restore/log_client_test.go b/br/pkg/restore/log_client/log_file_manager_test.go
similarity index 95%
rename from br/pkg/restore/log_restore/log_client_test.go
rename to br/pkg/restore/log_client/log_file_manager_test.go
index 14be787d6921a..f4c2906a9f03f 100644
--- a/br/pkg/restore/log_restore/log_client_test.go
+++ b/br/pkg/restore/log_client/log_file_manager_test.go
@@ -3,7 +3,7 @@
// NOTE: we need to create client with only `storage` field.
// However adding a public API for that is weird, so this test uses the `restore` package instead of `restore_test`.
// Maybe we should refactor these APIs when possible.
-package logrestore_test
+package logclient_test
import (
"context"
@@ -19,7 +19,7 @@ import (
"github.com/pingcap/errors"
backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/log"
- logrestore "github.com/pingcap/tidb/br/pkg/restore/log_restore"
+ logclient "github.com/pingcap/tidb/br/pkg/restore/log_client"
"github.com/pingcap/tidb/br/pkg/storage"
"github.com/pingcap/tidb/br/pkg/stream"
"github.com/pingcap/tidb/br/pkg/utils/iter"
@@ -147,7 +147,7 @@ func (b *mockMetaBuilder) build(temp string) (*storage.LocalStorage, error) {
return local, err
}
-func (b *mockMetaBuilder) b(useV2 bool) (*storage.LocalStorage, string) {
+func (b *mockMetaBuilder) b(_ bool) (*storage.LocalStorage, string) {
path, err := b.createTempDir()
if err != nil {
panic(err)
@@ -227,14 +227,14 @@ func testReadMetaBetweenTSWithVersion(t *testing.T, m metaMaker) {
os.RemoveAll(temp)
}
}()
- init := logrestore.LogFileManagerInit{
+ init := logclient.LogFileManagerInit{
StartTS: c.startTS,
RestoreTS: c.endTS,
Storage: loc,
MetadataDownloadBatchSize: 32,
}
- cli, err := logrestore.CreateLogFileManager(ctx, init)
+ cli, err := logclient.CreateLogFileManager(ctx, init)
req.Equal(cli.ShiftTS(), c.expectedShiftTS)
req.NoError(err)
metas, err := cli.ReadStreamMeta(ctx)
@@ -461,7 +461,7 @@ func testFileManagerWithMeta(t *testing.T, m metaMaker) {
}
}()
ctx := context.Background()
- fm, err := logrestore.CreateLogFileManager(ctx, logrestore.LogFileManagerInit{
+ fm, err := logclient.CreateLogFileManager(ctx, logclient.LogFileManagerInit{
StartTS: start,
RestoreTS: end,
Storage: loc,
@@ -478,7 +478,7 @@ func testFileManagerWithMeta(t *testing.T, m metaMaker) {
ctx,
iter.Map(
datas,
- func(d *logrestore.LogDataFileInfo) *backuppb.DataFileInfo {
+ func(d *logclient.LogDataFileInfo) *backuppb.DataFileInfo {
return d.DataFileInfo
},
),
@@ -520,7 +520,7 @@ func TestFilterDataFiles(t *testing.T) {
os.RemoveAll(temp)
}
}()
- fm, err := logrestore.CreateLogFileManager(ctx, logrestore.LogFileManagerInit{
+ fm, err := logclient.CreateLogFileManager(ctx, logclient.LogFileManagerInit{
StartTS: 0,
RestoreTS: 10,
Storage: loc,
@@ -535,7 +535,7 @@ func TestFilterDataFiles(t *testing.T) {
}
metaIter := iter.FromSlice(metas)
files := iter.CollectAll(ctx, fm.FilterDataFiles(metaIter)).Item
- check := func(file *logrestore.LogDataFileInfo, metaKey string, goff, foff int) {
+ check := func(file *logclient.LogDataFileInfo, metaKey string, goff, foff int) {
req.Equal(file.MetaDataGroupName, metaKey)
req.Equal(file.OffsetInMetaGroup, goff)
req.Equal(file.OffsetInMergedGroup, foff)
diff --git a/br/pkg/restore/utils/log_file_map.go b/br/pkg/restore/log_client/log_file_map.go
similarity index 99%
rename from br/pkg/restore/utils/log_file_map.go
rename to br/pkg/restore/log_client/log_file_map.go
index 026b66d6174a1..db50c8391418b 100644
--- a/br/pkg/restore/utils/log_file_map.go
+++ b/br/pkg/restore/log_client/log_file_map.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package utils
+package logclient
// each 64 items constitute a bitmap unit
type bitMap map[int]uint64
diff --git a/br/pkg/restore/utils/log_file_map_test.go b/br/pkg/restore/log_client/log_file_map_test.go
similarity index 94%
rename from br/pkg/restore/utils/log_file_map_test.go
rename to br/pkg/restore/log_client/log_file_map_test.go
index 9e43febf49f7f..ab0e8920eb102 100644
--- a/br/pkg/restore/utils/log_file_map_test.go
+++ b/br/pkg/restore/log_client/log_file_map_test.go
@@ -12,14 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package utils_test
+package logclient_test
import (
"fmt"
"math/rand"
"testing"
- "github.com/pingcap/tidb/br/pkg/restore/utils"
+ logclient "github.com/pingcap/tidb/br/pkg/restore/log_client"
"github.com/stretchr/testify/require"
)
@@ -33,7 +33,7 @@ func TestLogFilesSkipMap(t *testing.T) {
)
for ratio < 1 {
- skipmap := utils.NewLogFilesSkipMap()
+ skipmap := logclient.NewLogFilesSkipMap()
nativemap := make(map[string]map[int]map[int]struct{})
count := 0
for i := 0; i < int(ratio*float64(metaNum*groupNum*fileNum)); i++ {
diff --git a/br/pkg/restore/main_test.go b/br/pkg/restore/log_client/main_test.go
similarity index 96%
rename from br/pkg/restore/main_test.go
rename to br/pkg/restore/log_client/main_test.go
index c566b421500d9..1ffa12894b356 100644
--- a/br/pkg/restore/main_test.go
+++ b/br/pkg/restore/log_client/main_test.go
@@ -1,4 +1,4 @@
-// Copyright 2021 PingCAP, Inc.
+// Copyright 2024 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package restore_test
+package logclient_test
import (
"fmt"
diff --git a/br/pkg/restore/log_restore/BUILD.bazel b/br/pkg/restore/log_restore/BUILD.bazel
deleted file mode 100644
index a720b5c1298d9..0000000000000
--- a/br/pkg/restore/log_restore/BUILD.bazel
+++ /dev/null
@@ -1,64 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
-
-go_library(
- name = "log_restore",
- srcs = [
- "log_client.go",
- "split.go",
- ],
- importpath = "github.com/pingcap/tidb/br/pkg/restore/log_restore",
- visibility = ["//visibility:public"],
- deps = [
- "//br/pkg/errors",
- "//br/pkg/restore/split",
- "//br/pkg/restore/utils",
- "//br/pkg/rtree",
- "//br/pkg/storage",
- "//br/pkg/stream",
- "//br/pkg/utils/iter",
- "//pkg/kv",
- "//pkg/tablecodec",
- "//pkg/util",
- "//pkg/util/codec",
- "//pkg/util/redact",
- "@com_github_pingcap_errors//:errors",
- "@com_github_pingcap_kvproto//pkg/brpb",
- "@com_github_pingcap_log//:log",
- "@org_golang_x_sync//errgroup",
- "@org_uber_go_zap//:zap",
- ],
-)
-
-go_test(
- name = "log_restore_test",
- timeout = "short",
- srcs = [
- "export_test.go",
- "import_test.go",
- "log_client_test.go",
- "split_test.go",
- ],
- embed = [":log_restore"],
- flaky = True,
- shard_count = 9,
- deps = [
- "//br/pkg/errors",
- "//br/pkg/restore/file_importer",
- "//br/pkg/restore/split",
- "//br/pkg/restore/utils",
- "//br/pkg/storage",
- "//br/pkg/stream",
- "//br/pkg/utils/iter",
- "//pkg/kv",
- "//pkg/tablecodec",
- "//pkg/util/codec",
- "@com_github_pingcap_errors//:errors",
- "@com_github_pingcap_kvproto//pkg/brpb",
- "@com_github_pingcap_kvproto//pkg/import_sstpb",
- "@com_github_pingcap_kvproto//pkg/metapb",
- "@com_github_pingcap_log//:log",
- "@com_github_stretchr_testify//require",
- "@org_uber_go_zap//:zap",
- "@org_uber_go_zap//zapcore",
- ],
-)
diff --git a/br/pkg/restore/misc.go b/br/pkg/restore/misc.go
new file mode 100644
index 0000000000000..4836587e892cc
--- /dev/null
+++ b/br/pkg/restore/misc.go
@@ -0,0 +1,123 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package restore
+
+import (
+ "context"
+
+ "github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/log"
+ "github.com/pingcap/tidb/br/pkg/logutil"
+ "github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/pkg/domain"
+ "github.com/pingcap/tidb/pkg/parser/model"
+ tidbutil "github.com/pingcap/tidb/pkg/util"
+ "github.com/tikv/client-go/v2/oracle"
+ pd "github.com/tikv/pd/client"
+ "go.uber.org/zap"
+)
+
+// deprecated parameter
+type Granularity string
+
+const (
+ FineGrained Granularity = "fine-grained"
+ CoarseGrained Granularity = "coarse-grained"
+)
+
+type UniqueTableName struct {
+ DB string
+ Table string
+}
+
+func TransferBoolToValue(enable bool) string {
+ if enable {
+ return "ON"
+ }
+ return "OFF"
+}
+
+// GetTableSchema returns the schema of a table from TiDB.
+func GetTableSchema(
+ dom *domain.Domain,
+ dbName model.CIStr,
+ tableName model.CIStr,
+) (*model.TableInfo, error) {
+ info := dom.InfoSchema()
+ table, err := info.TableByName(dbName, tableName)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ return table.Meta(), nil
+}
+
+// GetExistedUserDBs get dbs created or modified by users
+func GetExistedUserDBs(dom *domain.Domain) []*model.DBInfo {
+ databases := dom.InfoSchema().AllSchemas()
+ existedDatabases := make([]*model.DBInfo, 0, 16)
+ for _, db := range databases {
+ dbName := db.Name.L
+ if tidbutil.IsMemOrSysDB(dbName) {
+ continue
+ } else if dbName == "test" && len(db.Tables) == 0 {
+ // tidb create test db on fresh cluster
+ // if it's empty we don't take it as user db
+ continue
+ }
+ existedDatabases = append(existedDatabases, db)
+ }
+
+ return existedDatabases
+}
+
+// GetTS gets a new timestamp from PD.
+func GetTS(ctx context.Context, pdClient pd.Client) (uint64, error) {
+ p, l, err := pdClient.GetTS(ctx)
+ if err != nil {
+ return 0, errors.Trace(err)
+ }
+ restoreTS := oracle.ComposeTS(p, l)
+ return restoreTS, nil
+}
+
+// GetTSWithRetry gets a new timestamp with retry from PD.
+func GetTSWithRetry(ctx context.Context, pdClient pd.Client) (uint64, error) {
+ var (
+ startTS uint64
+ getTSErr error
+ retry uint
+ )
+
+ err := utils.WithRetry(ctx, func() error {
+ startTS, getTSErr = GetTS(ctx, pdClient)
+ failpoint.Inject("get-ts-error", func(val failpoint.Value) {
+ if val.(bool) && retry < 3 {
+ getTSErr = errors.Errorf("rpc error: code = Unknown desc = [PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster")
+ }
+ })
+
+ retry++
+ if getTSErr != nil {
+ log.Warn("failed to get TS, retry it", zap.Uint("retry time", retry), logutil.ShortError(getTSErr))
+ }
+ return getTSErr
+ }, utils.NewPDReqBackoffer())
+
+ if err != nil {
+ log.Error("failed to get TS", zap.Error(err))
+ }
+ return startTS, errors.Trace(err)
+}
diff --git a/br/pkg/restore/misc_test.go b/br/pkg/restore/misc_test.go
new file mode 100644
index 0000000000000..3e63753b0cb36
--- /dev/null
+++ b/br/pkg/restore/misc_test.go
@@ -0,0 +1,75 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package restore_test
+
+import (
+ "math"
+ "testing"
+
+ "github.com/pingcap/tidb/br/pkg/mock"
+ "github.com/pingcap/tidb/br/pkg/restore"
+ "github.com/pingcap/tidb/pkg/infoschema"
+ "github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/stretchr/testify/require"
+)
+
+func TestGetExistedUserDBs(t *testing.T) {
+ m, err := mock.NewCluster()
+ require.Nil(t, err)
+ defer m.Stop()
+ dom := m.Domain
+
+ dbs := restore.GetExistedUserDBs(dom)
+ require.Equal(t, 0, len(dbs))
+
+ builder, err := infoschema.NewBuilder(dom, nil, nil).InitWithDBInfos(
+ []*model.DBInfo{
+ {Name: model.NewCIStr("mysql")},
+ {Name: model.NewCIStr("test")},
+ },
+ nil, nil, 1)
+ require.Nil(t, err)
+ dom.MockInfoCacheAndLoadInfoSchema(builder.Build(math.MaxUint64))
+ dbs = restore.GetExistedUserDBs(dom)
+ require.Equal(t, 0, len(dbs))
+
+ builder, err = infoschema.NewBuilder(dom, nil, nil).InitWithDBInfos(
+ []*model.DBInfo{
+ {Name: model.NewCIStr("mysql")},
+ {Name: model.NewCIStr("test")},
+ {Name: model.NewCIStr("d1")},
+ },
+ nil, nil, 1)
+ require.Nil(t, err)
+ dom.MockInfoCacheAndLoadInfoSchema(builder.Build(math.MaxUint64))
+ dbs = restore.GetExistedUserDBs(dom)
+ require.Equal(t, 1, len(dbs))
+
+ builder, err = infoschema.NewBuilder(dom, nil, nil).InitWithDBInfos(
+ []*model.DBInfo{
+ {Name: model.NewCIStr("mysql")},
+ {Name: model.NewCIStr("d1")},
+ {
+ Name: model.NewCIStr("test"),
+ Tables: []*model.TableInfo{{ID: 1, Name: model.NewCIStr("t1"), State: model.StatePublic}},
+ State: model.StatePublic,
+ },
+ },
+ nil, nil, 1)
+ require.Nil(t, err)
+ dom.MockInfoCacheAndLoadInfoSchema(builder.Build(math.MaxUint64))
+ dbs = restore.GetExistedUserDBs(dom)
+ require.Equal(t, 2, len(dbs))
+}
diff --git a/br/pkg/restore/pipeline_items.go b/br/pkg/restore/pipeline_items.go
deleted file mode 100644
index 58c7b9b100a91..0000000000000
--- a/br/pkg/restore/pipeline_items.go
+++ /dev/null
@@ -1,442 +0,0 @@
-// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
-
-package restore
-
-import (
- "context"
- "sync"
- "time"
-
- "github.com/pingcap/errors"
- backuppb "github.com/pingcap/kvproto/pkg/brpb"
- "github.com/pingcap/log"
- "github.com/pingcap/tidb/br/pkg/glue"
- "github.com/pingcap/tidb/br/pkg/logutil"
- "github.com/pingcap/tidb/br/pkg/metautil"
- "github.com/pingcap/tidb/br/pkg/restore/utils"
- "github.com/pingcap/tidb/br/pkg/rtree"
- "github.com/pingcap/tidb/br/pkg/summary"
- "github.com/pingcap/tidb/pkg/parser/model"
- "github.com/pingcap/tidb/pkg/util"
- "go.uber.org/zap"
- "golang.org/x/sync/errgroup"
-)
-
-const (
- defaultChannelSize = 1024
-)
-
-// TableSink is the 'sink' of restored data by a sender.
-type TableSink interface {
- EmitTables(tables ...CreatedTable)
- EmitError(error)
- Close()
-}
-
-type chanTableSink struct {
- outCh chan<- []CreatedTable
- errCh chan<- error
-}
-
-func (sink chanTableSink) EmitTables(tables ...CreatedTable) {
- sink.outCh <- tables
-}
-
-func (sink chanTableSink) EmitError(err error) {
- sink.errCh <- err
-}
-
-func (sink chanTableSink) Close() {
- // ErrCh may has multi sender part, don't close it.
- close(sink.outCh)
-}
-
-// ContextManager is the struct to manage a TiKV 'context' for restore.
-// Batcher will call Enter when any table should be restore on batch,
-// so you can do some prepare work here(e.g. set placement rules for online restore).
-type ContextManager interface {
- // Enter make some tables 'enter' this context(a.k.a., prepare for restore).
- Enter(ctx context.Context, tables []CreatedTable) error
- // Leave make some tables 'leave' this context(a.k.a., restore is done, do some post-works).
- Leave(ctx context.Context, tables []CreatedTable) error
- // Close closes the context manager, sometimes when the manager is 'killed' and should do some cleanup
- // it would be call.
- Close(ctx context.Context)
-}
-
-// NewBRContextManager makes a BR context manager, that is,
-// set placement rules for online restore when enter(see ),
-// unset them when leave.
-func NewBRContextManager(client *Client) ContextManager {
- return &brContextManager{
- client: client,
-
- hasTable: make(map[int64]CreatedTable),
- }
-}
-
-type brContextManager struct {
- client *Client
-
- // This 'set' of table ID allow us to handle each table just once.
- hasTable map[int64]CreatedTable
- mu sync.Mutex
-}
-
-func (manager *brContextManager) Close(ctx context.Context) {
- tbls := make([]*model.TableInfo, 0, len(manager.hasTable))
- for _, tbl := range manager.hasTable {
- tbls = append(tbls, tbl.Table)
- }
- splitPostWork(ctx, manager.client, tbls)
-}
-
-func (manager *brContextManager) Enter(ctx context.Context, tables []CreatedTable) error {
- placementRuleTables := make([]*model.TableInfo, 0, len(tables))
- manager.mu.Lock()
- defer manager.mu.Unlock()
-
- for _, tbl := range tables {
- if _, ok := manager.hasTable[tbl.Table.ID]; !ok {
- placementRuleTables = append(placementRuleTables, tbl.Table)
- }
- manager.hasTable[tbl.Table.ID] = tbl
- }
-
- return splitPrepareWork(ctx, manager.client, placementRuleTables)
-}
-
-func (manager *brContextManager) Leave(ctx context.Context, tables []CreatedTable) error {
- manager.mu.Lock()
- defer manager.mu.Unlock()
- placementRuleTables := make([]*model.TableInfo, 0, len(tables))
-
- for _, table := range tables {
- placementRuleTables = append(placementRuleTables, table.Table)
- }
-
- splitPostWork(ctx, manager.client, placementRuleTables)
- log.Info("restore table done", ZapTables(tables))
- for _, tbl := range placementRuleTables {
- delete(manager.hasTable, tbl.ID)
- }
- return nil
-}
-
-func splitPostWork(ctx context.Context, client *Client, tables []*model.TableInfo) {
- err := client.ResetPlacementRules(ctx, tables)
- if err != nil {
- log.Warn("reset placement rules failed", zap.Error(err))
- return
- }
-}
-
-func splitPrepareWork(ctx context.Context, client *Client, tables []*model.TableInfo) error {
- err := client.SetupPlacementRules(ctx, tables)
- if err != nil {
- log.Error("setup placement rules failed", zap.Error(err))
- return errors.Trace(err)
- }
-
- err = client.WaitPlacementSchedule(ctx, tables)
- if err != nil {
- log.Error("wait placement schedule failed", zap.Error(err))
- return errors.Trace(err)
- }
- return nil
-}
-
-// CreatedTable is a table created on restore process,
-// but not yet filled with data.
-type CreatedTable struct {
- RewriteRule *utils.RewriteRules
- Table *model.TableInfo
- OldTable *metautil.Table
-}
-
-func DefaultOutputTableChan() chan *CreatedTable {
- return make(chan *CreatedTable, defaultChannelSize)
-}
-
-// TableWithRange is a CreatedTable that has been bind to some of key ranges.
-type TableWithRange struct {
- CreatedTable
-
- // Range has been rewrited by rewrite rules.
- Range []rtree.Range
-}
-
-type TableIDWithFiles struct {
- TableID int64
-
- Files []*backuppb.File
- // RewriteRules is the rewrite rules for the specify table.
- // because these rules belongs to the *one table*.
- // we can hold them here.
- RewriteRules *utils.RewriteRules
-}
-
-// Exhaust drains all remaining errors in the channel, into a slice of errors.
-func Exhaust(ec <-chan error) []error {
- out := make([]error, 0, len(ec))
- for {
- select {
- case err := <-ec:
- out = append(out, err)
- default:
- // errCh will NEVER be closed(ya see, it has multi sender-part),
- // so we just consume the current backlog of this channel, then return.
- return out
- }
- }
-}
-
-// BatchSender is the abstract of how the batcher send a batch.
-type BatchSender interface {
- // PutSink sets the sink of this sender, user to this interface promise
- // call this function at least once before first call to `RestoreBatch`.
- PutSink(sink TableSink)
- // RestoreBatch will send the restore request.
- RestoreBatch(ranges DrainResult)
- Close()
-}
-
-// TiKVRestorer is the minimal methods required for restoring.
-// It contains the primitive APIs extract from `restore.Client`, so some of arguments may seem redundant.
-// Maybe TODO: make a better abstraction?
-type TiKVRestorer interface {
- // SplitRanges split regions implicated by the ranges and rewrite rules.
- // After spliting, it also scatters the fresh regions.
- SplitRanges(ctx context.Context,
- ranges []rtree.Range,
- updateCh glue.Progress,
- isRawKv bool) error
- // RestoreSSTFiles import the files to the TiKV.
- RestoreSSTFiles(ctx context.Context,
- tableIDWithFiles []TableIDWithFiles,
- updateCh glue.Progress) error
-}
-
-type tikvSender struct {
- client TiKVRestorer
-
- updateCh glue.Progress
-
- sink TableSink
- inCh chan<- DrainResult
-
- wg *sync.WaitGroup
-
- tableWaiters *sync.Map
-}
-
-func (b *tikvSender) PutSink(sink TableSink) {
- // don't worry about visibility, since we will call this before first call to
- // RestoreBatch, which is a sync point.
- b.sink = sink
-}
-
-func (b *tikvSender) RestoreBatch(ranges DrainResult) {
- log.Info("restore batch: waiting ranges", zap.Int("range", len(b.inCh)))
- b.inCh <- ranges
-}
-
-// NewTiKVSender make a sender that send restore requests to TiKV.
-func NewTiKVSender(
- ctx context.Context,
- cli TiKVRestorer,
- updateCh glue.Progress,
- splitConcurrency uint,
- granularity string,
-) (BatchSender, error) {
- inCh := make(chan DrainResult, defaultChannelSize)
- midCh := make(chan drainResultAndDone, defaultChannelSize)
-
- sender := &tikvSender{
- client: cli,
- updateCh: updateCh,
- inCh: inCh,
- wg: new(sync.WaitGroup),
- tableWaiters: new(sync.Map),
- }
-
- sender.wg.Add(2)
- go sender.splitWorker(ctx, inCh, midCh, splitConcurrency)
- if granularity == string(CoarseGrained) {
- outCh := make(chan drainResultAndDone, defaultChannelSize)
- // block on splitting and scattering regions.
- // in coarse-grained mode, wait all regions are split and scattered is
- // no longer a time-consuming operation, then we can batch download files
- // as much as enough and reduce the time of blocking restore.
- go sender.blockPipelineWorker(ctx, midCh, outCh)
- go sender.restoreWorker(ctx, outCh)
- } else {
- go sender.restoreWorker(ctx, midCh)
- }
- return sender, nil
-}
-
-func (b *tikvSender) Close() {
- close(b.inCh)
- b.wg.Wait()
- log.Debug("tikv sender closed")
-}
-
-type drainResultAndDone struct {
- result DrainResult
- done func()
-}
-
-func (b *tikvSender) blockPipelineWorker(ctx context.Context,
- inCh <-chan drainResultAndDone,
- outCh chan<- drainResultAndDone,
-) {
- defer close(outCh)
- res := make([]drainResultAndDone, 0, defaultChannelSize)
- for dr := range inCh {
- res = append(res, dr)
- }
-
- for _, dr := range res {
- select {
- case <-ctx.Done():
- return
- default:
- outCh <- dr
- }
- }
-}
-
-func (b *tikvSender) splitWorker(ctx context.Context,
- ranges <-chan DrainResult,
- next chan<- drainResultAndDone,
- concurrency uint,
-) {
- defer log.Debug("split worker closed")
- eg, ectx := errgroup.WithContext(ctx)
- defer func() {
- b.wg.Done()
- if err := eg.Wait(); err != nil {
- b.sink.EmitError(err)
- }
- close(next)
- log.Info("TiKV Sender: split worker exits.")
- }()
-
- start := time.Now()
- defer func() {
- elapsed := time.Since(start)
- summary.CollectDuration("split region", elapsed)
- }()
-
- pool := util.NewWorkerPool(concurrency, "split")
- for {
- select {
- case <-ectx.Done():
- return
- case result, ok := <-ranges:
- if !ok {
- return
- }
- // When the batcher has sent all ranges from a table, it would
- // mark this table 'all done'(BlankTablesAfterSend), and then we can send it to checksum.
- //
- // When there a sole worker sequentially running those batch tasks, everything is fine, however,
- // in the context of multi-workers, that become buggy, for example:
- // |------table 1, ranges 1------|------table 1, ranges 2------|
- // The batcher send batches: [
- // {Ranges: ranges 1},
- // {Ranges: ranges 2, BlankTablesAfterSend: table 1}
- // ]
- // And there are two workers runs concurrently:
- // worker 1: {Ranges: ranges 1}
- // worker 2: {Ranges: ranges 2, BlankTablesAfterSend: table 1}
- // And worker 2 finished its job before worker 1 done. Note the table wasn't restored fully,
- // hence the checksum would fail.
- done := b.registerTableIsRestoring(result.TablesToSend)
- pool.ApplyOnErrorGroup(eg, func() error {
- err := b.client.SplitRanges(ectx, result.Ranges, b.updateCh, false)
- if err != nil {
- log.Error("failed on split range", rtree.ZapRanges(result.Ranges), zap.Error(err))
- return err
- }
- next <- drainResultAndDone{
- result: result,
- done: done,
- }
- return nil
- })
- }
- }
-}
-
-// registerTableIsRestoring marks some tables as 'current restoring'.
-// Returning a function that mark the restore has been done.
-func (b *tikvSender) registerTableIsRestoring(ts []CreatedTable) func() {
- wgs := make([]*sync.WaitGroup, 0, len(ts))
- for _, t := range ts {
- i, _ := b.tableWaiters.LoadOrStore(t.Table.ID, new(sync.WaitGroup))
- wg := i.(*sync.WaitGroup)
- wg.Add(1)
- wgs = append(wgs, wg)
- }
- return func() {
- for _, wg := range wgs {
- wg.Done()
- }
- }
-}
-
-// waitTablesDone block the current goroutine,
-// till all tables provided are no more ‘current restoring’.
-func (b *tikvSender) waitTablesDone(ts []CreatedTable) {
- for _, t := range ts {
- wg, ok := b.tableWaiters.LoadAndDelete(t.Table.ID)
- if !ok {
- log.Panic("bug! table done before register!",
- zap.Any("wait-table-map", b.tableWaiters),
- zap.Stringer("table", t.Table.Name))
- }
- wg.(*sync.WaitGroup).Wait()
- }
-}
-
-func (b *tikvSender) restoreWorker(ctx context.Context, ranges <-chan drainResultAndDone) {
- eg, ectx := errgroup.WithContext(ctx)
- defer func() {
- log.Info("TiKV Sender: restore worker prepare to close.")
- if err := eg.Wait(); err != nil {
- b.sink.EmitError(err)
- }
- b.sink.Close()
- b.wg.Done()
- log.Info("TiKV Sender: restore worker exits.")
- }()
- for {
- select {
- case <-ectx.Done():
- return
- case r, ok := <-ranges:
- if !ok {
- return
- }
-
- files := r.result.Files()
- // There has been a worker in the `RestoreSSTFiles` procedure.
- // Spawning a raw goroutine won't make too many requests to TiKV.
- eg.Go(func() error {
- e := b.client.RestoreSSTFiles(ectx, files, b.updateCh)
- if e != nil {
- log.Error("restore batch meet error", logutil.ShortError(e), zapTableIDWithFiles(files))
- r.done()
- return e
- }
- log.Info("restore batch done", rtree.ZapRanges(r.result.Ranges))
- r.done()
- b.waitTablesDone(r.result.BlankTablesAfterSend)
- b.sink.EmitTables(r.result.BlankTablesAfterSend...)
- return nil
- })
- }
- }
-}
diff --git a/br/pkg/restore/snap_client/BUILD.bazel b/br/pkg/restore/snap_client/BUILD.bazel
new file mode 100644
index 0000000000000..d8ff0e9f84bf0
--- /dev/null
+++ b/br/pkg/restore/snap_client/BUILD.bazel
@@ -0,0 +1,122 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+ name = "snap_client",
+ srcs = [
+ "batcher.go",
+ "client.go",
+ "context_manager.go",
+ "import.go",
+ "pipeline_items.go",
+ "systable_restore.go",
+ "zap.go",
+ ],
+ importpath = "github.com/pingcap/tidb/br/pkg/restore/snap_client",
+ visibility = ["//visibility:public"],
+ deps = [
+ "//br/pkg/checkpoint",
+ "//br/pkg/checksum",
+ "//br/pkg/conn",
+ "//br/pkg/conn/util",
+ "//br/pkg/errors",
+ "//br/pkg/glue",
+ "//br/pkg/logutil",
+ "//br/pkg/metautil",
+ "//br/pkg/pdutil",
+ "//br/pkg/restore",
+ "//br/pkg/restore/internal/import_client",
+ "//br/pkg/restore/internal/prealloc_db",
+ "//br/pkg/restore/internal/prealloc_table_id",
+ "//br/pkg/restore/internal/utils",
+ "//br/pkg/restore/split",
+ "//br/pkg/restore/utils",
+ "//br/pkg/rtree",
+ "//br/pkg/storage",
+ "//br/pkg/summary",
+ "//br/pkg/utils",
+ "//br/pkg/version",
+ "//pkg/bindinfo",
+ "//pkg/domain",
+ "//pkg/domain/infosync",
+ "//pkg/kv",
+ "//pkg/meta",
+ "//pkg/parser/model",
+ "//pkg/parser/mysql",
+ "//pkg/tablecodec",
+ "//pkg/util",
+ "//pkg/util/codec",
+ "//pkg/util/engine",
+ "//pkg/util/redact",
+ "//pkg/util/table-filter",
+ "@com_github_google_uuid//:uuid",
+ "@com_github_opentracing_opentracing_go//:opentracing-go",
+ "@com_github_pingcap_errors//:errors",
+ "@com_github_pingcap_failpoint//:failpoint",
+ "@com_github_pingcap_kvproto//pkg/brpb",
+ "@com_github_pingcap_kvproto//pkg/import_sstpb",
+ "@com_github_pingcap_kvproto//pkg/kvrpcpb",
+ "@com_github_pingcap_kvproto//pkg/metapb",
+ "@com_github_pingcap_log//:log",
+ "@com_github_tikv_client_go_v2//util",
+ "@com_github_tikv_pd_client//:client",
+ "@com_github_tikv_pd_client//http",
+ "@org_golang_google_grpc//codes",
+ "@org_golang_google_grpc//keepalive",
+ "@org_golang_google_grpc//status",
+ "@org_golang_x_exp//maps",
+ "@org_golang_x_sync//errgroup",
+ "@org_uber_go_multierr//:multierr",
+ "@org_uber_go_zap//:zap",
+ "@org_uber_go_zap//zapcore",
+ ],
+)
+
+go_test(
+ name = "snap_client_test",
+ timeout = "short",
+ srcs = [
+ "batcher_test.go",
+ "client_test.go",
+ "export_test.go",
+ "import_test.go",
+ "main_test.go",
+ "pipeline_items_test.go",
+ "systable_restore_test.go",
+ ],
+ embed = [":snap_client"],
+ flaky = True,
+ shard_count = 17,
+ deps = [
+ "//br/pkg/errors",
+ "//br/pkg/glue",
+ "//br/pkg/gluetidb",
+ "//br/pkg/logutil",
+ "//br/pkg/metautil",
+ "//br/pkg/mock",
+ "//br/pkg/restore",
+ "//br/pkg/restore/internal/import_client",
+ "//br/pkg/restore/utils",
+ "//br/pkg/rtree",
+ "//br/pkg/utils",
+ "//br/pkg/utiltest",
+ "//pkg/domain",
+ "//pkg/parser/model",
+ "//pkg/parser/mysql",
+ "//pkg/session",
+ "//pkg/tablecodec",
+ "//pkg/testkit/testsetup",
+ "//pkg/types",
+ "//pkg/util",
+ "//pkg/util/codec",
+ "@com_github_pingcap_errors//:errors",
+ "@com_github_pingcap_failpoint//:failpoint",
+ "@com_github_pingcap_kvproto//pkg/brpb",
+ "@com_github_pingcap_kvproto//pkg/import_sstpb",
+ "@com_github_pingcap_kvproto//pkg/metapb",
+ "@com_github_pingcap_log//:log",
+ "@com_github_stretchr_testify//require",
+ "@org_golang_x_exp//slices",
+ "@org_uber_go_goleak//:goleak",
+ "@org_uber_go_zap//:zap",
+ ],
+)
diff --git a/br/pkg/restore/batcher.go b/br/pkg/restore/snap_client/batcher.go
similarity index 99%
rename from br/pkg/restore/batcher.go
rename to br/pkg/restore/snap_client/batcher.go
index cf7c41a915209..8795a3044397c 100644
--- a/br/pkg/restore/batcher.go
+++ b/br/pkg/restore/snap_client/batcher.go
@@ -1,6 +1,6 @@
// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
-package restore
+package snapclient
import (
"context"
@@ -111,7 +111,7 @@ func NewBatcher(
errCh chan<- error,
updateCh glue.Progress,
) (*Batcher, chan *CreatedTable) {
- outCh := DefaultOutputTableChan()
+ outCh := defaultOutputTableChan()
sendChan := make(chan SendType, 2)
b := &Batcher{
sendErr: errCh,
@@ -401,7 +401,7 @@ func (b *Batcher) Send(ctx context.Context) {
drainResult := b.drainRanges()
tbs := drainResult.TablesToSend
ranges := drainResult.Ranges
- log.Info("restore batch start", rtree.ZapRanges(ranges), ZapTables(tbs))
+ log.Info("restore batch start", rtree.ZapRanges(ranges), zapTables(tbs))
// Leave is called at b.contextCleaner
if err := b.manager.Enter(ctx, drainResult.TablesToSend); err != nil {
b.sendErr <- err
diff --git a/br/pkg/restore/batcher_test.go b/br/pkg/restore/snap_client/batcher_test.go
similarity index 89%
rename from br/pkg/restore/batcher_test.go
rename to br/pkg/restore/snap_client/batcher_test.go
index e1ef4ab8b6a71..f18bf9ade1adb 100644
--- a/br/pkg/restore/batcher_test.go
+++ b/br/pkg/restore/snap_client/batcher_test.go
@@ -1,6 +1,6 @@
// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
-package restore_test
+package snapclient_test
import (
"bytes"
@@ -13,7 +13,7 @@ import (
"github.com/pingcap/kvproto/pkg/import_sstpb"
"github.com/pingcap/log"
"github.com/pingcap/tidb/br/pkg/metautil"
- "github.com/pingcap/tidb/br/pkg/restore"
+ snapclient "github.com/pingcap/tidb/br/pkg/restore/snap_client"
"github.com/pingcap/tidb/br/pkg/restore/utils"
"github.com/pingcap/tidb/br/pkg/rtree"
"github.com/pingcap/tidb/pkg/parser/model"
@@ -28,14 +28,14 @@ type drySender struct {
ranges []rtree.Range
nBatch int
- sink restore.TableSink
+ sink snapclient.TableSink
}
-func (sender *drySender) PutSink(sink restore.TableSink) {
+func (sender *drySender) PutSink(sink snapclient.TableSink) {
sender.sink = sink
}
-func (sender *drySender) RestoreBatch(ranges restore.DrainResult) {
+func (sender *drySender) RestoreBatch(ranges snapclient.DrainResult) {
sender.mu.Lock()
defer sender.mu.Unlock()
log.Info("fake restore range", rtree.ZapRanges(ranges.Ranges))
@@ -87,7 +87,7 @@ func newMockManager() *recordCurrentTableManager {
}
}
-func (manager *recordCurrentTableManager) Enter(_ context.Context, tables []restore.CreatedTable) error {
+func (manager *recordCurrentTableManager) Enter(_ context.Context, tables []snapclient.CreatedTable) error {
manager.lock.Lock()
defer manager.lock.Unlock()
for _, t := range tables {
@@ -97,7 +97,7 @@ func (manager *recordCurrentTableManager) Enter(_ context.Context, tables []rest
return nil
}
-func (manager *recordCurrentTableManager) Leave(_ context.Context, tables []restore.CreatedTable) error {
+func (manager *recordCurrentTableManager) Leave(_ context.Context, tables []snapclient.CreatedTable) error {
manager.lock.Lock()
defer manager.lock.Unlock()
for _, t := range tables {
@@ -110,7 +110,7 @@ func (manager *recordCurrentTableManager) Leave(_ context.Context, tables []rest
return nil
}
-func (manager *recordCurrentTableManager) Has(tables ...restore.TableWithRange) bool {
+func (manager *recordCurrentTableManager) Has(tables ...snapclient.TableWithRange) bool {
manager.lock.Lock()
defer manager.lock.Unlock()
ids := make([]int64, 0, len(tables))
@@ -153,15 +153,15 @@ func (sender *drySender) BatchCount() int {
return sender.nBatch
}
-func fakeTableWithRange(id int64, rngs []rtree.Range) restore.TableWithRange {
+func fakeTableWithRange(id int64, rngs []rtree.Range) snapclient.TableWithRange {
tbl := &metautil.Table{
DB: &model.DBInfo{},
Info: &model.TableInfo{
ID: id,
},
}
- tblWithRng := restore.TableWithRange{
- CreatedTable: restore.CreatedTable{
+ tblWithRng := snapclient.TableWithRange{
+ CreatedTable: snapclient.CreatedTable{
RewriteRule: utils.EmptyRewriteRule(),
Table: tbl.Info,
OldTable: tbl,
@@ -202,7 +202,7 @@ func TestBasic(t *testing.T) {
errCh := make(chan error, 8)
sender := newDrySender()
manager := newMockManager()
- batcher, _ := restore.NewBatcher(ctx, sender, manager, errCh, nil)
+ batcher, _ := snapclient.NewBatcher(ctx, sender, manager, errCh, nil)
batcher.SetThreshold(2)
tableRanges := [][]rtree.Range{
@@ -211,7 +211,7 @@ func TestBasic(t *testing.T) {
{fakeRange("caa", "cab"), fakeRange("cac", "cad")},
}
- simpleTables := []restore.TableWithRange{}
+ simpleTables := []snapclient.TableWithRange{}
for i, ranges := range tableRanges {
simpleTables = append(simpleTables, fakeTableWithRange(int64(i), ranges))
}
@@ -235,7 +235,7 @@ func TestAutoSend(t *testing.T) {
errCh := make(chan error, 8)
sender := newDrySender()
manager := newMockManager()
- batcher, _ := restore.NewBatcher(ctx, sender, manager, errCh, nil)
+ batcher, _ := snapclient.NewBatcher(ctx, sender, manager, errCh, nil)
batcher.SetThreshold(1024)
simpleTable := fakeTableWithRange(1, []rtree.Range{fakeRange("caa", "cab"), fakeRange("cac", "cad")})
@@ -266,7 +266,7 @@ func TestSplitRangeOnSameTable(t *testing.T) {
errCh := make(chan error, 8)
sender := newDrySender()
manager := newMockManager()
- batcher, _ := restore.NewBatcher(ctx, sender, manager, errCh, nil)
+ batcher, _ := snapclient.NewBatcher(ctx, sender, manager, errCh, nil)
batcher.SetThreshold(2)
simpleTable := fakeTableWithRange(1, []rtree.Range{
@@ -306,7 +306,7 @@ func TestRewriteRules(t *testing.T) {
fakeRewriteRules("c", "cpp"),
}
- tables := make([]restore.TableWithRange, 0, len(tableRanges))
+ tables := make([]snapclient.TableWithRange, 0, len(tableRanges))
for i, ranges := range tableRanges {
table := fakeTableWithRange(int64(i), ranges)
table.RewriteRule = rewriteRules[i]
@@ -317,7 +317,7 @@ func TestRewriteRules(t *testing.T) {
errCh := make(chan error, 8)
sender := newDrySender()
manager := newMockManager()
- batcher, _ := restore.NewBatcher(ctx, sender, manager, errCh, nil)
+ batcher, _ := snapclient.NewBatcher(ctx, sender, manager, errCh, nil)
batcher.SetThreshold(2)
batcher.Add(tables[0])
@@ -348,7 +348,7 @@ func TestBatcherLen(t *testing.T) {
errCh := make(chan error, 8)
sender := newDrySender()
manager := newMockManager()
- batcher, _ := restore.NewBatcher(ctx, sender, manager, errCh, nil)
+ batcher, _ := snapclient.NewBatcher(ctx, sender, manager, errCh, nil)
batcher.SetThreshold(15)
simpleTable := fakeTableWithRange(1, []rtree.Range{
diff --git a/br/pkg/restore/snap_client/client.go b/br/pkg/restore/snap_client/client.go
new file mode 100644
index 0000000000000..ac43584eba951
--- /dev/null
+++ b/br/pkg/restore/snap_client/client.go
@@ -0,0 +1,1227 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package snapclient
+
+import (
+ "bytes"
+ "cmp"
+ "context"
+ "crypto/tls"
+ "encoding/json"
+ "fmt"
+ "slices"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/opentracing/opentracing-go"
+ "github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
+ backuppb "github.com/pingcap/kvproto/pkg/brpb"
+ "github.com/pingcap/log"
+ "github.com/pingcap/tidb/br/pkg/checkpoint"
+ "github.com/pingcap/tidb/br/pkg/checksum"
+ "github.com/pingcap/tidb/br/pkg/conn"
+ "github.com/pingcap/tidb/br/pkg/conn/util"
+ berrors "github.com/pingcap/tidb/br/pkg/errors"
+ "github.com/pingcap/tidb/br/pkg/glue"
+ "github.com/pingcap/tidb/br/pkg/logutil"
+ "github.com/pingcap/tidb/br/pkg/metautil"
+ "github.com/pingcap/tidb/br/pkg/pdutil"
+ "github.com/pingcap/tidb/br/pkg/restore"
+ importclient "github.com/pingcap/tidb/br/pkg/restore/internal/import_client"
+ tidallocdb "github.com/pingcap/tidb/br/pkg/restore/internal/prealloc_db"
+ tidalloc "github.com/pingcap/tidb/br/pkg/restore/internal/prealloc_table_id"
+ internalutils "github.com/pingcap/tidb/br/pkg/restore/internal/utils"
+ "github.com/pingcap/tidb/br/pkg/restore/split"
+ restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
+ "github.com/pingcap/tidb/br/pkg/rtree"
+ "github.com/pingcap/tidb/br/pkg/storage"
+ "github.com/pingcap/tidb/br/pkg/summary"
+ "github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/br/pkg/version"
+ "github.com/pingcap/tidb/pkg/domain"
+ "github.com/pingcap/tidb/pkg/kv"
+ "github.com/pingcap/tidb/pkg/meta"
+ "github.com/pingcap/tidb/pkg/parser/model"
+ tidbutil "github.com/pingcap/tidb/pkg/util"
+ "github.com/pingcap/tidb/pkg/util/redact"
+ kvutil "github.com/tikv/client-go/v2/util"
+ pd "github.com/tikv/pd/client"
+ pdhttp "github.com/tikv/pd/client/http"
+ "go.uber.org/zap"
+ "golang.org/x/sync/errgroup"
+ "google.golang.org/grpc/keepalive"
+)
+
+const (
+ strictPlacementPolicyMode = "STRICT"
+ ignorePlacementPolicyMode = "IGNORE"
+
+ defaultDDLConcurrency = 16
+ maxSplitKeysOnce = 10240
+)
+
+const minBatchDdlSize = 1
+
+type SnapClient struct {
+ // Tool clients used by SnapClient
+ fileImporter *SnapFileImporter
+ pdClient pd.Client
+ pdHTTPClient pdhttp.Client
+
+ // User configurable parameters
+ cipher *backuppb.CipherInfo
+ concurrencyPerStore uint
+ keepaliveConf keepalive.ClientParameters
+ rateLimit uint64
+ tlsConf *tls.Config
+
+ switchCh chan struct{}
+
+ storeCount int
+ supportPolicy bool
+ workerPool *tidbutil.WorkerPool
+
+ noSchema bool
+ hasSpeedLimited bool
+
+ databases map[string]*metautil.Database
+ ddlJobs []*model.Job
+
+ // store tables need to rebase info like auto id and random id and so on after create table
+ rebasedTablesMap map[restore.UniqueTableName]bool
+
+ backupMeta *backuppb.BackupMeta
+
+ // TODO Remove this field or replace it with a []*DB,
+ // since https://github.com/pingcap/br/pull/377 needs more DBs to speed up DDL execution.
+ // And for now, we must inject a pool of DBs to `Client.GoCreateTables`, otherwise there would be a race condition.
+ // This is dirty: why we need DBs from different sources?
+ // By replace it with a []*DB, we can remove the dirty parameter of `Client.GoCreateTable`,
+ // along with them in some private functions.
+ // Before you do it, you can firstly read discussions at
+ // https://github.com/pingcap/br/pull/377#discussion_r446594501,
+ // this probably isn't as easy as it seems like (however, not hard, too :D)
+ db *tidallocdb.DB
+
+ // use db pool to speed up restoration in BR binary mode.
+ dbPool []*tidallocdb.DB
+
+ dom *domain.Domain
+
+ // correspond to --tidb-placement-mode config.
+ // STRICT(default) means policy related SQL can be executed in tidb.
+ // IGNORE means policy related SQL will be ignored.
+ policyMode string
+
+ // policy name -> policy info
+ policyMap *sync.Map
+
+ batchDdlSize uint
+
+ // if fullClusterRestore = true:
+ // - if there's system tables in the backup(backup data since br 5.1.0), the cluster should be a fresh cluster
+ // without user database or table. and system tables about privileges is restored together with user data.
+ // - if there no system tables in the backup(backup data from br < 5.1.0), restore all user data just like
+ // previous version did.
+ // if fullClusterRestore = false, restore all user data just like previous version did.
+ // fullClusterRestore = true when there is no explicit filter setting, and it's full restore or point command
+ // with a full backup data.
+ // todo: maybe change to an enum
+ // this feature is controlled by flag with-sys-table
+ fullClusterRestore bool
+
+ // see RestoreCommonConfig.WithSysTable
+ withSysTable bool
+
+ // the rewrite mode of the downloaded SST files in TiKV.
+ rewriteMode RewriteMode
+
+ // checkpoint information for snapshot restore
+ checkpointRunner *checkpoint.CheckpointRunner[checkpoint.RestoreKeyType, checkpoint.RestoreValueType]
+ checkpointChecksum map[int64]*checkpoint.ChecksumItem
+}
+
+// NewRestoreClient returns a new RestoreClient.
+func NewRestoreClient(
+ pdClient pd.Client,
+ pdHTTPCli pdhttp.Client,
+ tlsConf *tls.Config,
+ keepaliveConf keepalive.ClientParameters,
+) *SnapClient {
+ return &SnapClient{
+ pdClient: pdClient,
+ pdHTTPClient: pdHTTPCli,
+ tlsConf: tlsConf,
+ keepaliveConf: keepaliveConf,
+ switchCh: make(chan struct{}),
+ }
+}
+
+func (rc *SnapClient) closeConn() {
+ // rc.db can be nil in raw kv mode.
+ if rc.db != nil {
+ rc.db.Close()
+ }
+ for _, db := range rc.dbPool {
+ db.Close()
+ }
+}
+
+// Close a client.
+func (rc *SnapClient) Close() {
+ // close the connection, and it must be succeed when in SQL mode.
+ rc.closeConn()
+
+ if err := rc.fileImporter.Close(); err != nil {
+ log.Warn("failed to close file improter")
+ }
+
+ log.Info("Restore client closed")
+}
+
+func (rc *SnapClient) SetRateLimit(rateLimit uint64) {
+ rc.rateLimit = rateLimit
+}
+
+func (rc *SnapClient) SetCrypter(crypter *backuppb.CipherInfo) {
+ rc.cipher = crypter
+}
+
+// GetClusterID gets the cluster id from down-stream cluster.
+func (rc *SnapClient) GetClusterID(ctx context.Context) uint64 {
+ return rc.pdClient.GetClusterID(ctx)
+}
+
+func (rc *SnapClient) GetDomain() *domain.Domain {
+ return rc.dom
+}
+
+// GetTLSConfig returns the tls config.
+func (rc *SnapClient) GetTLSConfig() *tls.Config {
+ return rc.tlsConf
+}
+
+// GetSupportPolicy tells whether target tidb support placement policy.
+func (rc *SnapClient) GetSupportPolicy() bool {
+ return rc.supportPolicy
+}
+
+func (rc *SnapClient) updateConcurrency() {
+ // we believe 32 is large enough for download worker pool.
+ // it won't reach the limit if sst files distribute evenly.
+ // when restore memory usage is still too high, we should reduce concurrencyPerStore
+ // to sarifice some speed to reduce memory usage.
+ count := uint(rc.storeCount) * rc.concurrencyPerStore * 32
+ log.Info("download coarse worker pool", zap.Uint("size", count))
+ rc.workerPool = tidbutil.NewWorkerPool(count, "file")
+}
+
+// SetConcurrencyPerStore sets the concurrency of download files for each store.
+func (rc *SnapClient) SetConcurrencyPerStore(c uint) {
+ log.Info("per-store download worker pool", zap.Uint("size", c))
+ rc.concurrencyPerStore = c
+}
+
+func (rc *SnapClient) SetBatchDdlSize(batchDdlsize uint) {
+ rc.batchDdlSize = batchDdlsize
+}
+
+func (rc *SnapClient) GetBatchDdlSize() uint {
+ return rc.batchDdlSize
+}
+
+func (rc *SnapClient) SetWithSysTable(withSysTable bool) {
+ rc.withSysTable = withSysTable
+}
+
+// TODO: remove this check and return RewriteModeKeyspace
+func (rc *SnapClient) SetRewriteMode(ctx context.Context) {
+ if err := version.CheckClusterVersion(ctx, rc.pdClient, version.CheckVersionForKeyspaceBR); err != nil {
+ log.Warn("Keyspace BR is not supported in this cluster, fallback to legacy restore", zap.Error(err))
+ rc.rewriteMode = RewriteModeLegacy
+ } else {
+ rc.rewriteMode = RewriteModeKeyspace
+ }
+}
+
+func (rc *SnapClient) GetRewriteMode() RewriteMode {
+ return rc.rewriteMode
+}
+
+// SetPlacementPolicyMode to policy mode.
+func (rc *SnapClient) SetPlacementPolicyMode(withPlacementPolicy string) {
+ switch strings.ToUpper(withPlacementPolicy) {
+ case strictPlacementPolicyMode:
+ rc.policyMode = strictPlacementPolicyMode
+ case ignorePlacementPolicyMode:
+ rc.policyMode = ignorePlacementPolicyMode
+ default:
+ rc.policyMode = strictPlacementPolicyMode
+ }
+ log.Info("set placement policy mode", zap.String("mode", rc.policyMode))
+}
+
+// AllocTableIDs would pre-allocate the table's origin ID if exists, so that the TiKV doesn't need to rewrite the key in
+// the download stage.
+func (rc *SnapClient) AllocTableIDs(ctx context.Context, tables []*metautil.Table) error {
+ preallocedTableIDs := tidalloc.New(tables)
+ ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnBR)
+ err := kv.RunInNewTxn(ctx, rc.GetDomain().Store(), true, func(_ context.Context, txn kv.Transaction) error {
+ return preallocedTableIDs.Alloc(meta.NewMeta(txn))
+ })
+ if err != nil {
+ return err
+ }
+
+ log.Info("registering the table IDs", zap.Stringer("ids", preallocedTableIDs))
+ for i := range rc.dbPool {
+ rc.dbPool[i].RegisterPreallocatedIDs(preallocedTableIDs)
+ }
+ if rc.db != nil {
+ rc.db.RegisterPreallocatedIDs(preallocedTableIDs)
+ }
+ return nil
+}
+
+// InitCheckpoint initialize the checkpoint status for the cluster. If the cluster is
+// restored for the first time, it will initialize the checkpoint metadata. Otherwrise,
+// it will load checkpoint metadata and checkpoint ranges/checksum from the external
+// storage.
+func (rc *SnapClient) InitCheckpoint(
+ ctx context.Context,
+ s storage.ExternalStorage,
+ taskName string,
+ config *pdutil.ClusterConfig,
+ checkpointFirstRun bool,
+) (map[int64]map[string]struct{}, *pdutil.ClusterConfig, error) {
+ var (
+ // checkpoint sets distinguished by range key
+ checkpointSetWithTableID = make(map[int64]map[string]struct{})
+
+ checkpointClusterConfig *pdutil.ClusterConfig
+
+ err error
+ )
+
+ if !checkpointFirstRun {
+ // load the checkpoint since this is not the first time to restore
+ meta, err := checkpoint.LoadCheckpointMetadataForRestore(ctx, s, taskName)
+ if err != nil {
+ return checkpointSetWithTableID, nil, errors.Trace(err)
+ }
+
+ // The schedulers config is nil, so the restore-schedulers operation is just nil.
+ // Then the undo function would use the result undo of `remove schedulers` operation,
+ // instead of that in checkpoint meta.
+ if meta.SchedulersConfig != nil {
+ checkpointClusterConfig = meta.SchedulersConfig
+ }
+
+ // t1 is the latest time the checkpoint ranges persisted to the external storage.
+ t1, err := checkpoint.WalkCheckpointFileForRestore(ctx, s, rc.cipher, taskName, func(tableID int64, rangeKey checkpoint.RestoreValueType) {
+ checkpointSet, exists := checkpointSetWithTableID[tableID]
+ if !exists {
+ checkpointSet = make(map[string]struct{})
+ checkpointSetWithTableID[tableID] = checkpointSet
+ }
+ checkpointSet[rangeKey.RangeKey] = struct{}{}
+ })
+ if err != nil {
+ return checkpointSetWithTableID, nil, errors.Trace(err)
+ }
+ // t2 is the latest time the checkpoint checksum persisted to the external storage.
+ checkpointChecksum, t2, err := checkpoint.LoadCheckpointChecksumForRestore(ctx, s, taskName)
+ if err != nil {
+ return checkpointSetWithTableID, nil, errors.Trace(err)
+ }
+ rc.checkpointChecksum = checkpointChecksum
+ // use the later time to adjust the summary elapsed time.
+ if t1 > t2 {
+ summary.AdjustStartTimeToEarlierTime(t1)
+ } else {
+ summary.AdjustStartTimeToEarlierTime(t2)
+ }
+ } else {
+ // initialize the checkpoint metadata since it is the first time to restore.
+ meta := &checkpoint.CheckpointMetadataForRestore{}
+ // a nil config means undo function
+ if config != nil {
+ meta.SchedulersConfig = &pdutil.ClusterConfig{Schedulers: config.Schedulers, ScheduleCfg: config.ScheduleCfg}
+ }
+ if err = checkpoint.SaveCheckpointMetadataForRestore(ctx, s, meta, taskName); err != nil {
+ return checkpointSetWithTableID, nil, errors.Trace(err)
+ }
+ }
+
+ rc.checkpointRunner, err = checkpoint.StartCheckpointRunnerForRestore(ctx, s, rc.cipher, taskName)
+ return checkpointSetWithTableID, checkpointClusterConfig, errors.Trace(err)
+}
+
+func (rc *SnapClient) WaitForFinishCheckpoint(ctx context.Context, flush bool) {
+ if rc.checkpointRunner != nil {
+ rc.checkpointRunner.WaitForFinish(ctx, flush)
+ }
+}
+
+// makeDBPool makes a session pool with specficated size by sessionFactory.
+func makeDBPool(size uint, dbFactory func() (*tidallocdb.DB, error)) ([]*tidallocdb.DB, error) {
+ dbPool := make([]*tidallocdb.DB, 0, size)
+ for i := uint(0); i < size; i++ {
+ db, e := dbFactory()
+ if e != nil {
+ return dbPool, e
+ }
+ if db != nil {
+ dbPool = append(dbPool, db)
+ }
+ }
+ return dbPool, nil
+}
+
+// Init create db connection and domain for storage.
+func (rc *SnapClient) Init(g glue.Glue, store kv.Storage) error {
+ // setDB must happen after set PolicyMode.
+ // we will use policyMode to set session variables.
+ var err error
+ rc.db, rc.supportPolicy, err = tidallocdb.NewDB(g, store, rc.policyMode)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ rc.dom, err = g.GetDomain(store)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ // init backupMeta only for passing unit test
+ if rc.backupMeta == nil {
+ rc.backupMeta = new(backuppb.BackupMeta)
+ }
+
+ // There are different ways to create session between in binary and in SQL.
+ //
+ // Maybe allow user modify the DDL concurrency isn't necessary,
+ // because executing DDL is really I/O bound (or, algorithm bound?),
+ // and we cost most of time at waiting DDL jobs be enqueued.
+ // So these jobs won't be faster or slower when machine become faster or slower,
+ // hence make it a fixed value would be fine.
+ rc.dbPool, err = makeDBPool(defaultDDLConcurrency, func() (*tidallocdb.DB, error) {
+ db, _, err := tidallocdb.NewDB(g, store, rc.policyMode)
+ return db, err
+ })
+ if err != nil {
+ log.Warn("create session pool failed, we will send DDLs only by created sessions",
+ zap.Error(err),
+ zap.Int("sessionCount", len(rc.dbPool)),
+ )
+ }
+ return errors.Trace(err)
+}
+
+func (rc *SnapClient) initClients(ctx context.Context, backend *backuppb.StorageBackend, isRawKvMode bool, isTxnKvMode bool) error {
+ stores, err := conn.GetAllTiKVStoresWithRetry(ctx, rc.pdClient, util.SkipTiFlash)
+ if err != nil {
+ return errors.Annotate(err, "failed to get stores")
+ }
+ rc.storeCount = len(stores)
+ rc.updateConcurrency()
+
+ var splitClientOpts []split.ClientOptionalParameter
+ if isRawKvMode {
+ splitClientOpts = append(splitClientOpts, split.WithRawKV())
+ }
+ metaClient := split.NewClient(rc.pdClient, rc.pdHTTPClient, rc.tlsConf, maxSplitKeysOnce, rc.storeCount+1, splitClientOpts...)
+ importCli := importclient.NewImportClient(metaClient, rc.tlsConf, rc.keepaliveConf)
+ rc.fileImporter, err = NewSnapFileImporter(ctx, metaClient, importCli, backend, isRawKvMode, isTxnKvMode, stores, rc.rewriteMode, rc.concurrencyPerStore)
+ return errors.Trace(err)
+}
+
+func (rc *SnapClient) needLoadSchemas(backupMeta *backuppb.BackupMeta) bool {
+ return !(backupMeta.IsRawKv || backupMeta.IsTxnKv)
+}
+
+// InitBackupMeta loads schemas from BackupMeta to initialize RestoreClient.
+func (rc *SnapClient) InitBackupMeta(
+ c context.Context,
+ backupMeta *backuppb.BackupMeta,
+ backend *backuppb.StorageBackend,
+ reader *metautil.MetaReader,
+ loadStats bool) error {
+ if rc.needLoadSchemas(backupMeta) {
+ databases, err := metautil.LoadBackupTables(c, reader, loadStats)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ rc.databases = databases
+
+ var ddlJobs []*model.Job
+ // ddls is the bytes of json.Marshal
+ ddls, err := reader.ReadDDLs(c)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if len(ddls) != 0 {
+ err = json.Unmarshal(ddls, &ddlJobs)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ }
+ rc.ddlJobs = ddlJobs
+ }
+ rc.backupMeta = backupMeta
+ log.Info("load backupmeta", zap.Int("databases", len(rc.databases)), zap.Int("jobs", len(rc.ddlJobs)))
+
+ return rc.initClients(c, backend, backupMeta.IsRawKv, backupMeta.IsTxnKv)
+}
+
+// IsRawKvMode checks whether the backup data is in raw kv format, in which case transactional recover is forbidden.
+func (rc *SnapClient) IsRawKvMode() bool {
+ return rc.backupMeta.IsRawKv
+}
+
+// GetFilesInRawRange gets all files that are in the given range or intersects with the given range.
+func (rc *SnapClient) GetFilesInRawRange(startKey []byte, endKey []byte, cf string) ([]*backuppb.File, error) {
+ if !rc.IsRawKvMode() {
+ return nil, errors.Annotate(berrors.ErrRestoreModeMismatch, "the backup data is not in raw kv mode")
+ }
+
+ for _, rawRange := range rc.backupMeta.RawRanges {
+ // First check whether the given range is backup-ed. If not, we cannot perform the restore.
+ if rawRange.Cf != cf {
+ continue
+ }
+
+ if (len(rawRange.EndKey) > 0 && bytes.Compare(startKey, rawRange.EndKey) >= 0) ||
+ (len(endKey) > 0 && bytes.Compare(rawRange.StartKey, endKey) >= 0) {
+ // The restoring range is totally out of the current range. Skip it.
+ continue
+ }
+
+ if bytes.Compare(startKey, rawRange.StartKey) < 0 ||
+ utils.CompareEndKey(endKey, rawRange.EndKey) > 0 {
+ // Only partial of the restoring range is in the current backup-ed range. So the given range can't be fully
+ // restored.
+ return nil, errors.Annotatef(berrors.ErrRestoreRangeMismatch,
+ "the given range to restore [%s, %s) is not fully covered by the range that was backed up [%s, %s)",
+ redact.Key(startKey), redact.Key(endKey), redact.Key(rawRange.StartKey), redact.Key(rawRange.EndKey),
+ )
+ }
+
+ // We have found the range that contains the given range. Find all necessary files.
+ files := make([]*backuppb.File, 0)
+
+ for _, file := range rc.backupMeta.Files {
+ if file.Cf != cf {
+ continue
+ }
+
+ if len(file.EndKey) > 0 && bytes.Compare(file.EndKey, startKey) < 0 {
+ // The file is before the range to be restored.
+ continue
+ }
+ if len(endKey) > 0 && bytes.Compare(endKey, file.StartKey) <= 0 {
+ // The file is after the range to be restored.
+ // The specified endKey is exclusive, so when it equals to a file's startKey, the file is still skipped.
+ continue
+ }
+
+ files = append(files, file)
+ }
+
+ // There should be at most one backed up range that covers the restoring range.
+ return files, nil
+ }
+
+ return nil, errors.Annotate(berrors.ErrRestoreRangeMismatch, "no backup data in the range")
+}
+
+// ResetTS resets the timestamp of PD to a bigger value.
+func (rc *SnapClient) ResetTS(ctx context.Context, pdCtrl *pdutil.PdController) error {
+ restoreTS := rc.backupMeta.GetEndVersion()
+ log.Info("reset pd timestamp", zap.Uint64("ts", restoreTS))
+ return utils.WithRetry(ctx, func() error {
+ return pdCtrl.ResetTS(ctx, restoreTS)
+ }, utils.NewPDReqBackoffer())
+}
+
+// GetDatabases returns all databases.
+func (rc *SnapClient) GetDatabases() []*metautil.Database {
+ dbs := make([]*metautil.Database, 0, len(rc.databases))
+ for _, db := range rc.databases {
+ dbs = append(dbs, db)
+ }
+ return dbs
+}
+
+// HasBackedUpSysDB whether we have backed up system tables
+// br backs system tables up since 5.1.0
+func (rc *SnapClient) HasBackedUpSysDB() bool {
+ sysDBs := []string{"mysql", "sys"}
+ for _, db := range sysDBs {
+ temporaryDB := utils.TemporaryDBName(db)
+ _, backedUp := rc.databases[temporaryDB.O]
+ if backedUp {
+ return true
+ }
+ }
+ return false
+}
+
+// GetPlacementPolicies returns policies.
+func (rc *SnapClient) GetPlacementPolicies() (*sync.Map, error) {
+ policies := &sync.Map{}
+ for _, p := range rc.backupMeta.Policies {
+ policyInfo := &model.PolicyInfo{}
+ err := json.Unmarshal(p.Info, policyInfo)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ policies.Store(policyInfo.Name.L, policyInfo)
+ }
+ return policies, nil
+}
+
+// GetDDLJobs returns ddl jobs.
+func (rc *SnapClient) GetDDLJobs() []*model.Job {
+ return rc.ddlJobs
+}
+
+// SetPolicyMap set policyMap.
+func (rc *SnapClient) SetPolicyMap(p *sync.Map) {
+ rc.policyMap = p
+}
+
+// CreatePolicies creates all policies in full restore.
+func (rc *SnapClient) CreatePolicies(ctx context.Context, policyMap *sync.Map) error {
+ var err error
+ policyMap.Range(func(key, value any) bool {
+ e := rc.db.CreatePlacementPolicy(ctx, value.(*model.PolicyInfo))
+ if e != nil {
+ err = e
+ return false
+ }
+ return true
+ })
+ return err
+}
+
+// CreateDatabases creates databases. If the client has the db pool, it would create it.
+func (rc *SnapClient) CreateDatabases(ctx context.Context, dbs []*metautil.Database) error {
+ if rc.IsSkipCreateSQL() {
+ log.Info("skip create database")
+ return nil
+ }
+
+ if len(rc.dbPool) == 0 {
+ log.Info("create databases sequentially")
+ for _, db := range dbs {
+ err := rc.db.CreateDatabase(ctx, db.Info, rc.supportPolicy, rc.policyMap)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ }
+ return nil
+ }
+
+ log.Info("create databases in db pool", zap.Int("pool size", len(rc.dbPool)))
+ eg, ectx := errgroup.WithContext(ctx)
+ workers := tidbutil.NewWorkerPool(uint(len(rc.dbPool)), "DB DDL workers")
+ for _, db_ := range dbs {
+ db := db_
+ workers.ApplyWithIDInErrorGroup(eg, func(id uint64) error {
+ conn := rc.dbPool[id%uint64(len(rc.dbPool))]
+ return conn.CreateDatabase(ectx, db.Info, rc.supportPolicy, rc.policyMap)
+ })
+ }
+ return eg.Wait()
+}
+
+// generateRebasedTables generate a map[UniqueTableName]bool to represent tables that haven't updated table info.
+// there are two situations:
+// 1. tables that already exists in the restored cluster.
+// 2. tables that are created by executing ddl jobs.
+// so, only tables in incremental restoration will be added to the map
+func (rc *SnapClient) generateRebasedTables(tables []*metautil.Table) {
+ if !rc.IsIncremental() {
+ // in full restoration, all tables are created by Session.CreateTable, and all tables' info is updated.
+ rc.rebasedTablesMap = make(map[restore.UniqueTableName]bool)
+ return
+ }
+
+ rc.rebasedTablesMap = make(map[restore.UniqueTableName]bool, len(tables))
+ for _, table := range tables {
+ rc.rebasedTablesMap[restore.UniqueTableName{DB: table.DB.Name.String(), Table: table.Info.Name.String()}] = true
+ }
+}
+
+// getRebasedTables returns tables that may need to be rebase auto increment id or auto random id
+func (rc *SnapClient) getRebasedTables() map[restore.UniqueTableName]bool {
+ return rc.rebasedTablesMap
+}
+
+func (rc *SnapClient) createTables(
+ ctx context.Context,
+ db *tidallocdb.DB,
+ tables []*metautil.Table,
+ newTS uint64,
+) ([]CreatedTable, error) {
+ log.Info("client to create tables")
+ if rc.IsSkipCreateSQL() {
+ log.Info("skip create table and alter autoIncID")
+ } else {
+ err := db.CreateTables(ctx, tables, rc.getRebasedTables(), rc.supportPolicy, rc.policyMap)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ }
+ cts := make([]CreatedTable, 0, len(tables))
+ for _, table := range tables {
+ newTableInfo, err := restore.GetTableSchema(rc.dom, table.DB.Name, table.Info.Name)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ if newTableInfo.IsCommonHandle != table.Info.IsCommonHandle {
+ return nil, errors.Annotatef(berrors.ErrRestoreModeMismatch,
+ "Clustered index option mismatch. Restored cluster's @@tidb_enable_clustered_index should be %v (backup table = %v, created table = %v).",
+ restore.TransferBoolToValue(table.Info.IsCommonHandle),
+ table.Info.IsCommonHandle,
+ newTableInfo.IsCommonHandle)
+ }
+ rules := restoreutils.GetRewriteRules(newTableInfo, table.Info, newTS, true)
+ ct := CreatedTable{
+ RewriteRule: rules,
+ Table: newTableInfo,
+ OldTable: table,
+ }
+ log.Debug("new created tables", zap.Any("table", ct))
+ cts = append(cts, ct)
+ }
+ return cts, nil
+}
+
+func (rc *SnapClient) createTablesInWorkerPool(ctx context.Context, tables []*metautil.Table, newTS uint64, outCh chan<- CreatedTable) error {
+ eg, ectx := errgroup.WithContext(ctx)
+ rater := logutil.TraceRateOver(logutil.MetricTableCreatedCounter)
+ workers := tidbutil.NewWorkerPool(uint(len(rc.dbPool)), "Create Tables Worker")
+ numOfTables := len(tables)
+
+ for lastSent := 0; lastSent < numOfTables; lastSent += int(rc.batchDdlSize) {
+ end := min(lastSent+int(rc.batchDdlSize), len(tables))
+ log.Info("create tables", zap.Int("table start", lastSent), zap.Int("table end", end))
+
+ tableSlice := tables[lastSent:end]
+ workers.ApplyWithIDInErrorGroup(eg, func(id uint64) error {
+ db := rc.dbPool[id%uint64(len(rc.dbPool))]
+ cts, err := rc.createTables(ectx, db, tableSlice, newTS) // ddl job for [lastSent:i)
+ failpoint.Inject("restore-createtables-error", func(val failpoint.Value) {
+ if val.(bool) {
+ err = errors.New("sample error without extra message")
+ }
+ })
+ if err != nil {
+ log.Error("create tables fail", zap.Error(err))
+ return err
+ }
+ for _, ct := range cts {
+ log.Debug("table created and send to next",
+ zap.Int("output chan size", len(outCh)),
+ zap.Stringer("table", ct.OldTable.Info.Name),
+ zap.Stringer("database", ct.OldTable.DB.Name))
+ outCh <- ct
+ rater.Inc()
+ rater.L().Info("table created",
+ zap.Stringer("table", ct.OldTable.Info.Name),
+ zap.Stringer("database", ct.OldTable.DB.Name))
+ }
+ return err
+ })
+ }
+ return eg.Wait()
+}
+
+func (rc *SnapClient) createTable(
+ ctx context.Context,
+ db *tidallocdb.DB,
+ table *metautil.Table,
+ newTS uint64,
+) (CreatedTable, error) {
+ if rc.IsSkipCreateSQL() {
+ log.Info("skip create table and alter autoIncID", zap.Stringer("table", table.Info.Name))
+ } else {
+ err := db.CreateTable(ctx, table, rc.getRebasedTables(), rc.supportPolicy, rc.policyMap)
+ if err != nil {
+ return CreatedTable{}, errors.Trace(err)
+ }
+ }
+ newTableInfo, err := restore.GetTableSchema(rc.dom, table.DB.Name, table.Info.Name)
+ if err != nil {
+ return CreatedTable{}, errors.Trace(err)
+ }
+ if newTableInfo.IsCommonHandle != table.Info.IsCommonHandle {
+ return CreatedTable{}, errors.Annotatef(berrors.ErrRestoreModeMismatch,
+ "Clustered index option mismatch. Restored cluster's @@tidb_enable_clustered_index should be %v (backup table = %v, created table = %v).",
+ restore.TransferBoolToValue(table.Info.IsCommonHandle),
+ table.Info.IsCommonHandle,
+ newTableInfo.IsCommonHandle)
+ }
+ rules := restoreutils.GetRewriteRules(newTableInfo, table.Info, newTS, true)
+ et := CreatedTable{
+ RewriteRule: rules,
+ Table: newTableInfo,
+ OldTable: table,
+ }
+ return et, nil
+}
+
+func (rc *SnapClient) createTablesWithSoleDB(ctx context.Context,
+ createOneTable func(ctx context.Context, db *tidallocdb.DB, t *metautil.Table) error,
+ tables []*metautil.Table) error {
+ for _, t := range tables {
+ if err := createOneTable(ctx, rc.db, t); err != nil {
+ return errors.Trace(err)
+ }
+ }
+ return nil
+}
+
+func (rc *SnapClient) createTablesWithDBPool(ctx context.Context,
+ createOneTable func(ctx context.Context, db *tidallocdb.DB, t *metautil.Table) error,
+ tables []*metautil.Table) error {
+ eg, ectx := errgroup.WithContext(ctx)
+ workers := tidbutil.NewWorkerPool(uint(len(rc.dbPool)), "DDL workers")
+ for _, t := range tables {
+ table := t
+ workers.ApplyWithIDInErrorGroup(eg, func(id uint64) error {
+ db := rc.dbPool[id%uint64(len(rc.dbPool))]
+ return createOneTable(ectx, db, table)
+ })
+ }
+ return eg.Wait()
+}
+
+// InitFullClusterRestore init fullClusterRestore and set SkipGrantTable as needed
+func (rc *SnapClient) InitFullClusterRestore(explicitFilter bool) {
+ rc.fullClusterRestore = !explicitFilter && rc.IsFull()
+
+ log.Info("full cluster restore", zap.Bool("value", rc.fullClusterRestore))
+}
+
+func (rc *SnapClient) IsFullClusterRestore() bool {
+ return rc.fullClusterRestore
+}
+
+// IsFull returns whether this backup is full.
+func (rc *SnapClient) IsFull() bool {
+ failpoint.Inject("mock-incr-backup-data", func() {
+ failpoint.Return(false)
+ })
+ return !rc.IsIncremental()
+}
+
+// IsIncremental returns whether this backup is incremental.
+func (rc *SnapClient) IsIncremental() bool {
+ return !(rc.backupMeta.StartVersion == rc.backupMeta.EndVersion ||
+ rc.backupMeta.StartVersion == 0)
+}
+
+// NeedCheckFreshCluster is every time. except restore from a checkpoint or user has not set filter argument.
+func (rc *SnapClient) NeedCheckFreshCluster(ExplicitFilter bool, firstRun bool) bool {
+ return rc.IsFull() && !ExplicitFilter && firstRun
+}
+
+// EnableSkipCreateSQL sets switch of skip create schema and tables.
+func (rc *SnapClient) EnableSkipCreateSQL() {
+ rc.noSchema = true
+}
+
+// IsSkipCreateSQL returns whether we need skip create schema and tables in restore.
+func (rc *SnapClient) IsSkipCreateSQL() bool {
+ return rc.noSchema
+}
+
+// CheckTargetClusterFresh check whether the target cluster is fresh or not
+// if there's no user dbs or tables, we take it as a fresh cluster, although
+// user may have created some users or made other changes.
+func (rc *SnapClient) CheckTargetClusterFresh(ctx context.Context) error {
+ log.Info("checking whether target cluster is fresh")
+ userDBs := restore.GetExistedUserDBs(rc.dom)
+ if len(userDBs) == 0 {
+ return nil
+ }
+
+ const maxPrintCount = 10
+ userTableOrDBNames := make([]string, 0, maxPrintCount+1)
+ addName := func(name string) bool {
+ if len(userTableOrDBNames) == maxPrintCount {
+ userTableOrDBNames = append(userTableOrDBNames, "...")
+ return false
+ }
+ userTableOrDBNames = append(userTableOrDBNames, name)
+ return true
+ }
+outer:
+ for _, db := range userDBs {
+ if !addName(db.Name.L) {
+ break outer
+ }
+ for _, tbl := range db.Tables {
+ if !addName(tbl.Name.L) {
+ break outer
+ }
+ }
+ }
+ log.Error("not fresh cluster", zap.Strings("user tables", userTableOrDBNames))
+ return errors.Annotate(berrors.ErrRestoreNotFreshCluster, "user db/tables: "+strings.Join(userTableOrDBNames, ", "))
+}
+
+// ExecDDLs executes the queries of the ddl jobs.
+func (rc *SnapClient) ExecDDLs(ctx context.Context, ddlJobs []*model.Job) error {
+ // Sort the ddl jobs by schema version in ascending order.
+ slices.SortFunc(ddlJobs, func(i, j *model.Job) int {
+ return cmp.Compare(i.BinlogInfo.SchemaVersion, j.BinlogInfo.SchemaVersion)
+ })
+
+ for _, job := range ddlJobs {
+ err := rc.db.ExecDDL(ctx, job)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ log.Info("execute ddl query",
+ zap.String("db", job.SchemaName),
+ zap.String("query", job.Query),
+ zap.Int64("historySchemaVersion", job.BinlogInfo.SchemaVersion))
+ }
+ return nil
+}
+
+func (rc *SnapClient) ResetSpeedLimit(ctx context.Context) error {
+ rc.hasSpeedLimited = false
+ err := rc.setSpeedLimit(ctx, 0)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ return nil
+}
+
+func (rc *SnapClient) setSpeedLimit(ctx context.Context, rateLimit uint64) error {
+ if !rc.hasSpeedLimited {
+ stores, err := util.GetAllTiKVStores(ctx, rc.pdClient, util.SkipTiFlash)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ eg, ectx := errgroup.WithContext(ctx)
+ for _, store := range stores {
+ if err := ectx.Err(); err != nil {
+ return errors.Trace(err)
+ }
+
+ finalStore := store
+ rc.workerPool.ApplyOnErrorGroup(eg,
+ func() error {
+ err := rc.fileImporter.SetDownloadSpeedLimit(ectx, finalStore.GetId(), rateLimit)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ return nil
+ })
+ }
+
+ if err := eg.Wait(); err != nil {
+ return errors.Trace(err)
+ }
+ rc.hasSpeedLimited = true
+ }
+ return nil
+}
+
+func getFileRangeKey(f string) string {
+ // the backup date file pattern is `{store_id}_{region_id}_{epoch_version}_{key}_{ts}_{cf}.sst`
+ // so we need to compare with out the `_{cf}.sst` suffix
+ idx := strings.LastIndex(f, "_")
+ if idx < 0 {
+ panic(fmt.Sprintf("invalid backup data file name: '%s'", f))
+ }
+
+ return f[:idx]
+}
+
+// isFilesBelongToSameRange check whether two files are belong to the same range with different cf.
+func isFilesBelongToSameRange(f1, f2 string) bool {
+ return getFileRangeKey(f1) == getFileRangeKey(f2)
+}
+
+func drainFilesByRange(files []*backuppb.File) ([]*backuppb.File, []*backuppb.File) {
+ if len(files) == 0 {
+ return nil, nil
+ }
+ idx := 1
+ for idx < len(files) {
+ if !isFilesBelongToSameRange(files[idx-1].Name, files[idx].Name) {
+ break
+ }
+ idx++
+ }
+
+ return files[:idx], files[idx:]
+}
+
+// RestoreSSTFiles tries to restore the files.
+func (rc *SnapClient) RestoreSSTFiles(
+ ctx context.Context,
+ tableIDWithFiles []TableIDWithFiles,
+ updateCh glue.Progress,
+) (err error) {
+ start := time.Now()
+ fileCount := 0
+ defer func() {
+ elapsed := time.Since(start)
+ if err == nil {
+ log.Info("Restore files", zap.Duration("take", elapsed))
+ summary.CollectSuccessUnit("files", fileCount, elapsed)
+ }
+ }()
+
+ log.Debug("start to restore files", zap.Int("files", fileCount))
+
+ if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
+ span1 := span.Tracer().StartSpan("Client.RestoreSSTFiles", opentracing.ChildOf(span.Context()))
+ defer span1.Finish()
+ ctx = opentracing.ContextWithSpan(ctx, span1)
+ }
+
+ eg, ectx := errgroup.WithContext(ctx)
+ err = rc.setSpeedLimit(ctx, rc.rateLimit)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ var rangeFiles []*backuppb.File
+ var leftFiles []*backuppb.File
+LOOPFORTABLE:
+ for _, tableIDWithFile := range tableIDWithFiles {
+ tableID := tableIDWithFile.TableID
+ files := tableIDWithFile.Files
+ rules := tableIDWithFile.RewriteRules
+ fileCount += len(files)
+ for rangeFiles, leftFiles = drainFilesByRange(files); len(rangeFiles) != 0; rangeFiles, leftFiles = drainFilesByRange(leftFiles) {
+ if ectx.Err() != nil {
+ log.Warn("Restoring encountered error and already stopped, give up remained files.",
+ zap.Int("remained", len(leftFiles)),
+ logutil.ShortError(ectx.Err()))
+ // We will fetch the error from the errgroup then (If there were).
+ // Also note if the parent context has been canceled or something,
+ // breaking here directly is also a reasonable behavior.
+ break LOOPFORTABLE
+ }
+ filesReplica := rangeFiles
+ rc.fileImporter.WaitUntilUnblock()
+ rc.workerPool.ApplyOnErrorGroup(eg, func() (restoreErr error) {
+ fileStart := time.Now()
+ defer func() {
+ if restoreErr == nil {
+ log.Info("import files done", logutil.Files(filesReplica),
+ zap.Duration("take", time.Since(fileStart)))
+ updateCh.Inc()
+ }
+ }()
+ if importErr := rc.fileImporter.ImportSSTFiles(ectx, filesReplica, rules, rc.cipher, rc.dom.Store().GetCodec().GetAPIVersion()); importErr != nil {
+ return errors.Trace(importErr)
+ }
+
+ // the data of this range has been import done
+ if rc.checkpointRunner != nil && len(filesReplica) > 0 {
+ rangeKey := getFileRangeKey(filesReplica[0].Name)
+ // The checkpoint range shows this ranges of kvs has been restored into
+ // the table corresponding to the table-id.
+ if err := checkpoint.AppendRangesForRestore(ectx, rc.checkpointRunner, tableID, rangeKey); err != nil {
+ return errors.Trace(err)
+ }
+ }
+ return nil
+ })
+ }
+ }
+
+ if err := eg.Wait(); err != nil {
+ summary.CollectFailureUnit("file", err)
+ log.Error(
+ "restore files failed",
+ zap.Error(err),
+ )
+ return errors.Trace(err)
+ }
+ // Once the parent context canceled and there is no task running in the errgroup,
+ // we may break the for loop without error in the errgroup. (Will this happen?)
+ // At that time, return the error in the context here.
+ return ctx.Err()
+}
+
+func (rc *SnapClient) execChecksum(
+ ctx context.Context,
+ tbl *CreatedTable,
+ kvClient kv.Client,
+ concurrency uint,
+) error {
+ logger := log.L().With(
+ zap.String("db", tbl.OldTable.DB.Name.O),
+ zap.String("table", tbl.OldTable.Info.Name.O),
+ )
+
+ if tbl.OldTable.NoChecksum() {
+ logger.Warn("table has no checksum, skipping checksum")
+ return nil
+ }
+
+ if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
+ span1 := span.Tracer().StartSpan("Client.execChecksum", opentracing.ChildOf(span.Context()))
+ defer span1.Finish()
+ ctx = opentracing.ContextWithSpan(ctx, span1)
+ }
+
+ item, exists := rc.checkpointChecksum[tbl.Table.ID]
+ if !exists {
+ startTS, err := restore.GetTSWithRetry(ctx, rc.pdClient)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ exe, err := checksum.NewExecutorBuilder(tbl.Table, startTS).
+ SetOldTable(tbl.OldTable).
+ SetConcurrency(concurrency).
+ SetOldKeyspace(tbl.RewriteRule.OldKeyspace).
+ SetNewKeyspace(tbl.RewriteRule.NewKeyspace).
+ SetExplicitRequestSourceType(kvutil.ExplicitTypeBR).
+ Build()
+ if err != nil {
+ return errors.Trace(err)
+ }
+ checksumResp, err := exe.Execute(ctx, kvClient, func() {
+ // TODO: update progress here.
+ })
+ if err != nil {
+ return errors.Trace(err)
+ }
+ item = &checkpoint.ChecksumItem{
+ TableID: tbl.Table.ID,
+ Crc64xor: checksumResp.Checksum,
+ TotalKvs: checksumResp.TotalKvs,
+ TotalBytes: checksumResp.TotalBytes,
+ }
+ if rc.checkpointRunner != nil {
+ err = rc.checkpointRunner.FlushChecksumItem(ctx, item)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ }
+ }
+ table := tbl.OldTable
+ if item.Crc64xor != table.Crc64Xor ||
+ item.TotalKvs != table.TotalKvs ||
+ item.TotalBytes != table.TotalBytes {
+ logger.Error("failed in validate checksum",
+ zap.Uint64("origin tidb crc64", table.Crc64Xor),
+ zap.Uint64("calculated crc64", item.Crc64xor),
+ zap.Uint64("origin tidb total kvs", table.TotalKvs),
+ zap.Uint64("calculated total kvs", item.TotalKvs),
+ zap.Uint64("origin tidb total bytes", table.TotalBytes),
+ zap.Uint64("calculated total bytes", item.TotalBytes),
+ )
+ return errors.Annotate(berrors.ErrRestoreChecksumMismatch, "failed to validate checksum")
+ }
+ logger.Info("success in validate checksum")
+ return nil
+}
+
+func (rc *SnapClient) WaitForFilesRestored(ctx context.Context, files []*backuppb.File, updateCh glue.Progress) error {
+ errCh := make(chan error, len(files))
+ eg, ectx := errgroup.WithContext(ctx)
+ defer close(errCh)
+
+ for _, file := range files {
+ fileReplica := file
+ rc.workerPool.ApplyOnErrorGroup(eg,
+ func() error {
+ defer func() {
+ log.Info("import sst files done", logutil.Files(files))
+ updateCh.Inc()
+ }()
+ return rc.fileImporter.ImportSSTFiles(ectx, []*backuppb.File{fileReplica}, restoreutils.EmptyRewriteRule(), rc.cipher, rc.backupMeta.ApiVersion)
+ })
+ }
+ if err := eg.Wait(); err != nil {
+ return errors.Trace(err)
+ }
+ return nil
+}
+
+// RestoreRaw tries to restore raw keys in the specified range.
+func (rc *SnapClient) RestoreRaw(
+ ctx context.Context, startKey []byte, endKey []byte, files []*backuppb.File, updateCh glue.Progress,
+) error {
+ start := time.Now()
+ defer func() {
+ elapsed := time.Since(start)
+ log.Info("Restore Raw",
+ logutil.Key("startKey", startKey),
+ logutil.Key("endKey", endKey),
+ zap.Duration("take", elapsed))
+ }()
+ err := rc.fileImporter.SetRawRange(startKey, endKey)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ err = rc.WaitForFilesRestored(ctx, files, updateCh)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ log.Info(
+ "finish to restore raw range",
+ logutil.Key("startKey", startKey),
+ logutil.Key("endKey", endKey),
+ )
+ return nil
+}
+
+// SplitRanges implements TiKVRestorer. It splits region by
+// data range after rewrite.
+func (rc *SnapClient) SplitRanges(
+ ctx context.Context,
+ ranges []rtree.Range,
+ updateCh glue.Progress,
+ isRawKv bool,
+) error {
+ splitClientOpts := make([]split.ClientOptionalParameter, 0, 2)
+ splitClientOpts = append(splitClientOpts, split.WithOnSplit(func(keys [][]byte) {
+ for range keys {
+ updateCh.Inc()
+ }
+ }))
+ if isRawKv {
+ splitClientOpts = append(splitClientOpts, split.WithRawKV())
+ }
+
+ splitter := internalutils.NewRegionSplitter(split.NewClient(
+ rc.pdClient,
+ rc.pdHTTPClient,
+ rc.tlsConf,
+ maxSplitKeysOnce,
+ rc.storeCount+1,
+ splitClientOpts...,
+ ))
+
+ return splitter.ExecuteSplit(ctx, ranges)
+}
diff --git a/br/pkg/restore/snap_client/client_test.go b/br/pkg/restore/snap_client/client_test.go
new file mode 100644
index 0000000000000..12b91b4937556
--- /dev/null
+++ b/br/pkg/restore/snap_client/client_test.go
@@ -0,0 +1,350 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package snapclient_test
+
+import (
+ "context"
+ "fmt"
+ "math"
+ "slices"
+ "sort"
+ "strconv"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/kvproto/pkg/import_sstpb"
+ "github.com/pingcap/kvproto/pkg/metapb"
+ berrors "github.com/pingcap/tidb/br/pkg/errors"
+ "github.com/pingcap/tidb/br/pkg/gluetidb"
+ "github.com/pingcap/tidb/br/pkg/metautil"
+ "github.com/pingcap/tidb/br/pkg/mock"
+ importclient "github.com/pingcap/tidb/br/pkg/restore/internal/import_client"
+ snapclient "github.com/pingcap/tidb/br/pkg/restore/snap_client"
+ "github.com/pingcap/tidb/br/pkg/utiltest"
+ "github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/pingcap/tidb/pkg/parser/mysql"
+ "github.com/pingcap/tidb/pkg/tablecodec"
+ "github.com/pingcap/tidb/pkg/types"
+ "github.com/stretchr/testify/require"
+)
+
+var mc *mock.Cluster
+
+func TestCreateTables(t *testing.T) {
+ m := mc
+ g := gluetidb.New()
+ client := snapclient.NewRestoreClient(m.PDClient, m.PDHTTPCli, nil, utiltest.DefaultTestKeepaliveCfg)
+ err := client.Init(g, m.Storage)
+ require.NoError(t, err)
+
+ info, err := m.Domain.GetSnapshotInfoSchema(math.MaxUint64)
+ require.NoError(t, err)
+ dbSchema, isExist := info.SchemaByName(model.NewCIStr("test"))
+ require.True(t, isExist)
+
+ client.SetBatchDdlSize(1)
+ tables := make([]*metautil.Table, 4)
+ intField := types.NewFieldType(mysql.TypeLong)
+ intField.SetCharset("binary")
+ for i := len(tables) - 1; i >= 0; i-- {
+ tables[i] = &metautil.Table{
+ DB: dbSchema,
+ Info: &model.TableInfo{
+ ID: int64(i),
+ Name: model.NewCIStr("test" + strconv.Itoa(i)),
+ Columns: []*model.ColumnInfo{{
+ ID: 1,
+ Name: model.NewCIStr("id"),
+ FieldType: *intField,
+ State: model.StatePublic,
+ }},
+ Charset: "utf8mb4",
+ Collate: "utf8mb4_bin",
+ },
+ }
+ }
+ rules, newTables, err := client.CreateTables(m.Domain, tables, 0)
+ require.NoError(t, err)
+ // make sure tables and newTables have same order
+ for i, tbl := range tables {
+ require.Equal(t, tbl.Info.Name, newTables[i].Name)
+ }
+ for _, nt := range newTables {
+ require.Regexp(t, "test[0-3]", nt.Name.String())
+ }
+ oldTableIDExist := make(map[int64]bool)
+ newTableIDExist := make(map[int64]bool)
+ for _, tr := range rules.Data {
+ oldTableID := tablecodec.DecodeTableID(tr.GetOldKeyPrefix())
+ require.False(t, oldTableIDExist[oldTableID], "table rule duplicate old table id")
+ oldTableIDExist[oldTableID] = true
+
+ newTableID := tablecodec.DecodeTableID(tr.GetNewKeyPrefix())
+ require.False(t, newTableIDExist[newTableID], "table rule duplicate new table id")
+ newTableIDExist[newTableID] = true
+ }
+
+ for i := 0; i < len(tables); i++ {
+ require.True(t, oldTableIDExist[int64(i)], "table rule does not exist")
+ }
+}
+
+func getStartedMockedCluster(t *testing.T) *mock.Cluster {
+ t.Helper()
+ cluster, err := mock.NewCluster()
+ require.NoError(t, err)
+ err = cluster.Start()
+ require.NoError(t, err)
+ return cluster
+}
+
+func TestNeedCheckTargetClusterFresh(t *testing.T) {
+ // cannot use shared `mc`, other parallel case may change it.
+ cluster := getStartedMockedCluster(t)
+ defer cluster.Stop()
+
+ g := gluetidb.New()
+ client := snapclient.NewRestoreClient(cluster.PDClient, cluster.PDHTTPCli, nil, utiltest.DefaultTestKeepaliveCfg)
+ err := client.Init(g, cluster.Storage)
+ require.NoError(t, err)
+
+ // not set filter and first run with checkpoint
+ require.True(t, client.NeedCheckFreshCluster(false, true))
+
+ // skip check when has checkpoint
+ require.False(t, client.NeedCheckFreshCluster(false, false))
+
+ // skip check when set --filter
+ require.False(t, client.NeedCheckFreshCluster(true, false))
+
+ // skip check when has set --filter and has checkpoint
+ require.False(t, client.NeedCheckFreshCluster(true, true))
+
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/br/pkg/restore/snap_client/mock-incr-backup-data", "return(false)"))
+ defer func() {
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/br/pkg/restore/snap_client/mock-incr-backup-data"))
+ }()
+ // skip check when increment backup
+ require.False(t, client.NeedCheckFreshCluster(false, true))
+}
+
+func TestCheckTargetClusterFresh(t *testing.T) {
+ // cannot use shared `mc`, other parallel case may change it.
+ cluster := getStartedMockedCluster(t)
+ defer cluster.Stop()
+
+ g := gluetidb.New()
+ client := snapclient.NewRestoreClient(cluster.PDClient, cluster.PDHTTPCli, nil, utiltest.DefaultTestKeepaliveCfg)
+ err := client.Init(g, cluster.Storage)
+ require.NoError(t, err)
+
+ ctx := context.Background()
+ require.NoError(t, client.CheckTargetClusterFresh(ctx))
+
+ require.NoError(t, client.CreateDatabases(ctx, []*metautil.Database{{Info: &model.DBInfo{Name: model.NewCIStr("user_db")}}}))
+ require.True(t, berrors.ErrRestoreNotFreshCluster.Equal(client.CheckTargetClusterFresh(ctx)))
+}
+
+func TestCheckTargetClusterFreshWithTable(t *testing.T) {
+ // cannot use shared `mc`, other parallel case may change it.
+ cluster := getStartedMockedCluster(t)
+ defer cluster.Stop()
+
+ g := gluetidb.New()
+ client := snapclient.NewRestoreClient(cluster.PDClient, cluster.PDHTTPCli, nil, utiltest.DefaultTestKeepaliveCfg)
+ err := client.Init(g, cluster.Storage)
+ require.NoError(t, err)
+
+ ctx := context.Background()
+ info, err := cluster.Domain.GetSnapshotInfoSchema(math.MaxUint64)
+ require.NoError(t, err)
+ dbSchema, isExist := info.SchemaByName(model.NewCIStr("test"))
+ require.True(t, isExist)
+ intField := types.NewFieldType(mysql.TypeLong)
+ intField.SetCharset("binary")
+ table := &metautil.Table{
+ DB: dbSchema,
+ Info: &model.TableInfo{
+ ID: int64(1),
+ Name: model.NewCIStr("t"),
+ Columns: []*model.ColumnInfo{{
+ ID: 1,
+ Name: model.NewCIStr("id"),
+ FieldType: *intField,
+ State: model.StatePublic,
+ }},
+ Charset: "utf8mb4",
+ Collate: "utf8mb4_bin",
+ },
+ }
+ _, _, err = client.CreateTables(cluster.Domain, []*metautil.Table{table}, 0)
+ require.NoError(t, err)
+
+ require.True(t, berrors.ErrRestoreNotFreshCluster.Equal(client.CheckTargetClusterFresh(ctx)))
+}
+
+func TestInitFullClusterRestore(t *testing.T) {
+ cluster := mc
+ g := gluetidb.New()
+ client := snapclient.NewRestoreClient(cluster.PDClient, cluster.PDHTTPCli, nil, utiltest.DefaultTestKeepaliveCfg)
+ err := client.Init(g, cluster.Storage)
+ require.NoError(t, err)
+
+ // explicit filter
+ client.InitFullClusterRestore(true)
+ require.False(t, client.IsFullClusterRestore())
+
+ client.InitFullClusterRestore(false)
+ require.True(t, client.IsFullClusterRestore())
+ // set it to false again
+ client.InitFullClusterRestore(true)
+ require.False(t, client.IsFullClusterRestore())
+
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/br/pkg/restore/snap_client/mock-incr-backup-data", "return(true)"))
+ defer func() {
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/br/pkg/restore/snap_client/mock-incr-backup-data"))
+ }()
+ client.InitFullClusterRestore(false)
+ require.False(t, client.IsFullClusterRestore())
+}
+
+// Mock ImporterClient interface
+type FakeImporterClient struct {
+ importclient.ImporterClient
+}
+
+// Record the stores that have communicated
+type RecordStores struct {
+ mu sync.Mutex
+ stores []uint64
+}
+
+func NewRecordStores() RecordStores {
+ return RecordStores{stores: make([]uint64, 0)}
+}
+
+func (r *RecordStores) put(id uint64) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ r.stores = append(r.stores, id)
+}
+
+func (r *RecordStores) sort() {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ slices.Sort(r.stores)
+}
+
+func (r *RecordStores) len() int {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ return len(r.stores)
+}
+
+func (r *RecordStores) get(i int) uint64 {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ return r.stores[i]
+}
+
+func (r *RecordStores) toString() string {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ return fmt.Sprintf("%v", r.stores)
+}
+
+var recordStores RecordStores
+
+const (
+ SET_SPEED_LIMIT_ERROR = 999999
+ WORKING_TIME = 100
+)
+
+func (fakeImportCli FakeImporterClient) SetDownloadSpeedLimit(
+ ctx context.Context,
+ storeID uint64,
+ req *import_sstpb.SetDownloadSpeedLimitRequest,
+) (*import_sstpb.SetDownloadSpeedLimitResponse, error) {
+ if storeID == SET_SPEED_LIMIT_ERROR {
+ return nil, fmt.Errorf("storeID:%v ERROR", storeID)
+ }
+
+ time.Sleep(WORKING_TIME * time.Millisecond) // simulate doing 100 ms work
+ recordStores.put(storeID)
+ return nil, nil
+}
+
+func (fakeImportCli FakeImporterClient) CheckMultiIngestSupport(ctx context.Context, stores []uint64) error {
+ return nil
+}
+
+func TestSetSpeedLimit(t *testing.T) {
+ mockStores := []*metapb.Store{
+ {Id: 1},
+ {Id: 2},
+ {Id: 3},
+ {Id: 4},
+ {Id: 5},
+ {Id: 6},
+ {Id: 7},
+ {Id: 8},
+ {Id: 9},
+ {Id: 10},
+ }
+
+ // 1. The cost of concurrent communication is expected to be less than the cost of serial communication.
+ client := snapclient.NewRestoreClient(
+ utiltest.NewFakePDClient(mockStores, false, nil), nil, nil, utiltest.DefaultTestKeepaliveCfg)
+ ctx := context.Background()
+
+ recordStores = NewRecordStores()
+ start := time.Now()
+ err := snapclient.MockCallSetSpeedLimit(ctx, FakeImporterClient{}, client, 10)
+ cost := time.Since(start)
+ require.NoError(t, err)
+
+ recordStores.sort()
+ t.Logf("Total Cost: %v\n", cost)
+ t.Logf("Has Communicated: %v\n", recordStores.toString())
+
+ serialCost := len(mockStores) * WORKING_TIME
+ require.Less(t, cost, time.Duration(serialCost)*time.Millisecond)
+ require.Equal(t, len(mockStores), recordStores.len())
+ for i := 0; i < recordStores.len(); i++ {
+ require.Equal(t, mockStores[i].Id, recordStores.get(i))
+ }
+
+ // 2. Expect the number of communicated stores to be less than the length of the mockStore
+ // Because subsequent unstarted communications are aborted when an error is encountered.
+ recordStores = NewRecordStores()
+ mockStores[5].Id = SET_SPEED_LIMIT_ERROR // setting a fault store
+ client = snapclient.NewRestoreClient(
+ utiltest.NewFakePDClient(mockStores, false, nil), nil, nil, utiltest.DefaultTestKeepaliveCfg)
+
+ // Concurrency needs to be less than the number of stores
+ err = snapclient.MockCallSetSpeedLimit(ctx, FakeImporterClient{}, client, 2)
+ require.Error(t, err)
+ t.Log(err)
+
+ recordStores.sort()
+ sort.Slice(mockStores, func(i, j int) bool { return mockStores[i].Id < mockStores[j].Id })
+ t.Logf("Has Communicated: %v\n", recordStores.toString())
+ require.Less(t, recordStores.len(), len(mockStores))
+ for i := 0; i < recordStores.len(); i++ {
+ require.Equal(t, mockStores[i].Id, recordStores.get(i))
+ }
+}
diff --git a/br/pkg/restore/snap_client/context_manager.go b/br/pkg/restore/snap_client/context_manager.go
new file mode 100644
index 0000000000000..649f5b8b87980
--- /dev/null
+++ b/br/pkg/restore/snap_client/context_manager.go
@@ -0,0 +1,285 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package snapclient
+
+import (
+ "context"
+ "crypto/tls"
+ "encoding/hex"
+ "fmt"
+ "strconv"
+ "sync"
+ "time"
+
+ "github.com/pingcap/errors"
+ "github.com/pingcap/kvproto/pkg/metapb"
+ "github.com/pingcap/log"
+ "github.com/pingcap/tidb/br/pkg/conn"
+ "github.com/pingcap/tidb/br/pkg/conn/util"
+ berrors "github.com/pingcap/tidb/br/pkg/errors"
+ "github.com/pingcap/tidb/br/pkg/restore/split"
+ "github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/pingcap/tidb/pkg/tablecodec"
+ "github.com/pingcap/tidb/pkg/util/codec"
+ pd "github.com/tikv/pd/client"
+ pdhttp "github.com/tikv/pd/client/http"
+ "go.uber.org/zap"
+)
+
+// ContextManager is the struct to manage a TiKV 'context' for restore.
+// Batcher will call Enter when any table should be restore on batch,
+// so you can do some prepare work here(e.g. set placement rules for online restore).
+type ContextManager interface {
+ // Enter make some tables 'enter' this context(a.k.a., prepare for restore).
+ Enter(ctx context.Context, tables []CreatedTable) error
+ // Leave make some tables 'leave' this context(a.k.a., restore is done, do some post-works).
+ Leave(ctx context.Context, tables []CreatedTable) error
+ // Close closes the context manager, sometimes when the manager is 'killed' and should do some cleanup
+ // it would be call.
+ Close(ctx context.Context)
+}
+
+// NewBRContextManager makes a BR context manager, that is,
+// set placement rules for online restore when enter(see ),
+// unset them when leave.
+func NewBRContextManager(ctx context.Context, pdClient pd.Client, pdHTTPCli pdhttp.Client, tlsConf *tls.Config, isOnline bool) (ContextManager, error) {
+ manager := &brContextManager{
+ // toolClient reuse the split.SplitClient to do miscellaneous things. It doesn't
+ // call split related functions so set the arguments to arbitrary values.
+ toolClient: split.NewClient(pdClient, pdHTTPCli, tlsConf, maxSplitKeysOnce, 3),
+ isOnline: isOnline,
+
+ hasTable: make(map[int64]CreatedTable),
+ }
+
+ err := manager.loadRestoreStores(ctx, pdClient)
+ return manager, errors.Trace(err)
+}
+
+type brContextManager struct {
+ toolClient split.SplitClient
+ restoreStores []uint64
+ isOnline bool
+
+ // This 'set' of table ID allow us to handle each table just once.
+ hasTable map[int64]CreatedTable
+ mu sync.Mutex
+}
+
+func (manager *brContextManager) Close(ctx context.Context) {
+ tbls := make([]*model.TableInfo, 0, len(manager.hasTable))
+ for _, tbl := range manager.hasTable {
+ tbls = append(tbls, tbl.Table)
+ }
+ manager.splitPostWork(ctx, tbls)
+}
+
+func (manager *brContextManager) Enter(ctx context.Context, tables []CreatedTable) error {
+ placementRuleTables := make([]*model.TableInfo, 0, len(tables))
+ manager.mu.Lock()
+ defer manager.mu.Unlock()
+
+ for _, tbl := range tables {
+ if _, ok := manager.hasTable[tbl.Table.ID]; !ok {
+ placementRuleTables = append(placementRuleTables, tbl.Table)
+ }
+ manager.hasTable[tbl.Table.ID] = tbl
+ }
+
+ return manager.splitPrepareWork(ctx, placementRuleTables)
+}
+
+func (manager *brContextManager) Leave(ctx context.Context, tables []CreatedTable) error {
+ manager.mu.Lock()
+ defer manager.mu.Unlock()
+ placementRuleTables := make([]*model.TableInfo, 0, len(tables))
+
+ for _, table := range tables {
+ placementRuleTables = append(placementRuleTables, table.Table)
+ }
+
+ manager.splitPostWork(ctx, placementRuleTables)
+ log.Info("restore table done", zapTables(tables))
+ for _, tbl := range placementRuleTables {
+ delete(manager.hasTable, tbl.ID)
+ }
+ return nil
+}
+
+func (manager *brContextManager) splitPostWork(ctx context.Context, tables []*model.TableInfo) {
+ err := manager.resetPlacementRules(ctx, tables)
+ if err != nil {
+ log.Warn("reset placement rules failed", zap.Error(err))
+ return
+ }
+}
+
+func (manager *brContextManager) splitPrepareWork(ctx context.Context, tables []*model.TableInfo) error {
+ err := manager.setupPlacementRules(ctx, tables)
+ if err != nil {
+ log.Error("setup placement rules failed", zap.Error(err))
+ return errors.Trace(err)
+ }
+
+ err = manager.waitPlacementSchedule(ctx, tables)
+ if err != nil {
+ log.Error("wait placement schedule failed", zap.Error(err))
+ return errors.Trace(err)
+ }
+ return nil
+}
+
+const (
+ restoreLabelKey = "exclusive"
+ restoreLabelValue = "restore"
+)
+
+// loadRestoreStores loads the stores used to restore data. This function is called only when is online.
+func (manager *brContextManager) loadRestoreStores(ctx context.Context, pdClient util.StoreMeta) error {
+ if !manager.isOnline {
+ return nil
+ }
+ stores, err := conn.GetAllTiKVStoresWithRetry(ctx, pdClient, util.SkipTiFlash)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ for _, s := range stores {
+ if s.GetState() != metapb.StoreState_Up {
+ continue
+ }
+ for _, l := range s.GetLabels() {
+ if l.GetKey() == restoreLabelKey && l.GetValue() == restoreLabelValue {
+ manager.restoreStores = append(manager.restoreStores, s.GetId())
+ break
+ }
+ }
+ }
+ log.Info("load restore stores", zap.Uint64s("store-ids", manager.restoreStores))
+ return nil
+}
+
+// SetupPlacementRules sets rules for the tables' regions.
+func (manager *brContextManager) setupPlacementRules(ctx context.Context, tables []*model.TableInfo) error {
+ if !manager.isOnline || len(manager.restoreStores) == 0 {
+ return nil
+ }
+ log.Info("start setting placement rules")
+ rule, err := manager.toolClient.GetPlacementRule(ctx, "pd", "default")
+ if err != nil {
+ return errors.Trace(err)
+ }
+ rule.Index = 100
+ rule.Override = true
+ rule.LabelConstraints = append(rule.LabelConstraints, pdhttp.LabelConstraint{
+ Key: restoreLabelKey,
+ Op: "in",
+ Values: []string{restoreLabelValue},
+ })
+ for _, t := range tables {
+ rule.ID = getRuleID(t.ID)
+ rule.StartKeyHex = hex.EncodeToString(codec.EncodeBytes([]byte{}, tablecodec.EncodeTablePrefix(t.ID)))
+ rule.EndKeyHex = hex.EncodeToString(codec.EncodeBytes([]byte{}, tablecodec.EncodeTablePrefix(t.ID+1)))
+ err = manager.toolClient.SetPlacementRule(ctx, rule)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ }
+ log.Info("finish setting placement rules")
+ return nil
+}
+
+func (manager *brContextManager) checkRegions(ctx context.Context, tables []*model.TableInfo) (bool, string, error) {
+ for i, t := range tables {
+ start := codec.EncodeBytes([]byte{}, tablecodec.EncodeTablePrefix(t.ID))
+ end := codec.EncodeBytes([]byte{}, tablecodec.EncodeTablePrefix(t.ID+1))
+ ok, regionProgress, err := manager.checkRange(ctx, start, end)
+ if err != nil {
+ return false, "", errors.Trace(err)
+ }
+ if !ok {
+ return false, fmt.Sprintf("table %v/%v, %s", i, len(tables), regionProgress), nil
+ }
+ }
+ return true, "", nil
+}
+
+func (manager *brContextManager) checkRange(ctx context.Context, start, end []byte) (bool, string, error) {
+ regions, err := manager.toolClient.ScanRegions(ctx, start, end, -1)
+ if err != nil {
+ return false, "", errors.Trace(err)
+ }
+ for i, r := range regions {
+ NEXT_PEER:
+ for _, p := range r.Region.GetPeers() {
+ for _, storeID := range manager.restoreStores {
+ if p.GetStoreId() == storeID {
+ continue NEXT_PEER
+ }
+ }
+ return false, fmt.Sprintf("region %v/%v", i, len(regions)), nil
+ }
+ }
+ return true, "", nil
+}
+
+// waitPlacementSchedule waits PD to move tables to restore stores.
+func (manager *brContextManager) waitPlacementSchedule(ctx context.Context, tables []*model.TableInfo) error {
+ if !manager.isOnline || len(manager.restoreStores) == 0 {
+ return nil
+ }
+ log.Info("start waiting placement schedule")
+ ticker := time.NewTicker(time.Second * 10)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ticker.C:
+ ok, progress, err := manager.checkRegions(ctx, tables)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if ok {
+ log.Info("finish waiting placement schedule")
+ return nil
+ }
+ log.Info("placement schedule progress: " + progress)
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ }
+}
+
+func getRuleID(tableID int64) string {
+ return "restore-t" + strconv.FormatInt(tableID, 10)
+}
+
+// resetPlacementRules removes placement rules for tables.
+func (manager *brContextManager) resetPlacementRules(ctx context.Context, tables []*model.TableInfo) error {
+ if !manager.isOnline || len(manager.restoreStores) == 0 {
+ return nil
+ }
+ log.Info("start resetting placement rules")
+ var failedTables []int64
+ for _, t := range tables {
+ err := manager.toolClient.DeletePlacementRule(ctx, "pd", getRuleID(t.ID))
+ if err != nil {
+ log.Info("failed to delete placement rule for table", zap.Int64("table-id", t.ID))
+ failedTables = append(failedTables, t.ID)
+ }
+ }
+ if len(failedTables) > 0 {
+ return errors.Annotatef(berrors.ErrPDInvalidResponse, "failed to delete placement rules for tables %v", failedTables)
+ }
+ return nil
+}
diff --git a/br/pkg/restore/snap_client/export_test.go b/br/pkg/restore/snap_client/export_test.go
new file mode 100644
index 0000000000000..88d32d61d9c57
--- /dev/null
+++ b/br/pkg/restore/snap_client/export_test.go
@@ -0,0 +1,88 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package snapclient
+
+import (
+ "cmp"
+ "context"
+
+ "github.com/pingcap/errors"
+ "github.com/pingcap/kvproto/pkg/import_sstpb"
+ "github.com/pingcap/tidb/br/pkg/metautil"
+ importclient "github.com/pingcap/tidb/br/pkg/restore/internal/import_client"
+ restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
+ "github.com/pingcap/tidb/pkg/domain"
+ "github.com/pingcap/tidb/pkg/parser/model"
+ tidbutil "github.com/pingcap/tidb/pkg/util"
+ "golang.org/x/exp/slices"
+)
+
+var GetSSTMetaFromFile = getSSTMetaFromFile
+
+var GetKeyRangeByMode = getKeyRangeByMode
+
+// MockClient create a fake Client used to test.
+func MockClient(dbs map[string]*metautil.Database) *SnapClient {
+ return &SnapClient{databases: dbs}
+}
+
+// Mock the call of setSpeedLimit function
+func MockCallSetSpeedLimit(ctx context.Context, fakeImportClient importclient.ImporterClient, rc *SnapClient, concurrency uint) (err error) {
+ rc.SetRateLimit(42)
+ rc.workerPool = tidbutil.NewWorkerPool(128, "set-speed-limit")
+ rc.hasSpeedLimited = false
+ rc.fileImporter, err = NewSnapFileImporter(ctx, nil, fakeImportClient, nil, false, false, nil, rc.rewriteMode, 128)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ return rc.setSpeedLimit(ctx, rc.rateLimit)
+}
+
+// CreateTables creates multiple tables, and returns their rewrite rules.
+func (rc *SnapClient) CreateTables(
+ dom *domain.Domain,
+ tables []*metautil.Table,
+ newTS uint64,
+) (*restoreutils.RewriteRules, []*model.TableInfo, error) {
+ rc.dom = dom
+ rewriteRules := &restoreutils.RewriteRules{
+ Data: make([]*import_sstpb.RewriteRule, 0),
+ }
+ newTables := make([]*model.TableInfo, 0, len(tables))
+ errCh := make(chan error, 1)
+ tbMapping := map[string]int{}
+ for i, t := range tables {
+ tbMapping[t.Info.Name.String()] = i
+ }
+ dataCh := rc.GoCreateTables(context.TODO(), tables, newTS, errCh)
+ for et := range dataCh {
+ rules := et.RewriteRule
+ rewriteRules.Data = append(rewriteRules.Data, rules.Data...)
+ newTables = append(newTables, et.Table)
+ }
+ // Let's ensure that it won't break the original order.
+ slices.SortFunc(newTables, func(i, j *model.TableInfo) int {
+ return cmp.Compare(tbMapping[i.Name.String()], tbMapping[j.Name.String()])
+ })
+
+ select {
+ case err, ok := <-errCh:
+ if ok {
+ return nil, nil, errors.Trace(err)
+ }
+ default:
+ }
+ return rewriteRules, newTables, nil
+}
diff --git a/br/pkg/restore/snap_client/import.go b/br/pkg/restore/snap_client/import.go
new file mode 100644
index 0000000000000..cdab5a678628a
--- /dev/null
+++ b/br/pkg/restore/snap_client/import.go
@@ -0,0 +1,846 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package snapclient
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "math/rand"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
+ backuppb "github.com/pingcap/kvproto/pkg/brpb"
+ "github.com/pingcap/kvproto/pkg/import_sstpb"
+ "github.com/pingcap/kvproto/pkg/kvrpcpb"
+ "github.com/pingcap/kvproto/pkg/metapb"
+ "github.com/pingcap/log"
+ berrors "github.com/pingcap/tidb/br/pkg/errors"
+ "github.com/pingcap/tidb/br/pkg/logutil"
+ importclient "github.com/pingcap/tidb/br/pkg/restore/internal/import_client"
+ "github.com/pingcap/tidb/br/pkg/restore/split"
+ restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
+ "github.com/pingcap/tidb/br/pkg/summary"
+ "github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/pkg/kv"
+ "github.com/pingcap/tidb/pkg/util/codec"
+ kvutil "github.com/tikv/client-go/v2/util"
+ "go.uber.org/multierr"
+ "go.uber.org/zap"
+ "golang.org/x/exp/maps"
+ "golang.org/x/sync/errgroup"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+)
+
+type KvMode int
+
+const (
+ TiDB KvMode = iota
+ Raw
+ Txn
+)
+
+const (
+ // Todo: make it configable
+ gRPCTimeOut = 25 * time.Minute
+)
+
+// RewriteMode is a mode flag that tells the TiKV how to handle the rewrite rules.
+type RewriteMode int
+
+const (
+ // RewriteModeLegacy means no rewrite rule is applied.
+ RewriteModeLegacy RewriteMode = iota
+
+ // RewriteModeKeyspace means the rewrite rule could be applied to keyspace.
+ RewriteModeKeyspace
+)
+
+type storeTokenChannelMap struct {
+ sync.RWMutex
+ tokens map[uint64]chan struct{}
+}
+
+func (s *storeTokenChannelMap) acquireTokenCh(storeID uint64, bufferSize uint) chan struct{} {
+ s.RLock()
+ tokenCh, ok := s.tokens[storeID]
+ // handle the case that the store is new-scaled in the cluster
+ if !ok {
+ s.RUnlock()
+ s.Lock()
+ // Notice: worker channel can't replaced, because it is still used after unlock.
+ if tokenCh, ok = s.tokens[storeID]; !ok {
+ tokenCh = utils.BuildWorkerTokenChannel(bufferSize)
+ s.tokens[storeID] = tokenCh
+ }
+ s.Unlock()
+ } else {
+ s.RUnlock()
+ }
+ return tokenCh
+}
+
+func (s *storeTokenChannelMap) ShouldBlock() bool {
+ s.RLock()
+ defer s.RUnlock()
+ if len(s.tokens) == 0 {
+ // never block if there is no store worker pool
+ return false
+ }
+ for _, pool := range s.tokens {
+ if len(pool) > 0 {
+ // At least one store worker pool has available worker
+ return false
+ }
+ }
+ return true
+}
+
+func newStoreTokenChannelMap(stores []*metapb.Store, bufferSize uint) *storeTokenChannelMap {
+ storeTokenChannelMap := &storeTokenChannelMap{
+ sync.RWMutex{},
+ make(map[uint64]chan struct{}),
+ }
+ if bufferSize == 0 {
+ return storeTokenChannelMap
+ }
+ for _, store := range stores {
+ ch := utils.BuildWorkerTokenChannel(bufferSize)
+ storeTokenChannelMap.tokens[store.Id] = ch
+ }
+ return storeTokenChannelMap
+}
+
+type SnapFileImporter struct {
+ metaClient split.SplitClient
+ importClient importclient.ImporterClient
+ backend *backuppb.StorageBackend
+
+ downloadTokensMap *storeTokenChannelMap
+ ingestTokensMap *storeTokenChannelMap
+
+ concurrencyPerStore uint
+
+ kvMode KvMode
+ rawStartKey []byte
+ rawEndKey []byte
+ rewriteMode RewriteMode
+
+ cacheKey string
+ cond *sync.Cond
+}
+
+func NewSnapFileImporter(
+ ctx context.Context,
+ metaClient split.SplitClient,
+ importClient importclient.ImporterClient,
+ backend *backuppb.StorageBackend,
+ isRawKvMode bool,
+ isTxnKvMode bool,
+ tikvStores []*metapb.Store,
+ rewriteMode RewriteMode,
+ concurrencyPerStore uint,
+) (*SnapFileImporter, error) {
+ kvMode := TiDB
+ if isRawKvMode {
+ kvMode = Raw
+ }
+ if isTxnKvMode {
+ kvMode = Txn
+ }
+
+ fileImporter := &SnapFileImporter{
+ metaClient: metaClient,
+ backend: backend,
+ importClient: importClient,
+ downloadTokensMap: newStoreTokenChannelMap(tikvStores, concurrencyPerStore),
+ ingestTokensMap: newStoreTokenChannelMap(tikvStores, concurrencyPerStore),
+ kvMode: kvMode,
+ rewriteMode: rewriteMode,
+ cacheKey: fmt.Sprintf("BR-%s-%d", time.Now().Format("20060102150405"), rand.Int63()),
+ concurrencyPerStore: concurrencyPerStore,
+ cond: sync.NewCond(new(sync.Mutex)),
+ }
+
+ err := fileImporter.checkMultiIngestSupport(ctx, tikvStores)
+ return fileImporter, errors.Trace(err)
+}
+
+func (importer *SnapFileImporter) WaitUntilUnblock() {
+ importer.cond.L.Lock()
+ for importer.ShouldBlock() {
+ // wait for download worker notified
+ importer.cond.Wait()
+ }
+ importer.cond.L.Unlock()
+}
+
+func (importer *SnapFileImporter) ShouldBlock() bool {
+ if importer != nil {
+ return importer.downloadTokensMap.ShouldBlock() || importer.ingestTokensMap.ShouldBlock()
+ }
+ return false
+}
+
+func (importer *SnapFileImporter) releaseToken(tokenCh chan struct{}) {
+ tokenCh <- struct{}{}
+ // finish the task, notify the main goroutine to continue
+ importer.cond.L.Lock()
+ importer.cond.Signal()
+ importer.cond.L.Unlock()
+}
+
+func (importer *SnapFileImporter) Close() error {
+ if importer != nil && importer.importClient != nil {
+ return importer.importClient.CloseGrpcClient()
+ }
+ return nil
+}
+
+func (importer *SnapFileImporter) SetDownloadSpeedLimit(ctx context.Context, storeID, rateLimit uint64) error {
+ req := &import_sstpb.SetDownloadSpeedLimitRequest{
+ SpeedLimit: rateLimit,
+ }
+ _, err := importer.importClient.SetDownloadSpeedLimit(ctx, storeID, req)
+ return errors.Trace(err)
+}
+
+// checkMultiIngestSupport checks whether all stores support multi-ingest
+func (importer *SnapFileImporter) checkMultiIngestSupport(ctx context.Context, tikvStores []*metapb.Store) error {
+ storeIDs := make([]uint64, 0, len(tikvStores))
+ for _, s := range tikvStores {
+ if s.State != metapb.StoreState_Up {
+ continue
+ }
+ storeIDs = append(storeIDs, s.Id)
+ }
+
+ if err := importer.importClient.CheckMultiIngestSupport(ctx, storeIDs); err != nil {
+ return errors.Trace(err)
+ }
+ return nil
+}
+
+// SetRawRange sets the range to be restored in raw kv mode.
+func (importer *SnapFileImporter) SetRawRange(startKey, endKey []byte) error {
+ if importer.kvMode != Raw {
+ return errors.Annotate(berrors.ErrRestoreModeMismatch, "file importer is not in raw kv mode")
+ }
+ importer.rawStartKey = startKey
+ importer.rawEndKey = endKey
+ return nil
+}
+
+func getKeyRangeByMode(mode KvMode) func(f *backuppb.File, rules *restoreutils.RewriteRules) ([]byte, []byte, error) {
+ switch mode {
+ case Raw:
+ return func(f *backuppb.File, rules *restoreutils.RewriteRules) ([]byte, []byte, error) {
+ return f.GetStartKey(), f.GetEndKey(), nil
+ }
+ case Txn:
+ return func(f *backuppb.File, rules *restoreutils.RewriteRules) ([]byte, []byte, error) {
+ start, end := f.GetStartKey(), f.GetEndKey()
+ if len(start) != 0 {
+ start = codec.EncodeBytes([]byte{}, f.GetStartKey())
+ }
+ if len(end) != 0 {
+ end = codec.EncodeBytes([]byte{}, f.GetEndKey())
+ }
+ return start, end, nil
+ }
+ default:
+ return func(f *backuppb.File, rules *restoreutils.RewriteRules) ([]byte, []byte, error) {
+ return restoreutils.GetRewriteRawKeys(f, rules)
+ }
+ }
+}
+
+// getKeyRangeForFiles gets the maximum range on files.
+func (importer *SnapFileImporter) getKeyRangeForFiles(
+ files []*backuppb.File,
+ rewriteRules *restoreutils.RewriteRules,
+) ([]byte, []byte, error) {
+ var (
+ startKey, endKey []byte
+ start, end []byte
+ err error
+ )
+ getRangeFn := getKeyRangeByMode(importer.kvMode)
+ for _, f := range files {
+ start, end, err = getRangeFn(f, rewriteRules)
+ if err != nil {
+ return nil, nil, errors.Trace(err)
+ }
+ if len(startKey) == 0 || bytes.Compare(start, startKey) < 0 {
+ startKey = start
+ }
+ if len(endKey) == 0 || bytes.Compare(endKey, end) < 0 {
+ endKey = end
+ }
+ }
+
+ log.Debug("rewrite file keys", logutil.Files(files),
+ logutil.Key("startKey", startKey), logutil.Key("endKey", endKey))
+ return startKey, endKey, nil
+}
+
+// ImportSSTFiles tries to import a file.
+// All rules must contain encoded keys.
+func (importer *SnapFileImporter) ImportSSTFiles(
+ ctx context.Context,
+ files []*backuppb.File,
+ rewriteRules *restoreutils.RewriteRules,
+ cipher *backuppb.CipherInfo,
+ apiVersion kvrpcpb.APIVersion,
+) error {
+ start := time.Now()
+ log.Debug("import file", logutil.Files(files))
+
+ // Rewrite the start key and end key of file to scan regions
+ startKey, endKey, err := importer.getKeyRangeForFiles(files, rewriteRules)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ err = utils.WithRetry(ctx, func() error {
+ // Scan regions covered by the file range
+ regionInfos, errScanRegion := split.PaginateScanRegion(
+ ctx, importer.metaClient, startKey, endKey, split.ScanRegionPaginationLimit)
+ if errScanRegion != nil {
+ return errors.Trace(errScanRegion)
+ }
+
+ log.Debug("scan regions", logutil.Files(files), zap.Int("count", len(regionInfos)))
+ // Try to download and ingest the file in every region
+ regionLoop:
+ for _, regionInfo := range regionInfos {
+ info := regionInfo
+ // Try to download file.
+ downloadMetas, errDownload := importer.download(ctx, info, files, rewriteRules, cipher, apiVersion)
+ if errDownload != nil {
+ for _, e := range multierr.Errors(errDownload) {
+ switch errors.Cause(e) { // nolint:errorlint
+ case berrors.ErrKVRewriteRuleNotFound, berrors.ErrKVRangeIsEmpty:
+ // Skip this region
+ log.Warn("download file skipped",
+ logutil.Files(files),
+ logutil.Region(info.Region),
+ logutil.Key("startKey", startKey),
+ logutil.Key("endKey", endKey),
+ logutil.Key("file-simple-start", files[0].StartKey),
+ logutil.Key("file-simple-end", files[0].EndKey),
+ logutil.ShortError(e))
+ continue regionLoop
+ }
+ }
+ log.Warn("download file failed, retry later",
+ logutil.Files(files),
+ logutil.Region(info.Region),
+ logutil.Key("startKey", startKey),
+ logutil.Key("endKey", endKey),
+ logutil.ShortError(errDownload))
+ return errors.Trace(errDownload)
+ }
+ log.Debug("download file done",
+ zap.String("file-sample", files[0].Name), zap.Stringer("take", time.Since(start)),
+ logutil.Key("start", files[0].StartKey), logutil.Key("end", files[0].EndKey))
+ start = time.Now()
+ if errIngest := importer.ingest(ctx, files, info, downloadMetas); errIngest != nil {
+ log.Warn("ingest file failed, retry later",
+ logutil.Files(files),
+ logutil.SSTMetas(downloadMetas),
+ logutil.Region(info.Region),
+ zap.Error(errIngest))
+ return errors.Trace(errIngest)
+ }
+ log.Debug("ingest file done", zap.String("file-sample", files[0].Name), zap.Stringer("take", time.Since(start)))
+ }
+
+ for _, f := range files {
+ summary.CollectSuccessUnit(summary.TotalKV, 1, f.TotalKvs)
+ summary.CollectSuccessUnit(summary.TotalBytes, 1, f.TotalBytes)
+ }
+ return nil
+ }, utils.NewImportSSTBackoffer())
+ if err != nil {
+ log.Error("import sst file failed after retry, stop the whole progress", logutil.Files(files), zap.Error(err))
+ return errors.Trace(err)
+ }
+ return nil
+}
+
+// getSSTMetaFromFile compares the keys in file, region and rewrite rules, then returns a sst conn.
+// The range of the returned sst meta is [regionRule.NewKeyPrefix, append(regionRule.NewKeyPrefix, 0xff)].
+func getSSTMetaFromFile(
+ id []byte,
+ file *backuppb.File,
+ region *metapb.Region,
+ regionRule *import_sstpb.RewriteRule,
+ rewriteMode RewriteMode,
+) (meta *import_sstpb.SSTMeta, err error) {
+ r := *region
+ // If the rewrite mode is for keyspace, then the region bound should be decoded.
+ if rewriteMode == RewriteModeKeyspace {
+ if len(region.GetStartKey()) > 0 {
+ _, r.StartKey, err = codec.DecodeBytes(region.GetStartKey(), nil)
+ if err != nil {
+ return
+ }
+ }
+ if len(region.GetEndKey()) > 0 {
+ _, r.EndKey, err = codec.DecodeBytes(region.GetEndKey(), nil)
+ if err != nil {
+ return
+ }
+ }
+ }
+
+ // Get the column family of the file by the file name.
+ var cfName string
+ if strings.Contains(file.GetName(), restoreutils.DefaultCFName) {
+ cfName = restoreutils.DefaultCFName
+ } else if strings.Contains(file.GetName(), restoreutils.WriteCFName) {
+ cfName = restoreutils.WriteCFName
+ }
+ // Find the overlapped part between the file and the region.
+ // Here we rewrites the keys to compare with the keys of the region.
+ rangeStart := regionRule.GetNewKeyPrefix()
+ // rangeStart = max(rangeStart, region.StartKey)
+ if bytes.Compare(rangeStart, r.GetStartKey()) < 0 {
+ rangeStart = r.GetStartKey()
+ }
+
+ // Append 10 * 0xff to make sure rangeEnd cover all file key
+ // If choose to regionRule.NewKeyPrefix + 1, it may cause WrongPrefix here
+ // https://github.com/tikv/tikv/blob/970a9bf2a9ea782a455ae579ad237aaf6cb1daec/
+ // components/sst_importer/src/sst_importer.rs#L221
+ suffix := []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
+ rangeEnd := append(append([]byte{}, regionRule.GetNewKeyPrefix()...), suffix...)
+ // rangeEnd = min(rangeEnd, region.EndKey)
+ if len(r.GetEndKey()) > 0 && bytes.Compare(rangeEnd, r.GetEndKey()) > 0 {
+ rangeEnd = r.GetEndKey()
+ }
+
+ if bytes.Compare(rangeStart, rangeEnd) > 0 {
+ log.Panic("range start exceed range end",
+ logutil.File(file),
+ logutil.Key("startKey", rangeStart),
+ logutil.Key("endKey", rangeEnd))
+ }
+
+ log.Debug("get sstMeta",
+ logutil.Region(region),
+ logutil.File(file),
+ logutil.Key("startKey", rangeStart),
+ logutil.Key("endKey", rangeEnd))
+
+ return &import_sstpb.SSTMeta{
+ Uuid: id,
+ CfName: cfName,
+ Range: &import_sstpb.Range{
+ Start: rangeStart,
+ End: rangeEnd,
+ },
+ Length: file.GetSize_(),
+ RegionId: region.GetId(),
+ RegionEpoch: region.GetRegionEpoch(),
+ CipherIv: file.GetCipherIv(),
+ }, nil
+}
+
+// a new way to download ssts files
+// 1. download write + default sst files at peer level.
+// 2. control the download concurrency per store.
+func (importer *SnapFileImporter) download(
+ ctx context.Context,
+ regionInfo *split.RegionInfo,
+ files []*backuppb.File,
+ rewriteRules *restoreutils.RewriteRules,
+ cipher *backuppb.CipherInfo,
+ apiVersion kvrpcpb.APIVersion,
+) ([]*import_sstpb.SSTMeta, error) {
+ var (
+ downloadMetas = make([]*import_sstpb.SSTMeta, 0, len(files))
+ )
+ errDownload := utils.WithRetry(ctx, func() error {
+ var e error
+ // we treat Txn kv file as Raw kv file. because we don't have table id to decode
+ if importer.kvMode == Raw || importer.kvMode == Txn {
+ downloadMetas, e = importer.downloadRawKVSST(ctx, regionInfo, files, cipher, apiVersion)
+ } else {
+ downloadMetas, e = importer.downloadSST(ctx, regionInfo, files, rewriteRules, cipher, apiVersion)
+ }
+
+ failpoint.Inject("restore-storage-error", func(val failpoint.Value) {
+ msg := val.(string)
+ log.Debug("failpoint restore-storage-error injected.", zap.String("msg", msg))
+ e = errors.Annotate(e, msg)
+ })
+ failpoint.Inject("restore-gRPC-error", func(_ failpoint.Value) {
+ log.Warn("the connection to TiKV has been cut by a neko, meow :3")
+ e = status.Error(codes.Unavailable, "the connection to TiKV has been cut by a neko, meow :3")
+ })
+ if isDecryptSstErr(e) {
+ log.Info("fail to decrypt when download sst, try again with no-crypt", logutil.Files(files))
+ if importer.kvMode == Raw || importer.kvMode == Txn {
+ downloadMetas, e = importer.downloadRawKVSST(ctx, regionInfo, files, nil, apiVersion)
+ } else {
+ downloadMetas, e = importer.downloadSST(ctx, regionInfo, files, rewriteRules, nil, apiVersion)
+ }
+ }
+ if e != nil {
+ return errors.Trace(e)
+ }
+
+ return nil
+ }, utils.NewDownloadSSTBackoffer())
+
+ return downloadMetas, errDownload
+}
+
+func (importer *SnapFileImporter) buildDownloadRequest(
+ file *backuppb.File,
+ rewriteRules *restoreutils.RewriteRules,
+ regionInfo *split.RegionInfo,
+ cipher *backuppb.CipherInfo,
+) (*import_sstpb.DownloadRequest, import_sstpb.SSTMeta, error) {
+ uid := uuid.New()
+ id := uid[:]
+ // Get the rewrite rule for the file.
+ fileRule := restoreutils.FindMatchedRewriteRule(file, rewriteRules)
+ if fileRule == nil {
+ return nil, import_sstpb.SSTMeta{}, errors.Trace(berrors.ErrKVRewriteRuleNotFound)
+ }
+
+ // For the legacy version of TiKV, we need to encode the key prefix, since in the legacy
+ // version, the TiKV will rewrite the key with the encoded prefix without decoding the keys in
+ // the SST file. For the new version of TiKV that support keyspace rewrite, we don't need to
+ // encode the key prefix. The TiKV will decode the keys in the SST file and rewrite the keys
+ // with the plain prefix and encode the keys before writing to SST.
+
+ // for the keyspace rewrite mode
+ rule := *fileRule
+ // for the legacy rewrite mode
+ if importer.rewriteMode == RewriteModeLegacy {
+ rule.OldKeyPrefix = restoreutils.EncodeKeyPrefix(fileRule.GetOldKeyPrefix())
+ rule.NewKeyPrefix = restoreutils.EncodeKeyPrefix(fileRule.GetNewKeyPrefix())
+ }
+
+ sstMeta, err := getSSTMetaFromFile(id, file, regionInfo.Region, &rule, importer.rewriteMode)
+ if err != nil {
+ return nil, import_sstpb.SSTMeta{}, err
+ }
+
+ req := &import_sstpb.DownloadRequest{
+ Sst: *sstMeta,
+ StorageBackend: importer.backend,
+ Name: file.GetName(),
+ RewriteRule: rule,
+ CipherInfo: cipher,
+ StorageCacheId: importer.cacheKey,
+ // For the older version of TiDB, the request type will be default to `import_sstpb.RequestType_Legacy`
+ RequestType: import_sstpb.DownloadRequestType_Keyspace,
+ Context: &kvrpcpb.Context{
+ ResourceControlContext: &kvrpcpb.ResourceControlContext{
+ ResourceGroupName: "", // TODO,
+ },
+ RequestSource: kvutil.BuildRequestSource(true, kv.InternalTxnBR, kvutil.ExplicitTypeBR),
+ },
+ }
+ return req, *sstMeta, nil
+}
+
+func (importer *SnapFileImporter) downloadSST(
+ ctx context.Context,
+ regionInfo *split.RegionInfo,
+ files []*backuppb.File,
+ rewriteRules *restoreutils.RewriteRules,
+ cipher *backuppb.CipherInfo,
+ apiVersion kvrpcpb.APIVersion,
+) ([]*import_sstpb.SSTMeta, error) {
+ var mu sync.Mutex
+ downloadMetasMap := make(map[string]import_sstpb.SSTMeta)
+ resultMetasMap := make(map[string]*import_sstpb.SSTMeta)
+ downloadReqsMap := make(map[string]*import_sstpb.DownloadRequest)
+ for _, file := range files {
+ req, sstMeta, err := importer.buildDownloadRequest(file, rewriteRules, regionInfo, cipher)
+ if err != nil {
+ return nil, errors.Trace(err)
+ }
+ sstMeta.ApiVersion = apiVersion
+ downloadMetasMap[file.Name] = sstMeta
+ downloadReqsMap[file.Name] = req
+ }
+
+ eg, ectx := errgroup.WithContext(ctx)
+ for _, p := range regionInfo.Region.GetPeers() {
+ peer := p
+ eg.Go(func() error {
+ tokenCh := importer.downloadTokensMap.acquireTokenCh(peer.GetStoreId(), importer.concurrencyPerStore)
+ select {
+ case <-ectx.Done():
+ return ectx.Err()
+ case <-tokenCh:
+ }
+ defer func() {
+ importer.releaseToken(tokenCh)
+ }()
+ for _, file := range files {
+ req, ok := downloadReqsMap[file.Name]
+ if !ok {
+ return errors.New("not found file key for download request")
+ }
+ var err error
+ var resp *import_sstpb.DownloadResponse
+ resp, err = utils.WithRetryV2(ectx, utils.NewDownloadSSTBackoffer(), func(ctx context.Context) (*import_sstpb.DownloadResponse, error) {
+ dctx, cancel := context.WithTimeout(ctx, gRPCTimeOut)
+ defer cancel()
+ return importer.importClient.DownloadSST(dctx, peer.GetStoreId(), req)
+ })
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if resp.GetError() != nil {
+ return errors.Annotate(berrors.ErrKVDownloadFailed, resp.GetError().GetMessage())
+ }
+ if resp.GetIsEmpty() {
+ return errors.Trace(berrors.ErrKVRangeIsEmpty)
+ }
+
+ mu.Lock()
+ sstMeta, ok := downloadMetasMap[file.Name]
+ if !ok {
+ mu.Unlock()
+ return errors.Errorf("not found file %s for download sstMeta", file.Name)
+ }
+ sstMeta.Range = &import_sstpb.Range{
+ Start: restoreutils.TruncateTS(resp.Range.GetStart()),
+ End: restoreutils.TruncateTS(resp.Range.GetEnd()),
+ }
+ resultMetasMap[file.Name] = &sstMeta
+ mu.Unlock()
+
+ log.Debug("download from peer",
+ logutil.Region(regionInfo.Region),
+ logutil.File(file),
+ logutil.Peer(peer),
+ logutil.Key("resp-range-start", resp.Range.Start),
+ logutil.Key("resp-range-end", resp.Range.End),
+ zap.Bool("resp-isempty", resp.IsEmpty),
+ zap.Uint32("resp-crc32", resp.Crc32),
+ zap.Int("len files", len(files)),
+ )
+ }
+ return nil
+ })
+ }
+ if err := eg.Wait(); err != nil {
+ return nil, err
+ }
+ return maps.Values(resultMetasMap), nil
+}
+
+func (importer *SnapFileImporter) downloadRawKVSST(
+ ctx context.Context,
+ regionInfo *split.RegionInfo,
+ files []*backuppb.File,
+ cipher *backuppb.CipherInfo,
+ apiVersion kvrpcpb.APIVersion,
+) ([]*import_sstpb.SSTMeta, error) {
+ downloadMetas := make([]*import_sstpb.SSTMeta, 0, len(files))
+ for _, file := range files {
+ uid := uuid.New()
+ id := uid[:]
+ // Empty rule
+ var rule import_sstpb.RewriteRule
+ sstMeta, err := getSSTMetaFromFile(id, file, regionInfo.Region, &rule, RewriteModeLegacy)
+ if err != nil {
+ return nil, err
+ }
+
+ // Cut the SST file's range to fit in the restoring range.
+ if bytes.Compare(importer.rawStartKey, sstMeta.Range.GetStart()) > 0 {
+ sstMeta.Range.Start = importer.rawStartKey
+ }
+ if len(importer.rawEndKey) > 0 &&
+ (len(sstMeta.Range.GetEnd()) == 0 || bytes.Compare(importer.rawEndKey, sstMeta.Range.GetEnd()) <= 0) {
+ sstMeta.Range.End = importer.rawEndKey
+ sstMeta.EndKeyExclusive = true
+ }
+ if bytes.Compare(sstMeta.Range.GetStart(), sstMeta.Range.GetEnd()) > 0 {
+ return nil, errors.Trace(berrors.ErrKVRangeIsEmpty)
+ }
+
+ req := &import_sstpb.DownloadRequest{
+ Sst: *sstMeta,
+ StorageBackend: importer.backend,
+ Name: file.GetName(),
+ RewriteRule: rule,
+ IsRawKv: true,
+ CipherInfo: cipher,
+ StorageCacheId: importer.cacheKey,
+ }
+ log.Debug("download SST", logutil.SSTMeta(sstMeta), logutil.Region(regionInfo.Region))
+
+ var atomicResp atomic.Pointer[import_sstpb.DownloadResponse]
+ eg, ectx := errgroup.WithContext(ctx)
+ for _, p := range regionInfo.Region.GetPeers() {
+ peer := p
+ eg.Go(func() error {
+ resp, err := importer.importClient.DownloadSST(ectx, peer.GetStoreId(), req)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if resp.GetError() != nil {
+ return errors.Annotate(berrors.ErrKVDownloadFailed, resp.GetError().GetMessage())
+ }
+ if resp.GetIsEmpty() {
+ return errors.Trace(berrors.ErrKVRangeIsEmpty)
+ }
+
+ atomicResp.Store(resp)
+ return nil
+ })
+ }
+
+ if err := eg.Wait(); err != nil {
+ return nil, err
+ }
+
+ downloadResp := atomicResp.Load()
+ sstMeta.Range.Start = downloadResp.Range.GetStart()
+ sstMeta.Range.End = downloadResp.Range.GetEnd()
+ sstMeta.ApiVersion = apiVersion
+ downloadMetas = append(downloadMetas, sstMeta)
+ }
+ return downloadMetas, nil
+}
+
+func (importer *SnapFileImporter) ingest(
+ ctx context.Context,
+ files []*backuppb.File,
+ info *split.RegionInfo,
+ downloadMetas []*import_sstpb.SSTMeta,
+) error {
+ tokenCh := importer.ingestTokensMap.acquireTokenCh(info.Leader.GetStoreId(), importer.concurrencyPerStore)
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-tokenCh:
+ }
+ defer func() {
+ importer.releaseToken(tokenCh)
+ }()
+ for {
+ ingestResp, errIngest := importer.ingestSSTs(ctx, downloadMetas, info)
+ if errIngest != nil {
+ return errors.Trace(errIngest)
+ }
+
+ errPb := ingestResp.GetError()
+ switch {
+ case errPb == nil:
+ return nil
+ case errPb.NotLeader != nil:
+ // If error is `NotLeader`, update the region info and retry
+ var newInfo *split.RegionInfo
+ if newLeader := errPb.GetNotLeader().GetLeader(); newLeader != nil {
+ newInfo = &split.RegionInfo{
+ Leader: newLeader,
+ Region: info.Region,
+ }
+ } else {
+ for {
+ // Slow path, get region from PD
+ newInfo, errIngest = importer.metaClient.GetRegion(
+ ctx, info.Region.GetStartKey())
+ if errIngest != nil {
+ return errors.Trace(errIngest)
+ }
+ if newInfo != nil {
+ break
+ }
+ // do not get region info, wait a second and GetRegion() again.
+ log.Warn("ingest get region by key return nil", logutil.Region(info.Region),
+ logutil.Files(files),
+ logutil.SSTMetas(downloadMetas),
+ )
+ time.Sleep(time.Second)
+ }
+ }
+
+ if !split.CheckRegionEpoch(newInfo, info) {
+ return errors.Trace(berrors.ErrKVEpochNotMatch)
+ }
+ log.Debug("ingest sst returns not leader error, retry it",
+ logutil.Files(files),
+ logutil.SSTMetas(downloadMetas),
+ logutil.Region(info.Region),
+ zap.Stringer("newLeader", newInfo.Leader))
+ info = newInfo
+ case errPb.EpochNotMatch != nil:
+ // TODO handle epoch not match error
+ // 1. retry download if needed
+ // 2. retry ingest
+ return errors.Trace(berrors.ErrKVEpochNotMatch)
+ case errPb.KeyNotInRegion != nil:
+ return errors.Trace(berrors.ErrKVKeyNotInRegion)
+ default:
+ // Other errors like `ServerIsBusy`, `RegionNotFound`, etc. should be retryable
+ return errors.Annotatef(berrors.ErrKVIngestFailed, "ingest error %s", errPb)
+ }
+ }
+}
+
+func (importer *SnapFileImporter) ingestSSTs(
+ ctx context.Context,
+ sstMetas []*import_sstpb.SSTMeta,
+ regionInfo *split.RegionInfo,
+) (*import_sstpb.IngestResponse, error) {
+ leader := regionInfo.Leader
+ if leader == nil {
+ return nil, errors.Annotatef(berrors.ErrPDLeaderNotFound,
+ "region id %d has no leader", regionInfo.Region.Id)
+ }
+ reqCtx := &kvrpcpb.Context{
+ RegionId: regionInfo.Region.GetId(),
+ RegionEpoch: regionInfo.Region.GetRegionEpoch(),
+ Peer: leader,
+ ResourceControlContext: &kvrpcpb.ResourceControlContext{
+ ResourceGroupName: "", // TODO,
+ },
+ RequestSource: kvutil.BuildRequestSource(true, kv.InternalTxnBR, kvutil.ExplicitTypeBR),
+ }
+
+ req := &import_sstpb.MultiIngestRequest{
+ Context: reqCtx,
+ Ssts: sstMetas,
+ }
+ log.Debug("ingest SSTs", logutil.SSTMetas(sstMetas), logutil.Leader(leader))
+ resp, err := importer.importClient.MultiIngest(ctx, leader.GetStoreId(), req)
+ return resp, errors.Trace(err)
+}
+
+func isDecryptSstErr(err error) bool {
+ return err != nil &&
+ strings.Contains(err.Error(), "Engine Engine") &&
+ strings.Contains(err.Error(), "Corruption: Bad table magic number")
+}
diff --git a/br/pkg/restore/file_importer/import_test.go b/br/pkg/restore/snap_client/import_test.go
similarity index 88%
rename from br/pkg/restore/file_importer/import_test.go
rename to br/pkg/restore/snap_client/import_test.go
index 3d90711baa117..7ed64c1b96ea8 100644
--- a/br/pkg/restore/file_importer/import_test.go
+++ b/br/pkg/restore/snap_client/import_test.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package file_importer_test
+package snapclient_test
import (
"testing"
@@ -20,7 +20,7 @@ import (
backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/kvproto/pkg/import_sstpb"
"github.com/pingcap/kvproto/pkg/metapb"
- fileimporter "github.com/pingcap/tidb/br/pkg/restore/file_importer"
+ snapclient "github.com/pingcap/tidb/br/pkg/restore/snap_client"
restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
"github.com/pingcap/tidb/pkg/util/codec"
"github.com/stretchr/testify/require"
@@ -46,7 +46,7 @@ func TestGetKeyRangeByMode(t *testing.T) {
},
}
// raw kv
- testRawFn := fileimporter.GetKeyRangeByModeForTest(fileimporter.Raw)
+ testRawFn := snapclient.GetKeyRangeByMode(snapclient.Raw)
start, end, err := testRawFn(file, rule)
require.NoError(t, err)
require.Equal(t, []byte("t1a"), start)
@@ -58,7 +58,7 @@ func TestGetKeyRangeByMode(t *testing.T) {
require.Equal(t, []byte(""), end)
// txn kv: the keys must be encoded.
- testTxnFn := fileimporter.GetKeyRangeByModeForTest(fileimporter.Txn)
+ testTxnFn := snapclient.GetKeyRangeByMode(snapclient.Txn)
start, end, err = testTxnFn(file, rule)
require.NoError(t, err)
require.Equal(t, codec.EncodeBytes(nil, []byte("t1a")), start)
@@ -70,7 +70,7 @@ func TestGetKeyRangeByMode(t *testing.T) {
require.Equal(t, []byte(""), end)
// normal kv: the keys must be encoded.
- testFn := fileimporter.GetKeyRangeByModeForTest(fileimporter.TiDB)
+ testFn := snapclient.GetKeyRangeByMode(snapclient.TiDB)
start, end, err = testFn(file, rule)
require.NoError(t, err)
require.Equal(t, codec.EncodeBytes(nil, []byte("t2a")), start)
@@ -99,7 +99,7 @@ func TestGetSSTMetaFromFile(t *testing.T) {
StartKey: []byte("t2abc"),
EndKey: []byte("t3a"),
}
- sstMeta, err := fileimporter.GetSSTMetaFromFile([]byte{}, file, region, rule, fileimporter.RewriteModeLegacy)
+ sstMeta, err := snapclient.GetSSTMetaFromFile([]byte{}, file, region, rule, snapclient.RewriteModeLegacy)
require.Nil(t, err)
require.Equal(t, "t2abc", string(sstMeta.GetRange().GetStart()))
require.Equal(t, "t2\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", string(sstMeta.GetRange().GetEnd()))
diff --git a/br/pkg/restore/snap_client/main_test.go b/br/pkg/restore/snap_client/main_test.go
new file mode 100644
index 0000000000000..b6904103e9969
--- /dev/null
+++ b/br/pkg/restore/snap_client/main_test.go
@@ -0,0 +1,57 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package snapclient_test
+
+import (
+ "fmt"
+ "os"
+ "testing"
+
+ "github.com/pingcap/tidb/br/pkg/mock"
+ "github.com/pingcap/tidb/pkg/testkit/testsetup"
+ "go.uber.org/goleak"
+)
+
+func TestMain(m *testing.M) {
+ testsetup.SetupForCommonTest()
+ opts := []goleak.Option{
+ goleak.IgnoreTopFunction("github.com/golang/glog.(*fileSink).flushDaemon"),
+ goleak.IgnoreTopFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"),
+ goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"),
+ goleak.IgnoreTopFunction("github.com/klauspost/compress/zstd.(*blockDec).startDecoder"),
+ goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"),
+ goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"),
+ goleak.IgnoreTopFunction("github.com/pingcap/goleveldb/leveldb.(*DB).mpoolDrain"),
+ }
+
+ var err error
+ mc, err = mock.NewCluster()
+ if err != nil {
+ panic(err)
+ }
+ err = mc.Start()
+ if err != nil {
+ panic(err)
+ }
+ exitCode := m.Run()
+ mc.Stop()
+ if exitCode == 0 {
+ if err := goleak.Find(opts...); err != nil {
+ fmt.Fprintf(os.Stderr, "goleak: Errors on successful test run: %v\n", err)
+ exitCode = 1
+ }
+ }
+ os.Exit(exitCode)
+}
diff --git a/br/pkg/restore/snap_client/pipeline_items.go b/br/pkg/restore/snap_client/pipeline_items.go
new file mode 100644
index 0000000000000..79f47d3a9c71c
--- /dev/null
+++ b/br/pkg/restore/snap_client/pipeline_items.go
@@ -0,0 +1,746 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package snapclient
+
+import (
+ "context"
+ "sort"
+ "sync"
+ "time"
+
+ "github.com/opentracing/opentracing-go"
+ "github.com/pingcap/errors"
+ backuppb "github.com/pingcap/kvproto/pkg/brpb"
+ "github.com/pingcap/log"
+ "github.com/pingcap/tidb/br/pkg/glue"
+ "github.com/pingcap/tidb/br/pkg/logutil"
+ "github.com/pingcap/tidb/br/pkg/metautil"
+ tidallocdb "github.com/pingcap/tidb/br/pkg/restore/internal/prealloc_db"
+ restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
+ "github.com/pingcap/tidb/br/pkg/rtree"
+ "github.com/pingcap/tidb/br/pkg/storage"
+ "github.com/pingcap/tidb/br/pkg/summary"
+ "github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/pkg/domain/infosync"
+ "github.com/pingcap/tidb/pkg/kv"
+ "github.com/pingcap/tidb/pkg/parser/model"
+ tidbutil "github.com/pingcap/tidb/pkg/util"
+ "github.com/pingcap/tidb/pkg/util/engine"
+ pdhttp "github.com/tikv/pd/client/http"
+ "go.uber.org/zap"
+ "golang.org/x/sync/errgroup"
+)
+
+const defaultChannelSize = 1024
+
+// defaultChecksumConcurrency is the default number of the concurrent
+// checksum tasks.
+const defaultChecksumConcurrency = 64
+
+// TableSink is the 'sink' of restored data by a sender.
+type TableSink interface {
+ EmitTables(tables ...CreatedTable)
+ EmitError(error)
+ Close()
+}
+
+type chanTableSink struct {
+ outCh chan<- []CreatedTable
+ errCh chan<- error
+}
+
+func (sink chanTableSink) EmitTables(tables ...CreatedTable) {
+ sink.outCh <- tables
+}
+
+func (sink chanTableSink) EmitError(err error) {
+ sink.errCh <- err
+}
+
+func (sink chanTableSink) Close() {
+ // ErrCh may has multi sender part, don't close it.
+ close(sink.outCh)
+}
+
+// CreatedTable is a table created on restore process,
+// but not yet filled with data.
+type CreatedTable struct {
+ RewriteRule *restoreutils.RewriteRules
+ Table *model.TableInfo
+ OldTable *metautil.Table
+}
+
+func defaultOutputTableChan() chan *CreatedTable {
+ return make(chan *CreatedTable, defaultChannelSize)
+}
+
+// TableWithRange is a CreatedTable that has been bind to some of key ranges.
+type TableWithRange struct {
+ CreatedTable
+
+ // Range has been rewrited by rewrite rules.
+ Range []rtree.Range
+}
+
+type TableIDWithFiles struct {
+ TableID int64
+
+ Files []*backuppb.File
+ // RewriteRules is the rewrite rules for the specify table.
+ // because these rules belongs to the *one table*.
+ // we can hold them here.
+ RewriteRules *restoreutils.RewriteRules
+}
+
+// BatchSender is the abstract of how the batcher send a batch.
+type BatchSender interface {
+ // PutSink sets the sink of this sender, user to this interface promise
+ // call this function at least once before first call to `RestoreBatch`.
+ PutSink(sink TableSink)
+ // RestoreBatch will send the restore request.
+ RestoreBatch(ranges DrainResult)
+ Close()
+}
+
+// TiKVRestorer is the minimal methods required for restoring.
+// It contains the primitive APIs extract from `restore.Client`, so some of arguments may seem redundant.
+// Maybe TODO: make a better abstraction?
+type TiKVRestorer interface {
+ // SplitRanges split regions implicated by the ranges and rewrite rules.
+ // After spliting, it also scatters the fresh regions.
+ SplitRanges(ctx context.Context,
+ ranges []rtree.Range,
+ updateCh glue.Progress,
+ isRawKv bool) error
+ // RestoreSSTFiles import the files to the TiKV.
+ RestoreSSTFiles(ctx context.Context,
+ tableIDWithFiles []TableIDWithFiles,
+ updateCh glue.Progress) error
+}
+
+type tikvSender struct {
+ client TiKVRestorer
+
+ updateCh glue.Progress
+
+ sink TableSink
+ inCh chan<- DrainResult
+
+ wg *sync.WaitGroup
+
+ tableWaiters *sync.Map
+}
+
+func (b *tikvSender) PutSink(sink TableSink) {
+ // don't worry about visibility, since we will call this before first call to
+ // RestoreBatch, which is a sync point.
+ b.sink = sink
+}
+
+func (b *tikvSender) RestoreBatch(ranges DrainResult) {
+ log.Info("restore batch: waiting ranges", zap.Int("range", len(b.inCh)))
+ b.inCh <- ranges
+}
+
+// NewTiKVSender make a sender that send restore requests to TiKV.
+func NewTiKVSender(
+ ctx context.Context,
+ cli TiKVRestorer,
+ updateCh glue.Progress,
+ splitConcurrency uint,
+) (BatchSender, error) {
+ inCh := make(chan DrainResult, defaultChannelSize)
+ midCh := make(chan drainResultAndDone, defaultChannelSize)
+
+ sender := &tikvSender{
+ client: cli,
+ updateCh: updateCh,
+ inCh: inCh,
+ wg: new(sync.WaitGroup),
+ tableWaiters: new(sync.Map),
+ }
+
+ sender.wg.Add(2)
+ go sender.splitWorker(ctx, inCh, midCh, splitConcurrency)
+ outCh := make(chan drainResultAndDone, defaultChannelSize)
+ // block on splitting and scattering regions.
+ // in coarse-grained mode, wait all regions are split and scattered is
+ // no longer a time-consuming operation, then we can batch download files
+ // as much as enough and reduce the time of blocking restore.
+ go sender.blockPipelineWorker(ctx, midCh, outCh)
+ go sender.restoreWorker(ctx, outCh)
+ return sender, nil
+}
+
+func (b *tikvSender) Close() {
+ close(b.inCh)
+ b.wg.Wait()
+ log.Debug("tikv sender closed")
+}
+
+type drainResultAndDone struct {
+ result DrainResult
+ done func()
+}
+
+func (b *tikvSender) blockPipelineWorker(ctx context.Context,
+ inCh <-chan drainResultAndDone,
+ outCh chan<- drainResultAndDone,
+) {
+ defer close(outCh)
+ res := make([]drainResultAndDone, 0, defaultChannelSize)
+ for dr := range inCh {
+ res = append(res, dr)
+ }
+
+ for _, dr := range res {
+ select {
+ case <-ctx.Done():
+ return
+ default:
+ outCh <- dr
+ }
+ }
+}
+
+func (b *tikvSender) splitWorker(ctx context.Context,
+ ranges <-chan DrainResult,
+ next chan<- drainResultAndDone,
+ concurrency uint,
+) {
+ defer log.Debug("split worker closed")
+ eg, ectx := errgroup.WithContext(ctx)
+ defer func() {
+ b.wg.Done()
+ if err := eg.Wait(); err != nil {
+ b.sink.EmitError(err)
+ }
+ close(next)
+ log.Info("TiKV Sender: split worker exits.")
+ }()
+
+ start := time.Now()
+ defer func() {
+ elapsed := time.Since(start)
+ summary.CollectDuration("split region", elapsed)
+ }()
+
+ pool := tidbutil.NewWorkerPool(concurrency, "split")
+ for {
+ select {
+ case <-ectx.Done():
+ return
+ case result, ok := <-ranges:
+ if !ok {
+ return
+ }
+ // When the batcher has sent all ranges from a table, it would
+ // mark this table 'all done'(BlankTablesAfterSend), and then we can send it to checksum.
+ //
+ // When there a sole worker sequentially running those batch tasks, everything is fine, however,
+ // in the context of multi-workers, that become buggy, for example:
+ // |------table 1, ranges 1------|------table 1, ranges 2------|
+ // The batcher send batches: [
+ // {Ranges: ranges 1},
+ // {Ranges: ranges 2, BlankTablesAfterSend: table 1}
+ // ]
+ // And there are two workers runs concurrently:
+ // worker 1: {Ranges: ranges 1}
+ // worker 2: {Ranges: ranges 2, BlankTablesAfterSend: table 1}
+ // And worker 2 finished its job before worker 1 done. Note the table wasn't restored fully,
+ // hence the checksum would fail.
+ done := b.registerTableIsRestoring(result.TablesToSend)
+ pool.ApplyOnErrorGroup(eg, func() error {
+ err := b.client.SplitRanges(ectx, result.Ranges, b.updateCh, false)
+ if err != nil {
+ log.Error("failed on split range", rtree.ZapRanges(result.Ranges), zap.Error(err))
+ return err
+ }
+ next <- drainResultAndDone{
+ result: result,
+ done: done,
+ }
+ return nil
+ })
+ }
+ }
+}
+
+// registerTableIsRestoring marks some tables as 'current restoring'.
+// Returning a function that mark the restore has been done.
+func (b *tikvSender) registerTableIsRestoring(ts []CreatedTable) func() {
+ wgs := make([]*sync.WaitGroup, 0, len(ts))
+ for _, t := range ts {
+ i, _ := b.tableWaiters.LoadOrStore(t.Table.ID, new(sync.WaitGroup))
+ wg := i.(*sync.WaitGroup)
+ wg.Add(1)
+ wgs = append(wgs, wg)
+ }
+ return func() {
+ for _, wg := range wgs {
+ wg.Done()
+ }
+ }
+}
+
+// waitTablesDone block the current goroutine,
+// till all tables provided are no more ‘current restoring’.
+func (b *tikvSender) waitTablesDone(ts []CreatedTable) {
+ for _, t := range ts {
+ wg, ok := b.tableWaiters.LoadAndDelete(t.Table.ID)
+ if !ok {
+ log.Panic("bug! table done before register!",
+ zap.Any("wait-table-map", b.tableWaiters),
+ zap.Stringer("table", t.Table.Name))
+ }
+ wg.(*sync.WaitGroup).Wait()
+ }
+}
+
+func (b *tikvSender) restoreWorker(ctx context.Context, ranges <-chan drainResultAndDone) {
+ eg, ectx := errgroup.WithContext(ctx)
+ defer func() {
+ log.Info("TiKV Sender: restore worker prepare to close.")
+ if err := eg.Wait(); err != nil {
+ b.sink.EmitError(err)
+ }
+ b.sink.Close()
+ b.wg.Done()
+ log.Info("TiKV Sender: restore worker exits.")
+ }()
+ for {
+ select {
+ case <-ectx.Done():
+ return
+ case r, ok := <-ranges:
+ if !ok {
+ return
+ }
+
+ files := r.result.Files()
+ // There has been a worker in the `RestoreSSTFiles` procedure.
+ // Spawning a raw goroutine won't make too many requests to TiKV.
+ eg.Go(func() error {
+ e := b.client.RestoreSSTFiles(ectx, files, b.updateCh)
+ if e != nil {
+ log.Error("restore batch meet error", logutil.ShortError(e), zapTableIDWithFiles(files))
+ r.done()
+ return e
+ }
+ log.Info("restore batch done", rtree.ZapRanges(r.result.Ranges), zapTableIDWithFiles(files))
+ r.done()
+ b.waitTablesDone(r.result.BlankTablesAfterSend)
+ b.sink.EmitTables(r.result.BlankTablesAfterSend...)
+ return nil
+ })
+ }
+ }
+}
+
+func concurrentHandleTablesCh(
+ ctx context.Context,
+ inCh <-chan *CreatedTable,
+ outCh chan<- *CreatedTable,
+ errCh chan<- error,
+ workers *tidbutil.WorkerPool,
+ processFun func(context.Context, *CreatedTable) error,
+ deferFun func()) {
+ eg, ectx := errgroup.WithContext(ctx)
+ defer func() {
+ if err := eg.Wait(); err != nil {
+ errCh <- err
+ }
+ close(outCh)
+ deferFun()
+ }()
+
+ for {
+ select {
+ // if we use ectx here, maybe canceled will mask real error.
+ case <-ctx.Done():
+ errCh <- ctx.Err()
+ case tbl, ok := <-inCh:
+ if !ok {
+ return
+ }
+ cloneTable := tbl
+ worker := workers.ApplyWorker()
+ eg.Go(func() error {
+ defer workers.RecycleWorker(worker)
+ err := processFun(ectx, cloneTable)
+ if err != nil {
+ return err
+ }
+ outCh <- cloneTable
+ return nil
+ })
+ }
+ }
+}
+
+// GoCreateTables create tables, and generate their information.
+// this function will use workers as the same number of sessionPool,
+// leave sessionPool nil to send DDLs sequential.
+func (rc *SnapClient) GoCreateTables(
+ ctx context.Context,
+ tables []*metautil.Table,
+ newTS uint64,
+ errCh chan<- error,
+) <-chan CreatedTable {
+ // Could we have a smaller size of tables?
+ log.Info("start create tables")
+
+ rc.generateRebasedTables(tables)
+ if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
+ span1 := span.Tracer().StartSpan("Client.GoCreateTables", opentracing.ChildOf(span.Context()))
+ defer span1.Finish()
+ ctx = opentracing.ContextWithSpan(ctx, span1)
+ }
+ outCh := make(chan CreatedTable, len(tables))
+ rater := logutil.TraceRateOver(logutil.MetricTableCreatedCounter)
+
+ var err error
+
+ if rc.batchDdlSize > minBatchDdlSize && len(rc.dbPool) > 0 {
+ err = rc.createTablesInWorkerPool(ctx, tables, newTS, outCh)
+ if err == nil {
+ defer log.Debug("all tables are created")
+ close(outCh)
+ return outCh
+ } else if !utils.FallBack2CreateTable(err) {
+ errCh <- err
+ close(outCh)
+ return outCh
+ }
+ // fall back to old create table (sequential create table)
+ log.Info("fall back to the sequential create table")
+ }
+
+ createOneTable := func(c context.Context, db *tidallocdb.DB, t *metautil.Table) error {
+ select {
+ case <-c.Done():
+ return c.Err()
+ default:
+ }
+ rt, err := rc.createTable(c, db, t, newTS)
+ if err != nil {
+ log.Error("create table failed",
+ zap.Error(err),
+ zap.Stringer("db", t.DB.Name),
+ zap.Stringer("table", t.Info.Name))
+ return errors.Trace(err)
+ }
+ log.Debug("table created and send to next",
+ zap.Int("output chan size", len(outCh)),
+ zap.Stringer("table", t.Info.Name),
+ zap.Stringer("database", t.DB.Name))
+ outCh <- rt
+ rater.Inc()
+ rater.L().Info("table created",
+ zap.Stringer("table", t.Info.Name),
+ zap.Stringer("database", t.DB.Name))
+ return nil
+ }
+ go func() {
+ defer close(outCh)
+ defer log.Debug("all tables are created")
+ var err error
+ if len(rc.dbPool) > 0 {
+ err = rc.createTablesWithDBPool(ctx, createOneTable, tables)
+ } else {
+ err = rc.createTablesWithSoleDB(ctx, createOneTable, tables)
+ }
+ if err != nil {
+ errCh <- err
+ }
+ }()
+
+ return outCh
+}
+
+func (rc *SnapClient) GoBlockCreateTablesPipeline(ctx context.Context, sz int, inCh <-chan CreatedTable) <-chan CreatedTable {
+ outCh := make(chan CreatedTable, sz)
+
+ go func() {
+ defer close(outCh)
+ cachedTables := make([]CreatedTable, 0, sz)
+ for tbl := range inCh {
+ cachedTables = append(cachedTables, tbl)
+ }
+
+ sort.Slice(cachedTables, func(a, b int) bool {
+ return cachedTables[a].Table.ID < cachedTables[b].Table.ID
+ })
+
+ for _, tbl := range cachedTables {
+ select {
+ case <-ctx.Done():
+ return
+ default:
+ outCh <- tbl
+ }
+ }
+ }()
+ return outCh
+}
+
+// GoValidateFileRanges validate files by a stream of tables and yields
+// tables with range.
+func (rc *SnapClient) GoValidateFileRanges(
+ ctx context.Context,
+ tableStream <-chan CreatedTable,
+ fileOfTable map[int64][]*backuppb.File,
+ splitSizeBytes, splitKeyCount uint64,
+ errCh chan<- error,
+) <-chan TableWithRange {
+ // Could we have a smaller outCh size?
+ outCh := make(chan TableWithRange, len(fileOfTable))
+ go func() {
+ defer close(outCh)
+ defer log.Info("all range generated")
+ for {
+ select {
+ case <-ctx.Done():
+ errCh <- ctx.Err()
+ return
+ case t, ok := <-tableStream:
+ if !ok {
+ return
+ }
+ files := fileOfTable[t.OldTable.Info.ID]
+ if partitions := t.OldTable.Info.Partition; partitions != nil {
+ log.Debug("table partition",
+ zap.Stringer("database", t.OldTable.DB.Name),
+ zap.Stringer("table", t.Table.Name),
+ zap.Any("partition info", partitions),
+ )
+ for _, partition := range partitions.Definitions {
+ files = append(files, fileOfTable[partition.ID]...)
+ }
+ }
+ for _, file := range files {
+ err := restoreutils.ValidateFileRewriteRule(file, t.RewriteRule)
+ if err != nil {
+ errCh <- err
+ return
+ }
+ }
+ // Merge small ranges to reduce split and scatter regions.
+ ranges, stat, err := restoreutils.MergeAndRewriteFileRanges(
+ files, t.RewriteRule, splitSizeBytes, splitKeyCount)
+ if err != nil {
+ errCh <- err
+ return
+ }
+ log.Info("merge and validate file",
+ zap.Stringer("database", t.OldTable.DB.Name),
+ zap.Stringer("table", t.Table.Name),
+ zap.Int("Files(total)", stat.TotalFiles),
+ zap.Int("File(write)", stat.TotalWriteCFFile),
+ zap.Int("File(default)", stat.TotalDefaultCFFile),
+ zap.Int("Region(total)", stat.TotalRegions),
+ zap.Int("Regoin(keys avg)", stat.RegionKeysAvg),
+ zap.Int("Region(bytes avg)", stat.RegionBytesAvg),
+ zap.Int("Merged(regions)", stat.MergedRegions),
+ zap.Int("Merged(keys avg)", stat.MergedRegionKeysAvg),
+ zap.Int("Merged(bytes avg)", stat.MergedRegionBytesAvg))
+
+ tableWithRange := TableWithRange{
+ CreatedTable: t,
+ Range: ranges,
+ }
+ log.Debug("sending range info",
+ zap.Stringer("table", t.Table.Name),
+ zap.Int("files", len(files)),
+ zap.Int("range size", len(ranges)),
+ zap.Int("output channel size", len(outCh)))
+ outCh <- tableWithRange
+ }
+ }
+ }()
+ return outCh
+}
+
+// GoValidateChecksum forks a goroutine to validate checksum after restore.
+// it returns a channel fires a struct{} when all things get done.
+func (rc *SnapClient) GoValidateChecksum(
+ ctx context.Context,
+ inCh <-chan *CreatedTable,
+ kvClient kv.Client,
+ errCh chan<- error,
+ updateCh glue.Progress,
+ concurrency uint,
+) chan *CreatedTable {
+ log.Info("Start to validate checksum")
+ outCh := defaultOutputTableChan()
+ workers := tidbutil.NewWorkerPool(defaultChecksumConcurrency, "RestoreChecksum")
+ go concurrentHandleTablesCh(ctx, inCh, outCh, errCh, workers, func(c context.Context, tbl *CreatedTable) error {
+ start := time.Now()
+ defer func() {
+ elapsed := time.Since(start)
+ summary.CollectSuccessUnit("table checksum", 1, elapsed)
+ }()
+ err := rc.execChecksum(c, tbl, kvClient, concurrency)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ updateCh.Inc()
+ return nil
+ }, func() {
+ log.Info("all checksum ended")
+ })
+ return outCh
+}
+
+func (rc *SnapClient) GoUpdateMetaAndLoadStats(
+ ctx context.Context,
+ s storage.ExternalStorage,
+ inCh <-chan *CreatedTable,
+ errCh chan<- error,
+ statsConcurrency uint,
+ loadStats bool,
+) chan *CreatedTable {
+ log.Info("Start to update meta then load stats")
+ outCh := defaultOutputTableChan()
+ workers := tidbutil.NewWorkerPool(statsConcurrency, "UpdateStats")
+ statsHandler := rc.dom.StatsHandle()
+
+ go concurrentHandleTablesCh(ctx, inCh, outCh, errCh, workers, func(c context.Context, tbl *CreatedTable) error {
+ oldTable := tbl.OldTable
+ var statsErr error = nil
+ if loadStats && oldTable.Stats != nil {
+ log.Info("start loads analyze after validate checksum",
+ zap.Int64("old id", oldTable.Info.ID),
+ zap.Int64("new id", tbl.Table.ID),
+ )
+ start := time.Now()
+ // NOTICE: skip updating cache after load stats from json
+ if statsErr = statsHandler.LoadStatsFromJSONNoUpdate(ctx, rc.dom.InfoSchema(), oldTable.Stats, 0); statsErr != nil {
+ log.Error("analyze table failed", zap.Any("table", oldTable.Stats), zap.Error(statsErr))
+ }
+ log.Info("restore stat done",
+ zap.Stringer("table", oldTable.Info.Name),
+ zap.Stringer("db", oldTable.DB.Name),
+ zap.Duration("cost", time.Since(start)))
+ } else if loadStats && len(oldTable.StatsFileIndexes) > 0 {
+ log.Info("start to load statistic data for each partition",
+ zap.Int64("old id", oldTable.Info.ID),
+ zap.Int64("new id", tbl.Table.ID),
+ )
+ start := time.Now()
+ rewriteIDMap := restoreutils.GetTableIDMap(tbl.Table, tbl.OldTable.Info)
+ if statsErr = metautil.RestoreStats(ctx, s, rc.cipher, statsHandler, tbl.Table, oldTable.StatsFileIndexes, rewriteIDMap); statsErr != nil {
+ log.Error("analyze table failed", zap.Any("table", oldTable.StatsFileIndexes), zap.Error(statsErr))
+ }
+ log.Info("restore statistic data done",
+ zap.Stringer("table", oldTable.Info.Name),
+ zap.Stringer("db", oldTable.DB.Name),
+ zap.Duration("cost", time.Since(start)))
+ }
+
+ if statsErr != nil || !loadStats || (oldTable.Stats == nil && len(oldTable.StatsFileIndexes) == 0) {
+ // Not need to return err when failed because of update analysis-meta
+ log.Info("start update metas", zap.Stringer("table", oldTable.Info.Name), zap.Stringer("db", oldTable.DB.Name))
+ // the total kvs contains the index kvs, but the stats meta needs the count of rows
+ count := int64(oldTable.TotalKvs / uint64(len(oldTable.Info.Indices)+1))
+ if statsErr = statsHandler.SaveMetaToStorage(tbl.Table.ID, count, 0, "br restore"); statsErr != nil {
+ log.Error("update stats meta failed", zap.Any("table", tbl.Table), zap.Error(statsErr))
+ }
+ }
+ return nil
+ }, func() {
+ log.Info("all stats updated")
+ })
+ return outCh
+}
+
+func (rc *SnapClient) GoWaitTiFlashReady(
+ ctx context.Context,
+ inCh <-chan *CreatedTable,
+ updateCh glue.Progress,
+ errCh chan<- error,
+) chan *CreatedTable {
+ log.Info("Start to wait tiflash replica sync")
+ outCh := defaultOutputTableChan()
+ workers := tidbutil.NewWorkerPool(4, "WaitForTiflashReady")
+ // TODO support tiflash store changes
+ tikvStats, err := infosync.GetTiFlashStoresStat(context.Background())
+ if err != nil {
+ errCh <- err
+ }
+ tiFlashStores := make(map[int64]pdhttp.StoreInfo)
+ for _, store := range tikvStats.Stores {
+ if engine.IsTiFlashHTTPResp(&store.Store) {
+ tiFlashStores[store.Store.ID] = store
+ }
+ }
+ go concurrentHandleTablesCh(ctx, inCh, outCh, errCh, workers, func(c context.Context, tbl *CreatedTable) error {
+ if tbl.Table != nil && tbl.Table.TiFlashReplica == nil {
+ log.Info("table has no tiflash replica",
+ zap.Stringer("table", tbl.OldTable.Info.Name),
+ zap.Stringer("db", tbl.OldTable.DB.Name))
+ updateCh.Inc()
+ return nil
+ }
+ if rc.dom == nil {
+ // unreachable, current we have initial domain in mgr.
+ log.Fatal("unreachable, domain is nil")
+ }
+ log.Info("table has tiflash replica, start sync..",
+ zap.Stringer("table", tbl.OldTable.Info.Name),
+ zap.Stringer("db", tbl.OldTable.DB.Name))
+ for {
+ var progress float64
+ if pi := tbl.Table.GetPartitionInfo(); pi != nil && len(pi.Definitions) > 0 {
+ for _, p := range pi.Definitions {
+ progressOfPartition, err := infosync.MustGetTiFlashProgress(p.ID, tbl.Table.TiFlashReplica.Count, &tiFlashStores)
+ if err != nil {
+ log.Warn("failed to get progress for tiflash partition replica, retry it",
+ zap.Int64("tableID", tbl.Table.ID), zap.Int64("partitionID", p.ID), zap.Error(err))
+ time.Sleep(time.Second)
+ continue
+ }
+ progress += progressOfPartition
+ }
+ progress = progress / float64(len(pi.Definitions))
+ } else {
+ var err error
+ progress, err = infosync.MustGetTiFlashProgress(tbl.Table.ID, tbl.Table.TiFlashReplica.Count, &tiFlashStores)
+ if err != nil {
+ log.Warn("failed to get progress for tiflash replica, retry it",
+ zap.Int64("tableID", tbl.Table.ID), zap.Error(err))
+ time.Sleep(time.Second)
+ continue
+ }
+ }
+ // check until progress is 1
+ if progress == 1 {
+ log.Info("tiflash replica synced",
+ zap.Stringer("table", tbl.OldTable.Info.Name),
+ zap.Stringer("db", tbl.OldTable.DB.Name))
+ break
+ }
+ // just wait for next check
+ // tiflash check the progress every 2s
+ // we can wait 2.5x times
+ time.Sleep(5 * time.Second)
+ }
+ updateCh.Inc()
+ return nil
+ }, func() {
+ log.Info("all tiflash replica synced")
+ })
+ return outCh
+}
diff --git a/br/pkg/restore/util_test.go b/br/pkg/restore/snap_client/pipeline_items_test.go
similarity index 67%
rename from br/pkg/restore/util_test.go
rename to br/pkg/restore/snap_client/pipeline_items_test.go
index 1716bc61ccbba..97660996b116a 100644
--- a/br/pkg/restore/util_test.go
+++ b/br/pkg/restore/snap_client/pipeline_items_test.go
@@ -1,6 +1,18 @@
-// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
-
-package restore
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package snapclient_test
import (
"context"
@@ -10,12 +22,11 @@ import (
"github.com/pingcap/errors"
backuppb "github.com/pingcap/kvproto/pkg/brpb"
- "github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/log"
berrors "github.com/pingcap/tidb/br/pkg/errors"
"github.com/pingcap/tidb/br/pkg/glue"
"github.com/pingcap/tidb/br/pkg/logutil"
- "github.com/pingcap/tidb/br/pkg/restore/split"
+ snapclient "github.com/pingcap/tidb/br/pkg/restore/snap_client"
"github.com/pingcap/tidb/br/pkg/rtree"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/stretchr/testify/require"
@@ -46,7 +57,7 @@ func (f *fakeRestorer) SplitRanges(ctx context.Context, ranges []rtree.Range, up
return nil
}
-func (f *fakeRestorer) RestoreSSTFiles(ctx context.Context, tableIDWithFiles []TableIDWithFiles, updateCh glue.Progress) error {
+func (f *fakeRestorer) RestoreSSTFiles(ctx context.Context, tableIDWithFiles []snapclient.TableIDWithFiles, updateCh glue.Progress) error {
f.mu.Lock()
defer f.mu.Unlock()
@@ -64,7 +75,7 @@ func (f *fakeRestorer) RestoreSSTFiles(ctx context.Context, tableIDWithFiles []T
return err
}
-func fakeRanges(keys ...string) (r DrainResult) {
+func fakeRanges(keys ...string) (r snapclient.DrainResult) {
for i := range keys {
if i+1 == len(keys) {
return
@@ -75,7 +86,7 @@ func fakeRanges(keys ...string) (r DrainResult) {
Files: []*backuppb.File{{Name: "fake.sst"}},
})
r.TableEndOffsetInRanges = append(r.TableEndOffsetInRanges, len(r.Ranges))
- r.TablesToSend = append(r.TablesToSend, CreatedTable{
+ r.TablesToSend = append(r.TablesToSend, snapclient.CreatedTable{
Table: &model.TableInfo{
ID: int64(i),
},
@@ -90,7 +101,7 @@ type errorInTimeSink struct {
t *testing.T
}
-func (e errorInTimeSink) EmitTables(tables ...CreatedTable) {}
+func (e errorInTimeSink) EmitTables(tables ...snapclient.CreatedTable) {}
func (e errorInTimeSink) EmitError(err error) {
e.errCh <- err
@@ -116,16 +127,14 @@ func assertErrorEmitInTime(ctx context.Context, t *testing.T) errorInTimeSink {
}
}
-func TestRestoreFailed(t *testing.T) {
- ranges := []DrainResult{
+func TestSplitFailed(t *testing.T) {
+ ranges := []snapclient.DrainResult{
fakeRanges("aax", "abx", "abz"),
fakeRanges("abz", "bbz", "bcy"),
fakeRanges("bcy", "cad", "xxy"),
}
- r := &fakeRestorer{
- tableIDIsInsequence: true,
- }
- sender, err := NewTiKVSender(context.TODO(), r, nil, 1, string(FineGrained))
+ r := &fakeRestorer{errorInSplit: true, tableIDIsInsequence: true}
+ sender, err := snapclient.NewTiKVSender(context.TODO(), r, nil, 1)
require.NoError(t, err)
dctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
@@ -135,20 +144,22 @@ func TestRestoreFailed(t *testing.T) {
sender.RestoreBatch(r)
}
sink.Wait()
- sink.Close()
sender.Close()
- require.GreaterOrEqual(t, len(r.restoredFiles), 1)
+ require.GreaterOrEqual(t, len(r.splitRanges), 2)
+ require.Len(t, r.restoredFiles, 0)
require.True(t, r.tableIDIsInsequence)
}
-func TestSplitFailed(t *testing.T) {
- ranges := []DrainResult{
+func TestRestoreFailed(t *testing.T) {
+ ranges := []snapclient.DrainResult{
fakeRanges("aax", "abx", "abz"),
fakeRanges("abz", "bbz", "bcy"),
fakeRanges("bcy", "cad", "xxy"),
}
- r := &fakeRestorer{errorInSplit: true, tableIDIsInsequence: true}
- sender, err := NewTiKVSender(context.TODO(), r, nil, 1, string(FineGrained))
+ r := &fakeRestorer{
+ tableIDIsInsequence: true,
+ }
+ sender, err := snapclient.NewTiKVSender(context.TODO(), r, nil, 1)
require.NoError(t, err)
dctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
@@ -157,53 +168,15 @@ func TestSplitFailed(t *testing.T) {
for _, r := range ranges {
sender.RestoreBatch(r)
}
- sink.Wait()
+ var wg sync.WaitGroup
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ sink.Wait()
+ }()
+ sink.Close()
sender.Close()
- require.GreaterOrEqual(t, len(r.splitRanges), 2)
- require.Len(t, r.restoredFiles, 0)
+ wg.Wait()
+ require.GreaterOrEqual(t, len(r.restoredFiles), 1)
require.True(t, r.tableIDIsInsequence)
}
-
-func regionInfo(startKey, endKey string) *split.RegionInfo {
- return &split.RegionInfo{
- Region: &metapb.Region{
- StartKey: []byte(startKey),
- EndKey: []byte(endKey),
- },
- }
-}
-
-func TestSplitCheckPartRegionConsistency(t *testing.T) {
- var (
- startKey []byte = []byte("a")
- endKey []byte = []byte("f")
- err error
- )
- err = split.CheckPartRegionConsistency(startKey, endKey, nil)
- require.Error(t, err)
- err = split.CheckPartRegionConsistency(startKey, endKey, []*split.RegionInfo{
- regionInfo("b", "c"),
- })
- require.Error(t, err)
- err = split.CheckPartRegionConsistency(startKey, endKey, []*split.RegionInfo{
- regionInfo("a", "c"),
- regionInfo("d", "e"),
- })
- require.Error(t, err)
- err = split.CheckPartRegionConsistency(startKey, endKey, []*split.RegionInfo{
- regionInfo("a", "c"),
- regionInfo("c", "d"),
- })
- require.NoError(t, err)
- err = split.CheckPartRegionConsistency(startKey, endKey, []*split.RegionInfo{
- regionInfo("a", "c"),
- regionInfo("c", "d"),
- regionInfo("d", "f"),
- })
- require.NoError(t, err)
- err = split.CheckPartRegionConsistency(startKey, endKey, []*split.RegionInfo{
- regionInfo("a", "c"),
- regionInfo("c", "z"),
- })
- require.NoError(t, err)
-}
diff --git a/br/pkg/restore/systable_restore.go b/br/pkg/restore/snap_client/systable_restore.go
similarity index 61%
rename from br/pkg/restore/systable_restore.go
rename to br/pkg/restore/snap_client/systable_restore.go
index 93b503f51b5b0..6210c2fccab4a 100644
--- a/br/pkg/restore/systable_restore.go
+++ b/br/pkg/restore/snap_client/systable_restore.go
@@ -1,6 +1,6 @@
// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
-package restore
+package snapclient
import (
"context"
@@ -11,8 +11,11 @@ import (
"github.com/pingcap/log"
berrors "github.com/pingcap/tidb/br/pkg/errors"
"github.com/pingcap/tidb/br/pkg/logutil"
+ "github.com/pingcap/tidb/br/pkg/metautil"
+ "github.com/pingcap/tidb/br/pkg/restore"
"github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/pkg/bindinfo"
+ "github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
filter "github.com/pingcap/tidb/pkg/util/table-filter"
@@ -39,6 +42,18 @@ var statsTables = map[string]map[string]struct{}{
},
}
+// tables in this map is restored when fullClusterRestore=true
+var sysPrivilegeTableMap = map[string]string{
+ "user": "(user = '%s' and host = '%%')", // since v1.0.0
+ "db": "(user = '%s' and host = '%%')", // since v1.0.0
+ "tables_priv": "(user = '%s' and host = '%%')", // since v1.0.0
+ "columns_priv": "(user = '%s' and host = '%%')", // since v1.0.0
+ "default_roles": "(user = '%s' and host = '%%')", // since v3.0.0
+ "role_edges": "(to_user = '%s' and to_host = '%%')", // since v3.0.0
+ "global_priv": "(user = '%s' and host = '%%')", // since v3.0.8
+ "global_grants": "(user = '%s' and host = '%%')", // since v5.0.3
+}
+
var unRecoverableTable = map[string]map[string]struct{}{
"mysql": {
// some variables in tidb (e.g. gc_safe_point) cannot be recovered.
@@ -80,7 +95,7 @@ func isStatsTable(schemaName string, tableName string) bool {
// RestoreSystemSchemas restores the system schema(i.e. the `mysql` schema).
// Detail see https://github.com/pingcap/br/issues/679#issuecomment-762592254.
-func (rc *Client) RestoreSystemSchemas(ctx context.Context, f filter.Filter) (rerr error) {
+func (rc *SnapClient) RestoreSystemSchemas(ctx context.Context, f filter.Filter) (rerr error) {
sysDBs := []string{mysql.SystemDB, mysql.SysDB}
for _, sysDB := range sysDBs {
err := rc.restoreSystemSchema(ctx, f, sysDB)
@@ -93,7 +108,7 @@ func (rc *Client) RestoreSystemSchemas(ctx context.Context, f filter.Filter) (re
// restoreSystemSchema restores a system schema(i.e. the `mysql` or `sys` schema).
// Detail see https://github.com/pingcap/br/issues/679#issuecomment-762592254.
-func (rc *Client) restoreSystemSchema(ctx context.Context, f filter.Filter, sysDB string) (rerr error) {
+func (rc *SnapClient) restoreSystemSchema(ctx context.Context, f filter.Filter, sysDB string) (rerr error) {
temporaryDB := utils.TemporaryDBName(sysDB)
defer func() {
// Don't clean the temporary database for next restore with checkpoint.
@@ -147,7 +162,7 @@ type database struct {
}
// getDatabaseByName make a record of a database from info schema by its name.
-func (rc *Client) getDatabaseByName(name string) (*database, bool) {
+func (rc *SnapClient) getDatabaseByName(name string) (*database, bool) {
infoSchema := rc.dom.InfoSchema()
schema, ok := infoSchema.SchemaByName(model.NewCIStr(name))
if !ok {
@@ -166,7 +181,7 @@ func (rc *Client) getDatabaseByName(name string) (*database, bool) {
// afterSystemTablesReplaced do some extra work for special system tables.
// e.g. after inserting to the table mysql.user, we must execute `FLUSH PRIVILEGES` to allow it take effect.
-func (rc *Client) afterSystemTablesReplaced(ctx context.Context, db string, tables []string) error {
+func (rc *SnapClient) afterSystemTablesReplaced(ctx context.Context, db string, tables []string) error {
if db != mysql.SystemDB {
return nil
}
@@ -181,7 +196,7 @@ func (rc *Client) afterSystemTablesReplaced(ctx context.Context, db string, tabl
log.Info("privilege system table restored, please reconnect to make it effective")
}
} else if table == "bind_info" {
- if serr := rc.db.se.Execute(ctx, bindinfo.StmtRemoveDuplicatedPseudoBinding); serr != nil {
+ if serr := rc.db.Session().Execute(ctx, bindinfo.StmtRemoveDuplicatedPseudoBinding); serr != nil {
log.Warn("failed to delete duplicated pseudo binding", zap.Error(serr))
err = multierr.Append(err,
berrors.ErrUnknown.Wrap(serr).GenWithStack("failed to delete duplicated pseudo binding %s", bindinfo.StmtRemoveDuplicatedPseudoBinding))
@@ -194,12 +209,12 @@ func (rc *Client) afterSystemTablesReplaced(ctx context.Context, db string, tabl
}
// replaceTemporaryTableToSystable replaces the temporary table to real system table.
-func (rc *Client) replaceTemporaryTableToSystable(ctx context.Context, ti *model.TableInfo, db *database) error {
+func (rc *SnapClient) replaceTemporaryTableToSystable(ctx context.Context, ti *model.TableInfo, db *database) error {
dbName := db.Name.L
tableName := ti.Name.L
execSQL := func(sql string) error {
// SQLs here only contain table name and database name, seems it is no need to redact them.
- if err := rc.db.se.Execute(ctx, sql); err != nil {
+ if err := rc.db.Session().Execute(ctx, sql); err != nil {
log.Warn("failed to execute SQL restore system database",
zap.String("table", tableName),
zap.Stringer("database", db.Name),
@@ -272,14 +287,106 @@ func (rc *Client) replaceTemporaryTableToSystable(ctx context.Context, ti *model
return execSQL(renameSQL)
}
-func (rc *Client) cleanTemporaryDatabase(ctx context.Context, originDB string) {
+func (rc *SnapClient) cleanTemporaryDatabase(ctx context.Context, originDB string) {
database := utils.TemporaryDBName(originDB)
log.Debug("dropping temporary database", zap.Stringer("database", database))
sql := fmt.Sprintf("DROP DATABASE IF EXISTS %s", utils.EncloseName(database.L))
- if err := rc.db.se.Execute(ctx, sql); err != nil {
+ if err := rc.db.Session().Execute(ctx, sql); err != nil {
logutil.WarnTerm("failed to drop temporary database, it should be dropped manually",
zap.Stringer("database", database),
logutil.ShortError(err),
)
}
}
+
+func CheckSysTableCompatibility(dom *domain.Domain, tables []*metautil.Table) error {
+ log.Info("checking target cluster system table compatibility with backed up data")
+ privilegeTablesInBackup := make([]*metautil.Table, 0)
+ for _, table := range tables {
+ decodedSysDBName, ok := utils.GetSysDBCIStrName(table.DB.Name)
+ if ok && decodedSysDBName.L == mysql.SystemDB && sysPrivilegeTableMap[table.Info.Name.L] != "" {
+ privilegeTablesInBackup = append(privilegeTablesInBackup, table)
+ }
+ }
+ sysDB := model.NewCIStr(mysql.SystemDB)
+ for _, table := range privilegeTablesInBackup {
+ ti, err := restore.GetTableSchema(dom, sysDB, table.Info.Name)
+ if err != nil {
+ log.Error("missing table on target cluster", zap.Stringer("table", table.Info.Name))
+ return errors.Annotate(berrors.ErrRestoreIncompatibleSys, "missed system table: "+table.Info.Name.O)
+ }
+ backupTi := table.Info
+ // skip checking the number of columns in mysql.user table,
+ // because higher versions of TiDB may add new columns.
+ if len(ti.Columns) != len(backupTi.Columns) && backupTi.Name.L != sysUserTableName {
+ log.Error("column count mismatch",
+ zap.Stringer("table", table.Info.Name),
+ zap.Int("col in cluster", len(ti.Columns)),
+ zap.Int("col in backup", len(backupTi.Columns)))
+ return errors.Annotatef(berrors.ErrRestoreIncompatibleSys,
+ "column count mismatch, table: %s, col in cluster: %d, col in backup: %d",
+ table.Info.Name.O, len(ti.Columns), len(backupTi.Columns))
+ }
+ backupColMap := make(map[string]*model.ColumnInfo)
+ for i := range backupTi.Columns {
+ col := backupTi.Columns[i]
+ backupColMap[col.Name.L] = col
+ }
+ // order can be different but type must compatible
+ for i := range ti.Columns {
+ col := ti.Columns[i]
+ backupCol := backupColMap[col.Name.L]
+ if backupCol == nil {
+ // skip when the backed up mysql.user table is missing columns.
+ if backupTi.Name.L == sysUserTableName {
+ log.Warn("missing column in backup data",
+ zap.Stringer("table", table.Info.Name),
+ zap.String("col", fmt.Sprintf("%s %s", col.Name, col.FieldType.String())))
+ continue
+ }
+ log.Error("missing column in backup data",
+ zap.Stringer("table", table.Info.Name),
+ zap.String("col", fmt.Sprintf("%s %s", col.Name, col.FieldType.String())))
+ return errors.Annotatef(berrors.ErrRestoreIncompatibleSys,
+ "missing column in backup data, table: %s, col: %s %s",
+ table.Info.Name.O,
+ col.Name, col.FieldType.String())
+ }
+ if !utils.IsTypeCompatible(backupCol.FieldType, col.FieldType) {
+ log.Error("incompatible column",
+ zap.Stringer("table", table.Info.Name),
+ zap.String("col in cluster", fmt.Sprintf("%s %s", col.Name, col.FieldType.String())),
+ zap.String("col in backup", fmt.Sprintf("%s %s", backupCol.Name, backupCol.FieldType.String())))
+ return errors.Annotatef(berrors.ErrRestoreIncompatibleSys,
+ "incompatible column, table: %s, col in cluster: %s %s, col in backup: %s %s",
+ table.Info.Name.O,
+ col.Name, col.FieldType.String(),
+ backupCol.Name, backupCol.FieldType.String())
+ }
+ }
+
+ if backupTi.Name.L == sysUserTableName {
+ // check whether the columns of table in cluster are less than the backup data
+ clusterColMap := make(map[string]*model.ColumnInfo)
+ for i := range ti.Columns {
+ col := ti.Columns[i]
+ clusterColMap[col.Name.L] = col
+ }
+ // order can be different
+ for i := range backupTi.Columns {
+ col := backupTi.Columns[i]
+ clusterCol := clusterColMap[col.Name.L]
+ if clusterCol == nil {
+ log.Error("missing column in cluster data",
+ zap.Stringer("table", table.Info.Name),
+ zap.String("col", fmt.Sprintf("%s %s", col.Name, col.FieldType.String())))
+ return errors.Annotatef(berrors.ErrRestoreIncompatibleSys,
+ "missing column in cluster data, table: %s, col: %s %s",
+ table.Info.Name.O,
+ col.Name, col.FieldType.String())
+ }
+ }
+ }
+ }
+ return nil
+}
diff --git a/br/pkg/restore/snap_client/systable_restore_test.go b/br/pkg/restore/snap_client/systable_restore_test.go
new file mode 100644
index 0000000000000..2c2cf6938a83f
--- /dev/null
+++ b/br/pkg/restore/snap_client/systable_restore_test.go
@@ -0,0 +1,119 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package snapclient_test
+
+import (
+ "math"
+ "testing"
+
+ berrors "github.com/pingcap/tidb/br/pkg/errors"
+ "github.com/pingcap/tidb/br/pkg/gluetidb"
+ "github.com/pingcap/tidb/br/pkg/metautil"
+ "github.com/pingcap/tidb/br/pkg/restore"
+ snapclient "github.com/pingcap/tidb/br/pkg/restore/snap_client"
+ "github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/br/pkg/utiltest"
+ "github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/pingcap/tidb/pkg/parser/mysql"
+ "github.com/pingcap/tidb/pkg/session"
+ "github.com/stretchr/testify/require"
+)
+
+func TestCheckSysTableCompatibility(t *testing.T) {
+ cluster := mc
+ g := gluetidb.New()
+ client := snapclient.NewRestoreClient(cluster.PDClient, cluster.PDHTTPCli, nil, utiltest.DefaultTestKeepaliveCfg)
+ err := client.Init(g, cluster.Storage)
+ require.NoError(t, err)
+
+ info, err := cluster.Domain.GetSnapshotInfoSchema(math.MaxUint64)
+ require.NoError(t, err)
+ dbSchema, isExist := info.SchemaByName(model.NewCIStr(mysql.SystemDB))
+ require.True(t, isExist)
+ tmpSysDB := dbSchema.Clone()
+ tmpSysDB.Name = utils.TemporaryDBName(mysql.SystemDB)
+ sysDB := model.NewCIStr(mysql.SystemDB)
+ userTI, err := restore.GetTableSchema(cluster.Domain, sysDB, model.NewCIStr("user"))
+ require.NoError(t, err)
+
+ // user table in cluster have more columns(success)
+ mockedUserTI := userTI.Clone()
+ userTI.Columns = append(userTI.Columns, &model.ColumnInfo{Name: model.NewCIStr("new-name")})
+ err = snapclient.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{
+ DB: tmpSysDB,
+ Info: mockedUserTI,
+ }})
+ require.NoError(t, err)
+ userTI.Columns = userTI.Columns[:len(userTI.Columns)-1]
+
+ // user table in cluster have less columns(failed)
+ mockedUserTI = userTI.Clone()
+ mockedUserTI.Columns = append(mockedUserTI.Columns, &model.ColumnInfo{Name: model.NewCIStr("new-name")})
+ err = snapclient.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{
+ DB: tmpSysDB,
+ Info: mockedUserTI,
+ }})
+ require.True(t, berrors.ErrRestoreIncompatibleSys.Equal(err))
+
+ // column order mismatch(success)
+ mockedUserTI = userTI.Clone()
+ mockedUserTI.Columns[4], mockedUserTI.Columns[5] = mockedUserTI.Columns[5], mockedUserTI.Columns[4]
+ err = snapclient.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{
+ DB: tmpSysDB,
+ Info: mockedUserTI,
+ }})
+ require.NoError(t, err)
+
+ // incompatible column type
+ mockedUserTI = userTI.Clone()
+ mockedUserTI.Columns[0].FieldType.SetFlen(2000) // Columns[0] is `Host` char(255)
+ err = snapclient.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{
+ DB: tmpSysDB,
+ Info: mockedUserTI,
+ }})
+ require.True(t, berrors.ErrRestoreIncompatibleSys.Equal(err))
+
+ // compatible
+ mockedUserTI = userTI.Clone()
+ err = snapclient.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{
+ DB: tmpSysDB,
+ Info: mockedUserTI,
+ }})
+ require.NoError(t, err)
+
+ // use the mysql.db table to test for column count mismatch.
+ dbTI, err := restore.GetTableSchema(cluster.Domain, sysDB, model.NewCIStr("db"))
+ require.NoError(t, err)
+
+ // other system tables in cluster have more columns(failed)
+ mockedDBTI := dbTI.Clone()
+ dbTI.Columns = append(dbTI.Columns, &model.ColumnInfo{Name: model.NewCIStr("new-name")})
+ err = snapclient.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{
+ DB: tmpSysDB,
+ Info: mockedDBTI,
+ }})
+ require.True(t, berrors.ErrRestoreIncompatibleSys.Equal(err))
+}
+
+// NOTICE: Once there is a new system table, BR needs to ensure that it is correctly classified:
+//
+// - IF it is an unrecoverable table, please add the table name into `unRecoverableTable`.
+// - IF it is an system privilege table, please add the table name into `sysPrivilegeTableMap`.
+// - IF it is an statistics table, please add the table name into `statsTables`.
+//
+// The above variables are in the file br/pkg/restore/systable_restore.go
+func TestMonitorTheSystemTableIncremental(t *testing.T) {
+ require.Equal(t, int64(198), session.CurrentBootstrapVersion)
+}
diff --git a/br/pkg/restore/logutil.go b/br/pkg/restore/snap_client/zap.go
similarity index 66%
rename from br/pkg/restore/logutil.go
rename to br/pkg/restore/snap_client/zap.go
index bcc032a10bf01..453b3337d5e82 100644
--- a/br/pkg/restore/logutil.go
+++ b/br/pkg/restore/snap_client/zap.go
@@ -1,4 +1,4 @@
-// Copyright 2023 PingCAP, Inc.
+// Copyright 2024 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,15 +12,32 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package restore
+package snapclient
import (
+ "fmt"
+
"github.com/pingcap/errors"
"github.com/pingcap/tidb/br/pkg/logutil"
+ "github.com/pingcap/tidb/br/pkg/utils"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
+// ZapTables make zap field of table for debuging, including table names.
+func zapTables(tables []CreatedTable) zapcore.Field {
+ return logutil.AbbreviatedArray("tables", tables, func(input any) []string {
+ tables := input.([]CreatedTable)
+ names := make([]string, 0, len(tables))
+ for _, t := range tables {
+ names = append(names, fmt.Sprintf("%s.%s",
+ utils.EncloseName(t.OldTable.DB.Name.String()),
+ utils.EncloseName(t.OldTable.Info.Name.String())))
+ }
+ return names
+ })
+}
+
type zapTableIDWithFilesMarshaler []TableIDWithFiles
func zapTableIDWithFiles(fs []TableIDWithFiles) zap.Field {
diff --git a/br/pkg/restore/split/BUILD.bazel b/br/pkg/restore/split/BUILD.bazel
index 448f67d62a579..1d1c05350b8a9 100644
--- a/br/pkg/restore/split/BUILD.bazel
+++ b/br/pkg/restore/split/BUILD.bazel
@@ -7,7 +7,6 @@ go_library(
"mock_pd_client.go",
"region.go",
"split.go",
- "sum_sorted.go",
],
importpath = "github.com/pingcap/tidb/br/pkg/restore/split",
visibility = ["//visibility:public"],
@@ -26,7 +25,6 @@ go_library(
"//pkg/util/intest",
"//pkg/util/redact",
"@com_github_docker_go_units//:go-units",
- "@com_github_google_btree//:btree",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_kvproto//pkg/errorpb",
@@ -54,11 +52,10 @@ go_test(
srcs = [
"client_test.go",
"split_test.go",
- "sum_sorted_test.go",
],
embed = [":split"],
flaky = True,
- shard_count = 18,
+ shard_count = 17,
deps = [
"//br/pkg/errors",
"//br/pkg/utils",
diff --git a/br/pkg/restore/split/split.go b/br/pkg/restore/split/split.go
index c69e5959f9812..b41a0ec62fa94 100644
--- a/br/pkg/restore/split/split.go
+++ b/br/pkg/restore/split/split.go
@@ -6,6 +6,7 @@ import (
"bytes"
"context"
"encoding/hex"
+ goerrors "errors"
"time"
"github.com/pingcap/errors"
@@ -112,7 +113,7 @@ func PaginateScanRegion(
var batch []*RegionInfo
batch, err = client.ScanRegions(ctx, scanStartKey, endKey, limit)
if err != nil {
- err = errors.Annotatef(berrors.ErrPDBatchScanRegion, "scan regions from start-key:%s, err: %s",
+ err = errors.Annotatef(berrors.ErrPDBatchScanRegion.Wrap(err), "scan regions from start-key:%s, err: %s",
redact.Key(scanStartKey), err.Error())
return err
}
@@ -227,7 +228,8 @@ func NewWaitRegionOnlineBackoffer() *WaitRegionOnlineBackoffer {
// NextBackoff returns a duration to wait before retrying again
func (b *WaitRegionOnlineBackoffer) NextBackoff(err error) time.Duration {
// TODO(lance6716): why we only backoff when the error is ErrPDBatchScanRegion?
- if berrors.ErrPDBatchScanRegion.Equal(err) {
+ var perr *errors.Error
+ if goerrors.As(err, &perr) && berrors.ErrPDBatchScanRegion.ID() == perr.ID() {
// it needs more time to wait splitting the regions that contains data in PITR.
// 2s * 150
delayTime := b.Stat.ExponentialBackoff()
diff --git a/br/pkg/restore/util.go b/br/pkg/restore/util.go
deleted file mode 100644
index 99d6b00b273a5..0000000000000
--- a/br/pkg/restore/util.go
+++ /dev/null
@@ -1,152 +0,0 @@
-// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
-
-package restore
-
-import (
- "context"
- "fmt"
-
- _ "github.com/go-sql-driver/mysql" // mysql driver
- backuppb "github.com/pingcap/kvproto/pkg/brpb"
- "github.com/pingcap/log"
- "github.com/pingcap/tidb/br/pkg/glue"
- "github.com/pingcap/tidb/br/pkg/logutil"
- "github.com/pingcap/tidb/br/pkg/restore/split"
- restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
- "github.com/pingcap/tidb/br/pkg/rtree"
- "github.com/pingcap/tidb/br/pkg/utils"
- "go.uber.org/zap"
- "go.uber.org/zap/zapcore"
-)
-
-type Granularity string
-
-const (
- FineGrained Granularity = "fine-grained"
- CoarseGrained Granularity = "coarse-grained"
-
- maxSplitKeysOnce = 10240
-)
-
-// GoValidateFileRanges validate files by a stream of tables and yields
-// tables with range.
-func GoValidateFileRanges(
- ctx context.Context,
- tableStream <-chan CreatedTable,
- fileOfTable map[int64][]*backuppb.File,
- splitSizeBytes, splitKeyCount uint64,
- errCh chan<- error,
-) <-chan TableWithRange {
- // Could we have a smaller outCh size?
- outCh := make(chan TableWithRange, len(fileOfTable))
- go func() {
- defer close(outCh)
- defer log.Info("all range generated")
- for {
- select {
- case <-ctx.Done():
- errCh <- ctx.Err()
- return
- case t, ok := <-tableStream:
- if !ok {
- return
- }
- files := fileOfTable[t.OldTable.Info.ID]
- if partitions := t.OldTable.Info.Partition; partitions != nil {
- log.Debug("table partition",
- zap.Stringer("database", t.OldTable.DB.Name),
- zap.Stringer("table", t.Table.Name),
- zap.Any("partition info", partitions),
- )
- for _, partition := range partitions.Definitions {
- files = append(files, fileOfTable[partition.ID]...)
- }
- }
- for _, file := range files {
- err := restoreutils.ValidateFileRewriteRule(file, t.RewriteRule)
- if err != nil {
- errCh <- err
- return
- }
- }
- // Merge small ranges to reduce split and scatter regions.
- ranges, stat, err := restoreutils.MergeAndRewriteFileRanges(
- files, t.RewriteRule, splitSizeBytes, splitKeyCount)
- if err != nil {
- errCh <- err
- return
- }
- log.Info("merge and validate file",
- zap.Stringer("database", t.OldTable.DB.Name),
- zap.Stringer("table", t.Table.Name),
- zap.Int("Files(total)", stat.TotalFiles),
- zap.Int("File(write)", stat.TotalWriteCFFile),
- zap.Int("File(default)", stat.TotalDefaultCFFile),
- zap.Int("Region(total)", stat.TotalRegions),
- zap.Int("Regoin(keys avg)", stat.RegionKeysAvg),
- zap.Int("Region(bytes avg)", stat.RegionBytesAvg),
- zap.Int("Merged(regions)", stat.MergedRegions),
- zap.Int("Merged(keys avg)", stat.MergedRegionKeysAvg),
- zap.Int("Merged(bytes avg)", stat.MergedRegionBytesAvg))
-
- tableWithRange := TableWithRange{
- CreatedTable: t,
- Range: ranges,
- }
- log.Debug("sending range info",
- zap.Stringer("table", t.Table.Name),
- zap.Int("files", len(files)),
- zap.Int("range size", len(ranges)),
- zap.Int("output channel size", len(outCh)))
- outCh <- tableWithRange
- }
- }
- }()
- return outCh
-}
-
-// SplitRanges splits region by
-// 1. data range after rewrite.
-// 2. rewrite rules.
-func SplitRanges(
- ctx context.Context,
- client *Client,
- ranges []rtree.Range,
- updateCh glue.Progress,
- isRawKv bool,
-) error {
- splitClientOpts := make([]split.ClientOptionalParameter, 0, 2)
- splitClientOpts = append(splitClientOpts, split.WithOnSplit(func(keys [][]byte) {
- for range keys {
- updateCh.Inc()
- }
- }))
- if isRawKv {
- splitClientOpts = append(splitClientOpts, split.WithRawKV())
- }
-
- splitter := restoreutils.NewRegionSplitter(split.NewClient(
- client.GetPDClient(),
- client.pdHTTPClient,
- client.GetTLSConfig(),
- maxSplitKeysOnce,
- client.GetStoreCount()+1,
- splitClientOpts...,
- ))
-
- return splitter.ExecuteSplit(ctx, ranges)
-}
-
-// ZapTables make zap field of table for debuging, including table names.
-func ZapTables(tables []CreatedTable) zapcore.Field {
- return logutil.AbbreviatedArray("tables", tables, func(input any) []string {
- tables := input.([]CreatedTable)
- names := make([]string, 0, len(tables))
- for _, t := range tables {
- names = append(names, fmt.Sprintf("%s.%s",
- utils.EncloseName(t.OldTable.DB.Name.String()),
- utils.EncloseName(t.OldTable.Info.Name.String())))
- }
- return names
- })
-}
diff --git a/br/pkg/restore/utils/BUILD.bazel b/br/pkg/restore/utils/BUILD.bazel
index 002654bf00b6f..ec0b08d320555 100644
--- a/br/pkg/restore/utils/BUILD.bazel
+++ b/br/pkg/restore/utils/BUILD.bazel
@@ -3,26 +3,20 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "utils",
srcs = [
- "id.go",
- "key.go",
- "log_file_map.go",
"merge.go",
+ "misc.go",
"rewrite_rule.go",
- "split.go",
- "value.go",
],
importpath = "github.com/pingcap/tidb/br/pkg/restore/utils",
visibility = ["//visibility:public"],
deps = [
"//br/pkg/errors",
"//br/pkg/logutil",
- "//br/pkg/restore/split",
"//br/pkg/rtree",
"//pkg/parser/model",
"//pkg/tablecodec",
"//pkg/util/codec",
"//pkg/util/redact",
- "@com_github_opentracing_opentracing_go//:opentracing-go",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_kvproto//pkg/brpb",
"@com_github_pingcap_kvproto//pkg/import_sstpb",
@@ -35,26 +29,29 @@ go_test(
name = "utils_test",
timeout = "short",
srcs = [
- "log_file_map_test.go",
"merge_test.go",
+ "misc_test.go",
"rewrite_rule_test.go",
- "split_test.go",
],
flaky = True,
- shard_count = 12,
+ shard_count = 9,
deps = [
":utils",
"//br/pkg/conn",
"//br/pkg/errors",
+ "//br/pkg/restore",
"//br/pkg/restore/split",
"//br/pkg/rtree",
+ "//br/pkg/utiltest",
"//pkg/sessionctx/stmtctx",
"//pkg/tablecodec",
"//pkg/types",
"//pkg/util/codec",
"@com_github_pingcap_errors//:errors",
+ "@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_kvproto//pkg/brpb",
"@com_github_pingcap_kvproto//pkg/import_sstpb",
+ "@com_github_pingcap_kvproto//pkg/metapb",
"@com_github_stretchr_testify//require",
],
)
diff --git a/br/pkg/restore/utils/id.go b/br/pkg/restore/utils/misc.go
similarity index 75%
rename from br/pkg/restore/utils/id.go
rename to br/pkg/restore/utils/misc.go
index 02b72ef1c7dcc..ba146de5a53d4 100644
--- a/br/pkg/restore/utils/id.go
+++ b/br/pkg/restore/utils/misc.go
@@ -14,7 +14,15 @@
package utils
-import "github.com/pingcap/tidb/pkg/parser/model"
+import (
+ "github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/pingcap/tidb/pkg/util/codec"
+)
+
+const (
+ WriteCFName = "write"
+ DefaultCFName = "default"
+)
// GetPartitionIDMap creates a map maping old physical ID to new physical ID.
func GetPartitionIDMap(newTable, oldTable *model.TableInfo) map[int64]int64 {
@@ -56,3 +64,20 @@ func GetIndexIDMap(newTable, oldTable *model.TableInfo) map[int64]int64 {
return indexIDMap
}
+
+func TruncateTS(key []byte) []byte {
+ if len(key) == 0 {
+ return nil
+ }
+ if len(key) < 8 {
+ return key
+ }
+ return key[:len(key)-8]
+}
+
+func EncodeKeyPrefix(key []byte) []byte {
+ encodedPrefix := make([]byte, 0)
+ ungroupedLen := len(key) % 8
+ encodedPrefix = append(encodedPrefix, codec.EncodeBytes([]byte{}, key[:len(key)-ungroupedLen])...)
+ return append(encodedPrefix[:len(encodedPrefix)-9], key[len(key)-ungroupedLen:]...)
+}
diff --git a/br/pkg/restore/utils/misc_test.go b/br/pkg/restore/utils/misc_test.go
new file mode 100644
index 0000000000000..f7314829f3af9
--- /dev/null
+++ b/br/pkg/restore/utils/misc_test.go
@@ -0,0 +1,98 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils_test
+
+import (
+ "context"
+ "testing"
+
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/kvproto/pkg/metapb"
+ "github.com/pingcap/tidb/br/pkg/restore"
+ "github.com/pingcap/tidb/br/pkg/restore/split"
+ "github.com/pingcap/tidb/br/pkg/utiltest"
+ "github.com/stretchr/testify/require"
+)
+
+func TestGetTSWithRetry(t *testing.T) {
+ t.Run("PD leader is healthy:", func(t *testing.T) {
+ retryTimes := -1000
+ pDClient := utiltest.NewFakePDClient(nil, false, &retryTimes)
+ _, err := restore.GetTSWithRetry(context.Background(), pDClient)
+ require.NoError(t, err)
+ })
+
+ t.Run("PD leader failure:", func(t *testing.T) {
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/br/pkg/utils/set-attempt-to-one", "1*return(true)"))
+ defer func() {
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/br/pkg/utils/set-attempt-to-one"))
+ }()
+ retryTimes := -1000
+ pDClient := utiltest.NewFakePDClient(nil, true, &retryTimes)
+ _, err := restore.GetTSWithRetry(context.Background(), pDClient)
+ require.Error(t, err)
+ })
+
+ t.Run("PD leader switch successfully", func(t *testing.T) {
+ retryTimes := 0
+ pDClient := utiltest.NewFakePDClient(nil, true, &retryTimes)
+ _, err := restore.GetTSWithRetry(context.Background(), pDClient)
+ require.NoError(t, err)
+ })
+}
+
+func regionInfo(startKey, endKey string) *split.RegionInfo {
+ return &split.RegionInfo{
+ Region: &metapb.Region{
+ StartKey: []byte(startKey),
+ EndKey: []byte(endKey),
+ },
+ }
+}
+
+func TestSplitCheckPartRegionConsistency(t *testing.T) {
+ var (
+ startKey []byte = []byte("a")
+ endKey []byte = []byte("f")
+ err error
+ )
+ err = split.CheckPartRegionConsistency(startKey, endKey, nil)
+ require.Error(t, err)
+ err = split.CheckPartRegionConsistency(startKey, endKey, []*split.RegionInfo{
+ regionInfo("b", "c"),
+ })
+ require.Error(t, err)
+ err = split.CheckPartRegionConsistency(startKey, endKey, []*split.RegionInfo{
+ regionInfo("a", "c"),
+ regionInfo("d", "e"),
+ })
+ require.Error(t, err)
+ err = split.CheckPartRegionConsistency(startKey, endKey, []*split.RegionInfo{
+ regionInfo("a", "c"),
+ regionInfo("c", "d"),
+ })
+ require.NoError(t, err)
+ err = split.CheckPartRegionConsistency(startKey, endKey, []*split.RegionInfo{
+ regionInfo("a", "c"),
+ regionInfo("c", "d"),
+ regionInfo("d", "f"),
+ })
+ require.NoError(t, err)
+ err = split.CheckPartRegionConsistency(startKey, endKey, []*split.RegionInfo{
+ regionInfo("a", "c"),
+ regionInfo("c", "z"),
+ })
+ require.NoError(t, err)
+}
diff --git a/br/pkg/restore/utils/rewrite_rule_test.go b/br/pkg/restore/utils/rewrite_rule_test.go
index 03ac302cf4ed1..5a6fd7241417c 100644
--- a/br/pkg/restore/utils/rewrite_rule_test.go
+++ b/br/pkg/restore/utils/rewrite_rule_test.go
@@ -280,81 +280,3 @@ func TestGetRewriteTableID(t *testing.T) {
require.Equal(t, tableID, newTableID)
}
}
-
-func rangeEquals(t *testing.T, obtained, expected []rtree.Range) {
- require.Equal(t, len(expected), len(obtained))
- for i := range obtained {
- require.Equal(t, expected[i].StartKey, obtained[i].StartKey)
- require.Equal(t, expected[i].EndKey, obtained[i].EndKey)
- }
-}
-
-func TestSortRange(t *testing.T) {
- dataRules := []*import_sstpb.RewriteRule{
- {OldKeyPrefix: tablecodec.GenTableRecordPrefix(1), NewKeyPrefix: tablecodec.GenTableRecordPrefix(4)},
- {OldKeyPrefix: tablecodec.GenTableRecordPrefix(2), NewKeyPrefix: tablecodec.GenTableRecordPrefix(5)},
- }
- rewriteRules := &utils.RewriteRules{
- Data: dataRules,
- }
- ranges1 := []rtree.Range{
- {
- StartKey: append(tablecodec.GenTableRecordPrefix(1), []byte("aaa")...),
- EndKey: append(tablecodec.GenTableRecordPrefix(1), []byte("bbb")...), Files: nil,
- },
- }
- for i, rg := range ranges1 {
- tmp, _ := utils.RewriteRange(&rg, rewriteRules)
- ranges1[i] = *tmp
- }
- rs1, err := utils.SortRanges(ranges1)
- require.NoErrorf(t, err, "sort range1 failed: %v", err)
- rangeEquals(t, rs1, []rtree.Range{
- {
- StartKey: append(tablecodec.GenTableRecordPrefix(4), []byte("aaa")...),
- EndKey: append(tablecodec.GenTableRecordPrefix(4), []byte("bbb")...), Files: nil,
- },
- })
-
- ranges2 := []rtree.Range{
- {
- StartKey: append(tablecodec.GenTableRecordPrefix(1), []byte("aaa")...),
- EndKey: append(tablecodec.GenTableRecordPrefix(2), []byte("bbb")...), Files: nil,
- },
- }
- for _, rg := range ranges2 {
- _, err := utils.RewriteRange(&rg, rewriteRules)
- require.Error(t, err)
- require.Regexp(t, "table id mismatch.*", err.Error())
- }
-
- ranges3 := []rtree.Range{
- {StartKey: []byte("aaa"), EndKey: []byte("aae")},
- {StartKey: []byte("aae"), EndKey: []byte("aaz")},
- {StartKey: []byte("ccd"), EndKey: []byte("ccf")},
- {StartKey: []byte("ccf"), EndKey: []byte("ccj")},
- }
- rewriteRules1 := &utils.RewriteRules{
- Data: []*import_sstpb.RewriteRule{
- {
- OldKeyPrefix: []byte("aa"),
- NewKeyPrefix: []byte("xx"),
- }, {
- OldKeyPrefix: []byte("cc"),
- NewKeyPrefix: []byte("bb"),
- },
- },
- }
- for i, rg := range ranges3 {
- tmp, _ := utils.RewriteRange(&rg, rewriteRules1)
- ranges3[i] = *tmp
- }
- rs3, err := utils.SortRanges(ranges3)
- require.NoErrorf(t, err, "sort range1 failed: %v", err)
- rangeEquals(t, rs3, []rtree.Range{
- {StartKey: []byte("bbd"), EndKey: []byte("bbf"), Files: nil},
- {StartKey: []byte("bbf"), EndKey: []byte("bbj"), Files: nil},
- {StartKey: []byte("xxa"), EndKey: []byte("xxe"), Files: nil},
- {StartKey: []byte("xxe"), EndKey: []byte("xxz"), Files: nil},
- })
-}
diff --git a/br/pkg/rtree/rtree.go b/br/pkg/rtree/rtree.go
index ee4648e47bb9e..0c7d0ed5ce460 100644
--- a/br/pkg/rtree/rtree.go
+++ b/br/pkg/rtree/rtree.go
@@ -352,14 +352,13 @@ func (rangeTree *ProgressRangeTree) Insert(pr *ProgressRange) error {
// FindContained finds if there is a progress range containing the key range [startKey, endKey).
func (rangeTree *ProgressRangeTree) FindContained(startKey, endKey []byte) (*ProgressRange, error) {
- var ret *ProgressRange
- rangeTree.Descend(func(pr *ProgressRange) bool {
- if bytes.Compare(pr.Origin.StartKey, startKey) <= 0 {
- ret = pr
- return false
- }
- return true
- })
+ startPr := &ProgressRange{
+ Origin: Range{
+ StartKey: startKey,
+ EndKey: endKey,
+ },
+ }
+ ret := rangeTree.find(startPr)
if ret == nil {
return nil, errors.Errorf("Cannot find progress range that contains the start key: %s", redact.Key(startKey))
diff --git a/br/pkg/stream/search.go b/br/pkg/stream/search.go
index f788d335ab456..7cf940a42f135 100644
--- a/br/pkg/stream/search.go
+++ b/br/pkg/stream/search.go
@@ -22,11 +22,6 @@ import (
"golang.org/x/sync/errgroup"
)
-const (
- writeCFName = "write"
- defaultCFName = "default"
-)
-
// Comparator is used for comparing the relationship of src and dst
type Comparator interface {
Compare(src, dst []byte) bool
@@ -198,9 +193,9 @@ func (s *StreamBackupSearch) Search(ctx context.Context) ([]*StreamKVInfo, error
writeCFEntries := make(map[string]*StreamKVInfo, 64)
for entry := range entriesCh {
- if entry.CFName == writeCFName {
+ if entry.CFName == WriteCF {
writeCFEntries[entry.EncodedKey] = entry
- } else if entry.CFName == defaultCFName {
+ } else if entry.CFName == DefaultCF {
defaultCFEntries[entry.EncodedKey] = entry
}
}
@@ -246,7 +241,7 @@ func (s *StreamBackupSearch) searchFromDataFile(
return errors.Annotatef(err, "decode raw key error, file: %s", dataFile.Path)
}
- if dataFile.Cf == writeCFName {
+ if dataFile.Cf == WriteCF {
rawWriteCFValue := new(RawWriteCFValue)
if err := rawWriteCFValue.ParseFrom(v); err != nil {
return errors.Annotatef(err, "parse raw write cf value error, file: %s", dataFile.Path)
@@ -267,7 +262,7 @@ func (s *StreamBackupSearch) searchFromDataFile(
ShortValue: valueStr,
}
ch <- kvInfo
- } else if dataFile.Cf == defaultCFName {
+ } else if dataFile.Cf == DefaultCF {
kvInfo := &StreamKVInfo{
CFName: dataFile.Cf,
StartTs: ts,
diff --git a/br/pkg/stream/search_test.go b/br/pkg/stream/search_test.go
index 6759b37dd6425..224beb5ac7403 100644
--- a/br/pkg/stream/search_test.go
+++ b/br/pkg/stream/search_test.go
@@ -121,7 +121,7 @@ func fakeDataFile(t *testing.T, s storage.ExternalStorage) (defaultCFDataFile, w
defaultCFCheckSum := sha256.Sum256(defaultCFBuf.Bytes())
defaultCFDataFile = &backuppb.DataFileInfo{
Path: defaultCFFile,
- Cf: defaultCFName,
+ Cf: DefaultCF,
Sha256: defaultCFCheckSum[:],
}
@@ -135,7 +135,7 @@ func fakeDataFile(t *testing.T, s storage.ExternalStorage) (defaultCFDataFile, w
writeCFCheckSum := sha256.Sum256(writeCFBuf.Bytes())
writeCFDataFile = &backuppb.DataFileInfo{
Path: writeCFFile,
- Cf: writeCFName,
+ Cf: WriteCF,
Sha256: writeCFCheckSum[:],
}
@@ -178,7 +178,7 @@ func TestMergeCFEntries(t *testing.T) {
Key: hex.EncodeToString([]byte(defaultCF.key)),
EncodedKey: encodedKey,
StartTs: uint64(defaultCF.startTs),
- CFName: defaultCFName,
+ CFName: DefaultCF,
Value: defaultCF.val,
}
}
@@ -189,7 +189,7 @@ func TestMergeCFEntries(t *testing.T) {
EncodedKey: encodedKey,
StartTs: uint64(writeCF.startTs),
CommitTs: uint64(writeCF.commitTS),
- CFName: writeCFName,
+ CFName: WriteCF,
Value: writeCF.val,
}
}
diff --git a/br/pkg/streamhelper/BUILD.bazel b/br/pkg/streamhelper/BUILD.bazel
index 1eaf87efc9978..a823c1059b94c 100644
--- a/br/pkg/streamhelper/BUILD.bazel
+++ b/br/pkg/streamhelper/BUILD.bazel
@@ -69,7 +69,7 @@ go_test(
],
flaky = True,
race = "on",
- shard_count = 27,
+ shard_count = 28,
deps = [
":streamhelper",
"//br/pkg/errors",
diff --git a/br/pkg/streamhelper/advancer.go b/br/pkg/streamhelper/advancer.go
index 0ed0c79b70d01..47e075967f77c 100644
--- a/br/pkg/streamhelper/advancer.go
+++ b/br/pkg/streamhelper/advancer.go
@@ -434,6 +434,7 @@ func (c *CheckpointAdvancer) onTaskEvent(ctx context.Context, e TaskEvent) error
case EventDel:
utils.LogBackupTaskCountDec()
c.task = nil
+ c.isPaused.Store(false)
c.taskRange = nil
// This would be synced by `taskMu`, perhaps we'd better rename that to `tickMu`.
// Do the null check because some of test cases won't equip the advancer with subscriber.
@@ -450,11 +451,11 @@ func (c *CheckpointAdvancer) onTaskEvent(ctx context.Context, e TaskEvent) error
metrics.LastCheckpoint.DeleteLabelValues(e.Name)
case EventPause:
if c.task.GetName() == e.Name {
- c.isPaused.CompareAndSwap(false, true)
+ c.isPaused.Store(true)
}
case EventResume:
if c.task.GetName() == e.Name {
- c.isPaused.CompareAndSwap(true, false)
+ c.isPaused.Store(false)
}
case EventErr:
return e.Err
diff --git a/br/pkg/streamhelper/advancer_test.go b/br/pkg/streamhelper/advancer_test.go
index f8889c4f4e8a2..88971c7962de6 100644
--- a/br/pkg/streamhelper/advancer_test.go
+++ b/br/pkg/streamhelper/advancer_test.go
@@ -5,6 +5,7 @@ package streamhelper_test
import (
"context"
"fmt"
+ "strings"
"sync"
"testing"
"time"
@@ -541,6 +542,34 @@ func TestCheckPointResume(t *testing.T) {
require.ErrorContains(t, adv.OnTick(ctx), "lagged too large")
}
+func TestUnregisterAfterPause(t *testing.T) {
+ c := createFakeCluster(t, 4, false)
+ defer func() {
+ fmt.Println(c)
+ }()
+ c.splitAndScatter("01", "02", "022", "023", "033", "04", "043")
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ env := &testEnv{fakeCluster: c, testCtx: t}
+ adv := streamhelper.NewCheckpointAdvancer(env)
+ adv.UpdateConfigWith(func(c *config.Config) {
+ c.CheckPointLagLimit = 1 * time.Minute
+ })
+ adv.StartTaskListener(ctx)
+ c.advanceClusterTimeBy(1 * time.Minute)
+ require.NoError(t, adv.OnTick(ctx))
+ env.PauseTask(ctx, "whole")
+ time.Sleep(1 * time.Second)
+ c.advanceClusterTimeBy(1 * time.Minute)
+ require.NoError(t, adv.OnTick(ctx))
+ env.unregisterTask()
+ env.putTask()
+ require.Eventually(t, func() bool {
+ err := adv.OnTick(ctx)
+ return err != nil && strings.Contains(err.Error(), "check point lagged too large")
+ }, 5*time.Second, 300*time.Millisecond)
+}
+
func TestOwnershipLost(t *testing.T) {
c := createFakeCluster(t, 4, false)
c.splitAndScatter(manyRegions(0, 10240)...)
diff --git a/br/pkg/streamhelper/basic_lib_for_test.go b/br/pkg/streamhelper/basic_lib_for_test.go
index 82cdb709f11ca..00e4ed324deab 100644
--- a/br/pkg/streamhelper/basic_lib_for_test.go
+++ b/br/pkg/streamhelper/basic_lib_for_test.go
@@ -717,6 +717,22 @@ func (t *testEnv) unregisterTask() {
}
}
+func (t *testEnv) putTask() {
+ rngs := t.ranges
+ if len(rngs) == 0 {
+ rngs = []kv.KeyRange{{}}
+ }
+ tsk := streamhelper.TaskEvent{
+ Type: streamhelper.EventAdd,
+ Name: "whole",
+ Info: &backup.StreamBackupTaskInfo{
+ Name: "whole",
+ },
+ Ranges: rngs,
+ }
+ t.taskCh <- tsk
+}
+
func (t *testEnv) ScanLocksInOneRegion(bo *tikv.Backoffer, key []byte, maxVersion uint64, limit uint32) ([]*txnlock.Lock, *tikv.KeyLocation, error) {
for _, r := range t.regions {
if len(r.locks) != 0 {
diff --git a/br/pkg/task/BUILD.bazel b/br/pkg/task/BUILD.bazel
index 130256094c93a..2412923b352aa 100644
--- a/br/pkg/task/BUILD.bazel
+++ b/br/pkg/task/BUILD.bazel
@@ -35,8 +35,9 @@ go_library(
"//br/pkg/pdutil",
"//br/pkg/restore",
"//br/pkg/restore/data",
- "//br/pkg/restore/file_importer",
- "//br/pkg/restore/rawkv",
+ "//br/pkg/restore/ingestrec",
+ "//br/pkg/restore/log_client",
+ "//br/pkg/restore/snap_client",
"//br/pkg/restore/tiflashrec",
"//br/pkg/restore/utils",
"//br/pkg/rtree",
@@ -49,6 +50,7 @@ go_library(
"//br/pkg/utils",
"//br/pkg/version",
"//pkg/config",
+ "//pkg/domain",
"//pkg/kv",
"//pkg/parser/model",
"//pkg/parser/mysql",
@@ -59,6 +61,7 @@ go_library(
"//pkg/types",
"//pkg/util",
"//pkg/util/cdcutil",
+ "//pkg/util/collate",
"//pkg/util/mathutil",
"//pkg/util/table-filter",
"@com_github_docker_go_units//:go-units",
@@ -99,32 +102,45 @@ go_test(
"backup_ebs_test.go",
"backup_test.go",
"common_test.go",
+ "config_test.go",
"restore_test.go",
"stream_test.go",
],
embed = [":task"],
flaky = True,
- shard_count = 23,
+ shard_count = 29,
deps = [
+ "//br/pkg/backup",
"//br/pkg/config",
"//br/pkg/conn",
"//br/pkg/errors",
+ "//br/pkg/gluetidb",
+ "//br/pkg/gluetidb/mock",
"//br/pkg/metautil",
- "//br/pkg/restore",
+ "//br/pkg/mock",
+ "//br/pkg/restore/snap_client",
+ "//br/pkg/restore/tiflashrec",
"//br/pkg/storage",
"//br/pkg/stream",
"//br/pkg/utils",
+ "//br/pkg/utiltest",
"//pkg/config",
+ "//pkg/ddl",
"//pkg/parser/model",
+ "//pkg/parser/mysql",
"//pkg/statistics/handle/util",
"//pkg/tablecodec",
+ "//pkg/testkit",
+ "//pkg/types",
"//pkg/util/table-filter",
+ "@com_github_gogo_protobuf//proto",
"@com_github_golang_protobuf//proto",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_kvproto//pkg/brpb",
"@com_github_pingcap_kvproto//pkg/encryptionpb",
"@com_github_pingcap_kvproto//pkg/metapb",
"@com_github_spf13_pflag//:pflag",
+ "@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",
"@com_github_tikv_client_go_v2//oracle",
"@com_github_tikv_pd_client//:client",
diff --git a/br/pkg/task/common.go b/br/pkg/task/common.go
index 343f1c0e84b16..a38a90f596671 100644
--- a/br/pkg/task/common.go
+++ b/br/pkg/task/common.go
@@ -33,6 +33,7 @@ import (
filter "github.com/pingcap/tidb/pkg/util/table-filter"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
+ "github.com/tikv/client-go/v2/config"
pd "github.com/tikv/pd/client"
"go.etcd.io/etcd/client/pkg/v3/transport"
clientv3 "go.etcd.io/etcd/client/v3"
@@ -159,6 +160,15 @@ func (tls *TLSConfig) ToPDSecurityOption() pd.SecurityOption {
return securityOption
}
+// Convert the TLS config to the PD security option.
+func (tls *TLSConfig) ToKVSecurity() config.Security {
+ return config.Security{
+ ClusterSSLCA: tls.CA,
+ ClusterSSLCert: tls.Cert,
+ ClusterSSLKey: tls.Key,
+ }
+}
+
// ParseFromFlags parses the TLS config from the flag set.
func (tls *TLSConfig) ParseFromFlags(flags *pflag.FlagSet) (err error) {
tls.CA, tls.Cert, tls.Key, err = ParseTLSTripleFromFlags(flags)
@@ -405,12 +415,12 @@ func parseCipherType(t string) (encryptionpb.EncryptionMethod, error) {
func checkCipherKey(cipherKey, cipherKeyFile string) error {
if (len(cipherKey) == 0) == (len(cipherKeyFile) == 0) {
return errors.Annotate(berrors.ErrInvalidArgument,
- "exactly one of --crypter.key or --crypter.key-file should be provided")
+ "exactly one of cipher key or keyfile path should be provided")
}
return nil
}
-func getCipherKeyContent(cipherKey, cipherKeyFile string) ([]byte, error) {
+func GetCipherKeyContent(cipherKey, cipherKeyFile string) ([]byte, error) {
if err := checkCipherKey(cipherKey, cipherKeyFile); err != nil {
return nil, errors.Trace(err)
}
@@ -470,7 +480,7 @@ func (cfg *Config) parseCipherInfo(flags *pflag.FlagSet) error {
return errors.Trace(err)
}
- cfg.CipherInfo.CipherKey, err = getCipherKeyContent(key, keyFilePath)
+ cfg.CipherInfo.CipherKey, err = GetCipherKeyContent(key, keyFilePath)
if err != nil {
return errors.Trace(err)
}
diff --git a/br/pkg/task/config_test.go b/br/pkg/task/config_test.go
new file mode 100644
index 0000000000000..6a86b43dc24c2
--- /dev/null
+++ b/br/pkg/task/config_test.go
@@ -0,0 +1,325 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package task
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "testing"
+
+ "github.com/golang/protobuf/proto"
+ backuppb "github.com/pingcap/kvproto/pkg/brpb"
+ "github.com/pingcap/kvproto/pkg/encryptionpb"
+ "github.com/pingcap/kvproto/pkg/metapb"
+ "github.com/pingcap/tidb/br/pkg/conn"
+ "github.com/pingcap/tidb/br/pkg/metautil"
+ snapclient "github.com/pingcap/tidb/br/pkg/restore/snap_client"
+ "github.com/pingcap/tidb/br/pkg/storage"
+ "github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/pkg/parser/model"
+ "github.com/pingcap/tidb/pkg/statistics/handle/util"
+ "github.com/pingcap/tidb/pkg/tablecodec"
+ "github.com/stretchr/testify/require"
+ pd "github.com/tikv/pd/client"
+ "google.golang.org/grpc/keepalive"
+)
+
+func TestRestoreConfigAdjust(t *testing.T) {
+ cfg := &RestoreConfig{}
+ cfg.Adjust()
+
+ require.Equal(t, uint32(defaultRestoreConcurrency), cfg.Config.Concurrency)
+ require.Equal(t, defaultSwitchInterval, cfg.Config.SwitchModeInterval)
+ require.Equal(t, conn.DefaultMergeRegionKeyCount, cfg.MergeSmallRegionKeyCount.Value)
+ require.Equal(t, conn.DefaultMergeRegionSizeBytes, cfg.MergeSmallRegionSizeBytes.Value)
+}
+
+type mockPDClient struct {
+ pd.Client
+}
+
+func (m mockPDClient) GetClusterID(_ context.Context) uint64 {
+ return 1
+}
+
+func (m mockPDClient) GetAllStores(ctx context.Context, opts ...pd.GetStoreOption) ([]*metapb.Store, error) {
+ return []*metapb.Store{}, nil
+}
+
+func TestConfigureRestoreClient(t *testing.T) {
+ cfg := Config{
+ Concurrency: 1024,
+ }
+ restoreComCfg := RestoreCommonConfig{
+ Online: true,
+ }
+ restoreCfg := &RestoreConfig{
+ Config: cfg,
+ RestoreCommonConfig: restoreComCfg,
+ DdlBatchSize: 127,
+ }
+ client := snapclient.NewRestoreClient(mockPDClient{}, nil, nil, keepalive.ClientParameters{})
+ ctx := context.Background()
+ err := configureRestoreClient(ctx, client, restoreCfg)
+ require.NoError(t, err)
+ require.Equal(t, uint(128), client.GetBatchDdlSize())
+}
+
+func TestAdjustRestoreConfigForStreamRestore(t *testing.T) {
+ restoreCfg := RestoreConfig{}
+
+ restoreCfg.adjustRestoreConfigForStreamRestore()
+ require.Equal(t, restoreCfg.PitrBatchCount, uint32(defaultPiTRBatchCount))
+ require.Equal(t, restoreCfg.PitrBatchSize, uint32(defaultPiTRBatchSize))
+ require.Equal(t, restoreCfg.PitrConcurrency, uint32(defaultPiTRConcurrency))
+ require.Equal(t, restoreCfg.Concurrency, restoreCfg.PitrConcurrency)
+}
+
+func TestCheckRestoreDBAndTable(t *testing.T) {
+ cases := []struct {
+ cfgSchemas map[string]struct{}
+ cfgTables map[string]struct{}
+ backupDBs map[string]*metautil.Database
+ }{
+ {
+ cfgSchemas: map[string]struct{}{
+ utils.EncloseName("test"): {},
+ },
+ cfgTables: map[string]struct{}{
+ utils.EncloseDBAndTable("test", "t"): {},
+ utils.EncloseDBAndTable("test", "t2"): {},
+ },
+ backupDBs: mockReadSchemasFromBackupMeta(t, map[string][]string{
+ "test": {"T", "T2"},
+ }),
+ },
+ {
+ cfgSchemas: map[string]struct{}{
+ utils.EncloseName("mysql"): {},
+ },
+ cfgTables: map[string]struct{}{
+ utils.EncloseDBAndTable("mysql", "t"): {},
+ utils.EncloseDBAndTable("mysql", "t2"): {},
+ },
+ backupDBs: mockReadSchemasFromBackupMeta(t, map[string][]string{
+ "__TiDB_BR_Temporary_mysql": {"T", "T2"},
+ }),
+ },
+ {
+ cfgSchemas: map[string]struct{}{
+ utils.EncloseName("test"): {},
+ },
+ cfgTables: map[string]struct{}{
+ utils.EncloseDBAndTable("test", "T"): {},
+ utils.EncloseDBAndTable("test", "T2"): {},
+ },
+ backupDBs: mockReadSchemasFromBackupMeta(t, map[string][]string{
+ "test": {"t", "t2"},
+ }),
+ },
+ {
+ cfgSchemas: map[string]struct{}{
+ utils.EncloseName("TEST"): {},
+ },
+ cfgTables: map[string]struct{}{
+ utils.EncloseDBAndTable("TEST", "t"): {},
+ utils.EncloseDBAndTable("TEST", "T2"): {},
+ },
+ backupDBs: mockReadSchemasFromBackupMeta(t, map[string][]string{
+ "test": {"t", "t2"},
+ }),
+ },
+ {
+ cfgSchemas: map[string]struct{}{
+ utils.EncloseName("TeSt"): {},
+ },
+ cfgTables: map[string]struct{}{
+ utils.EncloseDBAndTable("TeSt", "tabLe"): {},
+ utils.EncloseDBAndTable("TeSt", "taBle2"): {},
+ },
+ backupDBs: mockReadSchemasFromBackupMeta(t, map[string][]string{
+ "TesT": {"TablE", "taBle2"},
+ }),
+ },
+ {
+ cfgSchemas: map[string]struct{}{
+ utils.EncloseName("TeSt"): {},
+ utils.EncloseName("MYSQL"): {},
+ },
+ cfgTables: map[string]struct{}{
+ utils.EncloseDBAndTable("TeSt", "tabLe"): {},
+ utils.EncloseDBAndTable("TeSt", "taBle2"): {},
+ utils.EncloseDBAndTable("MYSQL", "taBle"): {},
+ },
+ backupDBs: mockReadSchemasFromBackupMeta(t, map[string][]string{
+ "TesT": {"table", "TaBLE2"},
+ "__TiDB_BR_Temporary_mysql": {"tablE"},
+ }),
+ },
+ {
+ cfgSchemas: map[string]struct{}{
+ utils.EncloseName("sys"): {},
+ },
+ cfgTables: map[string]struct{}{
+ utils.EncloseDBAndTable("sys", "t"): {},
+ utils.EncloseDBAndTable("sys", "t2"): {},
+ },
+ backupDBs: mockReadSchemasFromBackupMeta(t, map[string][]string{
+ "__TiDB_BR_Temporary_sys": {"T", "T2"},
+ }),
+ },
+ }
+
+ cfg := &RestoreConfig{}
+ for _, ca := range cases {
+ cfg.Schemas = ca.cfgSchemas
+ cfg.Tables = ca.cfgTables
+
+ backupDBs := make([]*metautil.Database, 0, len(ca.backupDBs))
+ for _, db := range ca.backupDBs {
+ backupDBs = append(backupDBs, db)
+ }
+ err := CheckRestoreDBAndTable(backupDBs, cfg)
+ require.NoError(t, err)
+ }
+}
+
+func mockReadSchemasFromBackupMeta(t *testing.T, db2Tables map[string][]string) map[string]*metautil.Database {
+ testDir := t.TempDir()
+ store, err := storage.NewLocalStorage(testDir)
+ require.NoError(t, err)
+
+ mockSchemas := make([]*backuppb.Schema, 0)
+ var dbID int64 = 1
+ for db, tables := range db2Tables {
+ dbName := model.NewCIStr(db)
+ mockTblList := make([]*model.TableInfo, 0)
+ tblBytesList, statsBytesList := make([][]byte, 0), make([][]byte, 0)
+
+ for i, table := range tables {
+ tblName := model.NewCIStr(table)
+ mockTbl := &model.TableInfo{
+ ID: dbID*100 + int64(i),
+ Name: tblName,
+ }
+ mockTblList = append(mockTblList, mockTbl)
+
+ mockStats := util.JSONTable{
+ DatabaseName: dbName.String(),
+ TableName: tblName.String(),
+ }
+
+ tblBytes, err := json.Marshal(mockTbl)
+ require.NoError(t, err)
+ tblBytesList = append(tblBytesList, tblBytes)
+
+ statsBytes, err := json.Marshal(mockStats)
+ require.NoError(t, err)
+ statsBytesList = append(statsBytesList, statsBytes)
+ }
+
+ mockDB := model.DBInfo{
+ ID: dbID,
+ Name: dbName,
+ Tables: mockTblList,
+ }
+ dbID++
+ dbBytes, err := json.Marshal(mockDB)
+ require.NoError(t, err)
+
+ for i := 0; i < len(tblBytesList); i++ {
+ mockSchemas = append(mockSchemas, &backuppb.Schema{
+ Db: dbBytes,
+ Table: tblBytesList[i],
+ Stats: statsBytesList[i],
+ },
+ )
+ }
+ }
+
+ mockFiles := []*backuppb.File{
+ {
+ Name: fmt.Sprintf("%p.sst", &mockSchemas),
+ StartKey: tablecodec.EncodeRowKey(1, []byte("a")),
+ EndKey: tablecodec.EncodeRowKey(2, []byte("a")),
+ },
+ }
+
+ meta := mockBackupMeta(mockSchemas, mockFiles)
+ data, err := proto.Marshal(meta)
+ require.NoError(t, err)
+
+ ctx := context.Background()
+ err = store.WriteFile(ctx, metautil.MetaFile, data)
+ require.NoError(t, err)
+
+ dbs, err := metautil.LoadBackupTables(
+ ctx,
+ metautil.NewMetaReader(
+ meta,
+ store,
+ &backuppb.CipherInfo{
+ CipherType: encryptionpb.EncryptionMethod_PLAINTEXT,
+ }),
+ true,
+ )
+ require.NoError(t, err)
+ return dbs
+}
+
+func mockBackupMeta(mockSchemas []*backuppb.Schema, mockFiles []*backuppb.File) *backuppb.BackupMeta {
+ return &backuppb.BackupMeta{
+ Files: mockFiles,
+ Schemas: mockSchemas,
+ }
+}
+
+func TestMapTableToFiles(t *testing.T) {
+ filesOfTable1 := []*backuppb.File{
+ {
+ Name: "table1-1.sst",
+ StartKey: tablecodec.EncodeTablePrefix(1),
+ EndKey: tablecodec.EncodeTablePrefix(1),
+ },
+ {
+ Name: "table1-2.sst",
+ StartKey: tablecodec.EncodeTablePrefix(1),
+ EndKey: tablecodec.EncodeTablePrefix(1),
+ },
+ {
+ Name: "table1-3.sst",
+ StartKey: tablecodec.EncodeTablePrefix(1),
+ EndKey: tablecodec.EncodeTablePrefix(1),
+ },
+ }
+ filesOfTable2 := []*backuppb.File{
+ {
+ Name: "table2-1.sst",
+ StartKey: tablecodec.EncodeTablePrefix(2),
+ EndKey: tablecodec.EncodeTablePrefix(2),
+ },
+ {
+ Name: "table2-2.sst",
+ StartKey: tablecodec.EncodeTablePrefix(2),
+ EndKey: tablecodec.EncodeTablePrefix(2),
+ },
+ }
+
+ result := MapTableToFiles(append(filesOfTable2, filesOfTable1...))
+
+ require.Equal(t, filesOfTable1, result[1])
+ require.Equal(t, filesOfTable2, result[2])
+}
diff --git a/br/pkg/task/restore.go b/br/pkg/task/restore.go
index bebc5fc458795..050bd1748924c 100644
--- a/br/pkg/task/restore.go
+++ b/br/pkg/task/restore.go
@@ -3,9 +3,10 @@
package task
import (
+ "cmp"
"context"
"fmt"
- "sort"
+ "slices"
"strings"
"time"
@@ -18,25 +19,30 @@ import (
"github.com/pingcap/tidb/br/pkg/checkpoint"
pconfig "github.com/pingcap/tidb/br/pkg/config"
"github.com/pingcap/tidb/br/pkg/conn"
+ connutil "github.com/pingcap/tidb/br/pkg/conn/util"
berrors "github.com/pingcap/tidb/br/pkg/errors"
"github.com/pingcap/tidb/br/pkg/glue"
"github.com/pingcap/tidb/br/pkg/httputil"
"github.com/pingcap/tidb/br/pkg/logutil"
"github.com/pingcap/tidb/br/pkg/metautil"
- "github.com/pingcap/tidb/br/pkg/pdutil"
"github.com/pingcap/tidb/br/pkg/restore"
- fileimporter "github.com/pingcap/tidb/br/pkg/restore/file_importer"
+ snapclient "github.com/pingcap/tidb/br/pkg/restore/snap_client"
"github.com/pingcap/tidb/br/pkg/restore/tiflashrec"
"github.com/pingcap/tidb/br/pkg/summary"
"github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/br/pkg/version"
"github.com/pingcap/tidb/pkg/config"
+ "github.com/pingcap/tidb/pkg/domain"
+ "github.com/pingcap/tidb/pkg/kv"
+ "github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/util"
+ "github.com/pingcap/tidb/pkg/util/collate"
"github.com/pingcap/tidb/pkg/util/mathutil"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/tikv/client-go/v2/tikv"
+ pd "github.com/tikv/pd/client"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/multierr"
"go.uber.org/zap"
@@ -142,7 +148,7 @@ func (cfg *RestoreCommonConfig) adjust() {
func DefineRestoreCommonFlags(flags *pflag.FlagSet) {
// TODO remove experimental tag if it's stable
flags.Bool(flagOnline, false, "(experimental) Whether online when restore")
- flags.String(flagGranularity, string(restore.CoarseGrained), "Whether split & scatter regions using fine-grained way during restore")
+ flags.String(flagGranularity, string(restore.CoarseGrained), "(deprecated) Whether split & scatter regions using fine-grained way during restore")
flags.Uint(flagConcurrencyPerStore, 128, "The size of thread pool on each store that executes tasks, only enabled when `--granularity=coarse-grained`")
flags.Uint32(flagConcurrency, 128, "(deprecated) The size of thread pool on BR that executes tasks, "+
"where each task restores one SST file to TiKV")
@@ -519,43 +525,68 @@ func (cfg *RestoreConfig) generateSnapshotRestoreTaskName(clusterID uint64) stri
return cfg.checkpointSnapshotRestoreTaskName
}
-func configureRestoreClient(ctx context.Context, client *restore.Client, cfg *RestoreConfig) error {
+func configureRestoreClient(ctx context.Context, client *snapclient.SnapClient, cfg *RestoreConfig) error {
client.SetRateLimit(cfg.RateLimit)
client.SetCrypter(&cfg.CipherInfo)
- client.SetGranularity(cfg.Granularity)
- if cfg.Online {
- client.EnableOnline()
- }
if cfg.NoSchema {
client.EnableSkipCreateSQL()
}
- client.SetSwitchModeInterval(cfg.SwitchModeInterval)
client.SetBatchDdlSize(cfg.DdlBatchSize)
client.SetPlacementPolicyMode(cfg.WithPlacementPolicy)
client.SetWithSysTable(cfg.WithSysTable)
+ client.SetRewriteMode(ctx)
+ return nil
+}
- err := restore.CheckKeyspaceBREnable(ctx, client.GetPDClient())
+func CheckNewCollationEnable(
+ backupNewCollationEnable string,
+ g glue.Glue,
+ storage kv.Storage,
+ CheckRequirements bool,
+) (bool, error) {
+ se, err := g.CreateSession(storage)
if err != nil {
- log.Warn("Keyspace BR is not supported in this cluster, fallback to legacy restore", zap.Error(err))
- client.SetRewriteMode(fileimporter.RewriteModeLegacy)
- } else {
- client.SetRewriteMode(fileimporter.RewriteModeKeyspace)
+ return false, errors.Trace(err)
}
- err = client.LoadRestoreStores(ctx)
+ newCollationEnable, err := se.GetGlobalVariable(utils.GetTidbNewCollationEnabled())
if err != nil {
- return errors.Trace(err)
+ return false, errors.Trace(err)
+ }
+ // collate.newCollationEnabled is set to 1 when the collate package is initialized,
+ // so we need to modify this value according to the config of the cluster
+ // before using the collate package.
+ enabled := newCollationEnable == "True"
+ // modify collate.newCollationEnabled according to the config of the cluster
+ collate.SetNewCollationEnabledForTest(enabled)
+ log.Info(fmt.Sprintf("set %s", utils.TidbNewCollationEnabled), zap.Bool("new_collation_enabled", enabled))
+
+ if backupNewCollationEnable == "" {
+ if CheckRequirements {
+ return enabled, errors.Annotatef(berrors.ErrUnknown,
+ "the value '%s' not found in backupmeta. "+
+ "you can use \"SELECT VARIABLE_VALUE FROM mysql.tidb WHERE VARIABLE_NAME='%s';\" to manually check the config. "+
+ "if you ensure the value '%s' in backup cluster is as same as restore cluster, use --check-requirements=false to skip this check",
+ utils.TidbNewCollationEnabled, utils.TidbNewCollationEnabled, utils.TidbNewCollationEnabled)
+ }
+ log.Warn(fmt.Sprintf("the config '%s' is not in backupmeta", utils.TidbNewCollationEnabled))
+ return enabled, nil
}
- client.SetConcurrency(uint(cfg.Concurrency))
- return nil
+
+ if !strings.EqualFold(backupNewCollationEnable, newCollationEnable) {
+ return enabled, errors.Annotatef(berrors.ErrUnknown,
+ "the config '%s' not match, upstream:%v, downstream: %v",
+ utils.TidbNewCollationEnabled, backupNewCollationEnable, newCollationEnable)
+ }
+
+ return enabled, nil
}
// CheckRestoreDBAndTable is used to check whether the restore dbs or tables have been backup
-func CheckRestoreDBAndTable(client *restore.Client, cfg *RestoreConfig) error {
+func CheckRestoreDBAndTable(schemas []*metautil.Database, cfg *RestoreConfig) error {
if len(cfg.Schemas) == 0 && len(cfg.Tables) == 0 {
return nil
}
- schemas := client.GetDatabases()
schemasMap := make(map[string]struct{})
tablesMap := make(map[string]struct{})
for _, db := range schemas {
@@ -736,7 +767,7 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
mgr.ProcessTiKVConfigs(ctx, kvConfigs, httpCli)
keepaliveCfg.PermitWithoutStream = true
- client := restore.NewRestoreClient(mgr.GetPDClient(), mgr.GetPDHTTPClient(), mgr.GetTLSConfig(), keepaliveCfg)
+ client := snapclient.NewRestoreClient(mgr.GetPDClient(), mgr.GetPDHTTPClient(), mgr.GetTLSConfig(), keepaliveCfg)
// using tikv config to set the concurrency-per-store for client.
client.SetConcurrencyPerStore(kvConfigs.ImportGoroutines.Value)
err = configureRestoreClient(ctx, client, cfg)
@@ -754,6 +785,14 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
if err != nil {
return errors.Trace(err)
}
+ if cfg.CheckRequirements {
+ err := checkIncompatibleChangefeed(ctx, backupMeta.EndVersion, mgr.GetDomain().GetEtcdClient())
+ log.Info("Checking incompatible TiCDC changefeeds before restoring.",
+ logutil.ShortError(err), zap.Uint64("restore-ts", backupMeta.EndVersion))
+ if err != nil {
+ return errors.Trace(err)
+ }
+ }
backupVersion := version.NormalizeBackupVersion(backupMeta.ClusterVersion)
if cfg.CheckRequirements && backupVersion != nil {
@@ -761,7 +800,7 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
return errors.Trace(versionErr)
}
}
- if _, err = restore.CheckNewCollationEnable(backupMeta.GetNewCollationsEnabled(), g, mgr.GetStorage(), cfg.CheckRequirements); err != nil {
+ if _, err = CheckNewCollationEnable(backupMeta.GetNewCollationsEnabled(), g, mgr.GetStorage(), cfg.CheckRequirements); err != nil {
return errors.Trace(err)
}
@@ -773,7 +812,7 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
if client.IsRawKvMode() {
return errors.Annotate(berrors.ErrRestoreModeMismatch, "cannot do transactional restore from raw kv data")
}
- if err = CheckRestoreDBAndTable(client, cfg); err != nil {
+ if err = CheckRestoreDBAndTable(client.GetDatabases(), cfg); err != nil {
return err
}
files, tables, dbs := filterRestoreFiles(client, cfg)
@@ -785,7 +824,7 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
g.Record(summary.RestoreDataSize, archiveSize)
//restore from tidb will fetch a general Size issue https://github.com/pingcap/tidb/issues/27247
g.Record("Size", archiveSize)
- restoreTS, err := client.GetTSWithRetry(ctx)
+ restoreTS, err := restore.GetTSWithRetry(ctx, mgr.GetPDClient())
if err != nil {
return errors.Trace(err)
}
@@ -796,7 +835,8 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
cfg.UseCheckpoint = false
}
- restoreSchedulers, schedulersConfig, err := restorePreWork(ctx, client, mgr, true)
+ importModeSwitcher := restore.NewImportModeSwitcher(mgr.GetPDClient(), cfg.Config.SwitchModeInterval, mgr.GetTLSConfig())
+ restoreSchedulers, schedulersConfig, err := restore.RestorePreWork(ctx, mgr, importModeSwitcher, cfg.Online, true)
if err != nil {
return errors.Trace(err)
}
@@ -811,7 +851,7 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
log.Info("start to remove the pd scheduler")
// run the post-work to avoid being stuck in the import
// mode or emptied schedulers.
- restorePostWork(ctx, client, restoreSchedulers)
+ restore.RestorePostWork(ctx, importModeSwitcher, restoreSchedulers, cfg.Online)
log.Info("finish removing pd scheduler")
}()
@@ -844,7 +884,7 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
}
if client.IsFullClusterRestore() && client.HasBackedUpSysDB() {
- if err = client.CheckSysTableCompatibility(mgr.GetDomain(), tables); err != nil {
+ if err = snapclient.CheckSysTableCompatibility(mgr.GetDomain(), tables); err != nil {
return errors.Trace(err)
}
}
@@ -902,15 +942,15 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
if client.IsIncremental() {
newTS = restoreTS
}
- ddlJobs := restore.FilterDDLJobs(client.GetDDLJobs(), tables)
- ddlJobs = restore.FilterDDLJobByRules(ddlJobs, restore.DDLJobBlockListRule)
+ ddlJobs := FilterDDLJobs(client.GetDDLJobs(), tables)
+ ddlJobs = FilterDDLJobByRules(ddlJobs, DDLJobBlockListRule)
- err = client.PreCheckTableTiFlashReplica(ctx, tables, cfg.tiflashRecorder)
+ err = PreCheckTableTiFlashReplica(ctx, mgr.GetPDClient(), tables, cfg.tiflashRecorder)
if err != nil {
return errors.Trace(err)
}
- err = client.PreCheckTableClusterIndex(tables, ddlJobs, mgr.GetDomain())
+ err = PreCheckTableClusterIndex(tables, ddlJobs, mgr.GetDomain())
if err != nil {
return errors.Trace(err)
}
@@ -963,7 +1003,7 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
// We make bigger errCh so we won't block on multi-part failed.
errCh := make(chan error, 32)
- tableStream := client.GoCreateTables(ctx, mgr.GetDomain(), tables, newTS, errCh)
+ tableStream := client.GoCreateTables(ctx, tables, newTS, errCh)
if len(files) == 0 {
log.Info("no files, empty databases and tables are restored")
@@ -978,12 +1018,12 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
// If the API V2 data occurs in the restore process, the cluster must
// support the keyspace rewrite mode.
- if (len(oldKeyspace) > 0 || len(newKeyspace) > 0) && client.GetRewriteMode() == fileimporter.RewriteModeLegacy {
+ if (len(oldKeyspace) > 0 || len(newKeyspace) > 0) && client.GetRewriteMode() == snapclient.RewriteModeLegacy {
return errors.Annotate(berrors.ErrRestoreModeMismatch, "cluster only supports legacy rewrite mode")
}
// Hijack the tableStream and rewrite the rewrite rules.
- tableStream = util.ChanMap(tableStream, func(t restore.CreatedTable) restore.CreatedTable {
+ tableStream = util.ChanMap(tableStream, func(t snapclient.CreatedTable) snapclient.CreatedTable {
// Set the keyspace info for the checksum requests
t.RewriteRule.OldKeyspace = oldKeyspace
t.RewriteRule.NewKeyspace = newKeyspace
@@ -997,7 +1037,7 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
}
if cfg.tiflashRecorder != nil {
- tableStream = util.ChanMap(tableStream, func(t restore.CreatedTable) restore.CreatedTable {
+ tableStream = util.ChanMap(tableStream, func(t snapclient.CreatedTable) snapclient.CreatedTable {
if cfg.tiflashRecorder != nil {
cfg.tiflashRecorder.Rewrite(t.OldTable.Info.ID, t.Table.ID)
}
@@ -1006,12 +1046,12 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
}
// Block on creating tables before restore starts. since create table is no longer a heavy operation any more.
- tableStream = GoBlockCreateTablesPipeline(ctx, maxRestoreBatchSizeLimit, tableStream)
+ tableStream = client.GoBlockCreateTablesPipeline(ctx, maxRestoreBatchSizeLimit, tableStream)
tableFileMap := MapTableToFiles(files)
log.Debug("mapped table to files", zap.Any("result map", tableFileMap))
- rangeStream := restore.GoValidateFileRanges(
+ rangeStream := client.GoValidateFileRanges(
ctx, tableStream, tableFileMap, kvConfigs.MergeRegionSize.Value, kvConfigs.MergeRegionKeyCount.Value, errCh)
rangeSize := EstimateRangeSize(files)
@@ -1028,10 +1068,7 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
}
// Restore sst files in batch.
- batchSize := mathutil.Clamp(int(cfg.Concurrency), defaultRestoreConcurrency, maxRestoreBatchSizeLimit)
- if client.GetGranularity() == string(restore.CoarseGrained) {
- batchSize = mathutil.MaxInt
- }
+ batchSize := mathutil.MaxInt
failpoint.Inject("small-batch-size", func(v failpoint.Value) {
log.Info("failpoint small batch size is on", zap.Int("size", v.(int)))
batchSize = v.(int)
@@ -1052,12 +1089,15 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
progressLen,
!cfg.LogProgress)
defer updateCh.Close()
- sender, err := restore.NewTiKVSender(ctx, client, updateCh, cfg.PDConcurrency, cfg.Granularity)
+ sender, err := snapclient.NewTiKVSender(ctx, client, updateCh, cfg.PDConcurrency)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ manager, err := snapclient.NewBRContextManager(ctx, mgr.GetPDClient(), mgr.GetPDHTTPClient(), mgr.GetTLSConfig(), cfg.Online)
if err != nil {
return errors.Trace(err)
}
- manager := restore.NewBRContextManager(client)
- batcher, afterTableRestoredCh := restore.NewBatcher(ctx, sender, manager, errCh, updateCh)
+ batcher, afterTableRestoredCh := snapclient.NewBatcher(ctx, sender, manager, errCh, updateCh)
batcher.SetCheckpoint(checkpointSetWithTableID)
batcher.SetThreshold(batchSize)
batcher.EnableAutoCommit(ctx, cfg.BatchFlushInterval)
@@ -1073,7 +1113,7 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
}
// pipeline update meta and load stats
- postHandleCh = client.GoUpdateMetaAndLoadStats(ctx, s, &cfg.CipherInfo, postHandleCh, errCh, cfg.StatsConcurrency, cfg.LoadStats)
+ postHandleCh = client.GoUpdateMetaAndLoadStats(ctx, s, postHandleCh, errCh, cfg.StatsConcurrency, cfg.LoadStats)
// pipeline wait Tiflash synced
if cfg.WaitTiflashReady {
@@ -1103,7 +1143,7 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
select {
case err = <-errCh:
- err = multierr.Append(err, multierr.Combine(restore.Exhaust(errCh)...))
+ err = multierr.Append(err, multierr.Combine(Exhaust(errCh)...))
case <-finish:
}
@@ -1126,6 +1166,21 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf
return nil
}
+// Exhaust drains all remaining errors in the channel, into a slice of errors.
+func Exhaust(ec <-chan error) []error {
+ out := make([]error, 0, len(ec))
+ for {
+ select {
+ case err := <-ec:
+ out = append(out, err)
+ default:
+ // errCh will NEVER be closed(ya see, it has multi sender-part),
+ // so we just consume the current backlog of this channel, then return.
+ return out
+ }
+ }
+}
+
// EstimateRangeSize estimates the total range count by file.
func EstimateRangeSize(files []*backuppb.File) int {
result := 0
@@ -1165,7 +1220,7 @@ func MapTableToFiles(files []*backuppb.File) map[int64][]*backuppb.File {
// i.e. don't execute checksum, just increase the process anyhow.
func dropToBlackhole(
ctx context.Context,
- inCh <-chan *restore.CreatedTable,
+ inCh <-chan *snapclient.CreatedTable,
errCh chan<- error,
) <-chan struct{} {
outCh := make(chan struct{}, 1)
@@ -1191,7 +1246,7 @@ func dropToBlackhole(
// filterRestoreFiles filters tables that can't be processed after applying cfg.TableFilter.MatchTable.
// if the db has no table that can be processed, the db will be filtered too.
func filterRestoreFiles(
- client *restore.Client,
+ client *snapclient.SnapClient,
cfg *RestoreConfig,
) (files []*backuppb.File, tables []*metautil.Table, dbs []*metautil.Database) {
for _, db := range client.GetDatabases() {
@@ -1214,41 +1269,6 @@ func filterRestoreFiles(
return
}
-// restorePreWork executes some prepare work before restore.
-// TODO make this function returns a restore post work.
-func restorePreWork(ctx context.Context, client *restore.Client, mgr *conn.Mgr, switchToImport bool) (pdutil.UndoFunc, *pdutil.ClusterConfig, error) {
- if client.IsOnline() {
- return pdutil.Nop, nil, nil
- }
-
- if switchToImport {
- // Switch TiKV cluster to import mode (adjust rocksdb configuration).
- client.SwitchToImportMode(ctx)
- }
-
- return mgr.RemoveSchedulersWithConfig(ctx)
-}
-
-// restorePostWork executes some post work after restore.
-// TODO: aggregate all lifetime manage methods into batcher's context manager field.
-func restorePostWork(
- ctx context.Context, client *restore.Client, restoreSchedulers pdutil.UndoFunc,
-) {
- if ctx.Err() != nil {
- log.Warn("context canceled, try shutdown")
- ctx = context.Background()
- }
- if client.IsOnline() {
- return
- }
- if err := client.SwitchToNormalMode(ctx); err != nil {
- log.Warn("fail to switch to normal mode", zap.Error(err))
- }
- if err := restoreSchedulers(ctx); err != nil {
- log.Warn("failed to restore PD schedulers", zap.Error(err))
- }
-}
-
// enableTiDBConfig tweaks some of configs of TiDB to make the restore progress go well.
// return a function that could restore the config to origin.
func enableTiDBConfig() func() {
@@ -1271,8 +1291,8 @@ func enableTiDBConfig() func() {
// by send tables to batcher.
func restoreTableStream(
ctx context.Context,
- inputCh <-chan restore.TableWithRange,
- batcher *restore.Batcher,
+ inputCh <-chan snapclient.TableWithRange,
+ batcher *snapclient.Batcher,
errCh chan<- error,
) {
oldTableCount := 0
@@ -1300,28 +1320,194 @@ func restoreTableStream(
}
}
-func GoBlockCreateTablesPipeline(ctx context.Context, sz int, inCh <-chan restore.CreatedTable) <-chan restore.CreatedTable {
- outCh := make(chan restore.CreatedTable, sz)
+func getTiFlashNodeCount(ctx context.Context, pdClient pd.Client) (uint64, error) {
+ tiFlashStores, err := conn.GetAllTiKVStoresWithRetry(ctx, pdClient, connutil.TiFlashOnly)
+ if err != nil {
+ return 0, errors.Trace(err)
+ }
+ return uint64(len(tiFlashStores)), nil
+}
- go func() {
- defer close(outCh)
- cachedTables := make([]restore.CreatedTable, 0, sz)
- for tbl := range inCh {
- cachedTables = append(cachedTables, tbl)
+// PreCheckTableTiFlashReplica checks whether TiFlash replica is less than TiFlash node.
+func PreCheckTableTiFlashReplica(
+ ctx context.Context,
+ pdClient pd.Client,
+ tables []*metautil.Table,
+ recorder *tiflashrec.TiFlashRecorder,
+) error {
+ tiFlashStoreCount, err := getTiFlashNodeCount(ctx, pdClient)
+ if err != nil {
+ return err
+ }
+ for _, table := range tables {
+ if table.Info.TiFlashReplica != nil {
+ // we should not set available to true. because we cannot guarantee the raft log lag of tiflash when restore finished.
+ // just let tiflash ticker set it by checking lag of all related regions.
+ table.Info.TiFlashReplica.Available = false
+ table.Info.TiFlashReplica.AvailablePartitionIDs = nil
+ if recorder != nil {
+ recorder.AddTable(table.Info.ID, *table.Info.TiFlashReplica)
+ log.Info("record tiflash replica for table, to reset it by ddl later",
+ zap.Stringer("db", table.DB.Name),
+ zap.Stringer("table", table.Info.Name),
+ )
+ table.Info.TiFlashReplica = nil
+ } else if table.Info.TiFlashReplica.Count > tiFlashStoreCount {
+ // we cannot satisfy TiFlash replica in restore cluster. so we should
+ // set TiFlashReplica to unavailable in tableInfo, to avoid TiDB cannot sense TiFlash and make plan to TiFlash
+ // see details at https://github.com/pingcap/br/issues/931
+ // TODO maybe set table.Info.TiFlashReplica.Count to tiFlashStoreCount, but we need more tests about it.
+ log.Warn("table does not satisfy tiflash replica requirements, set tiflash replcia to unavailable",
+ zap.Stringer("db", table.DB.Name),
+ zap.Stringer("table", table.Info.Name),
+ zap.Uint64("expect tiflash replica", table.Info.TiFlashReplica.Count),
+ zap.Uint64("actual tiflash store", tiFlashStoreCount),
+ )
+ table.Info.TiFlashReplica = nil
+ }
}
+ }
+ return nil
+}
- sort.Slice(cachedTables, func(a, b int) bool {
- return cachedTables[a].Table.ID < cachedTables[b].Table.ID
- })
+// PreCheckTableClusterIndex checks whether backup tables and existed tables have different cluster index options。
+func PreCheckTableClusterIndex(
+ tables []*metautil.Table,
+ ddlJobs []*model.Job,
+ dom *domain.Domain,
+) error {
+ for _, table := range tables {
+ oldTableInfo, err := restore.GetTableSchema(dom, table.DB.Name, table.Info.Name)
+ // table exists in database
+ if err == nil {
+ if table.Info.IsCommonHandle != oldTableInfo.IsCommonHandle {
+ return errors.Annotatef(berrors.ErrRestoreModeMismatch,
+ "Clustered index option mismatch. Restored cluster's @@tidb_enable_clustered_index should be %v (backup table = %v, created table = %v).",
+ restore.TransferBoolToValue(table.Info.IsCommonHandle),
+ table.Info.IsCommonHandle,
+ oldTableInfo.IsCommonHandle)
+ }
+ }
+ }
+ for _, job := range ddlJobs {
+ if job.Type == model.ActionCreateTable {
+ tableInfo := job.BinlogInfo.TableInfo
+ if tableInfo != nil {
+ oldTableInfo, err := restore.GetTableSchema(dom, model.NewCIStr(job.SchemaName), tableInfo.Name)
+ // table exists in database
+ if err == nil {
+ if tableInfo.IsCommonHandle != oldTableInfo.IsCommonHandle {
+ return errors.Annotatef(berrors.ErrRestoreModeMismatch,
+ "Clustered index option mismatch. Restored cluster's @@tidb_enable_clustered_index should be %v (backup table = %v, created table = %v).",
+ restore.TransferBoolToValue(tableInfo.IsCommonHandle),
+ tableInfo.IsCommonHandle,
+ oldTableInfo.IsCommonHandle)
+ }
+ }
+ }
+ }
+ }
+ return nil
+}
- for _, tbl := range cachedTables {
- select {
- case <-ctx.Done():
- return
- default:
- outCh <- tbl
+func getDatabases(tables []*metautil.Table) (dbs []*model.DBInfo) {
+ dbIDs := make(map[int64]bool)
+ for _, table := range tables {
+ if !dbIDs[table.DB.ID] {
+ dbs = append(dbs, table.DB)
+ dbIDs[table.DB.ID] = true
+ }
+ }
+ return
+}
+
+// FilterDDLJobs filters ddl jobs.
+func FilterDDLJobs(allDDLJobs []*model.Job, tables []*metautil.Table) (ddlJobs []*model.Job) {
+ // Sort the ddl jobs by schema version in descending order.
+ slices.SortFunc(allDDLJobs, func(i, j *model.Job) int {
+ return cmp.Compare(j.BinlogInfo.SchemaVersion, i.BinlogInfo.SchemaVersion)
+ })
+ dbs := getDatabases(tables)
+ for _, db := range dbs {
+ // These maps is for solving some corner case.
+ // e.g. let "t=2" indicates that the id of database "t" is 2, if the ddl execution sequence is:
+ // rename "a" to "b"(a=1) -> drop "b"(b=1) -> create "b"(b=2) -> rename "b" to "a"(a=2)
+ // Which we cannot find the "create" DDL by name and id directly.
+ // To cover †his case, we must find all names and ids the database/table ever had.
+ dbIDs := make(map[int64]bool)
+ dbIDs[db.ID] = true
+ dbNames := make(map[string]bool)
+ dbNames[db.Name.String()] = true
+ for _, job := range allDDLJobs {
+ if job.BinlogInfo.DBInfo != nil {
+ if dbIDs[job.SchemaID] || dbNames[job.BinlogInfo.DBInfo.Name.String()] {
+ ddlJobs = append(ddlJobs, job)
+ // The the jobs executed with the old id, like the step 2 in the example above.
+ dbIDs[job.SchemaID] = true
+ // For the jobs executed after rename, like the step 3 in the example above.
+ dbNames[job.BinlogInfo.DBInfo.Name.String()] = true
+ }
}
}
- }()
- return outCh
+ }
+
+ for _, table := range tables {
+ tableIDs := make(map[int64]bool)
+ tableIDs[table.Info.ID] = true
+ tableNames := make(map[restore.UniqueTableName]bool)
+ name := restore.UniqueTableName{DB: table.DB.Name.String(), Table: table.Info.Name.String()}
+ tableNames[name] = true
+ for _, job := range allDDLJobs {
+ if job.BinlogInfo.TableInfo != nil {
+ name = restore.UniqueTableName{DB: job.SchemaName, Table: job.BinlogInfo.TableInfo.Name.String()}
+ if tableIDs[job.TableID] || tableNames[name] {
+ ddlJobs = append(ddlJobs, job)
+ tableIDs[job.TableID] = true
+ // For truncate table, the id may be changed
+ tableIDs[job.BinlogInfo.TableInfo.ID] = true
+ tableNames[name] = true
+ }
+ }
+ }
+ }
+ return ddlJobs
+}
+
+// FilterDDLJobByRules if one of rules returns true, the job in srcDDLJobs will be filtered.
+func FilterDDLJobByRules(srcDDLJobs []*model.Job, rules ...DDLJobFilterRule) (dstDDLJobs []*model.Job) {
+ dstDDLJobs = make([]*model.Job, 0, len(srcDDLJobs))
+ for _, ddlJob := range srcDDLJobs {
+ passed := true
+ for _, rule := range rules {
+ if rule(ddlJob) {
+ passed = false
+ break
+ }
+ }
+
+ if passed {
+ dstDDLJobs = append(dstDDLJobs, ddlJob)
+ }
+ }
+
+ return
+}
+
+type DDLJobFilterRule func(ddlJob *model.Job) bool
+
+var incrementalRestoreActionBlockList = map[model.ActionType]struct{}{
+ model.ActionSetTiFlashReplica: {},
+ model.ActionUpdateTiFlashReplicaStatus: {},
+ model.ActionLockTable: {},
+ model.ActionUnlockTable: {},
+}
+
+// DDLJobBlockListRule rule for filter ddl job with type in block list.
+func DDLJobBlockListRule(ddlJob *model.Job) bool {
+ return checkIsInActions(ddlJob.Type, incrementalRestoreActionBlockList)
+}
+
+func checkIsInActions(action model.ActionType, actions map[model.ActionType]struct{}) bool {
+ _, ok := actions[action]
+ return ok
}
diff --git a/br/pkg/task/restore_data.go b/br/pkg/task/restore_data.go
index 8e992de1f9328..ecfbf77f870ce 100644
--- a/br/pkg/task/restore_data.go
+++ b/br/pkg/task/restore_data.go
@@ -14,12 +14,16 @@ import (
"github.com/pingcap/tidb/br/pkg/conn"
"github.com/pingcap/tidb/br/pkg/conn/util"
"github.com/pingcap/tidb/br/pkg/glue"
+ "github.com/pingcap/tidb/br/pkg/logutil"
"github.com/pingcap/tidb/br/pkg/restore"
"github.com/pingcap/tidb/br/pkg/restore/data"
+ "github.com/pingcap/tidb/br/pkg/restore/tiflashrec"
"github.com/pingcap/tidb/br/pkg/storage"
"github.com/pingcap/tidb/br/pkg/summary"
"github.com/pingcap/tidb/br/pkg/utils"
tidbconfig "github.com/pingcap/tidb/pkg/config"
+ "github.com/pingcap/tidb/pkg/kv"
+ pd "github.com/tikv/pd/client"
"go.uber.org/zap"
)
@@ -77,9 +81,7 @@ func RunResolveKvData(c context.Context, g glue.Glue, cmdName string, cfg *Resto
tc.EnableGlobalKill = false
tidbconfig.StoreGlobalConfig(tc)
- client := restore.NewRestoreClient(mgr.GetPDClient(), mgr.GetPDHTTPClient(), mgr.GetTLSConfig(), keepaliveCfg)
-
- restoreTS, err := client.GetTS(ctx)
+ restoreTS, err := restore.GetTSWithRetry(ctx, mgr.GetPDClient())
if err != nil {
return errors.Trace(err)
}
@@ -157,13 +159,8 @@ func RunResolveKvData(c context.Context, g glue.Glue, cmdName string, cfg *Resto
//TODO: restore volume type into origin type
//ModifyVolume(*ec2.ModifyVolumeInput) (*ec2.ModifyVolumeOutput, error) by backupmeta
- err = client.Init(g, mgr.GetStorage())
- if err != nil {
- return errors.Trace(err)
- }
- defer client.Close()
// since we cannot reset tiflash automaticlly. so we should start it manually
- if err = client.ResetTiFlashReplicas(ctx, g, mgr.GetStorage()); err != nil {
+ if err = resetTiFlashReplicas(ctx, g, mgr.GetStorage(), mgr.GetPDClient()); err != nil {
return errors.Trace(err)
}
@@ -172,3 +169,96 @@ func RunResolveKvData(c context.Context, g glue.Glue, cmdName string, cfg *Resto
summary.SetSuccessStatus(true)
return nil
}
+
+func resetTiFlashReplicas(ctx context.Context, g glue.Glue, storage kv.Storage, pdClient pd.Client) error {
+ dom, err := g.GetDomain(storage)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ info := dom.InfoSchema()
+ allSchemaName := info.AllSchemaNames()
+ recorder := tiflashrec.New()
+
+ expectTiFlashStoreCount := uint64(0)
+ needTiFlash := false
+ for _, s := range allSchemaName {
+ for _, t := range info.SchemaTables(s) {
+ t := t.Meta()
+ if t.TiFlashReplica != nil {
+ expectTiFlashStoreCount = max(expectTiFlashStoreCount, t.TiFlashReplica.Count)
+ recorder.AddTable(t.ID, *t.TiFlashReplica)
+ needTiFlash = true
+ }
+ }
+ }
+ if !needTiFlash {
+ log.Info("no need to set tiflash replica, since there is no tables enable tiflash replica")
+ return nil
+ }
+ // we wait for ten minutes to wait tiflash starts.
+ // since tiflash only starts when set unmark recovery mode finished.
+ timeoutCtx, cancel := context.WithTimeout(ctx, 10*time.Minute)
+ defer cancel()
+ err = utils.WithRetry(timeoutCtx, func() error {
+ tiFlashStoreCount, err := getTiFlashNodeCount(ctx, pdClient)
+ log.Info("get tiflash store count for resetting TiFlash Replica",
+ zap.Uint64("count", tiFlashStoreCount))
+ if err != nil {
+ return errors.Trace(err)
+ }
+ if tiFlashStoreCount < expectTiFlashStoreCount {
+ log.Info("still waiting for enough tiflash store start",
+ zap.Uint64("expect", expectTiFlashStoreCount),
+ zap.Uint64("actual", tiFlashStoreCount),
+ )
+ return errors.New("tiflash store count is less than expected")
+ }
+ return nil
+ }, &waitTiFlashBackoffer{
+ Attempts: 30,
+ BaseBackoff: 4 * time.Second,
+ })
+ if err != nil {
+ return err
+ }
+
+ sqls := recorder.GenerateResetAlterTableDDLs(info)
+ log.Info("Generating SQLs for resetting tiflash replica",
+ zap.Strings("sqls", sqls))
+
+ return g.UseOneShotSession(storage, false, func(se glue.Session) error {
+ for _, sql := range sqls {
+ if errExec := se.ExecuteInternal(ctx, sql); errExec != nil {
+ logutil.WarnTerm("Failed to restore tiflash replica config, you may execute the sql restore it manually.",
+ logutil.ShortError(errExec),
+ zap.String("sql", sql),
+ )
+ }
+ }
+ return nil
+ })
+}
+
+type waitTiFlashBackoffer struct {
+ Attempts int
+ BaseBackoff time.Duration
+}
+
+// NextBackoff returns a duration to wait before retrying again
+func (b *waitTiFlashBackoffer) NextBackoff(error) time.Duration {
+ bo := b.BaseBackoff
+ b.Attempts--
+ if b.Attempts == 0 {
+ return 0
+ }
+ b.BaseBackoff *= 2
+ if b.BaseBackoff > 32*time.Second {
+ b.BaseBackoff = 32 * time.Second
+ }
+ return bo
+}
+
+// Attempt returns the remain attempt times
+func (b *waitTiFlashBackoffer) Attempt() int {
+ return b.Attempts
+}
diff --git a/br/pkg/task/restore_raw.go b/br/pkg/task/restore_raw.go
index bfb2d612397b8..b53459a386a05 100644
--- a/br/pkg/task/restore_raw.go
+++ b/br/pkg/task/restore_raw.go
@@ -14,6 +14,7 @@ import (
"github.com/pingcap/tidb/br/pkg/httputil"
"github.com/pingcap/tidb/br/pkg/metautil"
"github.com/pingcap/tidb/br/pkg/restore"
+ snapclient "github.com/pingcap/tidb/br/pkg/restore/snap_client"
restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
"github.com/pingcap/tidb/br/pkg/summary"
"github.com/spf13/cobra"
@@ -92,14 +93,10 @@ func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreR
// sometimes we have pooled the connections.
// sending heartbeats in idle times is useful.
keepaliveCfg.PermitWithoutStream = true
- client := restore.NewRestoreClient(mgr.GetPDClient(), mgr.GetPDHTTPClient(), mgr.GetTLSConfig(), keepaliveCfg)
+ client := snapclient.NewRestoreClient(mgr.GetPDClient(), mgr.GetPDHTTPClient(), mgr.GetTLSConfig(), keepaliveCfg)
client.SetRateLimit(cfg.RateLimit)
client.SetCrypter(&cfg.CipherInfo)
- client.SetConcurrency(uint(cfg.Concurrency))
- if cfg.Online {
- client.EnableOnline()
- }
- client.SetSwitchModeInterval(cfg.SwitchModeInterval)
+ client.SetConcurrencyPerStore(cfg.ConcurrencyPerStore.Value)
err = client.Init(g, mgr.GetStorage())
defer client.Close()
if err != nil {
@@ -148,16 +145,17 @@ func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreR
!cfg.LogProgress)
// RawKV restore does not need to rewrite keys.
- err = restore.SplitRanges(ctx, client, ranges, updateCh, true)
+ err = client.SplitRanges(ctx, ranges, updateCh, true)
if err != nil {
return errors.Trace(err)
}
- restoreSchedulers, _, err := restorePreWork(ctx, client, mgr, true)
+ importModeSwitcher := restore.NewImportModeSwitcher(mgr.GetPDClient(), cfg.SwitchModeInterval, mgr.GetTLSConfig())
+ restoreSchedulers, _, err := restore.RestorePreWork(ctx, mgr, importModeSwitcher, cfg.Online, true)
if err != nil {
return errors.Trace(err)
}
- defer restorePostWork(ctx, client, restoreSchedulers)
+ defer restore.RestorePostWork(ctx, importModeSwitcher, restoreSchedulers, cfg.Online)
err = client.RestoreRaw(ctx, cfg.StartKey, cfg.EndKey, files, updateCh)
if err != nil {
diff --git a/br/pkg/task/restore_test.go b/br/pkg/task/restore_test.go
index 5eeedbbd1c965..f35cb3dc8bc8b 100644
--- a/br/pkg/task/restore_test.go
+++ b/br/pkg/task/restore_test.go
@@ -1,311 +1,419 @@
// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
-package task
+package task_test
import (
"context"
"encoding/json"
- "fmt"
+ "math"
+ "strconv"
"testing"
- "github.com/golang/protobuf/proto"
+ "github.com/gogo/protobuf/proto"
backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/kvproto/pkg/encryptionpb"
"github.com/pingcap/kvproto/pkg/metapb"
- "github.com/pingcap/tidb/br/pkg/conn"
+ "github.com/pingcap/tidb/br/pkg/backup"
+ "github.com/pingcap/tidb/br/pkg/gluetidb"
+ gluemock "github.com/pingcap/tidb/br/pkg/gluetidb/mock"
"github.com/pingcap/tidb/br/pkg/metautil"
- "github.com/pingcap/tidb/br/pkg/restore"
- "github.com/pingcap/tidb/br/pkg/storage"
- "github.com/pingcap/tidb/br/pkg/utils"
+ "github.com/pingcap/tidb/br/pkg/mock"
+ "github.com/pingcap/tidb/br/pkg/restore/tiflashrec"
+ "github.com/pingcap/tidb/br/pkg/task"
+ utiltest "github.com/pingcap/tidb/br/pkg/utiltest"
+ "github.com/pingcap/tidb/pkg/ddl"
"github.com/pingcap/tidb/pkg/parser/model"
- "github.com/pingcap/tidb/pkg/statistics/handle/util"
- "github.com/pingcap/tidb/pkg/tablecodec"
+ "github.com/pingcap/tidb/pkg/parser/mysql"
+ "github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/types"
+ "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- pd "github.com/tikv/pd/client"
- "google.golang.org/grpc/keepalive"
+ "github.com/tikv/client-go/v2/oracle"
)
-func TestRestoreConfigAdjust(t *testing.T) {
- cfg := &RestoreConfig{}
- cfg.Adjust()
+func TestPreCheckTableTiFlashReplicas(t *testing.T) {
+ mockStores := []*metapb.Store{
+ {
+ Id: 1,
+ Labels: []*metapb.StoreLabel{
+ {
+ Key: "engine",
+ Value: "tiflash",
+ },
+ },
+ },
+ {
+ Id: 2,
+ Labels: []*metapb.StoreLabel{
+ {
+ Key: "engine",
+ Value: "tiflash",
+ },
+ },
+ },
+ }
- require.Equal(t, uint32(defaultRestoreConcurrency), cfg.Config.Concurrency)
- require.Equal(t, defaultSwitchInterval, cfg.Config.SwitchModeInterval)
- require.Equal(t, conn.DefaultMergeRegionKeyCount, cfg.MergeSmallRegionKeyCount.Value)
- require.Equal(t, conn.DefaultMergeRegionSizeBytes, cfg.MergeSmallRegionSizeBytes.Value)
-}
+ pdClient := utiltest.NewFakePDClient(mockStores, false, nil)
-type mockPDClient struct {
- pd.Client
-}
+ tables := make([]*metautil.Table, 4)
+ for i := 0; i < len(tables); i++ {
+ tiflashReplica := &model.TiFlashReplicaInfo{
+ Count: uint64(i),
+ }
+ if i == 0 {
+ tiflashReplica = nil
+ }
-func (m mockPDClient) GetClusterID(_ context.Context) uint64 {
- return 1
-}
+ tables[i] = &metautil.Table{
+ DB: &model.DBInfo{Name: model.NewCIStr("test")},
+ Info: &model.TableInfo{
+ ID: int64(i),
+ Name: model.NewCIStr("test" + strconv.Itoa(i)),
+ TiFlashReplica: tiflashReplica,
+ },
+ }
+ }
+ ctx := context.Background()
+ require.Nil(t, task.PreCheckTableTiFlashReplica(ctx, pdClient, tables, nil))
-func (m mockPDClient) GetAllStores(ctx context.Context, opts ...pd.GetStoreOption) ([]*metapb.Store, error) {
- return []*metapb.Store{}, nil
-}
+ for i := 0; i < len(tables); i++ {
+ if i == 0 || i > 2 {
+ require.Nil(t, tables[i].Info.TiFlashReplica)
+ } else {
+ require.NotNil(t, tables[i].Info.TiFlashReplica)
+ obtainCount := int(tables[i].Info.TiFlashReplica.Count)
+ require.Equal(t, i, obtainCount)
+ }
+ }
-func TestConfigureRestoreClient(t *testing.T) {
- cfg := Config{
- Concurrency: 1024,
+ require.Nil(t, task.PreCheckTableTiFlashReplica(ctx, pdClient, tables, tiflashrec.New()))
+ for i := 0; i < len(tables); i++ {
+ require.Nil(t, tables[i].Info.TiFlashReplica)
}
- restoreComCfg := RestoreCommonConfig{
- Online: true,
+}
+
+func TestPreCheckTableClusterIndex(t *testing.T) {
+ ctx := context.Background()
+ m, err := mock.NewCluster()
+ if err != nil {
+ panic(err)
}
- restoreCfg := &RestoreConfig{
- Config: cfg,
- RestoreCommonConfig: restoreComCfg,
- DdlBatchSize: 128,
+ err = m.Start()
+ if err != nil {
+ panic(err)
}
- client := restore.NewRestoreClient(mockPDClient{}, nil, nil, keepalive.ClientParameters{})
- ctx := context.Background()
- err := configureRestoreClient(ctx, client, restoreCfg)
+ defer m.Stop()
+ g := gluetidb.New()
+ se, err := g.CreateSession(m.Storage)
require.NoError(t, err)
- require.Equal(t, uint(128), client.GetBatchDdlSize())
- require.True(t, client.IsOnline())
-}
-func TestAdjustRestoreConfigForStreamRestore(t *testing.T) {
- restoreCfg := RestoreConfig{}
+ info, err := m.Domain.GetSnapshotInfoSchema(math.MaxUint64)
+ require.NoError(t, err)
+ dbSchema, isExist := info.SchemaByName(model.NewCIStr("test"))
+ require.True(t, isExist)
+
+ tables := make([]*metautil.Table, 4)
+ intField := types.NewFieldType(mysql.TypeLong)
+ intField.SetCharset("binary")
+ for i := len(tables) - 1; i >= 0; i-- {
+ tables[i] = &metautil.Table{
+ DB: dbSchema,
+ Info: &model.TableInfo{
+ ID: int64(i),
+ Name: model.NewCIStr("test" + strconv.Itoa(i)),
+ Columns: []*model.ColumnInfo{{
+ ID: 1,
+ Name: model.NewCIStr("id"),
+ FieldType: *intField,
+ State: model.StatePublic,
+ }},
+ Charset: "utf8mb4",
+ Collate: "utf8mb4_bin",
+ },
+ }
+ err = se.CreateTable(ctx, tables[i].DB.Name, tables[i].Info, ddl.OnExistIgnore)
+ require.NoError(t, err)
+ }
- restoreCfg.adjustRestoreConfigForStreamRestore()
- require.Equal(t, restoreCfg.PitrBatchCount, uint32(defaultPiTRBatchCount))
- require.Equal(t, restoreCfg.PitrBatchSize, uint32(defaultPiTRBatchSize))
- require.Equal(t, restoreCfg.PitrConcurrency, uint32(defaultPiTRConcurrency))
- require.Equal(t, restoreCfg.Concurrency, restoreCfg.PitrConcurrency)
+ // exist different tables
+ tables[1].Info.IsCommonHandle = true
+ err = task.PreCheckTableClusterIndex(tables, nil, m.Domain)
+ require.Error(t, err)
+ require.Regexp(t, `.*@@tidb_enable_clustered_index should be ON \(backup table = true, created table = false\).*`, err.Error())
+
+ // exist different DDLs
+ jobs := []*model.Job{{
+ ID: 5,
+ Type: model.ActionCreateTable,
+ SchemaName: "test",
+ Query: "",
+ BinlogInfo: &model.HistoryInfo{
+ TableInfo: &model.TableInfo{
+ Name: model.NewCIStr("test1"),
+ IsCommonHandle: true,
+ },
+ },
+ }}
+ err = task.PreCheckTableClusterIndex(nil, jobs, m.Domain)
+ require.Error(t, err)
+ require.Regexp(t, `.*@@tidb_enable_clustered_index should be ON \(backup table = true, created table = false\).*`, err.Error())
+
+ // should pass pre-check cluster index
+ tables[1].Info.IsCommonHandle = false
+ jobs[0].BinlogInfo.TableInfo.IsCommonHandle = false
+ require.Nil(t, task.PreCheckTableClusterIndex(tables, jobs, m.Domain))
}
-func TestCheckRestoreDBAndTable(t *testing.T) {
- cases := []struct {
- cfgSchemas map[string]struct{}
- cfgTables map[string]struct{}
- backupDBs map[string]*metautil.Database
+func TestCheckNewCollationEnable(t *testing.T) {
+ caseList := []struct {
+ backupMeta *backuppb.BackupMeta
+ newCollationEnableInCluster string
+ CheckRequirements bool
+ isErr bool
}{
{
- cfgSchemas: map[string]struct{}{
- utils.EncloseName("test"): {},
- },
- cfgTables: map[string]struct{}{
- utils.EncloseDBAndTable("test", "t"): {},
- utils.EncloseDBAndTable("test", "t2"): {},
- },
- backupDBs: mockReadSchemasFromBackupMeta(t, map[string][]string{
- "test": {"T", "T2"},
- }),
+ backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "True"},
+ newCollationEnableInCluster: "True",
+ CheckRequirements: true,
+ isErr: false,
},
{
- cfgSchemas: map[string]struct{}{
- utils.EncloseName("mysql"): {},
- },
- cfgTables: map[string]struct{}{
- utils.EncloseDBAndTable("mysql", "t"): {},
- utils.EncloseDBAndTable("mysql", "t2"): {},
- },
- backupDBs: mockReadSchemasFromBackupMeta(t, map[string][]string{
- "__TiDB_BR_Temporary_mysql": {"T", "T2"},
- }),
+ backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "True"},
+ newCollationEnableInCluster: "False",
+ CheckRequirements: true,
+ isErr: true,
},
{
- cfgSchemas: map[string]struct{}{
- utils.EncloseName("test"): {},
- },
- cfgTables: map[string]struct{}{
- utils.EncloseDBAndTable("test", "T"): {},
- utils.EncloseDBAndTable("test", "T2"): {},
- },
- backupDBs: mockReadSchemasFromBackupMeta(t, map[string][]string{
- "test": {"t", "t2"},
- }),
+ backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "False"},
+ newCollationEnableInCluster: "True",
+ CheckRequirements: true,
+ isErr: true,
},
{
- cfgSchemas: map[string]struct{}{
- utils.EncloseName("TEST"): {},
- },
- cfgTables: map[string]struct{}{
- utils.EncloseDBAndTable("TEST", "t"): {},
- utils.EncloseDBAndTable("TEST", "T2"): {},
- },
- backupDBs: mockReadSchemasFromBackupMeta(t, map[string][]string{
- "test": {"t", "t2"},
- }),
+ backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "False"},
+ newCollationEnableInCluster: "false",
+ CheckRequirements: true,
+ isErr: false,
},
{
- cfgSchemas: map[string]struct{}{
- utils.EncloseName("TeSt"): {},
- },
- cfgTables: map[string]struct{}{
- utils.EncloseDBAndTable("TeSt", "tabLe"): {},
- utils.EncloseDBAndTable("TeSt", "taBle2"): {},
- },
- backupDBs: mockReadSchemasFromBackupMeta(t, map[string][]string{
- "TesT": {"TablE", "taBle2"},
- }),
+ backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "False"},
+ newCollationEnableInCluster: "True",
+ CheckRequirements: false,
+ isErr: true,
},
{
- cfgSchemas: map[string]struct{}{
- utils.EncloseName("TeSt"): {},
- utils.EncloseName("MYSQL"): {},
- },
- cfgTables: map[string]struct{}{
- utils.EncloseDBAndTable("TeSt", "tabLe"): {},
- utils.EncloseDBAndTable("TeSt", "taBle2"): {},
- utils.EncloseDBAndTable("MYSQL", "taBle"): {},
- },
- backupDBs: mockReadSchemasFromBackupMeta(t, map[string][]string{
- "TesT": {"table", "TaBLE2"},
- "__TiDB_BR_Temporary_mysql": {"tablE"},
- }),
+ backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: "True"},
+ newCollationEnableInCluster: "False",
+ CheckRequirements: false,
+ isErr: true,
},
{
- cfgSchemas: map[string]struct{}{
- utils.EncloseName("sys"): {},
- },
- cfgTables: map[string]struct{}{
- utils.EncloseDBAndTable("sys", "t"): {},
- utils.EncloseDBAndTable("sys", "t2"): {},
- },
- backupDBs: mockReadSchemasFromBackupMeta(t, map[string][]string{
- "__TiDB_BR_Temporary_sys": {"T", "T2"},
- }),
+ backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: ""},
+ newCollationEnableInCluster: "True",
+ CheckRequirements: false,
+ isErr: false,
+ },
+ {
+ backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: ""},
+ newCollationEnableInCluster: "True",
+ CheckRequirements: true,
+ isErr: true,
+ },
+ {
+ backupMeta: &backuppb.BackupMeta{NewCollationsEnabled: ""},
+ newCollationEnableInCluster: "False",
+ CheckRequirements: false,
+ isErr: false,
},
}
- cfg := &RestoreConfig{}
- for _, ca := range cases {
- cfg.Schemas = ca.cfgSchemas
- cfg.Tables = ca.cfgTables
- client := restore.MockClient(ca.backupDBs)
-
- err := CheckRestoreDBAndTable(client, cfg)
- require.NoError(t, err)
+ for i, ca := range caseList {
+ g := &gluemock.MockGlue{
+ GlobalVars: map[string]string{"new_collation_enabled": ca.newCollationEnableInCluster},
+ }
+ enabled, err := task.CheckNewCollationEnable(ca.backupMeta.GetNewCollationsEnabled(), g, nil, ca.CheckRequirements)
+ t.Logf("[%d] Got Error: %v\n", i, err)
+ if ca.isErr {
+ require.Error(t, err)
+ } else {
+ require.NoError(t, err)
+ }
+ require.Equal(t, ca.newCollationEnableInCluster == "True", enabled)
}
}
-func mockReadSchemasFromBackupMeta(t *testing.T, db2Tables map[string][]string) map[string]*metautil.Database {
- testDir := t.TempDir()
- store, err := storage.NewLocalStorage(testDir)
- require.NoError(t, err)
+func TestFilterDDLJobs(t *testing.T) {
+ s := utiltest.CreateRestoreSchemaSuite(t)
+ tk := testkit.NewTestKit(t, s.Mock.Storage)
+ tk.MustExec("CREATE DATABASE IF NOT EXISTS test_db;")
+ tk.MustExec("CREATE TABLE IF NOT EXISTS test_db.test_table (c1 INT);")
+ lastTS, err := s.Mock.GetOracle().GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope})
+ require.NoErrorf(t, err, "Error get last ts: %s", err)
+ tk.MustExec("RENAME TABLE test_db.test_table to test_db.test_table1;")
+ tk.MustExec("DROP TABLE test_db.test_table1;")
+ tk.MustExec("DROP DATABASE test_db;")
+ tk.MustExec("CREATE DATABASE test_db;")
+ tk.MustExec("USE test_db;")
+ tk.MustExec("CREATE TABLE test_table1 (c2 CHAR(255));")
+ tk.MustExec("RENAME TABLE test_table1 to test_table;")
+ tk.MustExec("TRUNCATE TABLE test_table;")
- mockSchemas := make([]*backuppb.Schema, 0)
- var dbID int64 = 1
- for db, tables := range db2Tables {
- dbName := model.NewCIStr(db)
- mockTblList := make([]*model.TableInfo, 0)
- tblBytesList, statsBytesList := make([][]byte, 0), make([][]byte, 0)
-
- for i, table := range tables {
- tblName := model.NewCIStr(table)
- mockTbl := &model.TableInfo{
- ID: dbID*100 + int64(i),
- Name: tblName,
- }
- mockTblList = append(mockTblList, mockTbl)
-
- mockStats := util.JSONTable{
- DatabaseName: dbName.String(),
- TableName: tblName.String(),
- }
-
- tblBytes, err := json.Marshal(mockTbl)
- require.NoError(t, err)
- tblBytesList = append(tblBytesList, tblBytes)
+ ts, err := s.Mock.GetOracle().GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope})
+ require.NoErrorf(t, err, "Error get ts: %s", err)
- statsBytes, err := json.Marshal(mockStats)
- require.NoError(t, err)
- statsBytesList = append(statsBytesList, statsBytes)
- }
+ cipher := backuppb.CipherInfo{
+ CipherType: encryptionpb.EncryptionMethod_PLAINTEXT,
+ }
- mockDB := model.DBInfo{
- ID: dbID,
- Name: dbName,
- Tables: mockTblList,
- }
- dbID++
- dbBytes, err := json.Marshal(mockDB)
- require.NoError(t, err)
+ metaWriter := metautil.NewMetaWriter(s.Storage, metautil.MetaFileSize, false, "", &cipher)
+ ctx := context.Background()
+ metaWriter.StartWriteMetasAsync(ctx, metautil.AppendDDL)
+ s.MockGlue.SetSession(tk.Session())
+ err = backup.WriteBackupDDLJobs(metaWriter, s.MockGlue, s.Mock.Storage, lastTS, ts, false)
+ require.NoErrorf(t, err, "Error get ddl jobs: %s", err)
+ err = metaWriter.FinishWriteMetas(ctx, metautil.AppendDDL)
+ require.NoErrorf(t, err, "Flush failed", err)
+ err = metaWriter.FlushBackupMeta(ctx)
+ require.NoErrorf(t, err, "Finially flush backupmeta failed", err)
+ infoSchema, err := s.Mock.Domain.GetSnapshotInfoSchema(ts)
+ require.NoErrorf(t, err, "Error get snapshot info schema: %s", err)
+ dbInfo, ok := infoSchema.SchemaByName(model.NewCIStr("test_db"))
+ require.Truef(t, ok, "DB info not exist")
+ tableInfo, err := infoSchema.TableByName(model.NewCIStr("test_db"), model.NewCIStr("test_table"))
+ require.NoErrorf(t, err, "Error get table info: %s", err)
+ tables := []*metautil.Table{{
+ DB: dbInfo,
+ Info: tableInfo.Meta(),
+ }}
+ metaBytes, err := s.Storage.ReadFile(ctx, metautil.MetaFile)
+ require.NoError(t, err)
+ mockMeta := &backuppb.BackupMeta{}
+ err = proto.Unmarshal(metaBytes, mockMeta)
+ require.NoError(t, err)
+ // check the schema version
+ require.Equal(t, int32(metautil.MetaV1), mockMeta.Version)
+ metaReader := metautil.NewMetaReader(mockMeta, s.Storage, &cipher)
+ allDDLJobsBytes, err := metaReader.ReadDDLs(ctx)
+ require.NoError(t, err)
+ var allDDLJobs []*model.Job
+ err = json.Unmarshal(allDDLJobsBytes, &allDDLJobs)
+ require.NoError(t, err)
- for i := 0; i < len(tblBytesList); i++ {
- mockSchemas = append(mockSchemas, &backuppb.Schema{
- Db: dbBytes,
- Table: tblBytesList[i],
- Stats: statsBytesList[i],
- },
- )
- }
+ ddlJobs := task.FilterDDLJobs(allDDLJobs, tables)
+ for _, job := range ddlJobs {
+ t.Logf("get ddl job: %s", job.Query)
}
+ require.Equal(t, 7, len(ddlJobs))
+}
- mockFiles := []*backuppb.File{
- {
- Name: fmt.Sprintf("%p.sst", &mockSchemas),
- StartKey: tablecodec.EncodeRowKey(1, []byte("a")),
- EndKey: tablecodec.EncodeRowKey(2, []byte("a")),
- },
- }
+func TestFilterDDLJobsV2(t *testing.T) {
+ s := utiltest.CreateRestoreSchemaSuite(t)
+ tk := testkit.NewTestKit(t, s.Mock.Storage)
+ tk.MustExec("CREATE DATABASE IF NOT EXISTS test_db;")
+ tk.MustExec("CREATE TABLE IF NOT EXISTS test_db.test_table (c1 INT);")
+ lastTS, err := s.Mock.GetOracle().GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope})
+ require.NoErrorf(t, err, "Error get last ts: %s", err)
+ tk.MustExec("RENAME TABLE test_db.test_table to test_db.test_table1;")
+ tk.MustExec("DROP TABLE test_db.test_table1;")
+ tk.MustExec("DROP DATABASE test_db;")
+ tk.MustExec("CREATE DATABASE test_db;")
+ tk.MustExec("USE test_db;")
+ tk.MustExec("CREATE TABLE test_table1 (c2 CHAR(255));")
+ tk.MustExec("RENAME TABLE test_table1 to test_table;")
+ tk.MustExec("TRUNCATE TABLE test_table;")
- meta := mockBackupMeta(mockSchemas, mockFiles)
- data, err := proto.Marshal(meta)
- require.NoError(t, err)
+ ts, err := s.Mock.GetOracle().GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope})
+ require.NoErrorf(t, err, "Error get ts: %s", err)
+
+ cipher := backuppb.CipherInfo{
+ CipherType: encryptionpb.EncryptionMethod_PLAINTEXT,
+ }
+ metaWriter := metautil.NewMetaWriter(s.Storage, metautil.MetaFileSize, true, "", &cipher)
ctx := context.Background()
- err = store.WriteFile(ctx, metautil.MetaFile, data)
- require.NoError(t, err)
+ metaWriter.StartWriteMetasAsync(ctx, metautil.AppendDDL)
+ s.MockGlue.SetSession(tk.Session())
+ err = backup.WriteBackupDDLJobs(metaWriter, s.MockGlue, s.Mock.Storage, lastTS, ts, false)
+ require.NoErrorf(t, err, "Error get ddl jobs: %s", err)
+ err = metaWriter.FinishWriteMetas(ctx, metautil.AppendDDL)
+ require.NoErrorf(t, err, "Flush failed", err)
+ err = metaWriter.FlushBackupMeta(ctx)
+ require.NoErrorf(t, err, "Flush BackupMeta failed", err)
- dbs, err := metautil.LoadBackupTables(
- ctx,
- metautil.NewMetaReader(
- meta,
- store,
- &backuppb.CipherInfo{
- CipherType: encryptionpb.EncryptionMethod_PLAINTEXT,
- }),
- true,
- )
+ infoSchema, err := s.Mock.Domain.GetSnapshotInfoSchema(ts)
+ require.NoErrorf(t, err, "Error get snapshot info schema: %s", err)
+ dbInfo, ok := infoSchema.SchemaByName(model.NewCIStr("test_db"))
+ require.Truef(t, ok, "DB info not exist")
+ tableInfo, err := infoSchema.TableByName(model.NewCIStr("test_db"), model.NewCIStr("test_table"))
+ require.NoErrorf(t, err, "Error get table info: %s", err)
+ tables := []*metautil.Table{{
+ DB: dbInfo,
+ Info: tableInfo.Meta(),
+ }}
+ metaBytes, err := s.Storage.ReadFile(ctx, metautil.MetaFile)
+ require.NoError(t, err)
+ mockMeta := &backuppb.BackupMeta{}
+ err = proto.Unmarshal(metaBytes, mockMeta)
+ require.NoError(t, err)
+ // check the schema version
+ require.Equal(t, int32(metautil.MetaV2), mockMeta.Version)
+ metaReader := metautil.NewMetaReader(mockMeta, s.Storage, &cipher)
+ allDDLJobsBytes, err := metaReader.ReadDDLs(ctx)
+ require.NoError(t, err)
+ var allDDLJobs []*model.Job
+ err = json.Unmarshal(allDDLJobsBytes, &allDDLJobs)
require.NoError(t, err)
- return dbs
-}
-func mockBackupMeta(mockSchemas []*backuppb.Schema, mockFiles []*backuppb.File) *backuppb.BackupMeta {
- return &backuppb.BackupMeta{
- Files: mockFiles,
- Schemas: mockSchemas,
+ ddlJobs := task.FilterDDLJobs(allDDLJobs, tables)
+ for _, job := range ddlJobs {
+ t.Logf("get ddl job: %s", job.Query)
}
+ require.Equal(t, 7, len(ddlJobs))
}
-func TestMapTableToFiles(t *testing.T) {
- filesOfTable1 := []*backuppb.File{
+func TestFilterDDLJobByRules(t *testing.T) {
+ ddlJobs := []*model.Job{
{
- Name: "table1-1.sst",
- StartKey: tablecodec.EncodeTablePrefix(1),
- EndKey: tablecodec.EncodeTablePrefix(1),
+ Type: model.ActionSetTiFlashReplica,
},
{
- Name: "table1-2.sst",
- StartKey: tablecodec.EncodeTablePrefix(1),
- EndKey: tablecodec.EncodeTablePrefix(1),
+ Type: model.ActionAddPrimaryKey,
},
{
- Name: "table1-3.sst",
- StartKey: tablecodec.EncodeTablePrefix(1),
- EndKey: tablecodec.EncodeTablePrefix(1),
+ Type: model.ActionUpdateTiFlashReplicaStatus,
},
- }
- filesOfTable2 := []*backuppb.File{
{
- Name: "table2-1.sst",
- StartKey: tablecodec.EncodeTablePrefix(2),
- EndKey: tablecodec.EncodeTablePrefix(2),
+ Type: model.ActionCreateTable,
},
{
- Name: "table2-2.sst",
- StartKey: tablecodec.EncodeTablePrefix(2),
- EndKey: tablecodec.EncodeTablePrefix(2),
+ Type: model.ActionLockTable,
+ },
+ {
+ Type: model.ActionAddIndex,
+ },
+ {
+ Type: model.ActionUnlockTable,
+ },
+ {
+ Type: model.ActionCreateSchema,
+ },
+ {
+ Type: model.ActionModifyColumn,
},
}
- result := MapTableToFiles(append(filesOfTable2, filesOfTable1...))
+ expectedDDLTypes := []model.ActionType{
+ model.ActionAddPrimaryKey,
+ model.ActionCreateTable,
+ model.ActionAddIndex,
+ model.ActionCreateSchema,
+ model.ActionModifyColumn,
+ }
- require.Equal(t, filesOfTable1, result[1])
- require.Equal(t, filesOfTable2, result[2])
+ ddlJobs = task.FilterDDLJobByRules(ddlJobs, task.DDLJobBlockListRule)
+
+ require.Equal(t, len(expectedDDLTypes), len(ddlJobs))
+ for i, ddlJob := range ddlJobs {
+ assert.Equal(t, expectedDDLTypes[i], ddlJob.Type)
+ }
}
diff --git a/br/pkg/task/restore_txn.go b/br/pkg/task/restore_txn.go
index 00255681854de..00039eb51370e 100644
--- a/br/pkg/task/restore_txn.go
+++ b/br/pkg/task/restore_txn.go
@@ -12,6 +12,7 @@ import (
"github.com/pingcap/tidb/br/pkg/glue"
"github.com/pingcap/tidb/br/pkg/metautil"
"github.com/pingcap/tidb/br/pkg/restore"
+ snapclient "github.com/pingcap/tidb/br/pkg/restore/snap_client"
restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
"github.com/pingcap/tidb/br/pkg/summary"
)
@@ -39,11 +40,10 @@ func RunRestoreTxn(c context.Context, g glue.Glue, cmdName string, cfg *Config)
// sometimes we have pooled the connections.
// sending heartbeats in idle times is useful.
keepaliveCfg.PermitWithoutStream = true
- client := restore.NewRestoreClient(mgr.GetPDClient(), mgr.GetPDHTTPClient(), mgr.GetTLSConfig(), keepaliveCfg)
+ client := snapclient.NewRestoreClient(mgr.GetPDClient(), mgr.GetPDHTTPClient(), mgr.GetTLSConfig(), keepaliveCfg)
client.SetRateLimit(cfg.RateLimit)
client.SetCrypter(&cfg.CipherInfo)
- client.SetConcurrency(uint(cfg.Concurrency))
- client.SetSwitchModeInterval(cfg.SwitchModeInterval)
+ client.SetConcurrencyPerStore(uint(cfg.Concurrency))
err = client.Init(g, mgr.GetStorage())
defer client.Close()
if err != nil {
@@ -88,16 +88,17 @@ func RunRestoreTxn(c context.Context, g glue.Glue, cmdName string, cfg *Config)
!cfg.LogProgress)
// RawKV restore does not need to rewrite keys.
- err = restore.SplitRanges(ctx, client, ranges, updateCh, false)
+ err = client.SplitRanges(ctx, ranges, updateCh, false)
if err != nil {
return errors.Trace(err)
}
- restoreSchedulers, _, err := restorePreWork(ctx, client, mgr, true)
+ importModeSwitcher := restore.NewImportModeSwitcher(mgr.GetPDClient(), cfg.SwitchModeInterval, mgr.GetTLSConfig())
+ restoreSchedulers, _, err := restore.RestorePreWork(ctx, mgr, importModeSwitcher, false, true)
if err != nil {
return errors.Trace(err)
}
- defer restorePostWork(ctx, client, restoreSchedulers)
+ defer restore.RestorePostWork(ctx, importModeSwitcher, restoreSchedulers, false)
err = client.WaitForFilesRestored(ctx, files, updateCh)
if err != nil {
diff --git a/br/pkg/task/stream.go b/br/pkg/task/stream.go
index d032b3c517c9e..c8d297232274d 100644
--- a/br/pkg/task/stream.go
+++ b/br/pkg/task/stream.go
@@ -42,7 +42,8 @@ import (
"github.com/pingcap/tidb/br/pkg/logutil"
"github.com/pingcap/tidb/br/pkg/metautil"
"github.com/pingcap/tidb/br/pkg/restore"
- "github.com/pingcap/tidb/br/pkg/restore/rawkv"
+ "github.com/pingcap/tidb/br/pkg/restore/ingestrec"
+ logclient "github.com/pingcap/tidb/br/pkg/restore/log_client"
"github.com/pingcap/tidb/br/pkg/restore/tiflashrec"
restoreutils "github.com/pingcap/tidb/br/pkg/restore/utils"
"github.com/pingcap/tidb/br/pkg/storage"
@@ -56,7 +57,6 @@ import (
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/util/cdcutil"
"github.com/spf13/pflag"
- "github.com/tikv/client-go/v2/config"
"github.com/tikv/client-go/v2/oracle"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
@@ -91,9 +91,6 @@ var (
StreamTruncate: {},
}
- // rawKVBatchCount specifies the count of entries that the rawkv client puts into TiKV.
- rawKVBatchCount = 64
-
streamShiftDuration = time.Hour
)
@@ -1116,15 +1113,16 @@ func checkTaskExists(ctx context.Context, cfg *RestoreConfig, etcdCLI *clientv3.
"create log-backup task again and create a full backup on this cluster", tasks[0].Info.Name)
}
- // check cdc changefeed
- if cfg.CheckRequirements {
- nameSet, err := cdcutil.GetCDCChangefeedNameSet(ctx, etcdCLI)
- if err != nil {
- return err
- }
- if !nameSet.Empty() {
- return errors.Errorf("%splease remove changefeed(s) before restore", nameSet.MessageToUser())
- }
+ return nil
+}
+
+func checkIncompatibleChangefeed(ctx context.Context, backupTS uint64, etcdCLI *clientv3.Client) error {
+ nameSet, err := cdcutil.GetIncompatibleChangefeedsWithSafeTS(ctx, etcdCLI, backupTS)
+ if err != nil {
+ return err
+ }
+ if !nameSet.Empty() {
+ return errors.Errorf("%splease remove changefeed(s) before restore", nameSet.MessageToUser())
}
return nil
}
@@ -1236,6 +1234,7 @@ func restoreStream(
totalSize uint64
checkpointTotalKVCount uint64
checkpointTotalSize uint64
+ currentTS uint64
mu sync.Mutex
startTime = time.Now()
)
@@ -1245,9 +1244,12 @@ func restoreStream(
} else {
totalDureTime := time.Since(startTime)
summary.Log("restore log success summary", zap.Duration("total-take", totalDureTime),
- zap.Uint64("restore-from", cfg.StartTS), zap.Uint64("restore-to", cfg.RestoreTS),
- zap.String("restore-from", stream.FormatDate(oracle.GetTimeFromTS(cfg.StartTS))),
- zap.String("restore-to", stream.FormatDate(oracle.GetTimeFromTS(cfg.RestoreTS))),
+ zap.Uint64("source-start-point", cfg.StartTS),
+ zap.Uint64("source-end-point", cfg.RestoreTS),
+ zap.Uint64("target-end-point", currentTS),
+ zap.String("source-start", stream.FormatDate(oracle.GetTimeFromTS(cfg.StartTS))),
+ zap.String("source-end", stream.FormatDate(oracle.GetTimeFromTS(cfg.RestoreTS))),
+ zap.String("target-end", stream.FormatDate(oracle.GetTimeFromTS(currentTS))),
zap.Uint64("total-kv-count", totalKVCount),
zap.Uint64("skipped-kv-count-by-checkpoint", checkpointTotalKVCount),
zap.String("total-size", units.HumanSize(float64(totalSize))),
@@ -1282,26 +1284,26 @@ func restoreStream(
}
defer client.Close()
- var currentTS uint64
if taskInfo != nil && taskInfo.RewriteTS > 0 {
// reuse the task's rewrite ts
log.Info("reuse the task's rewrite ts", zap.Uint64("rewrite-ts", taskInfo.RewriteTS))
currentTS = taskInfo.RewriteTS
} else {
- currentTS, err = client.GetTSWithRetry(ctx)
+ currentTS, err = restore.GetTSWithRetry(ctx, mgr.GetPDClient())
if err != nil {
return errors.Trace(err)
}
}
client.SetCurrentTS(currentTS)
- restoreSchedulers, _, err := restorePreWork(ctx, client, mgr, false)
+ importModeSwitcher := restore.NewImportModeSwitcher(mgr.GetPDClient(), cfg.Config.SwitchModeInterval, mgr.GetTLSConfig())
+ restoreSchedulers, _, err := restore.RestorePreWork(ctx, mgr, importModeSwitcher, cfg.Online, false)
if err != nil {
return errors.Trace(err)
}
// Always run the post-work even on error, so we don't stuck in the import
// mode or emptied schedulers
- defer restorePostWork(ctx, client, restoreSchedulers)
+ defer restore.RestorePostWork(ctx, importModeSwitcher, restoreSchedulers, cfg.Online)
// It need disable GC in TiKV when PiTR.
// because the process of PITR is concurrent and kv events isn't sorted by tso.
@@ -1360,7 +1362,7 @@ func restoreStream(
newTask = false
}
// get the schemas ID replace information.
- schemasReplace, err := client.InitSchemasReplaceForDDL(ctx, &restore.InitSchemaConfig{
+ schemasReplace, err := client.InitSchemasReplaceForDDL(ctx, &logclient.InitSchemaConfig{
IsNewTask: newTask,
HasFullRestore: len(cfg.FullBackupStorage) > 0,
TableFilter: cfg.TableFilter,
@@ -1404,7 +1406,7 @@ func restoreStream(
rewriteRules := initRewriteRules(schemasReplace)
ingestRecorder := schemasReplace.GetIngestRecorder()
- if err := client.RangeFilterFromIngestRecorder(ingestRecorder, rewriteRules); err != nil {
+ if err := rangeFilterFromIngestRecorder(ingestRecorder, rewriteRules); err != nil {
return errors.Trace(err)
}
@@ -1456,7 +1458,7 @@ func restoreStream(
return errors.Annotate(err, "failed to insert rows into gc_delete_range")
}
- if err = client.RepairIngestIndex(ctx, ingestRecorder, g, mgr.GetStorage(), taskName); err != nil {
+ if err = client.RepairIngestIndex(ctx, ingestRecorder, g, taskName); err != nil {
return errors.Annotate(err, "failed to repair ingest index")
}
@@ -1491,11 +1493,11 @@ func restoreStream(
return nil
}
-func createRestoreClient(ctx context.Context, g glue.Glue, cfg *RestoreConfig, mgr *conn.Mgr) (*restore.Client, error) {
+func createRestoreClient(ctx context.Context, g glue.Glue, cfg *RestoreConfig, mgr *conn.Mgr) (*logclient.LogClient, error) {
var err error
keepaliveCfg := GetKeepalive(&cfg.Config)
keepaliveCfg.PermitWithoutStream = true
- client := restore.NewRestoreClient(mgr.GetPDClient(), mgr.GetPDHTTPClient(), mgr.GetTLSConfig(), keepaliveCfg)
+ client := logclient.NewRestoreClient(mgr.GetPDClient(), mgr.GetPDHTTPClient(), mgr.GetTLSConfig(), keepaliveCfg)
err = client.Init(g, mgr.GetStorage())
if err != nil {
return nil, errors.Trace(err)
@@ -1515,19 +1517,11 @@ func createRestoreClient(ctx context.Context, g glue.Glue, cfg *RestoreConfig, m
if err = client.SetStorage(ctx, u, &opts); err != nil {
return nil, errors.Trace(err)
}
- client.SetRateLimit(cfg.RateLimit)
client.SetCrypter(&cfg.CipherInfo)
client.SetConcurrency(uint(cfg.Concurrency))
- client.SetSwitchModeInterval(cfg.SwitchModeInterval)
- client.InitClients(ctx, u, false, false)
-
- rawKVClient, err := newRawBatchClient(ctx, cfg.PD, cfg.TLS)
- if err != nil {
- return nil, errors.Trace(err)
- }
- client.SetRawKVClient(rawKVClient)
+ client.InitClients(ctx, u)
- err = client.LoadRestoreStores(ctx)
+ err = client.SetRawKVBatchClient(ctx, cfg.PD, cfg.TLS.ToKVSecurity())
if err != nil {
return nil, errors.Trace(err)
}
@@ -1535,6 +1529,24 @@ func createRestoreClient(ctx context.Context, g glue.Glue, cfg *RestoreConfig, m
return client, nil
}
+// rangeFilterFromIngestRecorder rewrites the table id of items in the ingestRecorder
+// TODO: need to implement the range filter out feature
+func rangeFilterFromIngestRecorder(recorder *ingestrec.IngestRecorder, rewriteRules map[int64]*restoreutils.RewriteRules) error {
+ err := recorder.RewriteTableID(func(tableID int64) (int64, bool, error) {
+ rewriteRule, exists := rewriteRules[tableID]
+ if !exists {
+ // since the table's files will be skipped restoring, here also skips.
+ return 0, true, nil
+ }
+ newTableID := restoreutils.GetRewriteTableID(tableID, rewriteRule)
+ if newTableID == 0 {
+ return 0, false, errors.Errorf("newTableID is 0, tableID: %d", tableID)
+ }
+ return newTableID, false, nil
+ })
+ return errors.Trace(err)
+}
+
func getExternalStorageOptions(cfg *Config, u *backuppb.StorageBackend) storage.ExternalStorageOptions {
var httpClient *http.Client
if u.GetGcs() == nil {
@@ -1673,7 +1685,7 @@ func getFullBackupTS(
func parseFullBackupTablesStorage(
cfg *RestoreConfig,
-) (*restore.FullBackupStorageConfig, error) {
+) (*logclient.FullBackupStorageConfig, error) {
var storageName string
if len(cfg.FullBackupStorage) > 0 {
storageName = cfg.FullBackupStorage
@@ -1684,7 +1696,7 @@ func parseFullBackupTablesStorage(
if err != nil {
return nil, errors.Trace(err)
}
- return &restore.FullBackupStorageConfig{
+ return &logclient.FullBackupStorageConfig{
Backend: u,
Opts: storageOpts(&cfg.Config),
}, nil
@@ -1725,24 +1737,6 @@ func initRewriteRules(schemasReplace *stream.SchemasReplace) map[int64]*restoreu
return rules
}
-func newRawBatchClient(
- ctx context.Context,
- pdAddrs []string,
- tlsConfig TLSConfig,
-) (*rawkv.RawKVBatchClient, error) {
- security := config.Security{
- ClusterSSLCA: tlsConfig.CA,
- ClusterSSLCert: tlsConfig.Cert,
- ClusterSSLKey: tlsConfig.Key,
- }
- rawkvClient, err := rawkv.NewRawkvClient(ctx, pdAddrs, security)
- if err != nil {
- return nil, errors.Trace(err)
- }
-
- return rawkv.NewRawKVBatchClient(rawkvClient, rawKVBatchCount), nil
-}
-
// ShiftTS gets a smaller shiftTS than startTS.
// It has a safe duration between shiftTS and startTS for trasaction.
func ShiftTS(startTS uint64) uint64 {
diff --git a/br/pkg/utils/safe_point.go b/br/pkg/utils/safe_point.go
index 93fa9415aa369..5c06961d69ed9 100644
--- a/br/pkg/utils/safe_point.go
+++ b/br/pkg/utils/safe_point.go
@@ -9,7 +9,6 @@ import (
"github.com/google/uuid"
"github.com/pingcap/errors"
- "github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/log"
berrors "github.com/pingcap/tidb/br/pkg/errors"
"github.com/tikv/client-go/v2/oracle"
@@ -143,13 +142,3 @@ func StartServiceSafePointKeeper(
}()
return nil
}
-
-type FakePDClient struct {
- pd.Client
- Stores []*metapb.Store
-}
-
-// GetAllStores return fake stores.
-func (c FakePDClient) GetAllStores(context.Context, ...pd.GetStoreOption) ([]*metapb.Store, error) {
- return append([]*metapb.Store{}, c.Stores...), nil
-}
diff --git a/br/pkg/utiltest/BUILD.bazel b/br/pkg/utiltest/BUILD.bazel
new file mode 100644
index 0000000000000..857171c570572
--- /dev/null
+++ b/br/pkg/utiltest/BUILD.bazel
@@ -0,0 +1,22 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "utiltest",
+ srcs = [
+ "fake.go",
+ "suite.go",
+ ],
+ importpath = "github.com/pingcap/tidb/br/pkg/utiltest",
+ visibility = ["//visibility:public"],
+ deps = [
+ "//br/pkg/gluetidb/mock",
+ "//br/pkg/mock",
+ "//br/pkg/restore/split",
+ "//br/pkg/storage",
+ "@com_github_pingcap_kvproto//pkg/metapb",
+ "@com_github_pkg_errors//:errors",
+ "@com_github_stretchr_testify//require",
+ "@com_github_tikv_pd_client//:client",
+ "@org_golang_google_grpc//keepalive",
+ ],
+)
diff --git a/br/pkg/utiltest/fake.go b/br/pkg/utiltest/fake.go
new file mode 100644
index 0000000000000..5bb2e339851fe
--- /dev/null
+++ b/br/pkg/utiltest/fake.go
@@ -0,0 +1,115 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utiltest
+
+import (
+ "bytes"
+ "context"
+ "time"
+
+ "github.com/pingcap/kvproto/pkg/metapb"
+ "github.com/pingcap/tidb/br/pkg/restore/split"
+ "github.com/pkg/errors"
+ pd "github.com/tikv/pd/client"
+ "google.golang.org/grpc/keepalive"
+)
+
+var DefaultTestKeepaliveCfg = keepalive.ClientParameters{
+ Time: 3 * time.Second,
+ Timeout: 10 * time.Second,
+}
+
+type FakePDClient struct {
+ pd.Client
+ stores []*metapb.Store
+
+ notLeader bool
+ retryTimes *int
+}
+
+func NewFakePDClient(stores []*metapb.Store, notLeader bool, retryTime *int) FakePDClient {
+ var retryTimeInternal int
+ if retryTime == nil {
+ retryTime = &retryTimeInternal
+ }
+ return FakePDClient{
+ stores: stores,
+
+ notLeader: notLeader,
+ retryTimes: retryTime,
+ }
+}
+
+func (fpdc FakePDClient) GetAllStores(context.Context, ...pd.GetStoreOption) ([]*metapb.Store, error) {
+ return append([]*metapb.Store{}, fpdc.stores...), nil
+}
+
+func (fpdc FakePDClient) GetTS(ctx context.Context) (int64, int64, error) {
+ (*fpdc.retryTimes)++
+ if *fpdc.retryTimes >= 3 { // the mock PD leader switched successfully
+ fpdc.notLeader = false
+ }
+
+ if fpdc.notLeader {
+ return 0, 0, errors.Errorf(
+ "rpc error: code = Unknown desc = [PD:tso:ErrGenerateTimestamp]generate timestamp failed, " +
+ "requested pd is not leader of cluster",
+ )
+ }
+ return 1, 1, nil
+}
+
+type FakeSplitClient struct {
+ split.SplitClient
+ regions []*split.RegionInfo
+}
+
+func NewFakeSplitClient() *FakeSplitClient {
+ return &FakeSplitClient{
+ regions: make([]*split.RegionInfo, 0),
+ }
+}
+
+func (f *FakeSplitClient) AppendRegion(startKey, endKey []byte) {
+ f.regions = append(f.regions, &split.RegionInfo{
+ Region: &metapb.Region{
+ StartKey: startKey,
+ EndKey: endKey,
+ },
+ })
+}
+
+func (f *FakeSplitClient) ScanRegions(
+ ctx context.Context,
+ startKey, endKey []byte,
+ limit int,
+) ([]*split.RegionInfo, error) {
+ result := make([]*split.RegionInfo, 0)
+ count := 0
+ for _, rng := range f.regions {
+ if bytes.Compare(rng.Region.StartKey, endKey) <= 0 && bytes.Compare(rng.Region.EndKey, startKey) > 0 {
+ result = append(result, rng)
+ count++
+ }
+ if count >= limit {
+ break
+ }
+ }
+ return result, nil
+}
+
+func (f *FakeSplitClient) WaitRegionsScattered(context.Context, []*split.RegionInfo) (int, error) {
+ return 0, nil
+}
diff --git a/br/pkg/utiltest/suite.go b/br/pkg/utiltest/suite.go
new file mode 100644
index 0000000000000..bb784cf8298e7
--- /dev/null
+++ b/br/pkg/utiltest/suite.go
@@ -0,0 +1,46 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utiltest
+
+import (
+ "testing"
+
+ gluemock "github.com/pingcap/tidb/br/pkg/gluetidb/mock"
+ "github.com/pingcap/tidb/br/pkg/mock"
+ "github.com/pingcap/tidb/br/pkg/storage"
+ "github.com/stretchr/testify/require"
+)
+
+type TestRestoreSchemaSuite struct {
+ Mock *mock.Cluster
+ MockGlue *gluemock.MockGlue
+ Storage storage.ExternalStorage
+}
+
+func CreateRestoreSchemaSuite(t *testing.T) *TestRestoreSchemaSuite {
+ var err error
+ s := new(TestRestoreSchemaSuite)
+ s.MockGlue = &gluemock.MockGlue{}
+ s.Mock, err = mock.NewCluster()
+ require.NoError(t, err)
+ base := t.TempDir()
+ s.Storage, err = storage.NewLocalStorage(base)
+ require.NoError(t, err)
+ require.NoError(t, s.Mock.Start())
+ t.Cleanup(func() {
+ s.Mock.Stop()
+ })
+ return s
+}
diff --git a/br/tests/br_full/run.sh b/br/tests/br_full/run.sh
index 3752ce40995b9..450064df3f317 100755
--- a/br/tests/br_full/run.sh
+++ b/br/tests/br_full/run.sh
@@ -80,8 +80,8 @@ for ct in limit lz4 zstd; do
# restore full
echo "restore with $ct backup start..."
- export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/restore/restore-storage-error=1*return(\"connection refused\");github.com/pingcap/tidb/br/pkg/restore/restore-gRPC-error=1*return(true)"
- export GO_FAILPOINTS=$GO_FAILPOINTS";github.com/pingcap/tidb/br/pkg/restore/no-leader-error=3*return(true)"
+ export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/restore/snap_client/restore-storage-error=1*return(\"connection refused\");github.com/pingcap/tidb/br/pkg/restore/restore-gRPC-error=1*return(true)"
+ export GO_FAILPOINTS=$GO_FAILPOINTS";github.com/pingcap/tidb/br/pkg/restore/split/no-leader-error=3*return(true)"
run_br restore full -s "local://$TEST_DIR/$DB-$ct" --pd $PD_ADDR --ratelimit 1024
export GO_FAILPOINTS=""
@@ -111,8 +111,8 @@ for granularity in "fine-grained" "coarse-grained"; do
# restore full
echo "restore with $ct backup start..."
- export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/restore/restore-storage-error=1*return(\"connection refused\");github.com/pingcap/tidb/br/pkg/restore/restore-gRPC-error=1*return(true)"
- export GO_FAILPOINTS="${GO_FAILPOINTS};github.com/pingcap/tidb/br/pkg/restore/no-leader-error=3*return(true)"
+ export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/restore/snap_client/restore-storage-error=1*return(\"connection refused\");github.com/pingcap/tidb/br/pkg/restore/restore-gRPC-error=1*return(true)"
+ export GO_FAILPOINTS="${GO_FAILPOINTS};github.com/pingcap/tidb/br/pkg/restore/split/no-leader-error=3*return(true)"
run_br restore full -s "local://$TEST_DIR/$DB-$ct" --pd $PD_ADDR --ratelimit 1024 --granularity=$granularity
export GO_FAILPOINTS=""
diff --git a/br/tests/br_full_ddl/run.sh b/br/tests/br_full_ddl/run.sh
index 370d77dca66dd..1572d0ef6fddd 100755
--- a/br/tests/br_full_ddl/run.sh
+++ b/br/tests/br_full_ddl/run.sh
@@ -157,7 +157,7 @@ run_sql "DROP DATABASE $DB;"
# restore full
echo "restore start..."
-export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/restore/restore-createtables-error=return(true)"
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/restore/snap_client/restore-createtables-error=return(true)"
run_br restore full -s "local://$TEST_DIR/$DB" --pd $PD_ADDR --log-file $RESTORE_LOG --ddl-batch-size=128 || { cat $RESTORE_LOG; }
export GO_FAILPOINTS=""
diff --git a/build/BUILD.bazel b/build/BUILD.bazel
index 1d3f33dd8448a..b4e6d1c408cbc 100644
--- a/build/BUILD.bazel
+++ b/build/BUILD.bazel
@@ -178,7 +178,6 @@ nogo(
}) +
select({
"//build:without_rbe": [
- "//build/linter/filepermission",
],
"//conditions:default": [],
}),
diff --git a/build/patches/com_github_golang_protobuf.patch b/build/patches/com_github_golang_protobuf.patch
deleted file mode 100644
index 41f0137881a67..0000000000000
--- a/build/patches/com_github_golang_protobuf.patch
+++ /dev/null
@@ -1,12 +0,0 @@
-diff -urN a/descriptor/BUILD.bazel b/descriptor/BUILD.bazel
---- a/descriptor/BUILD.bazel 1969-12-31 19:00:00.000000000 -0500
-+++ b/descriptor/BUILD.bazel 2000-01-01 00:00:00.000000000 -0000
-@@ -21,7 +21,7 @@
- visibility = ["//visibility:public"],
- deps = [
- "//proto:go_default_library",
-- "@io_bazel_rules_go//proto/wkt:descriptor_go_proto",
-+ "@com_github_golang_protobuf//protoc-gen-go/descriptor:go_default_library",
- "@org_golang_google_protobuf//reflect/protodesc:go_default_library",
- "@org_golang_google_protobuf//reflect/protoreflect:go_default_library",
- "@org_golang_google_protobuf//runtime/protoimpl:go_default_library",
diff --git a/build/patches/io_etcd_go_etcd_api_v3.patch b/build/patches/io_etcd_go_etcd_api_v3.patch
deleted file mode 100644
index 97cfc7c5ad9a5..0000000000000
--- a/build/patches/io_etcd_go_etcd_api_v3.patch
+++ /dev/null
@@ -1,12 +0,0 @@
-diff -uprN old/io_etcd_go_etcd_api_v3/etcdserverpb/gw/BUILD.bazel new/io_etcd_go_etcd_api_v3/etcdserverpb/gw/BUILD.bazel
---- old/io_etcd_go_etcd_api_v3/etcdserverpb/gw/BUILD.bazel 2022-04-12 02:12:01.000000000 +0800
-+++ new/io_etcd_go_etcd_api_v3/etcdserverpb/gw/BUILD.bazel 2022-04-12 02:09:59.000000000 +0800
-@@ -8,7 +8,7 @@ go_library(
- visibility = ["//visibility:public"],
- deps = [
- "//etcdserverpb",
-- "@com_github_golang_protobuf//descriptor:go_default_library_gen",
-+ "@com_github_golang_protobuf//descriptor:descriptor",
- "@com_github_golang_protobuf//proto:go_default_library",
- "@com_github_grpc_ecosystem_grpc_gateway//runtime:go_default_library",
- "@com_github_grpc_ecosystem_grpc_gateway//utilities:go_default_library",
diff --git a/cmd/mirror/mirror.go b/cmd/mirror/mirror.go
index 07cd182274588..8d13e75bf6244 100644
--- a/cmd/mirror/mirror.go
+++ b/cmd/mirror/mirror.go
@@ -288,19 +288,11 @@ func dumpPatchArgsForRepo(repoName string) error {
}
candidate := filepath.Join(runfiles, "build", "patches", repoName+".patch")
if _, err := os.Stat(candidate); err == nil {
- if repoName == "io_etcd_go_etcd_api_v3" {
- fmt.Printf(` patch_args = ["-p2"],
+ fmt.Printf(` patch_args = ["-p1"],
patches = [
"//build/patches:%s.patch",
],
`, repoName)
- } else {
- fmt.Printf(` patch_args = ["-p1"],
- patches = [
- "//build/patches:%s.patch",
- ],
-`, repoName)
- }
} else if !os.IsNotExist(err) {
return err
}
diff --git a/docs/design/2024-05-10-extension-authentication-plugin.md b/docs/design/2024-05-10-extension-authentication-plugin.md
new file mode 100644
index 0000000000000..a517c5599370d
--- /dev/null
+++ b/docs/design/2024-05-10-extension-authentication-plugin.md
@@ -0,0 +1,236 @@
+# Extension Authentication Plugin
+
+- Author: [Yaoming Zhan](http://github.com/yzhan1)
+- Discussion PR: TBD
+- Tracking Issue: https://github.com/pingcap/tidb/issues/53181
+
+## Table of Contents
+
+* [Introduction](#introduction)
+* [Motivation or Background](#motivation-or-background)
+* [Detailed Design](#detailed-design)
+* [Test Design](#test-design)
+ * [Functional Tests](#functional-tests)
+ * [Scenario Tests](#scenario-tests)
+ * [Compatibility Tests](#compatibility-tests)
+ * [Benchmark Tests](#benchmark-tests)
+* [Impacts & Risks](#impacts--risks)
+* [Investigation & Alternatives](#investigation--alternatives)
+* [Unresolved Questions](#unresolved-questions)
+
+## Introduction
+
+This design aims to introduce the extension auth plugin feature to TiDB. The extension auth plugin is a plugin that allows users to implement their own authentication and authorization logic. The plugin is loaded by TiDB and is responsible for authenticating users and authorizing their operations.
+
+[Auth plugin](https://dev.mysql.com/doc/extending-mysql/8.0/en/writing-authentication-plugins-server-side.html) is a feature supported in MySQL, so implementing this feature in TiDB will make it more compatible with MySQL.
+
+## Motivation or Background
+
+Currently, TiDB only supports the built-in authentication and authorization mechanism. The built-in mechanism is not flexible enough to meet the needs of some users. For example, some users may want to use their own authentication and authorization logic, or some users may want to integrate TiDB with their existing authentication and authorization system. Using the extension system to implement an auth plugin is a good way for users to plug in their own logic.
+
+### Current Code Path
+
+Here we only cover the critical paths involving authentication and authorization.
+
+Authentication:
++ `server.onConn`
+ + A new connection is created and handled using a separate goroutine
++ `conn.handshake`
+ + Perform handshake when a new handshake event is received
++ `conn.readOptionalSSLRequestAndHandshakeResponse`
+ + SSL handshake completed before running custom auth code
+ + This function handles validating and aligning the plugin used for connection on both client and server, and reading the password from client
+ + For plugins like `caching_sha2_password` and LDAP, at this stage, the server can communicate with the client by sending extra data back to the client (e.g. salt). Then the client can use this data to process the password before sending it back to the server
++ `conn.openSessionAndDoAuth`
+ + If `readOptionalSSLRequestAndHandshakeResponse` does not return error, we move forward to the session creation and authentication stage using the password read from the client
++ `session.Auth` -> `privileges.ConnectionVerification`
+ + Authenticate the user based on the plugin specified. For password based authentication such as `mysql_native_password`, TiDB will compare the password digest with the locally stored hash
+
+
+Authorization is more straight-forward (more details available in the [official developer guide](https://pingcap.github.io/tidb-dev-guide/understand-tidb/dql.html)):
++ `server.onConn`
++ `server.HandleStmt`
++ `server.ExecuteStmt`
++ `executor.Compile`
++ `Planner.optimize`
++ `privileges.RequestVerification`
+ + Note that this function will be called multiple times, one time for each privilege required by the SQL statements
+
+## Detailed Design
+
+We propose improving the extension system to support the following functionalities:
++ Allow user defined extra authentication check to be executed during connection establishment phase
+ + Current `SessionHandler` [support](https://github.com/pingcap/tidb/tree/master/pkg/extension) in the extension system only allows running observer functions that do not affect the connection attempt result
+ + Errors returned from this check will cause the connect attempt to fail
++ Allow user defined extra SQL privilege check to be executed before SQL statements are executed
+
+Note that the above plugin SQL privilege checks are only additional to the existing checks done by TiDB. If the existing checks done by TiDB fails, the plugin checks won’t be invoked. This means that the user should already have the corresponding SQL privileges granted to itself before we can trigger the plugin privilege check, otherwise the SQL request should be rejected beforehand since the user is missing the privileges.
+
+Users should be able to develop custom auth plugins which handle the above two checks. For example, if a user develops a plugin called `authentication_my_plugin`, then once the plugin is loaded, the user can run the following SQL statement:
+
+```sql
+CREATE USER 'my_user'@'%' IDENTIFIED WITH 'authentication_my_plugin' AS 'optional_authentication_pwd';
+```
+
+After this, `my_user`’s connection verification and privilege verification will be handled by the plugin in addition to the existing default TiDB checks. Users not identified with the plugin will not go through the plugin authorization and authentication checks.
+
+### Interface
+We propose a set of interfaces for users to implement an auth plugin that are similar to what is offered in MySQL’s authentication plugins. We will modify TiDB’s code to call these interfaces during the authentication/authorization flows. Most of these interfaces will receive parameters that are used in the existing auth flows in TiDB, so that the plugin developer can access as much information as possible.
+
+Following is the interface for an auth plugin:
+```go
+// AuthPlugin contains attributes needed for an authentication plugin.
+type AuthPlugin struct {
+ // Name is the name of the auth plugin. It will be registered as a system variable in TiDB which can be used inside the `CREATE USER ... IDENTIFIED WITH 'plugin_name'` statement.
+ Name string
+
+ // RequiredClientSidePlugin is the name of the client-side plugin required by the server-side plugin. It will be used to check if the client has the required plugin installed and require the client to use it if installed.
+ // The user can require default MySQL plugins such as 'caching_sha2_password' or 'mysql_native_password'.
+ RequiredClientSidePlugin string
+
+ // AuthenticateUser is called when a client connects to the server as a user and the server authenticates the user.
+ // If an error is returned, the login attempt fails, otherwise it succeeds.
+ // authUser: the username in the connect attempt
+ // storedPwd: the user's password stored in mysql.user table
+ // inputPwd (authentication): the user's password passed in from the connection attempt in bytes
+ // salt: randomly generated salt for the current connection
+ // connState: the TLS connection state (contains the TLS certificate) if client is using TLS. It will be nil if the client is not using TLS
+ // authConn: interface for the plugin to communicate with the client
+ AuthenticateUser func(authUser string, storedPwd string, inputPwd []byte, salt []byte, connState *tls.ConnectionState, authConn conn.AuthConn) error
+
+ // GenerateAuthString is a function for user to implement customized ways to encode the password (e.g. hash/salt/clear-text). The returned string will be stored as the encoded password in the mysql.user table.
+ // If the input password is considered as invalid, this should return an error.
+ // pwd: User's input password in CREATE/ALTER USER statements in clear-text
+ GenerateAuthString func(pwd string) (string, bool)
+
+ // ValidateAuthString checks if the password hash stored in the mysql.user table or passed in from `IDENTIFIED AS` is valid.
+ // This is called when retrieving an existing user to make sure the password stored is valid and not modified and make sure user is passing a valid password hash in `IDENTIFIED AS`.
+ // pwdHash: hash of the password stored in the internal user table
+ ValidateAuthString func(pwdHash string) bool
+
+ // VerifyPrivilege is called for each user queries, and serves as an extra check for privileges for the user.
+ // It will only be executed if the user has already been granted the privilege in SQL layer.
+ // Returns true if user has the requested privilege.
+ // activeRoles: list of active MySQL roles for the current user
+ // user: current user's name
+ // host: the host that the user is connecting from
+ // db: the database to check for privilege
+ // table: the table to check for privilege
+ // column: the column to check for privilege (currently just a placeholder in TiDB as column-level privilege is not supported by TiDB yet)
+ // priv: the privilege type of the SQL statement that will be executed
+ // connState: the TLS connection state (contains the TLS certificate) if client is using TLS. It will be nil if the client is not using TLS
+ VerifyPrivilege func(activeRoles []*auth.RoleIdentity, user, host, db, table, column string, priv mysql.PrivilegeType, connState *tls.ConnectionState) bool
+
+ // VerifyDynamicPrivilege is called for each user queries, and serves as an extra check for dynamic privileges for the user.
+ // It will only be executed if the user has already been granted the dynamic privilege in SQL layer.
+ // Returns true if user has the requested privilege.
+ // activeRoles: list of active MySQL roles for the current user
+ // user: current user's name
+ // host: current user's host
+ // priv: the dynamic privilege required by the user's SQL statement
+ // withGrant: whether the statement to be executed is granting the user privilege for executing GRANT statements
+ // connState: the TLS connection state (contains the TLS certificate) if client is using TLS. It will be nil if the client is not using TLS
+ VerifyDynamicPrivilege func(activeRoles []*auth.RoleIdentity, user, host, privName string, withGrant bool, connState *tls.ConnectionState) bool
+}
+```
+
+Note that we are using a `struct` instead of an `interface` to make sure that if in the future we add a new function/attribute required, existing user-implemented plugin code are still compatible.
+
+Let’s discuss each attributes/functions and why they are needed:
+
+`Name`: the name of the plugin. This will be used in the `CREATE USER ... IDENTIFIED WITH 'plugin_name'` statement. The plugin name will be registered as a system variable in TiDB.
+
+`RequiredClientSidePlugin`: the name of the client-side plugin required by the server-side plugin. This is used to check if the client has the required plugin installed and require the client to use it if installed. The user can require default MySQL plugins such as `caching_sha2_password` or `mysql_native_password`.
+
+`AuthenticateUser`: this should include the authentication check performed during connection attempts after the plugin/SSL information has been exchanged between the client and server. It should be called inside https://github.com/pingcap/tidb/blob/8d9e67b37dea759db0980aeddf4da967bf93e83e/pkg/privilege/privileges/privileges.go#L514 where the privilege manager checks the authentication plugin, and receives all the information it needs.
+For example, username, password, and the TLS connection state which includes the TLS certificates. If an error is returned, the connection attempt will fail.
+If the server would like to communicate with the client to send extra data (e.g. salt) to the client, it can use the `authConn` interface to do so. The client can then use this data to process the password before sending it back to the server.
+
+`ValidateAuthString`: after the connection attempt is granted, validates whether the password stored locally in TiDB is a valid password hash. This should be called in https://github.com/pingcap/tidb/blob/8d9e67b37dea759db0980aeddf4da967bf93e83e/pkg/privilege/privileges/privileges.go#L213.
+This is also called with the user is created with `CREATE USER .... IDENTIFIED AS 'my_auth_string'` where `my_auth_string` will be validated using this function to make sure it's a valid hash.
+
+`GenerateAuthString`: called when executing `CREATE`/`ALTER USER`/`SET PASSWORD` statements to encode the password. It should return the encoded password and whether the input password is legal. Should be called inside https://github.com/pingcap/tidb/blob/8d9e67b37dea759db0980aeddf4da967bf93e83e/pkg/parser/ast/misc.go#L1421. Password validation during creation/alter time should be done inside this function.
+
+`VerifyPrivilege`: is called to authorize users for executing SQL statements. This will be an extra check on top of the existing check done with the `MySQLPrivilege.RequestVerification`, so it should be called after the statement here: https://github.com/pingcap/tidb/blob/8d9e67b37dea759db0980aeddf4da967bf93e83e/pkg/privilege/privileges/privileges.go#L189.
+If the user does not have the privileges granted in MySQL, the overall approval will fail and the user will get a privilege error. The overall logic should be:
+```go
+func VerifyPrivilege(mysqlPriv *MySQLPrivilege, user string) bool {
+ return mysqlPriv.RequestVerification(...) && getPluginForUser(user).RequestVerification(...)
+}
+```
+
+`VerifyDynamicPrivilege`: similar to the above interface, but used for dynamic privileges. This will be an extra check on top of the existing check done with the `MySQLPrivilege.RequestDynamicVerification`, so it should be called after the statement here: https://github.com/pingcap/tidb/blob/8d9e67b37dea759db0980aeddf4da967bf93e83e/pkg/privilege/privileges/privileges.go#L129.
+If the user does not have the dynamic privileges granted in MySQL, the overall approval will fail and the user will get a privilege error. The overall logic should be:
+```go
+func VerifyDynamicPrivilege(mysqlPriv *MySQLPrivilege, user string) bool {
+ return mysqlPriv.RequestDynamicVerification(...) && getPluginForUser(user).RequestDynamicVerification(...)
+}
+```
+
+Note that for both the privilege checking functions above, they only receive one privilege in the parameter. This means if the transactions include multiple statements requiring multiple privileges, these functions will be called multiple times using the different privileges.
+
+The above interfaces and parameters should give the users all information they need to make a decision on the authentication and authorization. For authorizations before each request, the MySQL user should already be granted with the corresponding privileges in MySQL using `GRANT` statements.
+
+### Compatibility
+The feature needs to be compatible with existing user related statements and options, such as `CREATE USER`, `ALTER USER`, `SET PASSWORD`.
+However, it will not work with existing functionalities such as password expiry and password validation. Users can implement their own password validation logic inside the `GenerateAuthString` implementation.
+
+It will also not work with `CREATE VIEW SQL SECURITY DEFINER` since the view is created with the privileges of the view creator, and if the creator is an auth plugin user, the plugin will not be able to check the privileges of the view creator because it
+does not have the `tls.ConnectionState` information for the definer to pass into `VerifyPrivilege`. This means we should not allow any users to create a view with `SQL SECURITY DEFINER` where the definer is an auth plugin user.
+However, if the plugin implementation does not have `VerifyPrivilege` and `VerifyDynamicPrivilege` implemented, the view creation with an auth plugin user as the definer will still be allowed, since we will have everything needed for the authorization check.
+
+## Test Design
+
+This feature requires tests focusing on functional and compatibility. Performance should not be a concern, since that depends on the actual auth plugin implementation and it is up to the users to keep the implementation performant if needed.
+
+### Functional Tests
+
+We will implement unit tests for all of the auth plugin extension interfaces, and ensure that they are executed properly in the code path that we inject them.
+
+### Scenario Tests
+
+We will implement client to server tests to test out scenarios where MySQL users are created with and without using an auth plugin from the extension system. For example:
++ Creating a user using the plugin
+ + Connecting as the user and verify that the connection verification API is called and errors are correctly translated to connection rejections
+ + Execute SQL statements and verify that the request (dynamic) verification APIs are called and errors are correctly translated to permission rejections
+ + Alter user changes can be applied correctly
+ + Passwords can be encoded with the API defined in the plugin
++ When the connection changes between users, the system should behave correctly
+ + From non-plugin user to plugin user
+ + From plugin user to non-plugin user
+ + From plugin user to plugin user
+
+### Compatibility Tests
+
+This feature should integrate well with all the existing test cases since it is an additional functionality. Existing tests related to authentication and authorization should continue to pass since they are not using auth plugins.
+
+We should add tests to verify that users cannot create views with `SQL SECURITY DEFINER` where the definer is an auth plugin user and the plugin implementation has `VerifyPrivilege` and `VerifyDynamicPrivilege` set, as mentioned in the compatibility section.
+
+### Benchmark Tests
+
+This feature should not affect the performance of the TiDB as it simply provides interfaces for users to inject their own auth plugins via the extension system. The performance implication will come from the implementation of those plugins which depends on how the users of the framework want to use it.
+
+## Impacts & Risks
+
+If a user is using an auth plugin, the SQL privilege status in TiDB might not reflect the actual privileges, since there might be an outside system that decides whether the privilege check can actually pass. However, this is an intentional behavioral change when the user uses an auth plugin since we are providing users customizations on checks to run.
+
+The design does not account for implementing the same level of proxy user support as in MySQL auth plugin, since currently TiDB does not support [proxy users](https://dev.mysql.com/doc/extending-mysql/8.0/en/writing-authentication-plugins-proxy-users.html) yet. Also existing support for password expiry and validation will not work, as mentioned in the earlier section.
+
+Also, users cannot create views with `SQL SECURITY DEFINER` where the definer is an auth plugin user because we will lack the `tls.ConnectionState` information for the definer to pass into `VerifyPrivilege`, unless the plugin implementation does not have `VerifyPrivilege` and `VerifyDynamicPrivilege` implemented.
+
+## Investigation & Alternatives
+
+### MySQL
+
+MySQL supports customizing authentication methods with [authentication plugins](https://dev.mysql.com/doc/extending-mysql/8.0/en/writing-authentication-plugins-server-side.html). This design proposes a similar set of interfaces for users to define, compared to those in MySQL. The main differences are that we also allow defining authorization checks (SQL privileges).
+
+### PostgreSQL
+
+PostgreSQL supports [custom authentication mechanisms](https://www.google.com/url?q=https://www.postgresql.org/docs/current/auth-pam.html&sa=D&source=docs&ust=1715365735752899&usg=AOvVaw36GkfLatUo1zfhHAY65XLC) using Linux’s [Pluggable Authentication Module (PAM)](https://documentation.suse.com/sles/12-SP5/html/SLES-all/cha-pam.html) support. When a user connects to Postgres, the server will authenticate the user using the PAM service locally, which can trigger some customized authentication code. However, this does not allow plugging in customized authorization checks for SQL privileges.
+
+Technically, we could implement PAM support in TiDB instead of what is being proposed in this design, but it will just be one standard authentication option which does not utilize the existing extension framework. Implementing auth plugin extension support is more flexible as it allows many different customizations for both authentication and authorization. With the auth plugin extension support in place, we could even consider implementing the PAM auth scheme as an auth plugin.
+
+
+## Unresolved Questions
+
+None
diff --git a/docs/design/2024-05-11-hash-join-v2.md b/docs/design/2024-05-11-hash-join-v2.md
new file mode 100644
index 0000000000000..3a764e4570dc8
--- /dev/null
+++ b/docs/design/2024-05-11-hash-join-v2.md
@@ -0,0 +1,178 @@
+# Proposal: Hash Join V2
+
+- Author(s): [windtalker](https://github.com/windtalker)
+- Tracking Issue: https://github.com/pingcap/tidb/issues/53127
+
+## Introduction
+Hash join is a widely used join algorithm. TiDB supports hash join since its 1.0 version, however, the current implementation
+of hash join has some shortcomings:
+* At build stage, it does not support concurrent build, which may lead some performance issues when the build side is large.
+* At probe stage, the interface is not well-designed, which may cause redundant calculations in some cases: https://github.com/pingcap/tidb/issues/47424
+* In current implementation, there are some concepts that are actually not found in other database's implementation. For example, beside left side/right side and build side/right side, there is inner side/outer side , and `useOuterToBuild`, `outerIsRight`, `tryToMatchInners`, `tryToMatchOuters` are introduced to handle the extra complex. This makes the current code too complex to understand and more error-prone when we try to fix bug.
+
+Taking into account the above factors, we decided to do a complete refactoring of hash join.
+
+## Problems
+
+The basic idea of hash join is to divide the join into build stage and probe stage. On the build stage, a hash table is built
+based on the join key, on the probe stage, a lookup is made in the hash table using the join key, and the join result is
+generated based on the lookup result and the join type. The problems faced in the build stage mainly include the design of
+the hash table, the data organization on the build side and the concurrent build of the hash table. The problems faced in
+the probe stage include memory control during the probe process(especially when there are large number of duplicated join keys),
+the processing of hash table lookup results for various join types and the generation of the final join result.
+
+## Detailed Design
+
+### Build side
+The most intuitive problem in build side is the data organization of the build side data. The input data of build side is using
+column storage, and during build stage, we convert the column storage to row storage.
+#### Row storage memory layout
+```
+|---------------------|-----------------|----------------------|-------------------------------|
+ | | | |
+ V V V V
+ next_row_ptr null_map serialized_key/key_length row_data
+```
+* next_row_ptr: the ptr to link to the next row, used in hash table build, it will make all the rows of the same hash value form a linked list
+* null_map: null_map actually includes two parts: the null_flag for each column in current row, the used_flag which is used in
+right semi/outer join. If used_flag is needed, it will always be the first bit of null_map.
+* serialized_key/key_length(optional): if the join key is inlined, and the key has variable length, this field is used to record the key length
+of current row, if the join key is not inlined, this field is the serialized representation of the join keys, used to quick
+join key compare during probe stage. This field is optional, for join keys that can be inlined in the row_data(for example,
+join key with one integer) and has fixed length, this field is not needed.
+* row_data: the data for all the columns of current row. The columns in row_data is variable length. For elements that has fixed length(e.g. int64),
+it will be saved directly, for elements has variable length(e.g. string related elements), it will first save the size followed by the raw data.
+
+Since the row_data is variable length, it is by design that the column data in row_data should only be used by sequential access. In order to adopt
+this sequential access restrict, the columns order in row_data must be well-designed instead of just using its original order.
+
+The columns in the build side can be divided into 3 categories:
+* columns used as join key: if the join key is inlined, then it will be accessed first since it need to be used to compare the join key
+* columns used by non-equal conditions: it will be used after join key comparison
+* all the other columns: these columns will be used last to construct the join results
+
+The order of the columns in all columns data is as follows
+
+| | has non-equal condition | no non-equal condition |
+|-------------------------|---------------------------------------------------------------|--------------------------------------|
+| join key is inlined | join key columns + non-equal condition columns + rest columns | join key columns + rest column |
+| join key is not inlined | non-equal condition columns + rest columns | all columns in their original order |
+
+#### RowTable
+RowTable is mainly used to store data that has been converted to row storage. RowTable needs to store two parts of information: meta and data.
+```go
+type rowTable struct {
+ meta *tableMeta
+ segments []*rowTableSegment
+}
+```
+##### Meta
+`tableMeta` is used to record some meta-related information used in build, including at least
+```go
+type TableMeta struct {
+ // if the row has fixed length
+ isFixedLength bool
+ // the row length if the row is fixed length
+ rowLength int
+ // if the join keys has fixed length
+ isJoinKeysFixedLength bool
+ // the join keys length if it is fixed length
+ joinKeysLength int
+ // is the join key inlined in the row data
+ isJoinKeysInlined bool
+ // the length of null map, the null map include null bit for each column in the row and the used flag for right semi/outer join
+ nullMapLength int
+ // the column order in row layout, as described above, the save column order maybe different from the column order in build schema
+ // for example, the build schema maybe [col1, col2, col3], and the column order in row maybe [col2, col1, col3], then this array
+ // is [1, 0, 2]
+ rowColumnsOrder []int
+ // the column size of each column, -1 mean variable column, the order is the same as rowColumnsOrder
+ columnsSize []int
+ // the serialize mode for each key
+ serializeModes []codec.SerializeMode
+ // the first n columns in row is used for other condition, if a join has other condition, we only need to extract
+ // first n columns from the RowTable to evaluate other condition
+ columnCountNeededForOtherCondition int
+ // total column numbers for build side chunk, this is used to construct the chunk if there is join other condition
+ totalColumnNumber int
+ // column index offset in null map, will be 1 when if there is usedFlag and 0 if there is no usedFlag
+ colOffsetInNullMap int
+ // keyMode is the key mode, it can be OneInt/FixedSerializedKey/VariableSerializedKey
+ keyMode keyMode
+ // offset to rowData, -1 for variable length, non-inlined key
+ rowDataOffset int
+ // the mask of usedFlag in nullmap
+ usedFlagMask uint32
+}
+```
+#### Data
+`rowTableSegment` is used to save row storage data, including at least
+```go
+type rowTableSegment struct {
+ rawData []byte // the chunk of memory to save the row data
+ hashValues []uint64 // the hash value of each rows
+ rowLocations []unsafe.Pointer // the start address of each row
+ validJoinKeyPos []int // the pos of rows that need to be inserted into hash table, used in hash table build
+}
+```
+### Hash table design
+Hash table uses a chain hash table. The required space will be resized in advance before the hash table is built, and will
+not be resized during the entire build process. Therefore, there is no need to save hash value-related information in the
+hash table. The row address can be stored in each slot of the hash table. The structure of the entire hash table is as follows:
+
+
+
+### Hash table building
+The building of the hash table is divided into two stages:
+1. Pre-build stage: Receive data, physically partition the data, and convert the data from column storage to row storage. The number of partitions: partition_num = max (concurrency, 16)
+2. Build stage: After all the data is pre-built, the rowTable data is assigned to the build thread, and each build thread builds its target hash table independently. If there are more build threads than partitions, the conflict is handled by CAS during the hash table build. If the build thread is less than or equal to the number of partitions, a single hash table can only be written by a single thread, so there is no write conflict
+
+### Overall process of build stage
+
+
+
+### Probe side
+
+#### Hash table lookup
+After calculating the hash value of the join key, the hash table lookup can be done by directly accessing the corresponding
+slot of the target hash table
+#### Join key comparison
+Since chain hash table is used, after hash table lookup, we still need to compare the join key. The join key on the Build
+side is already in the row. The join key on the probe side is serialized to a byte buffer, TiDB will use memory compare for
+key comparison. For some simple cases, it can be improved to use some specific comparison: for example, if the join key is
+one int, TiDB can use int compare instead of memory compare
+#### Joined block generation and non-equal condition evaluation
+After join key compare, we need to generate a joined block by combining the probe columns and the build columns. For joins
+don't have other non-equal conditions, the joined block is the final join result. For joins that have some non-equal conditions,
+the joined block is an intermediate result, and we evaluate the non-equal condition based on the joined block, and then
+generate the final join result. By generating an intermediate joined block, we can evaluate the non-equal condition in vector
+mode which is faster and easier to support complex non-equal condition.
+
+When generating intermediate joined block, we try to only construct the minimum columns which are enough for non-equal condition:
+* For probe side, only the columns used by non-equal condition are constructed
+* For build side
+ * If the join key is not inlined, only the columns used by non-equal condition are constructed
+ * If the join key is inlined, the join key columns and the columns used by non-equal condition are constructed
+
+#### Probe interface
+```go
+type JoinProbe interface {
+ // SetChunkForProbe will do some pre-work when start probing a chunk
+ SetChunkForProbe(chunk *chunk.Chunk) error
+ // Probe is to probe current chunk, the result chunk is set in result.chk, and Probe need to make sure result.chk.NumRows() <= result.chk.RequiredRows()
+ Probe(joinResult *hashjoinWorkerResult, sqlKiller sqlkiller.SQLKiller) (ok bool, result *hashjoinWorkerResult)
+ // IsCurrentChunkProbeDone returns true if current probe chunk is all probed
+ IsCurrentChunkProbeDone() bool
+ // ScanRowTable is called after all the probe chunks are probed. It is used in some special joins, like left outer join with left side to build, after all
+ // the probe side chunks are handled, it needs to scan the row table to return the un-matched rows
+ ScanRowTable(joinResult *hashjoinWorkerResult, sqlKiller sqlkiller.SQLKiller) (result *hashjoinWorkerResult)
+ // IsScanRowTableDone returns true after scan row table is done
+ IsScanRowTableDone() bool
+ // NeedScanRowTable returns true if current join need to scan row table after all the probe side chunks are handled
+ NeedScanRowTable() bool
+ // InitForScanRowTable do some pre-work before ScanRowTable, it must be called before ScanRowTable
+ InitForScanRowTable()
+}
+```
+#### Overall process of probe stage
+
diff --git a/docs/design/imgs/build-overall-process.png b/docs/design/imgs/build-overall-process.png
new file mode 100644
index 0000000000000..24194556a8b6e
Binary files /dev/null and b/docs/design/imgs/build-overall-process.png differ
diff --git a/docs/design/imgs/hash-table.png b/docs/design/imgs/hash-table.png
new file mode 100644
index 0000000000000..da2656aea0567
Binary files /dev/null and b/docs/design/imgs/hash-table.png differ
diff --git a/docs/design/imgs/probe-overall-process.png b/docs/design/imgs/probe-overall-process.png
new file mode 100644
index 0000000000000..3912b29d870aa
Binary files /dev/null and b/docs/design/imgs/probe-overall-process.png differ
diff --git a/dumpling/export/dump.go b/dumpling/export/dump.go
index 2db3f7164e8f1..36964ea5e1cc8 100644
--- a/dumpling/export/dump.go
+++ b/dumpling/export/dump.go
@@ -1145,9 +1145,24 @@ func getListTableTypeByConf(conf *Config) listTableType {
}
func prepareTableListToDump(tctx *tcontext.Context, conf *Config, db *sql.Conn) error {
- if conf.SpecifiedTables || conf.SQL != "" {
+ if conf.SQL != "" {
return nil
}
+
+ ifSeqExists, err := CheckIfSeqExists(db)
+ if err != nil {
+ return err
+ }
+ var listType listTableType
+ if ifSeqExists {
+ listType = listTableByShowFullTables
+ } else {
+ listType = getListTableTypeByConf(conf)
+ }
+
+ if conf.SpecifiedTables {
+ return updateSpecifiedTablesMeta(tctx, db, conf.Tables, listType)
+ }
databases, err := prepareDumpingDatabases(tctx, conf, db)
if err != nil {
return err
@@ -1161,17 +1176,6 @@ func prepareTableListToDump(tctx *tcontext.Context, conf *Config, db *sql.Conn)
tableTypes = append(tableTypes, TableTypeSequence)
}
- ifSeqExists, err := CheckIfSeqExists(db)
- if err != nil {
- return err
- }
- var listType listTableType
- if ifSeqExists {
- listType = listTableByShowFullTables
- } else {
- listType = getListTableTypeByConf(conf)
- }
-
conf.Tables, err = ListAllDatabasesTables(tctx, db, databases, listType, tableTypes...)
if err != nil {
return err
diff --git a/dumpling/export/sql.go b/dumpling/export/sql.go
index 1d7ed2ec3e0f8..0918be1640629 100644
--- a/dumpling/export/sql.go
+++ b/dumpling/export/sql.go
@@ -246,6 +246,119 @@ func RestoreCharset(w io.StringWriter) {
_, _ = w.WriteString("SET collation_connection = @PREV_COLLATION_CONNECTION;\n")
}
+// updateSpecifiedTablesMeta updates DatabaseTables with correct table type and avg row size.
+func updateSpecifiedTablesMeta(tctx *tcontext.Context, db *sql.Conn, dbTables DatabaseTables, listType listTableType) error {
+ var (
+ schema, table, tableTypeStr string
+ tableType TableType
+ avgRowLength uint64
+ err error
+ )
+ switch listType {
+ case listTableByInfoSchema:
+ dbNames := make([]string, 0, len(dbTables))
+ for db := range dbTables {
+ dbNames = append(dbNames, fmt.Sprintf("'%s'", db))
+ }
+ query := fmt.Sprintf("SELECT TABLE_SCHEMA,TABLE_NAME,TABLE_TYPE,AVG_ROW_LENGTH FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA IN (%s)", strings.Join(dbNames, ","))
+ if err := simpleQueryWithArgs(tctx, db, func(rows *sql.Rows) error {
+ var (
+ sqlAvgRowLength sql.NullInt64
+ err2 error
+ )
+ if err2 = rows.Scan(&schema, &table, &tableTypeStr, &sqlAvgRowLength); err != nil {
+ return errors.Trace(err2)
+ }
+
+ tbls, ok := dbTables[schema]
+ if !ok {
+ return nil
+ }
+ for _, tbl := range tbls {
+ if tbl.Name == table {
+ tableType, err2 = ParseTableType(tableTypeStr)
+ if err2 != nil {
+ return errors.Trace(err2)
+ }
+ if sqlAvgRowLength.Valid {
+ avgRowLength = uint64(sqlAvgRowLength.Int64)
+ } else {
+ avgRowLength = 0
+ }
+ tbl.Type = tableType
+ tbl.AvgRowLength = avgRowLength
+ }
+ }
+ return nil
+ }, query); err != nil {
+ return errors.Annotatef(err, "sql: %s", query)
+ }
+ return nil
+ case listTableByShowFullTables:
+ for schema, tbls := range dbTables {
+ query := fmt.Sprintf("SHOW FULL TABLES FROM `%s`",
+ escapeString(schema))
+ if err := simpleQueryWithArgs(tctx, db, func(rows *sql.Rows) error {
+ var err2 error
+ if err2 = rows.Scan(&table, &tableTypeStr); err != nil {
+ return errors.Trace(err2)
+ }
+ for _, tbl := range tbls {
+ if tbl.Name == table {
+ tableType, err2 = ParseTableType(tableTypeStr)
+ if err2 != nil {
+ return errors.Trace(err2)
+ }
+ tbl.Type = tableType
+ }
+ }
+ return nil
+ }, query); err != nil {
+ return errors.Annotatef(err, "sql: %s", query)
+ }
+ }
+ return nil
+ default:
+ const queryTemplate = "SHOW TABLE STATUS FROM `%s`"
+ for schema, tbls := range dbTables {
+ query := fmt.Sprintf(queryTemplate, escapeString(schema))
+ rows, err := db.QueryContext(tctx, query)
+ if err != nil {
+ return errors.Annotatef(err, "sql: %s", query)
+ }
+ results, err := GetSpecifiedColumnValuesAndClose(rows, "NAME", "ENGINE", "AVG_ROW_LENGTH", "COMMENT")
+ if err != nil {
+ return errors.Annotatef(err, "sql: %s", query)
+ }
+ for _, oneRow := range results {
+ table, engine, avgRowLengthStr, comment := oneRow[0], oneRow[1], oneRow[2], oneRow[3]
+ for _, tbl := range tbls {
+ if tbl.Name == table {
+ if avgRowLengthStr != "" {
+ avgRowLength, err = strconv.ParseUint(avgRowLengthStr, 10, 64)
+ if err != nil {
+ return errors.Annotatef(err, "sql: %s", query)
+ }
+ } else {
+ avgRowLength = 0
+ }
+ tbl.AvgRowLength = avgRowLength
+ tableType = TableTypeBase
+ if engine == "" && (comment == "" || comment == TableTypeViewStr) {
+ tableType = TableTypeView
+ } else if engine == "" {
+ tctx.L().Warn("invalid table without engine found", zap.String("database", schema), zap.String("table", table))
+ continue
+ }
+ tbl.Type = tableType
+ }
+ }
+ }
+ }
+ return nil
+ }
+}
+
// ListAllDatabasesTables lists all the databases and tables from the database
// listTableByInfoSchema list tables by table information_schema in MySQL
// listTableByShowTableStatus has better performance than listTableByInfoSchema
diff --git a/dumpling/tests/specified_table_view/run.sh b/dumpling/tests/specified_table_view/run.sh
new file mode 100644
index 0000000000000..0892d623d3c83
--- /dev/null
+++ b/dumpling/tests/specified_table_view/run.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+#
+# Copyright 2024 PingCAP, Inc. Licensed under Apache-2.0.
+
+set -eu
+cur=$(cd `dirname $0`; pwd)
+
+DB_NAME="specified_table_view"
+TABLE_NAME="t"
+VIEW_NAME="v"
+
+run_sql "drop database if exists \`$DB_NAME\`;"
+run_sql "create database \`$DB_NAME\`;"
+run_sql "create table \`$DB_NAME\`.\`$TABLE_NAME\` (a int);"
+run_sql "create view \`$DB_NAME\`.\`$VIEW_NAME\` as select * from \`$DB_NAME\`.\`$TABLE_NAME\`;"
+
+set +e
+rm -rf "$DUMPLING_OUTPUT_DIR"
+run_dumpling --consistency=lock -T="$DB_NAME.$TABLE_NAME,$DB_NAME.$VIEW_NAME" -L ${DUMPLING_OUTPUT_DIR}/dumpling.log
+set -e
+
+file_should_exist "$DUMPLING_OUTPUT_DIR/$DB_NAME.$TABLE_NAME-schema.sql"
+file_should_exist "$DUMPLING_OUTPUT_DIR/$DB_NAME.$VIEW_NAME-schema-view.sql"
+
+set +e
+rm -rf "$DUMPLING_OUTPUT_DIR"
+run_dumpling --consistency=lock -T="$DB_NAME.$TABLE_NAME,$DB_NAME.$VIEW_NAME" -L ${DUMPLING_OUTPUT_DIR}/dumpling.log
+set -e
+
+file_should_exist "$DUMPLING_OUTPUT_DIR/$DB_NAME.$TABLE_NAME-schema.sql"
+file_should_exist "$DUMPLING_OUTPUT_DIR/$DB_NAME.$VIEW_NAME-schema-view.sql"
diff --git a/go.mod b/go.mod
index 712b2c3c661cb..4a40cb8242cb4 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,7 @@ module github.com/pingcap/tidb
go 1.21
require (
- cloud.google.com/go/storage v1.36.0
+ cloud.google.com/go/storage v1.38.0
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0
@@ -19,7 +19,7 @@ require (
github.com/bazelbuild/buildtools v0.0.0-20230926111657-7d855c59baeb
github.com/bazelbuild/rules_go v0.42.1-0.20231101215950-df20c987afcb
github.com/blacktear23/go-proxyprotocol v1.0.6
- github.com/butuzov/mirror v1.1.0
+ github.com/butuzov/mirror v1.2.0
github.com/carlmjohnson/flagext v0.21.0
github.com/charithe/durationcheck v0.0.10
github.com/cheggaaa/pb/v3 v3.0.8
@@ -30,7 +30,7 @@ require (
github.com/coocood/freecache v1.2.1
github.com/coreos/go-semver v0.3.1
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548
- github.com/daixiang0/gci v0.12.3
+ github.com/daixiang0/gci v0.13.4
github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37
github.com/dgraph-io/ristretto v0.1.1
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13
@@ -48,9 +48,9 @@ require (
github.com/golang/protobuf v1.5.4
github.com/golang/snappy v0.0.4
github.com/golangci/gofmt v0.0.0-20231019111953-be8c47862aaa
- github.com/golangci/golangci-lint v1.57.2
+ github.com/golangci/golangci-lint v1.58.1
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb
- github.com/golangci/misspell v0.4.1
+ github.com/golangci/misspell v0.5.1
github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21
github.com/google/btree v1.1.2
github.com/google/pprof v0.0.0-20240117000934-35fc243c5815
@@ -82,9 +82,9 @@ require (
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d
github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f
- github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c
+ github.com/pingcap/failpoint v0.0.0-20240527053858-9b3b6e34194a
github.com/pingcap/fn v1.0.0
- github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754
+ github.com/pingcap/kvproto v0.0.0-20240513094934-d9297553c900
github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d
github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5
github.com/pingcap/tidb/pkg/parser v0.0.0-20211011031125-9b13dc409c5e
@@ -93,9 +93,10 @@ require (
github.com/prometheus/client_model v0.6.1
github.com/prometheus/common v0.53.0
github.com/prometheus/prometheus v0.50.1
+ github.com/qri-io/jsonschema v0.2.1
github.com/robfig/cron/v3 v3.0.1
github.com/sasha-s/go-deadlock v0.3.1
- github.com/shirou/gopsutil/v3 v3.24.2
+ github.com/shirou/gopsutil/v3 v3.24.4
github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0
github.com/soheilhy/cmux v0.1.5
github.com/spf13/cobra v1.8.0
@@ -106,14 +107,14 @@ require (
github.com/tdakkota/asciicheck v0.2.0
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2
github.com/tidwall/btree v1.7.0
- github.com/tikv/client-go/v2 v2.0.8-0.20240430145241-6cb0704fce51
- github.com/tikv/pd/client v0.0.0-20240430080403-1679dbca25b3
+ github.com/tikv/client-go/v2 v2.0.8-0.20240531102121-cb580bc4ea29
+ github.com/tikv/pd/client v0.0.0-20240531082952-199b01792159
github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a
github.com/twmb/murmur3 v1.1.6
github.com/uber/jaeger-client-go v2.22.1+incompatible
github.com/vbauerster/mpb/v7 v7.5.3
github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f
- github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457
+ github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17
github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0
go.etcd.io/etcd/api/v3 v3.5.12
go.etcd.io/etcd/client/pkg/v3 v3.5.12
@@ -128,16 +129,16 @@ require (
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.27.0
golang.org/x/exp v0.0.0-20240404231335-c0f41cb1a7a0
- golang.org/x/net v0.24.0
- golang.org/x/oauth2 v0.18.0
+ golang.org/x/net v0.25.0
+ golang.org/x/oauth2 v0.20.0
golang.org/x/sync v0.7.0
- golang.org/x/sys v0.19.0
- golang.org/x/term v0.19.0
- golang.org/x/text v0.14.0
+ golang.org/x/sys v0.20.0
+ golang.org/x/term v0.20.0
+ golang.org/x/text v0.15.0
golang.org/x/time v0.5.0
- golang.org/x/tools v0.20.0
- google.golang.org/api v0.162.0
- google.golang.org/grpc v1.63.2
+ golang.org/x/tools v0.21.0
+ google.golang.org/api v0.169.0
+ google.golang.org/grpc v1.64.0
gopkg.in/yaml.v2 v2.4.0
honnef.co/go/tools v0.4.7
k8s.io/api v0.28.6
@@ -146,14 +147,24 @@ require (
)
require (
+ github.com/andybalholm/brotli v1.0.5 // indirect
+ github.com/apache/arrow/go/v12 v12.0.1 // indirect
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
github.com/getsentry/sentry-go v0.27.0 // indirect
+ github.com/goccy/go-reflect v1.2.0 // indirect
+ github.com/google/flatbuffers v2.0.8+incompatible // indirect
+ github.com/klauspost/asmfmt v1.3.2 // indirect
+ github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+ github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
+ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
+ github.com/pierrec/lz4/v4 v4.1.15 // indirect
+ github.com/qri-io/jsonpointer v0.1.1 // indirect
+ github.com/zeebo/xxh3 v1.0.2 // indirect
)
require (
- cloud.google.com/go v0.112.0 // indirect
- cloud.google.com/go/compute v1.24.0 // indirect
- cloud.google.com/go/compute/metadata v0.2.3 // indirect
+ cloud.google.com/go v0.112.1 // indirect
+ cloud.google.com/go/compute/metadata v0.3.0 // indirect
cloud.google.com/go/iam v1.1.6 // indirect
cloud.google.com/go/pubsub v1.36.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect
@@ -161,7 +172,7 @@ require (
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
github.com/DataDog/zstd v1.5.5 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
- github.com/Masterminds/semver/v3 v3.1.1 // indirect
+ github.com/Masterminds/semver/v3 v3.2.1 // indirect
github.com/Masterminds/sprig/v3 v3.2.2 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
@@ -202,7 +213,7 @@ require (
github.com/google/renameio/v2 v2.0.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
- github.com/googleapis/gax-go/v2 v2.12.0 // indirect
+ github.com/googleapis/gax-go/v2 v2.12.2 // indirect
github.com/gorilla/handlers v1.5.1 // indirect
github.com/gorilla/websocket v1.5.1 // indirect
github.com/gostaticanalysis/analysisutil v0.7.1 // indirect
@@ -250,11 +261,11 @@ require (
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
- github.com/pkg/errors v0.9.1 // indirect
+ github.com/pkg/errors v0.9.1
github.com/pkg/xattr v0.4.9 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
- github.com/prometheus/procfs v0.13.0 // indirect
+ github.com/prometheus/procfs v0.15.1 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.4.6 // indirect
@@ -280,25 +291,23 @@ require (
go.etcd.io/etcd/client/v2 v2.305.12 // indirect
go.etcd.io/etcd/pkg/v3 v3.5.12 // indirect
go.etcd.io/etcd/raft/v3 v3.5.12 // indirect
- go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect
- go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
- go.opentelemetry.io/otel v1.22.0 // indirect
+ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
+ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+ go.opentelemetry.io/otel v1.24.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 // indirect
- go.opentelemetry.io/otel/metric v1.22.0 // indirect
- go.opentelemetry.io/otel/sdk v1.22.0 // indirect
- go.opentelemetry.io/otel/trace v1.22.0 // indirect
+ go.opentelemetry.io/otel/metric v1.24.0 // indirect
+ go.opentelemetry.io/otel/sdk v1.24.0 // indirect
+ go.opentelemetry.io/otel/trace v1.24.0 // indirect
go.opentelemetry.io/proto/otlp v1.1.0 // indirect
- golang.org/x/crypto v0.22.0 // indirect
+ golang.org/x/crypto v0.23.0 // indirect
golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
- gonum.org/v1/gonum v0.8.2 // indirect
- google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
- google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 // indirect
- google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect
- google.golang.org/protobuf v1.33.0 // indirect
+ google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect
+ google.golang.org/protobuf v1.34.1 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
diff --git a/go.sum b/go.sum
index 902c6f4b7e8be..8e9095dcdf599 100644
--- a/go.sum
+++ b/go.sum
@@ -13,18 +13,16 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM=
-cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4=
+cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM=
+cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg=
-cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40=
-cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
-cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
+cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
+cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc=
@@ -42,9 +40,11 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-cloud.google.com/go/storage v1.36.0 h1:P0mOkAcaJxhCTvAkMhxMfrTKiNcub4YmmPBtlhAyTr8=
-cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8=
+cloud.google.com/go/storage v1.38.0 h1:Az68ZRGlnNTpIBbLjSMIV2BDcwwXYlRlQzis0llkpJg=
+cloud.google.com/go/storage v1.38.0/go.mod h1:tlUADB0mAb9BgYls9lq+8MGkfzOXuLrnHXlpHmvFJoY=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8=
+git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
@@ -67,12 +67,15 @@ github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ=
github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM=
github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
+github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU=
+github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
-github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
+github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
+github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8=
github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk=
github.com/Shopify/sarama v1.29.0 h1:ARid8o8oieau9XrHI55f/L3EoRAhm9px6sonbD7yuUE=
@@ -86,18 +89,25 @@ github.com/YangKeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 h1:+OqGGFc2YHFd
github.com/YangKeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117/go.mod h1:bMGIq3AGbytbaMwf8wdv5Phdxz0FWHTIYMSzyrYgnQs=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
+github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
+github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
+github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM=
github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs=
github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 h1:Q/yk4z/cHUVZfgTqtD09qeYBxHwshQAjVRX73qs8UH0=
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU=
+github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/apache/arrow/go/v12 v12.0.1 h1:JsR2+hzYYjgSUkBSaahpqCetqZMr76djX80fF/DiJbg=
+github.com/apache/arrow/go/v12 v12.0.1/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw=
github.com/apache/skywalking-eyes v0.4.0 h1:O13kdRU6FCEZevfD01mdhTgCZLLfPZIQ0GXZrLl7FpQ=
github.com/apache/skywalking-eyes v0.4.0/go.mod h1:WblDbBgOLsLN0FJEBa9xj6PhuUA/J6spKYVTG4/F8Ls=
github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
-github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY=
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@@ -124,8 +134,10 @@ github.com/blacktear23/go-proxyprotocol v1.0.6 h1:eTt6UMpEnq59NjON49b3Cay8Dm0sCs
github.com/blacktear23/go-proxyprotocol v1.0.6/go.mod h1:FSCbgnRZrQXazBLL5snfBbrcFSMtcmUDhSRb9OfFA1o=
github.com/bmatcuk/doublestar/v2 v2.0.4 h1:6I6oUiT/sU27eE2OFcWqBhL1SwjyvQuOssxT4a1yidI=
github.com/bmatcuk/doublestar/v2 v2.0.4/go.mod h1:QMmcs3H2AUQICWhfzLXz+IYln8lRQmTZRptLie8RgRw=
-github.com/butuzov/mirror v1.1.0 h1:ZqX54gBVMXu78QLoiqdwpl2mgmoOJTk7s4p4o+0avZI=
-github.com/butuzov/mirror v1.1.0/go.mod h1:8Q0BdQU6rC6WILDiBM60DBfvV78OLJmMmixe7GF45AE=
+github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
+github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
+github.com/butuzov/mirror v1.2.0 h1:9YVK1qIjNspaqWutSv8gsge2e/Xpq1eqEkslEUHy5cs=
+github.com/butuzov/mirror v1.2.0/go.mod h1:DqZZDtzm42wIAIyHXeN8W/qb1EPlb9Qn/if9icBOpdQ=
github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g=
github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets=
github.com/carlmjohnson/flagext v0.21.0 h1:/c4uK3ie786Z7caXLcIMvePNSSiH3bQVGDvmGLMme60=
@@ -152,8 +164,11 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cloudfoundry/gosigar v1.3.6 h1:gIc08FbB3QPb+nAQhINIK/qhf5REKkY0FTGgRGXkcVc=
github.com/cloudfoundry/gosigar v1.3.6/go.mod h1:lNWstu5g5gw59O09Y+wsMNFzBSnU8a0u+Sfx4dq360E=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ=
-github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
+github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8=
@@ -186,8 +201,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
-github.com/daixiang0/gci v0.12.3 h1:yOZI7VAxAGPQmkb1eqt5g/11SUlwoat1fSblGLmdiQc=
-github.com/daixiang0/gci v0.12.3/go.mod h1:xtHP9N7AHdNvtRNfcx9gwTDfw7FRJx4bZUsiEfiNNAI=
+github.com/daixiang0/gci v0.13.4 h1:61UGkmpoAcxHM2hhNkZEf5SzwQtWJXTSws7jaPyqwlw=
+github.com/daixiang0/gci v0.13.4/go.mod h1:12etP2OniiIdP4q+kjUGrC/rUagga7ODbqsom5Eo5Yk=
github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37 h1:X6mKGhCFOxrKeeHAjv/3UvT6e5RRxW6wRdlqlV6/H4w=
github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37/go.mod h1:DC3JtzuG7kxMvJ6dZmf2ymjNyoXwgtklr7FN+Um2B0U=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -208,6 +223,7 @@ github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ=
github.com/dolthub/maphash v0.1.0/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4=
github.com/dolthub/swiss v0.2.1 h1:gs2osYs5SJkAaH5/ggVJqXQxRXtWshF6uE0lgR/Y3Gw=
@@ -226,9 +242,9 @@ github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FM
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A=
-github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew=
github.com/fatanugraha/noloopclosure v0.1.1 h1:AhepjAikNpk50qTZoipHZqeZtnyKT/C2Tk5dGn7nC+A=
github.com/fatanugraha/noloopclosure v0.1.1/go.mod h1:Mi9CiG5QvEgvPLtZLsTzjYwjIDnWAbo10r0BG7JpJII=
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
@@ -242,6 +258,7 @@ github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
+github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
@@ -260,12 +277,19 @@ github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF
github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
+github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g=
+github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks=
+github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY=
+github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY=
+github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
+github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U=
+github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
@@ -277,14 +301,19 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
+github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M=
+github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M=
github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8=
github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/goccy/go-reflect v1.2.0 h1:O0T8rZCuNmGXewnATuKYnkL0xm6o8UNOJZd/gOkb9ms=
+github.com/goccy/go-reflect v1.2.0/go.mod h1:n0oYZn8VcV2CkWTxi8B9QjkCoq6GTtCEdfmR66YhFtE=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -340,18 +369,20 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golangci/gofmt v0.0.0-20231019111953-be8c47862aaa h1:L0Zq43Px2HrLroRKEgfCsQLMJUkjskJBB1kd1Zjcvvc=
github.com/golangci/gofmt v0.0.0-20231019111953-be8c47862aaa/go.mod h1:Pm5KhLPA8gSnQwrQ6ukebRcapGb/BG9iUkdaiCcGHJM=
-github.com/golangci/golangci-lint v1.57.2 h1:NNhxfZyL5He1WWDrIvl1a4n5bvWZBcgAqBwlJAAgLTw=
-github.com/golangci/golangci-lint v1.57.2/go.mod h1:ApiG3S3Ca23QyfGp5BmsorTiVxJpr5jGiNS0BkdSidg=
+github.com/golangci/golangci-lint v1.58.1 h1:IYKjkt7nofq/mYXiDUyJiBZQi5kxD0jPCjBy6VXxjz8=
+github.com/golangci/golangci-lint v1.58.1/go.mod h1:IX9uSbhwDDOVTcceKZWmshlally+fOQYv1pZhIJCMNw=
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb h1:Bi7BYmZVg4C+mKGi8LeohcP2GGUl2XJD4xCkJoZSaYc=
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb/go.mod h1:ON/c2UR0VAAv6ZEAFKhjCLplESSmRFfZcDLASbI1GWo=
-github.com/golangci/misspell v0.4.1 h1:+y73iSicVy2PqyX7kmUefHusENlrP9YwuHZHPLGQj/g=
-github.com/golangci/misspell v0.4.1/go.mod h1:9mAN1quEo3DlpbaIKKyEvRxK1pwqR9s/Sea1bJCtlNI=
+github.com/golangci/misspell v0.5.1 h1:/SjR1clj5uDjNLwYzCahHwIOPmQgoH04AyQIiWGbhCM=
+github.com/golangci/misspell v0.5.1/go.mod h1:keMNyY6R9isGaSAu+4Q8NMBwMPkh15Gtc8UCVoDtAWo=
github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us=
github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
+github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM=
+github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -364,6 +395,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@@ -399,14 +432,15 @@ github.com/google/skylark v0.0.0-20181101142754-a5f7082aabed h1:rZdD1GeRTHD1aG+V
github.com/google/skylark v0.0.0-20181101142754-a5f7082aabed/go.mod h1:CKSX6SxHW1vp20ZNaeGe3TFFBIwCG6vaYrpAiOzX+NA=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
-github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
+github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUhuHF+DA=
+github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc=
github.com/gordonklaus/ineffassign v0.1.0 h1:y2Gd/9I7MdY1oEIt+n+rowjBNDcLQq3RsH5hwJd0f9s=
github.com/gordonklaus/ineffassign v0.1.0/go.mod h1:Qcp2HIAYhR7mNUVSIxZww3Guk4it82ghYcEXIAk+QT0=
github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4=
@@ -506,24 +540,32 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/errcheck v1.7.0 h1:+SbscKmWJ5mOK/bO1zS60F5I9WwZDWOfRsC4RwfwRV0=
github.com/kisielk/errcheck v1.7.0/go.mod h1:1kLL+jV4e+CFfueBmI1dSK2ADDyQnlrnrY/FqKluHJQ=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
+github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
+github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
+github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -557,6 +599,7 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
@@ -564,9 +607,14 @@ github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRC
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mgechev/revive v1.3.7 h1:502QY0vQGe9KtYJ9FpxMz9rL+Fc/P13CI5POL4uHCcE=
github.com/mgechev/revive v1.3.7/go.mod h1:RJ16jUbF0OWC3co/+XTxmFNgEpUPwnnA0BRllX2aDNA=
+github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
+github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
+github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
+github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -622,9 +670,14 @@ github.com/petermattis/goid v0.0.0-20231207134359-e60b3f734c67 h1:jik8PHtAIsPlCR
github.com/petermattis/goid v0.0.0-20231207134359-e60b3f734c67/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
+github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY=
+github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
+github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
+github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d h1:AEcvKyVM8CUII3bYzgz8haFXtGiqcrtXW1csu/5UELY=
github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d/go.mod h1:p8QnkZnmyV8L/M/jzYb8rT7kv3bz9m7bn1Ju94wDifs=
github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
@@ -632,15 +685,15 @@ github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTw
github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f h1:FxA+NgsdHNOv+/hZGxUh8Gb3WuZqgqmxDwztEOiA1v4=
github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
-github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c h1:CgbKAHto5CQgWM9fSBIvaxsJHuGP0uM74HXtv3MyyGQ=
-github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew=
+github.com/pingcap/failpoint v0.0.0-20240527053858-9b3b6e34194a h1:UgrcL8INjEbPRKE2h8yVgZvjOn2OGkxK9CFvoBWzgbk=
+github.com/pingcap/failpoint v0.0.0-20240527053858-9b3b6e34194a/go.mod h1:gPdo4h708R0CrwKM/DO0/6xJ64fz9vxzp2yKE2QON+s=
github.com/pingcap/fn v1.0.0 h1:CyA6AxcOZkQh52wIqYlAmaVmF6EvrcqFywP463pjA8g=
github.com/pingcap/fn v1.0.0/go.mod h1:u9WZ1ZiOD1RpNhcI42RucFh/lBuzTu6rw88a+oF2Z24=
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E=
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
-github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 h1:nU9wDeMsID8EWawRQVdmRYcNhUrlI4TKogZhXleG4QQ=
-github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
+github.com/pingcap/kvproto v0.0.0-20240513094934-d9297553c900 h1:snIM8DC846ufdlRclITACXfr1kvVIPU4cuQ6w3JVVY4=
+github.com/pingcap/kvproto v0.0.0-20240513094934-d9297553c900/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM=
github.com/pingcap/log v1.1.0/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d h1:y3EueKVfVykdpTyfUnQGqft0ud+xVFuCdp1XkVL0X1E=
@@ -679,12 +732,17 @@ github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3
github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4=
github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGKX7o=
-github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/prometheus/prometheus v0.50.1 h1:N2L+DYrxqPh4WZStU+o1p/gQlBaqFbcLBTjlp3vpdXw=
github.com/prometheus/prometheus v0.50.1/go.mod h1:FvE8dtQ1Ww63IlyKBn1V4s+zMwF9kHkVNkQBR1pM4CU=
+github.com/qri-io/jsonpointer v0.1.1 h1:prVZBZLL6TW5vsSB9fFHFAMBLI4b0ri5vribQlTJiBA=
+github.com/qri-io/jsonpointer v0.1.1/go.mod h1:DnJPaYgiKu56EuDp8TU5wFLdZIcAnb/uH9v37ZaMV64=
+github.com/qri-io/jsonschema v0.2.1 h1:NNFoKms+kut6ABPf6xiKNM5214jzxAhDBrPHCJ97Wg0=
+github.com/qri-io/jsonschema v0.2.1/go.mod h1:g7DPkiOsK1xv6T/Ao5scXRkd+yTFygcANPBaaqW+VrI=
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -695,23 +753,28 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w=
+github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk=
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0=
github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM=
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI=
github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0=
github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA=
-github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y=
-github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk=
+github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU=
+github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8=
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
@@ -785,10 +848,10 @@ github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=
github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
-github.com/tikv/client-go/v2 v2.0.8-0.20240430145241-6cb0704fce51 h1:jAMHlJz3EOTqtVe5qE1f9YOfHQ3cnv5R4htXn75XOHo=
-github.com/tikv/client-go/v2 v2.0.8-0.20240430145241-6cb0704fce51/go.mod h1:/4cCrzN6+KY2U5JeR72TllnJTtA5hm9yI6zqlNrvJUo=
-github.com/tikv/pd/client v0.0.0-20240430080403-1679dbca25b3 h1:1MReqAoXNPyTarVZ3cihghWrmczF6yj+tmBLkj7g9Oc=
-github.com/tikv/pd/client v0.0.0-20240430080403-1679dbca25b3/go.mod h1:FaQnRpX8ZXMR8pih4xNdNxqDrwZvpu7GMsJKDZyF9pk=
+github.com/tikv/client-go/v2 v2.0.8-0.20240531102121-cb580bc4ea29 h1:jzA3rnUU/oij9J2fRaFojWeQl2wypBHP0nhlOTljTH4=
+github.com/tikv/client-go/v2 v2.0.8-0.20240531102121-cb580bc4ea29/go.mod h1:yun3/1k+uoSDMwLPHWEASLoaIdULQ+ejWrhMjEuOoaA=
+github.com/tikv/pd/client v0.0.0-20240531082952-199b01792159 h1:JdeShNYbZ+1+g3qS4pdScPabibOiiZmUz6BDi8H1yUc=
+github.com/tikv/pd/client v0.0.0-20240531082952-199b01792159/go.mod h1:kNRekhwXqjTjNHy+kPmbZvsMmvl42zOj/UW5IIG+nP0=
github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a h1:A6uKudFIfAEpoPdaal3aSqGxBzLyU8TqyXImLwo6dIo=
github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a/go.mod h1:mkjARE7Yr8qU23YcGMSALbIxTQ9r9QBVahQOBRfU460=
github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
@@ -815,8 +878,8 @@ github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0
github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk=
github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww=
-github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457 h1:tBbuFCtyJNKT+BFAv6qjvTFpVdy97IYNaBwGUXifIUs=
-github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457/go.mod h1:pheqtXeHQFzxJk45lRQ0UIGIivKnLXvialZSFWs81A8=
+github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 h1:mr+7gGPUasLmH3/5Iv1zwQwiY0WgGO21Ym7Q4FVw+xs=
+github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17/go.mod h1:u9udtIEWeBkphB2isZ8V8xVIMWgcUobH+7FRMO/Ld6c=
github.com/xitongsys/parquet-go-source v0.0.0-20190524061010-2b72cbee77d5/go.mod h1:xxCx7Wpym/3QCo6JhujJX51dzSXrwmb0oH6FQb39SEA=
github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 h1:a742S4V5A15F93smuVxA60LQWsrCnN8bKeWDBARU1/k=
github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE=
@@ -831,6 +894,10 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
+github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
+github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
go.einride.tech/aip v0.66.0 h1:XfV+NQX6L7EOYK11yoHHFtndeaWh3KbD9/cN/6iWEt8=
go.einride.tech/aip v0.66.0/go.mod h1:qAhMsfT7plxBX+Oy7Huol6YUvZ0ZzdUz26yZsQwfl1M=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
@@ -859,22 +926,23 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 h1:UNQQKPfTDe1J81ViolILjTKPr9WetKW6uei2hFgJmFs=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw=
-go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y=
-go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
+go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
+go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0/go.mod h1:noq80iT8rrHP1SfybmPiRGc9dc5M8RPmGvtwo7Oo7tc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 h1:H2JFgRcGiyHg7H7bwcwaQJYrNFqCqrbTQ8K4p1OvDu8=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0/go.mod h1:WfCWp1bGoYK8MeULtI15MmQVczfR+bFkk0DF3h06QmQ=
-go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg=
-go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY=
-go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw=
-go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc=
-go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0=
-go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo=
+go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
+go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
+go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw=
+go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg=
+go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
+go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI=
go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY=
go.starlark.net v0.0.0-20210223155950-e043a3d3c984/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0=
@@ -924,8 +992,8 @@ golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
-golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -933,12 +1001,14 @@ golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE=
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/exp v0.0.0-20240404231335-c0f41cb1a7a0 h1:985EYyeCOxTpcgOTJpflJUwOeEz0CQOdPt73OzpE9F8=
golang.org/x/exp v0.0.0-20240404231335-c0f41cb1a7a0/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI=
golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f h1:phY1HzDcf18Aq9A8KkmRtY9WvOFIxN8wgfvy6Zm1DV8=
@@ -946,6 +1016,16 @@ golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f/go.mod h1:AbB0pIl
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
+golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
+golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
+golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1016,22 +1096,23 @@ golang.org/x/net v0.0.0-20220517181318-183a9ca12b87/go.mod h1:CfG3xpIq0wQ8r1q4Su
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
-golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
-golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
+golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo=
+golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1082,7 +1163,9 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1110,9 +1193,9 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -1122,23 +1205,23 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
-golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
-golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
+golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1164,6 +1247,7 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -1194,8 +1278,10 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201125231158-b5590deeca9b/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.1-0.20210205202024-ef80cdb6ec6d/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU=
golang.org/x/tools v0.1.1-0.20210302220138-2ac05c832e1a/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU=
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
@@ -1207,22 +1293,27 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
-golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY=
-golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg=
+golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw=
+golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
+golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
-gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM=
gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
+gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0=
+gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E=
+gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA=
gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
+gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY=
+gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -1239,8 +1330,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.162.0 h1:Vhs54HkaEpkMBdgGdOT2P6F0csGG/vxDS0hWHJzmmps=
-google.golang.org/api v0.162.0/go.mod h1:6SulDkfoBIg4NFmCuZ39XeeAgSHCPecfSUuDyYlAHs0=
+google.golang.org/api v0.169.0 h1:QwWPy71FgMWqJN/l6jVlFHUa29a7dcUy02I8o799nPY=
+google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1248,8 +1339,6 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
-google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -1284,10 +1373,10 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY=
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo=
-google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 h1:8eadJkXbwDEMNwcB5O0s5Y5eCfyuCLdvaiOIaGTrWmQ=
-google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
+google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4=
+google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0=
google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
@@ -1304,8 +1393,10 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM=
-google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
+google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY=
+google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg=
google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9 h1:ATnmU8nL2NfIyTSiBvJVDIDIr3qBmeW+c7z7XU21eWs=
google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9/go.mod h1:j5uROIAAgi3YmtiETMt1LW0d/lHqQ7wwrIY4uGRXLQ4=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
@@ -1320,9 +1411,11 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
-google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
+google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1363,6 +1456,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
honnef.co/go/tools v0.4.7 h1:9MDAWxMoSnB6QoSqiVr7P5mtkT9pOc1kSxchzPCnqJs=
honnef.co/go/tools v0.4.7/go.mod h1:+rnGS1THNh8zMwnd2oVOTL9QF6vmfyG6ZXBULae2uc0=
k8s.io/api v0.28.6 h1:yy6u9CuIhmg55YvF/BavPBBXB+5QicB64njJXxVnzLo=
@@ -1375,6 +1469,52 @@ k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw=
k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
+lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
+modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
+modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
+modernc.org/cc/v3 v3.37.0/go.mod h1:vtL+3mdHx/wcj3iEGz84rQa8vEqR6XM84v5Lcvfph20=
+modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
+modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc=
+modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw=
+modernc.org/ccgo/v3 v3.0.0-20220904174949-82d86e1b6d56/go.mod h1:YSXjPL62P2AMSxBphRHPn7IkzhVHqkvOnRKAKh+W6ZI=
+modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
+modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
+modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws=
+modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo=
+modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8/go.mod h1:fUB3Vn0nVPReA+7IG7yZDfjv1TMWjhQP8gCxrFAtL5g=
+modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
+modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
+modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
+modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA=
+modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A=
+modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU=
+modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU=
+modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA=
+modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0=
+modernc.org/libc v1.17.4/go.mod h1:WNg2ZH56rDEwdropAJeZPQkXmDwh+JCA1s/htl6r2fA=
+modernc.org/libc v1.18.0/go.mod h1:vj6zehR5bfc98ipowQOM2nIDUZnVew/wNC/2tOGS+q0=
+modernc.org/libc v1.20.3/go.mod h1:ZRfIaEkgrYgZDl6pa4W39HgN5G/yDW+NRmNKZBDFrk0=
+modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI=
+modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug=
+modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw=
+modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw=
+modernc.org/memory v1.3.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
+modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
+modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
+modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
+modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
+modernc.org/sqlite v1.18.2/go.mod h1:kvrTLEWgxUcHa2GfHBQtanR1H9ht3hTJNtKpzH9k1u0=
+modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
+modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
+modernc.org/tcl v1.13.2/go.mod h1:7CLiGIPo1M8Rv1Mitpv5akc2+8fxUd2y2UzC/MfMzy0=
+modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
diff --git a/lightning/cmd/tidb-lightning/main.go b/lightning/cmd/tidb-lightning/main.go
index 53c4fe30bef32..d68d6033acc4f 100644
--- a/lightning/cmd/tidb-lightning/main.go
+++ b/lightning/cmd/tidb-lightning/main.go
@@ -99,7 +99,9 @@ func main() {
finished := true
if common.IsContextCanceledError(err) {
err = nil
- finished = false
+ if app.TaskCanceled() {
+ finished = false
+ }
}
if err != nil {
logger.Error("tidb lightning encountered error stack info", zap.Error(err))
@@ -121,7 +123,7 @@ func main() {
}
}
- if err != nil {
+ if err != nil || (globalCfg.App.ServerMode && !finished) {
exit(1)
}
}
diff --git a/lightning/pkg/importer/chunk_process.go b/lightning/pkg/importer/chunk_process.go
index 4f071ccae038a..802e7c15d4b6c 100644
--- a/lightning/pkg/importer/chunk_process.go
+++ b/lightning/pkg/importer/chunk_process.go
@@ -84,7 +84,9 @@ func openParser(
tblInfo *model.TableInfo,
) (mydump.Parser, error) {
blockBufSize := int64(cfg.Mydumper.ReadBlockSize)
- reader, err := mydump.OpenReader(ctx, &chunk.FileMeta, store, storage.DecompressConfig{})
+ reader, err := mydump.OpenReader(ctx, &chunk.FileMeta, store, storage.DecompressConfig{
+ ZStdDecodeConcurrency: 1,
+ })
if err != nil {
return nil, err
}
@@ -724,7 +726,7 @@ func (cr *chunkProcessor) deliverLoop(
rc.status.FinishedFileSize.Add(delta)
}
} else {
- deliverLogger.Warn("offset go back", zap.Int64("curr", highOffset),
+ deliverLogger.Error("offset go back", zap.Int64("curr", highOffset),
zap.Int64("start", lowOffset))
}
}
diff --git a/lightning/pkg/importer/get_pre_info.go b/lightning/pkg/importer/get_pre_info.go
index 8bcb02d075ee3..5e34c6bf36186 100644
--- a/lightning/pkg/importer/get_pre_info.go
+++ b/lightning/pkg/importer/get_pre_info.go
@@ -143,9 +143,6 @@ func NewTargetInfoGetterImpl(
case config.BackendTiDB:
backendTargetInfoGetter = tidb.NewTargetInfoGetter(targetDB)
case config.BackendLocal:
- if pdHTTPCli == nil {
- return nil, common.ErrUnknown.GenWithStack("pd HTTP client is required when using local backend")
- }
backendTargetInfoGetter = local.NewTargetInfoGetter(tls, targetDB, pdHTTPCli)
default:
return nil, common.ErrUnknownBackend.GenWithStackByArgs(cfg.TikvImporter.Backend)
@@ -464,7 +461,9 @@ func (p *PreImportInfoGetterImpl) ReadFirstNRowsByTableName(ctx context.Context,
// ReadFirstNRowsByFileMeta reads the first N rows of an data file.
// It implements the PreImportInfoGetter interface.
func (p *PreImportInfoGetterImpl) ReadFirstNRowsByFileMeta(ctx context.Context, dataFileMeta mydump.SourceFileMeta, n int) ([]string, [][]types.Datum, error) {
- reader, err := mydump.OpenReader(ctx, &dataFileMeta, p.srcStorage, storage.DecompressConfig{})
+ reader, err := mydump.OpenReader(ctx, &dataFileMeta, p.srcStorage, storage.DecompressConfig{
+ ZStdDecodeConcurrency: 1,
+ })
if err != nil {
return nil, nil, errors.Trace(err)
}
@@ -613,7 +612,9 @@ func (p *PreImportInfoGetterImpl) sampleDataFromTable(
return resultIndexRatio, isRowOrdered, nil
}
sampleFile := tableMeta.DataFiles[0].FileMeta
- reader, err := mydump.OpenReader(ctx, &sampleFile, p.srcStorage, storage.DecompressConfig{})
+ reader, err := mydump.OpenReader(ctx, &sampleFile, p.srcStorage, storage.DecompressConfig{
+ ZStdDecodeConcurrency: 1,
+ })
if err != nil {
return 0.0, false, errors.Trace(err)
}
diff --git a/lightning/pkg/importer/get_pre_info_test.go b/lightning/pkg/importer/get_pre_info_test.go
index bee6416adfcf5..ca5b0cfbf337a 100644
--- a/lightning/pkg/importer/get_pre_info_test.go
+++ b/lightning/pkg/importer/get_pre_info_test.go
@@ -753,16 +753,22 @@ func TestGetPreInfoEstimateSourceSize(t *testing.T) {
}
func TestGetPreInfoIsTableEmpty(t *testing.T) {
- ctx := context.TODO()
+ ctx := context.Background()
db, mock, err := sqlmock.New()
require.NoError(t, err)
lnConfig := config.NewConfig()
lnConfig.TikvImporter.Backend = config.BackendLocal
- _, err = NewTargetInfoGetterImpl(lnConfig, db, nil)
- require.ErrorContains(t, err, "pd HTTP client is required when using local backend")
- lnConfig.TikvImporter.Backend = config.BackendTiDB
targetGetter, err := NewTargetInfoGetterImpl(lnConfig, db, nil)
require.NoError(t, err)
+ mock.ExpectQuery("SELECT version()").
+ WillReturnRows(sqlmock.NewRows([]string{"version()"}).AddRow("8.0.11-TiDB-v8.2.0-alpha-256-qweqweqw"))
+ err = targetGetter.CheckVersionRequirements(ctx)
+ require.ErrorContains(t, err, "pd HTTP client is required for component version check in local backend")
+ require.NoError(t, mock.ExpectationsWereMet())
+
+ lnConfig.TikvImporter.Backend = config.BackendTiDB
+ targetGetter, err = NewTargetInfoGetterImpl(lnConfig, db, nil)
+ require.NoError(t, err)
require.Equal(t, lnConfig, targetGetter.cfg)
mock.ExpectQuery("SELECT 1 FROM `test_db`.`test_tbl` USE INDEX\\(\\) LIMIT 1").
diff --git a/lightning/pkg/importer/precheck_impl.go b/lightning/pkg/importer/precheck_impl.go
index 04948dc12bdb6..3327813e701fa 100644
--- a/lightning/pkg/importer/precheck_impl.go
+++ b/lightning/pkg/importer/precheck_impl.go
@@ -837,7 +837,7 @@ func (ci *CDCPITRCheckItem) Check(ctx context.Context) (*precheck.CheckResult, e
errorMsg = append(errorMsg, fmt.Sprintf("found PiTR log streaming task(s): %v,", names))
}
- nameSet, err := cdcutil.GetCDCChangefeedNameSet(ctx, ci.etcdCli)
+ nameSet, err := cdcutil.GetRunningChangefeeds(ctx, ci.etcdCli)
if err != nil {
return nil, errors.Trace(err)
}
diff --git a/lightning/pkg/importer/precheck_impl_test.go b/lightning/pkg/importer/precheck_impl_test.go
index 6280b583af944..e35535ef6be69 100644
--- a/lightning/pkg/importer/precheck_impl_test.go
+++ b/lightning/pkg/importer/precheck_impl_test.go
@@ -641,7 +641,7 @@ func (s *precheckImplSuite) TestCDCPITRCheckItem() {
)
checkEtcdPut(
"/tidb/cdc/default/default/changefeed/info/test-1",
- `{"upstream-id":7195826648407968958,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306?time-zone=","create-time":"2023-02-03T15:23:34.773768+08:00","start-ts":439198420741652483,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":true,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"none","enable-partition-separator":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"storage":""},"scheduler":{"region-per-span":0}},"state":"failed","error":null,"creator-version":"v6.5.0-master-dirty"}`,
+ `{"upstream-id":7195826648407968958,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306?time-zone=","create-time":"2023-02-03T15:23:34.773768+08:00","start-ts":439198420741652483,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":true,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"transaction-atomicity":"","protocol":"","dispatchers":null,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false},"column-selectors":null,"schema-registry":"","encoder-concurrency":16,"terminator":"\r\n","date-separator":"none","enable-partition-separator":false},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"storage":""},"scheduler":{"region-per-span":0}},"state":"finished","error":null,"creator-version":"v6.5.0-master-dirty"}`,
)
checkEtcdPut("/tidb/cdc/default/default/changefeed/status/test")
checkEtcdPut("/tidb/cdc/default/default/changefeed/status/test-1")
diff --git a/lightning/pkg/importer/table_import.go b/lightning/pkg/importer/table_import.go
index dd6de8667427e..a08fe4695636f 100644
--- a/lightning/pkg/importer/table_import.go
+++ b/lightning/pkg/importer/table_import.go
@@ -1188,7 +1188,9 @@ func getChunkCompressedSizeForParquet(
chunk *checkpoints.ChunkCheckpoint,
store storage.ExternalStorage,
) (int64, error) {
- reader, err := mydump.OpenReader(ctx, &chunk.FileMeta, store, storage.DecompressConfig{})
+ reader, err := mydump.OpenReader(ctx, &chunk.FileMeta, store, storage.DecompressConfig{
+ ZStdDecodeConcurrency: 1,
+ })
if err != nil {
return 0, errors.Trace(err)
}
diff --git a/lightning/pkg/server/lightning.go b/lightning/pkg/server/lightning.go
index 6520e849fb6c8..3e4ed74f3913b 100644
--- a/lightning/pkg/server/lightning.go
+++ b/lightning/pkg/server/lightning.go
@@ -92,6 +92,8 @@ type Lightning struct {
cancelLock sync.Mutex
curTask *config.Config
cancel context.CancelFunc // for per task context, which maybe different from lightning context
+
+ taskCanceled bool
}
func initEnv(cfg *config.GlobalConfig) error {
@@ -604,6 +606,7 @@ func (l *Lightning) run(taskCtx context.Context, taskCfg *config.Config, o *opti
func (l *Lightning) Stop() {
l.cancelLock.Lock()
if l.cancel != nil {
+ l.taskCanceled = true
l.cancel()
}
l.cancelLock.Unlock()
@@ -613,6 +616,13 @@ func (l *Lightning) Stop() {
l.shutdown()
}
+// TaskCanceled return whether the current task is canceled.
+func (l *Lightning) TaskCanceled() bool {
+ l.cancelLock.Lock()
+ defer l.cancelLock.Unlock()
+ return l.taskCanceled
+}
+
// Status return the sum size of file which has been imported to TiKV and the total size of source file.
func (l *Lightning) Status() (finished int64, total int64) {
finished = l.status.FinishedFileSize.Load()
diff --git a/lightning/tests/lightning_column_permutation/config.toml b/lightning/tests/lightning_column_permutation/config.toml
index 7bbff7f805a46..845873d84c8b9 100644
--- a/lightning/tests/lightning_column_permutation/config.toml
+++ b/lightning/tests/lightning_column_permutation/config.toml
@@ -1,3 +1,6 @@
[mydumper]
strict-format = true
max-region-size = 200
+
+[mydumper.csv]
+terminator = "\n"
diff --git a/lightning/tests/lightning_config_max_error/run.sh b/lightning/tests/lightning_config_max_error/run.sh
index b43f886a50197..d0464245a573e 100755
--- a/lightning/tests/lightning_config_max_error/run.sh
+++ b/lightning/tests/lightning_config_max_error/run.sh
@@ -29,6 +29,7 @@ remaining_row_count=$(( ${uniq_row_count} + ${duplicated_row_count}/2 ))
run_sql 'DROP TABLE IF EXISTS mytest.testtbl'
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v3'
+run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
stderr_file="/tmp/${TEST_NAME}.stderr"
@@ -57,6 +58,7 @@ check_contains "COUNT(*): ${duplicated_row_count}"
run_sql 'DROP TABLE IF EXISTS mytest.testtbl'
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v3'
+run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
run_lightning --backend local --config "${mydir}/normal_config.toml"
@@ -71,6 +73,7 @@ check_contains "COUNT(*): ${remaining_row_count}"
run_sql 'DROP TABLE IF EXISTS mytest.testtbl'
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v3'
+run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
run_lightning --backend local --config "${mydir}/normal_config_old_style.toml"
@@ -83,12 +86,14 @@ check_contains "COUNT(*): ${remaining_row_count}"
# import a fourth time
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_records'
+run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
! run_lightning --backend local --config "${mydir}/ignore_config.toml"
[ $? -eq 0 ]
tail -n 10 $TEST_DIR/lightning.log | grep "ERROR" | tail -n 1 | grep -Fq "[Lightning:Config:ErrInvalidConfig]conflict.strategy cannot be set to \\\"ignore\\\" when use tikv-importer.backend = \\\"local\\\""
# Check tidb backend record duplicate entry in conflict_records table
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_records'
+run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
run_lightning --backend tidb --config "${mydir}/tidb.toml"
run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_records'
check_contains "COUNT(*): 15"
@@ -99,7 +104,7 @@ check_contains "row_data: ('5','bbb05')"
# Check max-error-record can limit the size of conflict_records table
run_sql 'DROP DATABASE IF EXISTS lightning_task_info'
run_sql 'DROP DATABASE IF EXISTS mytest'
-run_lightning --backend tidb --config "${mydir}/tidb-limit-record.toml" 2>&1 | grep "\`lightning_task_info\`.\`conflict_records\`" | grep -q "5"
+run_lightning --backend tidb --config "${mydir}/tidb-limit-record.toml" 2>&1 | grep "\`lightning_task_info\`.\`conflict_view\`" | grep -q "5"
run_sql 'SELECT COUNT(*) FROM lightning_task_info.conflict_records'
check_contains "COUNT(*): 5"
diff --git a/lightning/tests/lightning_duplicate_resolution_error/run.sh b/lightning/tests/lightning_duplicate_resolution_error/run.sh
index 164ae140cc10b..bd2978802923f 100644
--- a/lightning/tests/lightning_duplicate_resolution_error/run.sh
+++ b/lightning/tests/lightning_duplicate_resolution_error/run.sh
@@ -22,6 +22,7 @@ mydir=$(dirname "${BASH_SOURCE[0]}")
run_sql 'DROP TABLE IF EXISTS dup_resolve.a'
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v3'
+run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
! run_lightning --backend local --config "${mydir}/config.toml"
[ $? -eq 0 ]
diff --git a/lightning/tests/lightning_duplicate_resolution_error_pk_multiple_files/run.sh b/lightning/tests/lightning_duplicate_resolution_error_pk_multiple_files/run.sh
index b207b55ef8cbb..e6988d22b9699 100644
--- a/lightning/tests/lightning_duplicate_resolution_error_pk_multiple_files/run.sh
+++ b/lightning/tests/lightning_duplicate_resolution_error_pk_multiple_files/run.sh
@@ -22,6 +22,7 @@ mydir=$(dirname "${BASH_SOURCE[0]}")
run_sql 'DROP TABLE IF EXISTS dup_resolve.a'
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v3'
+run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
! run_lightning --backend local --config "${mydir}/config.toml"
[ $? -eq 0 ]
diff --git a/lightning/tests/lightning_duplicate_resolution_error_uk_multiple_files/run.sh b/lightning/tests/lightning_duplicate_resolution_error_uk_multiple_files/run.sh
index 65a20892ce342..e346b3961977b 100644
--- a/lightning/tests/lightning_duplicate_resolution_error_uk_multiple_files/run.sh
+++ b/lightning/tests/lightning_duplicate_resolution_error_uk_multiple_files/run.sh
@@ -22,6 +22,7 @@ mydir=$(dirname "${BASH_SOURCE[0]}")
run_sql 'DROP TABLE IF EXISTS dup_resolve.a'
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v3'
+run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
! run_lightning --backend local --config "${mydir}/config.toml"
[ $? -eq 0 ]
diff --git a/lightning/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/run.sh b/lightning/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/run.sh
index ef72491c0a114..02b441d5ca058 100644
--- a/lightning/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/run.sh
+++ b/lightning/tests/lightning_duplicate_resolution_error_uk_multiple_files_multicol_index/run.sh
@@ -22,6 +22,7 @@ mydir=$(dirname "${BASH_SOURCE[0]}")
run_sql 'DROP TABLE IF EXISTS dup_resolve.a'
run_sql 'DROP TABLE IF EXISTS lightning_task_info.conflict_error_v3'
+run_sql 'DROP VIEW IF EXISTS lightning_task_info.conflict_view'
! run_lightning --backend local --config "${mydir}/config.toml"
[ $? -eq 0 ]
diff --git a/lightning/tests/lightning_duplicate_resolution_merge/run.sh b/lightning/tests/lightning_duplicate_resolution_merge/run.sh
index 8d12c4dc9af5f..9060067362876 100644
--- a/lightning/tests/lightning_duplicate_resolution_merge/run.sh
+++ b/lightning/tests/lightning_duplicate_resolution_merge/run.sh
@@ -24,7 +24,7 @@ LOG_FILE1="$TEST_DIR/lightning_duplicate_resolution_merge1.log"
LOG_FILE2="$TEST_DIR/lightning_duplicate_resolution_merge2.log"
# let lightning run a bit slow to avoid some table in the first lightning finish too fast.
-export GO_FAILPOINTS="github.com/pingcap/tidb/lightning/pkg/importer/SlowDownImport=sleep(250)"
+export GO_FAILPOINTS="github.com/pingcap/tidb/lightning/pkg/importer/SlowDownImport=sleep(1000)"
run_lightning --backend local --sorted-kv-dir "$TEST_DIR/lightning_duplicate_resolution_merge.sorted1" \
-d "$CUR/data1" --log-file "$LOG_FILE1" --config "$CUR/config.toml" &
diff --git a/lightning/tests/lightning_issue_40657/run.sh b/lightning/tests/lightning_issue_40657/run.sh
index 0f3d9ca5d15cb..a9b7d5baf8907 100644
--- a/lightning/tests/lightning_issue_40657/run.sh
+++ b/lightning/tests/lightning_issue_40657/run.sh
@@ -24,7 +24,7 @@ run_lightning -d "$CUR/data1"
run_sql 'admin check table test.t'
run_sql 'select count(*) from test.t'
check_contains 'count(*): 4'
-run_sql 'select count(*) from lightning_task_info.conflict_error_v3'
+run_sql 'select count(*) from lightning_task_info.conflict_view'
check_contains 'count(*): 2'
run_sql 'truncate table test.t'
diff --git a/lightning/tidb-lightning.toml b/lightning/tidb-lightning.toml
index e7945b1c4eafb..15791b7dc36f0 100644
--- a/lightning/tidb-lightning.toml
+++ b/lightning/tidb-lightning.toml
@@ -94,14 +94,14 @@ driver = "file"
# - "": in the physical import mode, TiDB Lightning does not detect or handle conflicting data. If the source file contains conflicting primary or unique key records, the subsequent step reports an error. In the logical import mode, TiDB Lightning converts the "" strategy to the "error" strategy for processing.
# - "error": when detecting conflicting primary or unique key records in the imported data, TiDB Lightning terminates the import and reports an error.
# - "replace": when encountering conflicting primary or unique key records, TiDB Lightning retains the latest data and overwrites the old data.
-# The conflicting data are recorded in the `lightning_task_info.conflict_error_v2` table (recording conflicting data detected by post-import conflict detection in the physical import mode) and the `conflict_records` table (recording conflicting data detected by preprocess conflict detection in both logical and physical import modes) of the target TiDB cluster.
-# If you set `conflict.strategy = "replace"` in physical import mode, the conflicting data can be checked in the `lightning_task_info.conflict_view` view.
+# The conflicting data are recorded in the `lightning_task_info.conflict_view` view of the target TiDB cluster.
+# If the value for column is_precheck_conflict is 0, it stands for conflicting data detected by post-import conflict detection in the physical import mode; If the value for column is_precheck_conflict is 1, it stands for conflicting data detected by preprocess conflict detection in both logical and physical import modes.
# You can manually insert the correct records into the target table based on your application requirements. Note that the target TiKV must be v5.2.0 or later versions.
# - "ignore": when encountering conflicting primary or unique key records, TiDB Lightning retains the old data and ignores the new data. This option can only be used in the logical import mode.
strategy = ""
-# Controls whether to enable preprocess conflict detection, which checks conflicts in data before importing it to TiDB. The default value is false, indicating that TiDB Lightning only checks conflicts after the import. If you set it to true, TiDB Lightning checks conflicts both before and after the import. This parameter can be used only in the physical import mode. It is not recommended to set `precheck-conflict-before-import = true` for now.
+# Controls whether to enable preprocess conflict detection, which checks conflicts in data before importing it to TiDB. The default value is false, indicating that TiDB Lightning only checks conflicts after the import. If you set it to true, TiDB Lightning checks conflicts both before and after the import. This parameter can be used only in the physical import mode. In scenarios where the number of conflict records is greater than 1,000,000, it is recommended to set `precheck-conflict-before-import = true` for better performance in conflict detection. In other scenarios, it is recommended to disable it.
# precheck-conflict-before-import = false
-# Controls the maximum number of conflict errors that can be handled when strategy is "replace" or "ignore". You can set it only when strategy is "replace" or "ignore". The default value is 10000. If you set the value larger than 10000, it is possible that the import will have performance degradation or fail due to potential errors.
+# Controls the maximum number of conflict errors that can be handled when strategy is "replace" or "ignore". You can set it only when the strategy is "replace" or "ignore". The default value is 10000. If you set a value larger than 10000, the import process might experience performance degradation.
# threshold = 10000
# Controls the maximum number of records in the `conflict_records` table. The default value is 10000.
# Starting from v8.1.0, there is no need to configure `max-record-rows` manually, because TiDB Lightning automatically assigns the value of `max-record-rows` with the value of `threshold`, regardless of the user input. `max-record-rows` will be deprecated in a future release.
@@ -114,6 +114,15 @@ strategy = ""
backend = "importer"
# Address of tikv-importer when the backend is 'importer'
addr = "127.0.0.1:8287"
+
+# The `duplicate-resolution` parameter is deprecated starting from v8.0.0 and will be removed in a future release. If you set `tikv-importer.duplicate-resolution = "remove"` and do not set `conflict.strategy`, TiDB Lightning will automatically assign `"replace"` to `conflict.strategy` and enable the new version of conflict detection. For more information, see .
+# Whether to detect and resolve duplicate records (unique key conflict) in the physical import mode.
+# The following resolution algorithms are supported:
+# - none: does not detect duplicate records.
+# If there are duplicate records in the data source, it might lead to inconsistent data in the target TiDB.
+# If you set `tikv-importer.duplicate-resolution = "none"` and do not set `conflict.strategy`, TiDB Lightning will automatically assign `""` to `conflict.strategy`.
+# The default value is 'none'.
+# duplicate-resolution = 'none'
# Maximum KV size of SST files produced in the 'local' backend. This should be the same as
# the TiKV region size to avoid further region splitting. The default value is 96 MiB.
#region-split-size = '96MiB'
diff --git a/pkg/autoid_service/autoid.go b/pkg/autoid_service/autoid.go
index 5fa3ea1cffba2..dbbe7b8ee3353 100644
--- a/pkg/autoid_service/autoid.go
+++ b/pkg/autoid_service/autoid.go
@@ -334,17 +334,9 @@ func newWithCli(selfAddr string, cli *clientv3.Client, store kv.Storage) *Servic
leaderShip: l,
store: store,
}
- l.SetBeOwnerHook(func() {
- // Reset the map to avoid a case that a node lose leadership and regain it, then
- // improperly use the stale map to serve the autoid requests.
- // See https://github.com/pingcap/tidb/issues/52600
- service.autoIDLock.Lock()
- clear(service.autoIDMap)
- service.autoIDLock.Unlock()
-
- logutil.BgLogger().Info("leader change of autoid service, this node become owner",
- zap.String("addr", selfAddr),
- zap.String("category", "autoid service"))
+ l.SetListener(&ownerListener{
+ Service: service,
+ selfAddr: selfAddr,
})
// 10 means that autoid service's etcd lease is 10s.
err := l.CampaignOwner(10)
@@ -389,22 +381,6 @@ func MockForTest(store kv.Storage) autoid.AutoIDAllocClient {
// Close closes the Service and clean up resource.
func (s *Service) Close() {
if s.leaderShip != nil && s.leaderShip.IsOwner() {
- s.autoIDLock.Lock()
- defer s.autoIDLock.Unlock()
- for k, v := range s.autoIDMap {
- v.Lock()
- if v.base > 0 {
- err := v.forceRebase(context.Background(), s.store, k.dbID, k.tblID, v.base, v.isUnsigned)
- if err != nil {
- logutil.BgLogger().Warn("save cached ID fail when service exit", zap.String("category", "autoid service"),
- zap.Int64("db id", k.dbID),
- zap.Int64("table id", k.tblID),
- zap.Int64("value", v.base),
- zap.Error(err))
- }
- }
- v.Unlock()
- }
s.leaderShip.Cancel()
}
}
@@ -608,6 +584,29 @@ func (s *Service) Rebase(ctx context.Context, req *autoid.RebaseRequest) (*autoi
return &autoid.RebaseResponse{}, nil
}
+type ownerListener struct {
+ *Service
+ selfAddr string
+}
+
+var _ owner.Listener = (*ownerListener)(nil)
+
+func (l *ownerListener) OnBecomeOwner() {
+ // Reset the map to avoid a case that a node lose leadership and regain it, then
+ // improperly use the stale map to serve the autoid requests.
+ // See https://github.com/pingcap/tidb/issues/52600
+ l.autoIDLock.Lock()
+ clear(l.autoIDMap)
+ l.autoIDLock.Unlock()
+
+ logutil.BgLogger().Info("leader change of autoid service, this node become owner",
+ zap.String("addr", l.selfAddr),
+ zap.String("category", "autoid service"))
+}
+
+func (*ownerListener) OnRetireOwner() {
+}
+
func init() {
autoid1.MockForTest = MockForTest
}
diff --git a/pkg/config/BUILD.bazel b/pkg/config/BUILD.bazel
index 11b8d63b2156a..f12196ecc9721 100644
--- a/pkg/config/BUILD.bazel
+++ b/pkg/config/BUILD.bazel
@@ -37,7 +37,7 @@ go_test(
data = glob(["**"]),
embed = [":config"],
flaky = True,
- shard_count = 24,
+ shard_count = 25,
deps = [
"//pkg/testkit/testsetup",
"//pkg/util/logutil",
diff --git a/pkg/config/config.go b/pkg/config/config.go
index 87d8ca379c151..fab07fad8f041 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -75,8 +75,8 @@ const (
DefTableColumnCountLimit = 1017
// DefMaxOfTableColumnCountLimit is maximum limitation of the number of columns in a table
DefMaxOfTableColumnCountLimit = 4096
- // DefStatsLoadConcurrencyLimit is limit of the concurrency of stats-load
- DefStatsLoadConcurrencyLimit = 1
+ // DefStatsLoadConcurrencyLimit is limit of the concurrency of stats-load. When it is set to 0, it will be set by syncload.GetSyncLoadConcurrencyByCPU.
+ DefStatsLoadConcurrencyLimit = 0
// DefMaxOfStatsLoadConcurrencyLimit is maximum limitation of the concurrency of stats-load
DefMaxOfStatsLoadConcurrencyLimit = 128
// DefStatsLoadQueueSizeLimit is limit of the size of stats-load request queue
@@ -97,6 +97,8 @@ const (
DefAuthTokenRefreshInterval = time.Hour
// EnvVarKeyspaceName is the system env name for keyspace name.
EnvVarKeyspaceName = "KEYSPACE_NAME"
+ // MaxTokenLimit is the max token limit value.
+ MaxTokenLimit = 1024 * 1024
)
// Valid config maps
@@ -720,7 +722,7 @@ type Performance struct {
PlanReplayerGCLease string `toml:"plan-replayer-gc-lease" json:"plan-replayer-gc-lease"`
GOGC int `toml:"gogc" json:"gogc"`
EnforceMPP bool `toml:"enforce-mpp" json:"enforce-mpp"`
- StatsLoadConcurrency uint `toml:"stats-load-concurrency" json:"stats-load-concurrency"`
+ StatsLoadConcurrency int `toml:"stats-load-concurrency" json:"stats-load-concurrency"`
StatsLoadQueueSize uint `toml:"stats-load-queue-size" json:"stats-load-queue-size"`
AnalyzePartitionConcurrencyQuota uint `toml:"analyze-partition-concurrency-quota" json:"analyze-partition-concurrency-quota"`
PlanReplayerDumpWorkerConcurrency uint `toml:"plan-replayer-dump-worker-concurrency" json:"plan-replayer-dump-worker-concurrency"`
@@ -1010,7 +1012,7 @@ var defaultConf = Config{
GOGC: 100,
EnforceMPP: false,
PlanReplayerGCLease: "10m",
- StatsLoadConcurrency: 5,
+ StatsLoadConcurrency: 0, // 0 is auto mode.
StatsLoadQueueSize: 1000,
AnalyzePartitionConcurrencyQuota: 16,
PlanReplayerDumpWorkerConcurrency: 1,
@@ -1275,6 +1277,8 @@ func (c *Config) Load(confFile string) error {
}
if c.TokenLimit == 0 {
c.TokenLimit = 1000
+ } else if c.TokenLimit > MaxTokenLimit {
+ c.TokenLimit = MaxTokenLimit
}
// If any items in confFile file are not mapped into the Config struct, issue
// an error and stop the server from starting.
diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go
index f222a419ea532..1df8bb4a10fbf 100644
--- a/pkg/config/config_test.go
+++ b/pkg/config/config_test.go
@@ -754,7 +754,6 @@ store-limit=0
ttl-refreshed-txn-size=8192
resolve-lock-lite-threshold = 16
copr-req-timeout = "120s"
-enable-replica-selector-v2 = false
[tikv-client.async-commit]
keys-limit=123
total-key-size-limit=1024
@@ -805,8 +804,6 @@ max_connections = 200
require.Equal(t, uint(6000), conf.TiKVClient.RegionCacheTTL)
require.Equal(t, int64(0), conf.TiKVClient.StoreLimit)
require.Equal(t, int64(8192), conf.TiKVClient.TTLRefreshedTxnSize)
- require.Equal(t, false, conf.TiKVClient.EnableReplicaSelectorV2)
- require.Equal(t, true, defaultConf.TiKVClient.EnableReplicaSelectorV2)
require.Equal(t, uint(1000), conf.TokenLimit)
require.True(t, conf.EnableTableLock)
require.Equal(t, uint64(5), conf.DelayCleanTableLock)
@@ -1173,6 +1170,44 @@ func TestTableColumnCountLimit(t *testing.T) {
checkValid(DefMaxOfTableColumnCountLimit+1, false)
}
+func TestTokenLimit(t *testing.T) {
+ storeDir := t.TempDir()
+ configFile := filepath.Join(storeDir, "config.toml")
+ f, err := os.Create(configFile)
+ require.NoError(t, err)
+ defer func(configFile string) {
+ require.NoError(t, os.Remove(configFile))
+ }(configFile)
+
+ tests := []struct {
+ tokenLimit uint
+ expectedTokenLimit uint
+ }{
+ {
+ 0,
+ 1000,
+ },
+ {
+ 99999999999,
+ MaxTokenLimit,
+ },
+ }
+
+ for _, test := range tests {
+ require.NoError(t, f.Truncate(0))
+ _, err = f.Seek(0, 0)
+ require.NoError(t, err)
+ _, err = f.WriteString(fmt.Sprintf(`
+token-limit = %d
+`, test.tokenLimit))
+ require.NoError(t, err)
+ require.NoError(t, f.Sync())
+ conf := NewConfig()
+ require.NoError(t, conf.Load(configFile))
+ require.Equal(t, test.expectedTokenLimit, conf.TokenLimit)
+ }
+}
+
func TestEncodeDefTempStorageDir(t *testing.T) {
tests := []struct {
host string
@@ -1290,7 +1325,7 @@ func TestConfigExample(t *testing.T) {
func TestStatsLoadLimit(t *testing.T) {
conf := NewConfig()
checkConcurrencyValid := func(concurrency int, shouldBeValid bool) {
- conf.Performance.StatsLoadConcurrency = uint(concurrency)
+ conf.Performance.StatsLoadConcurrency = concurrency
require.Equal(t, shouldBeValid, conf.Valid() == nil)
}
checkConcurrencyValid(DefStatsLoadConcurrencyLimit, true)
diff --git a/pkg/ddl/BUILD.bazel b/pkg/ddl/BUILD.bazel
index df643deea7e08..f5cddf95e7fad 100644
--- a/pkg/ddl/BUILD.bazel
+++ b/pkg/ddl/BUILD.bazel
@@ -78,6 +78,7 @@ go_library(
"//pkg/ddl/syncer",
"//pkg/ddl/util",
"//pkg/distsql",
+ "//pkg/distsql/context",
"//pkg/disttask/framework/handle",
"//pkg/disttask/framework/proto",
"//pkg/disttask/framework/scheduler",
@@ -89,6 +90,8 @@ go_library(
"//pkg/domain/resourcegroup",
"//pkg/errctx",
"//pkg/expression",
+ "//pkg/expression/context",
+ "//pkg/expression/contextstatic",
"//pkg/infoschema",
"//pkg/kv",
"//pkg/lightning/backend",
@@ -128,6 +131,7 @@ go_library(
"//pkg/table/tables",
"//pkg/tablecodec",
"//pkg/tidb-binlog/pump_client",
+ "//pkg/ttl/cache",
"//pkg/types",
"//pkg/types/parser_driver",
"//pkg/util",
@@ -140,12 +144,15 @@ go_library(
"//pkg/util/dbterror/exeerrors",
"//pkg/util/domainutil",
"//pkg/util/engine",
+ "//pkg/util/execdetails",
"//pkg/util/filter",
"//pkg/util/gcutil",
+ "//pkg/util/generic",
"//pkg/util/hack",
"//pkg/util/intest",
"//pkg/util/logutil",
"//pkg/util/mathutil",
+ "//pkg/util/memory",
"//pkg/util/mock",
"//pkg/util/ranger",
"//pkg/util/resourcegrouptag",
@@ -157,7 +164,7 @@ go_library(
"//pkg/util/sqlexec",
"//pkg/util/sqlkiller",
"//pkg/util/stringutil",
- "//pkg/util/syncutil",
+ "//pkg/util/tiflash",
"//pkg/util/timeutil",
"//pkg/util/topsql",
"//pkg/util/topsql/state",
@@ -272,8 +279,10 @@ go_test(
"//pkg/disttask/framework/storage",
"//pkg/domain",
"//pkg/domain/infosync",
+ "//pkg/errctx",
"//pkg/errno",
"//pkg/executor",
+ "//pkg/expression",
"//pkg/infoschema",
"//pkg/keyspace",
"//pkg/kv",
@@ -292,7 +301,6 @@ go_test(
"//pkg/session",
"//pkg/session/types",
"//pkg/sessionctx",
- "//pkg/sessionctx/stmtctx",
"//pkg/sessionctx/variable",
"//pkg/sessiontxn",
"//pkg/store/gcworker",
@@ -303,6 +311,7 @@ go_test(
"//pkg/tablecodec",
"//pkg/testkit",
"//pkg/testkit/external",
+ "//pkg/testkit/testfailpoint",
"//pkg/testkit/testsetup",
"//pkg/testkit/testutil",
"//pkg/types",
@@ -331,6 +340,7 @@ go_test(
"@com_github_tikv_client_go_v2//util",
"@io_etcd_go_etcd_client_v3//:client",
"@org_golang_google_grpc//:grpc",
+ "@org_golang_x_sync//errgroup",
"@org_uber_go_atomic//:atomic",
"@org_uber_go_goleak//:goleak",
"@org_uber_go_zap//:zap",
diff --git a/pkg/ddl/backfilling.go b/pkg/ddl/backfilling.go
index d18506d4aee85..ce33c8a005e7d 100644
--- a/pkg/ddl/backfilling.go
+++ b/pkg/ddl/backfilling.go
@@ -29,6 +29,7 @@ import (
"github.com/pingcap/tidb/pkg/ddl/logutil"
ddlutil "github.com/pingcap/tidb/pkg/ddl/util"
"github.com/pingcap/tidb/pkg/expression"
+ exprctx "github.com/pingcap/tidb/pkg/expression/context"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/metrics"
"github.com/pingcap/tidb/pkg/parser/model"
@@ -40,9 +41,9 @@ import (
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/util"
+ contextutil "github.com/pingcap/tidb/pkg/util/context"
"github.com/pingcap/tidb/pkg/util/dbterror"
decoder "github.com/pingcap/tidb/pkg/util/rowDecoder"
- "github.com/pingcap/tidb/pkg/util/timeutil"
"github.com/pingcap/tidb/pkg/util/topsql"
"github.com/prometheus/client_golang/prometheus"
"github.com/tikv/client-go/v2/tikv"
@@ -146,6 +147,10 @@ type backfillCtx struct {
id int
*ddlCtx
sessCtx sessionctx.Context
+ warnings contextutil.WarnHandlerExt
+ loc *time.Location
+ exprCtx exprctx.BuildContext
+ tblCtx table.MutateContext
schemaName string
table table.Table
batchCnt int
@@ -153,22 +158,33 @@ type backfillCtx struct {
metricCounter prometheus.Counter
}
-func newBackfillCtx(ctx *ddlCtx, id int, sessCtx sessionctx.Context,
- schemaName string, tbl table.Table, jobCtx *JobContext, label string, isDistributed bool) *backfillCtx {
+func newBackfillCtx(id int, rInfo *reorgInfo,
+ schemaName string, tbl table.Table, jobCtx *JobContext, label string, isDistributed bool) (*backfillCtx, error) {
+ sessCtx, err := newSessCtx(rInfo.d.store, rInfo.ReorgMeta)
+ if err != nil {
+ return nil, err
+ }
+
if isDistributed {
id = int(backfillContextID.Add(1))
}
+
+ exprCtx := sessCtx.GetExprCtx()
return &backfillCtx{
id: id,
- ddlCtx: ctx,
+ ddlCtx: rInfo.d,
sessCtx: sessCtx,
+ warnings: sessCtx.GetSessionVars().StmtCtx.WarnHandler,
+ exprCtx: exprCtx,
+ tblCtx: sessCtx.GetTableCtx(),
+ loc: exprCtx.GetEvalCtx().Location(),
schemaName: schemaName,
table: tbl,
batchCnt: int(variable.GetDDLReorgBatchSize()),
jobContext: jobCtx,
metricCounter: metrics.BackfillTotalCounter.WithLabelValues(
metrics.GenerateReorgLabel(label, schemaName, tbl.Meta().Name.String())),
- }
+ }, nil
}
func updateTxnEntrySizeLimitIfNeeded(txn kv.Transaction) {
@@ -527,12 +543,12 @@ func loadDDLReorgVars(ctx context.Context, sessPool *sess.Pool) error {
return ddlutil.LoadDDLReorgVars(ctx, sCtx)
}
-func makeupDecodeColMap(sessCtx sessionctx.Context, dbName model.CIStr, t table.Table) (map[int64]decoder.Column, error) {
+func makeupDecodeColMap(dbName model.CIStr, t table.Table) (map[int64]decoder.Column, error) {
writableColInfos := make([]*model.ColumnInfo, 0, len(t.WritableCols()))
for _, col := range t.WritableCols() {
writableColInfos = append(writableColInfos, col.ColumnInfo)
}
- exprCols, _, err := expression.ColumnInfos2ColumnsAndNames(sessCtx.GetExprCtx(), dbName, t.Meta().Name, writableColInfos, t.Meta())
+ exprCols, _, err := expression.ColumnInfos2ColumnsAndNames(newReorgExprCtx(), dbName, t.Meta().Name, writableColInfos, t.Meta())
if err != nil {
return nil, err
}
@@ -543,24 +559,6 @@ func makeupDecodeColMap(sessCtx sessionctx.Context, dbName model.CIStr, t table.
return decodeColMap, nil
}
-func setSessCtxLocation(sctx sessionctx.Context, tzLocation *model.TimeZoneLocation) error {
- // It is set to SystemLocation to be compatible with nil LocationInfo.
- tz := *timeutil.SystemLocation()
- if sctx.GetSessionVars().TimeZone == nil {
- sctx.GetSessionVars().TimeZone = &tz
- } else {
- *sctx.GetSessionVars().TimeZone = tz
- }
- if tzLocation != nil {
- loc, err := tzLocation.GetLocation()
- if err != nil {
- return errors.Trace(err)
- }
- *sctx.GetSessionVars().TimeZone = *loc
- }
- return nil
-}
-
var backfillTaskChanSize = 128
// SetBackfillTaskChanSizeForTest is only used for test.
@@ -584,6 +582,7 @@ func SetBackfillTaskChanSizeForTest(n int) {
// The above operations are completed in a transaction.
// Finally, update the concurrent processing of the total number of rows, and store the completed handle value.
func (dc *ddlCtx) writePhysicalTableRecord(
+ ctx context.Context,
sessPool *sess.Pool,
t table.PhysicalTable,
bfWorkerType backfillerType,
@@ -603,15 +602,20 @@ func (dc *ddlCtx) writePhysicalTableRecord(
})
jc := reorgInfo.NewJobContext()
- sessCtx := newReorgSessCtx(reorgInfo.d.store)
- eg, egCtx := util.NewErrorGroupWithRecoverWithCtx(dc.ctx)
+ eg, egCtx := util.NewErrorGroupWithRecoverWithCtx(ctx)
- scheduler, err := newBackfillScheduler(egCtx, reorgInfo, sessPool, bfWorkerType, t, sessCtx, jc)
+ scheduler, err := newBackfillScheduler(egCtx, reorgInfo, sessPool, bfWorkerType, t, jc)
if err != nil {
return errors.Trace(err)
}
defer scheduler.close(true)
+ if lit, ok := scheduler.(*ingestBackfillScheduler); ok {
+ if lit.importStarted() {
+ return nil
+ }
+ }
+
err = scheduler.setupWorkers()
if err != nil {
return errors.Trace(err)
diff --git a/pkg/ddl/backfilling_dist_executor.go b/pkg/ddl/backfilling_dist_executor.go
index afb440a2161a4..c245e9c018bf8 100644
--- a/pkg/ddl/backfilling_dist_executor.go
+++ b/pkg/ddl/backfilling_dist_executor.go
@@ -59,6 +59,7 @@ type BackfillSubTaskMeta struct {
RangeSplitKeys [][]byte `json:"range_split_keys,omitempty"`
DataFiles []string `json:"data-files,omitempty"`
StatFiles []string `json:"stat-files,omitempty"`
+ TS uint64 `json:"ts,omitempty"`
// Each group of MetaGroups represents a different index kvs meta.
MetaGroups []*external.SortedKVMeta `json:"meta_groups,omitempty"`
// Only used for adding one single index.
@@ -91,6 +92,8 @@ func (s *backfillDistExecutor) newBackfillSubtaskExecutor(
jobMeta := &s.taskMeta.Job
ddlObj := s.d
+ // TODO getTableByTxn is using DDL ctx which is never cancelled except when shutdown.
+ // we should move this operation out of GetStepExecutor, and put into Init.
_, tblIface, err := ddlObj.getTableByTxn((*asAutoIDRequirement)(ddlObj.ddlCtx), jobMeta.SchemaID, jobMeta.TableID)
if err != nil {
return nil, err
diff --git a/pkg/ddl/backfilling_dist_scheduler.go b/pkg/ddl/backfilling_dist_scheduler.go
index f36a510478eaa..afaab2c30bf1d 100644
--- a/pkg/ddl/backfilling_dist_scheduler.go
+++ b/pkg/ddl/backfilling_dist_scheduler.go
@@ -41,7 +41,7 @@ import (
"github.com/pingcap/tidb/pkg/store/helper"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/util/backoff"
- tidblogutil "github.com/pingcap/tidb/pkg/util/logutil"
+ "github.com/tikv/client-go/v2/oracle"
"github.com/tikv/client-go/v2/tikv"
"go.uber.org/zap"
)
@@ -88,7 +88,7 @@ func (sch *BackfillingSchedulerExt) OnNextSubtasksBatch(
return nil, err
}
job := &backfillMeta.Job
- tblInfo, err := getTblInfo(sch.d, job)
+ tblInfo, err := getTblInfo(ctx, sch.d, job)
if err != nil {
return nil, err
}
@@ -100,7 +100,7 @@ func (sch *BackfillingSchedulerExt) OnNextSubtasksBatch(
if tblInfo.Partition != nil {
return generatePartitionPlan(tblInfo)
}
- return generateNonPartitionPlan(sch.d, tblInfo, job, sch.GlobalSort, len(execIDs))
+ return generateNonPartitionPlan(ctx, sch.d, tblInfo, job, sch.GlobalSort, len(execIDs))
case proto.BackfillStepMergeSort:
return generateMergePlan(taskHandle, task, logger)
case proto.BackfillStepWriteAndIngest:
@@ -200,8 +200,8 @@ func (sch *LitBackfillScheduler) Close() {
sch.BaseScheduler.Close()
}
-func getTblInfo(d *ddl, job *model.Job) (tblInfo *model.TableInfo, err error) {
- err = kv.RunInNewTxn(d.ctx, d.store, true, func(_ context.Context, txn kv.Transaction) error {
+func getTblInfo(ctx context.Context, d *ddl, job *model.Job) (tblInfo *model.TableInfo, err error) {
+ err = kv.RunInNewTxn(ctx, d.store, true, func(_ context.Context, txn kv.Transaction) error {
tblInfo, err = meta.NewMeta(txn).GetTable(job.SchemaID, job.TableID)
return err
})
@@ -241,11 +241,13 @@ const (
)
func generateNonPartitionPlan(
+ ctx context.Context,
d *ddl,
tblInfo *model.TableInfo,
job *model.Job,
useCloud bool,
- instanceCnt int) (metas [][]byte, err error) {
+ instanceCnt int,
+) (metas [][]byte, err error) {
tbl, err := getTable((*asAutoIDRequirement)(d.ddlCtx), job.SchemaID, tblInfo)
if err != nil {
return nil, err
@@ -266,7 +268,7 @@ func generateNonPartitionPlan(
subTaskMetas := make([][]byte, 0, 4)
backoffer := backoff.NewExponential(scanRegionBackoffBase, 2, scanRegionBackoffMax)
- err = handle.RunWithRetry(d.ctx, 8, backoffer, tidblogutil.Logger(d.ctx), func(_ context.Context) (bool, error) {
+ err = handle.RunWithRetry(ctx, 8, backoffer, logutil.DDLLogger(), func(_ context.Context) (bool, error) {
regionCache := d.store.(helper.Storage).GetRegionCache()
recordRegionMetas, err := regionCache.LoadRegionsInKeyRange(tikv.NewBackofferWithVars(context.Background(), 20000, nil), startKey, endKey)
if err != nil {
@@ -405,6 +407,16 @@ func splitSubtaskMetaForOneKVMetaGroup(
// Skip global sort for empty table.
return nil, nil
}
+ pdCli := store.GetPDClient()
+ p, l, err := pdCli.GetTS(ctx)
+ if err != nil {
+ return nil, err
+ }
+ ts := oracle.ComposeTS(p, l)
+ failpoint.Inject("mockTSForGlobalSort", func(val failpoint.Value) {
+ i := val.(int)
+ ts = uint64(i)
+ })
splitter, err := getRangeSplitter(
ctx, store, cloudStorageURI, int64(kvMeta.TotalKVSize), instanceCnt, kvMeta.MultipleFilesStats, logger)
if err != nil {
@@ -446,6 +458,7 @@ func splitSubtaskMetaForOneKVMetaGroup(
DataFiles: dataFiles,
StatFiles: statFiles,
RangeSplitKeys: rangeSplitKeys,
+ TS: ts,
}
metaBytes, err := json.Marshal(m)
if err != nil {
diff --git a/pkg/ddl/backfilling_dist_scheduler_test.go b/pkg/ddl/backfilling_dist_scheduler_test.go
index b73bf3da310c8..07c749cef7881 100644
--- a/pkg/ddl/backfilling_dist_scheduler_test.go
+++ b/pkg/ddl/backfilling_dist_scheduler_test.go
@@ -60,7 +60,8 @@ func TestBackfillingSchedulerLocalMode(t *testing.T) {
task.Step = sch.GetNextStep(&task.TaskBase)
require.Equal(t, proto.BackfillStepReadIndex, task.Step)
execIDs := []string{":4000"}
- metas, err := sch.OnNextSubtasksBatch(context.Background(), nil, task, execIDs, task.Step)
+ ctx := util.WithInternalSourceType(context.Background(), "backfill")
+ metas, err := sch.OnNextSubtasksBatch(ctx, nil, task, execIDs, task.Step)
require.NoError(t, err)
require.Equal(t, len(tblInfo.Partition.Definitions), len(metas))
for i, par := range tblInfo.Partition.Definitions {
@@ -73,7 +74,7 @@ func TestBackfillingSchedulerLocalMode(t *testing.T) {
task.State = proto.TaskStateRunning
task.Step = sch.GetNextStep(&task.TaskBase)
require.Equal(t, proto.StepDone, task.Step)
- metas, err = sch.OnNextSubtasksBatch(context.Background(), nil, task, execIDs, task.Step)
+ metas, err = sch.OnNextSubtasksBatch(ctx, nil, task, execIDs, task.Step)
require.NoError(t, err)
require.Len(t, metas, 0)
@@ -85,7 +86,7 @@ func TestBackfillingSchedulerLocalMode(t *testing.T) {
// 2.1 empty table
tk.MustExec("create table t1(id int primary key, v int)")
task = createAddIndexTask(t, dom, "test", "t1", proto.Backfill, false)
- metas, err = sch.OnNextSubtasksBatch(context.Background(), nil, task, execIDs, task.Step)
+ metas, err = sch.OnNextSubtasksBatch(ctx, nil, task, execIDs, task.Step)
require.NoError(t, err)
require.Equal(t, 0, len(metas))
// 2.2 non empty table.
@@ -97,7 +98,7 @@ func TestBackfillingSchedulerLocalMode(t *testing.T) {
task = createAddIndexTask(t, dom, "test", "t2", proto.Backfill, false)
// 2.2.1 stepInit
task.Step = sch.GetNextStep(&task.TaskBase)
- metas, err = sch.OnNextSubtasksBatch(context.Background(), nil, task, execIDs, task.Step)
+ metas, err = sch.OnNextSubtasksBatch(ctx, nil, task, execIDs, task.Step)
require.NoError(t, err)
require.Equal(t, 1, len(metas))
require.Equal(t, proto.BackfillStepReadIndex, task.Step)
@@ -105,7 +106,7 @@ func TestBackfillingSchedulerLocalMode(t *testing.T) {
task.State = proto.TaskStateRunning
task.Step = sch.GetNextStep(&task.TaskBase)
require.Equal(t, proto.StepDone, task.Step)
- metas, err = sch.OnNextSubtasksBatch(context.Background(), nil, task, execIDs, task.Step)
+ metas, err = sch.OnNextSubtasksBatch(ctx, nil, task, execIDs, task.Step)
require.NoError(t, err)
require.Equal(t, 0, len(metas))
}
diff --git a/pkg/ddl/backfilling_import_cloud.go b/pkg/ddl/backfilling_import_cloud.go
index fd38a268a06cb..16d9c872e4152 100644
--- a/pkg/ddl/backfilling_import_cloud.go
+++ b/pkg/ddl/backfilling_import_cloud.go
@@ -95,6 +95,7 @@ func (m *cloudImportExecutor) RunSubtask(ctx context.Context, subtask *proto.Sub
TotalKVCount: 0,
CheckHotspot: true,
},
+ TS: sm.TS,
}, engineUUID)
if err != nil {
return err
diff --git a/pkg/ddl/backfilling_operators.go b/pkg/ddl/backfilling_operators.go
index a95dc8b426eef..f4e23d6bd586a 100644
--- a/pkg/ddl/backfilling_operators.go
+++ b/pkg/ddl/backfilling_operators.go
@@ -120,7 +120,6 @@ func NewAddIndexIngestPipeline(
sessPool opSessPool,
backendCtx ingest.BackendCtx,
engines []ingest.Engine,
- sessCtx sessionctx.Context,
jobID int64,
tbl table.PhysicalTable,
idxInfos []*model.IndexInfo,
@@ -137,7 +136,7 @@ func NewAddIndexIngestPipeline(
indexes = append(indexes, index)
}
reqSrc := getDDLRequestSource(model.ActionAddIndex)
- copCtx, err := copr.NewCopContext(tbl.Meta(), idxInfos, sessCtx, reqSrc)
+ copCtx, err := NewReorgCopContext(store, reorgMeta, tbl.Meta(), idxInfos, reqSrc)
if err != nil {
return nil, err
}
@@ -174,7 +173,6 @@ func NewWriteIndexToExternalStoragePipeline(
store kv.Storage,
extStoreURI string,
sessPool opSessPool,
- sessCtx sessionctx.Context,
jobID, subtaskID int64,
tbl table.PhysicalTable,
idxInfos []*model.IndexInfo,
@@ -193,7 +191,7 @@ func NewWriteIndexToExternalStoragePipeline(
indexes = append(indexes, index)
}
reqSrc := getDDLRequestSource(model.ActionAddIndex)
- copCtx, err := copr.NewCopContext(tbl.Meta(), idxInfos, sessCtx, reqSrc)
+ copCtx, err := NewReorgCopContext(store, reorgMeta, tbl.Meta(), idxInfos, reqSrc)
if err != nil {
return nil, err
}
@@ -444,9 +442,6 @@ func (w *tableScanWorker) Close() {
}
}
-// OperatorCallBackForTest is used for test to mock scan record error.
-var OperatorCallBackForTest func()
-
func (w *tableScanWorker) scanRecords(task TableScanTask, sender func(IndexRecordChunk)) {
logutil.Logger(w.ctx).Info("start a table scan task",
zap.Int("id", task.ID), zap.Stringer("task", task))
@@ -456,9 +451,7 @@ func (w *tableScanWorker) scanRecords(task TableScanTask, sender func(IndexRecor
failpoint.Inject("mockScanRecordError", func(_ failpoint.Value) {
failpoint.Return(errors.New("mock scan record error"))
})
- failpoint.Inject("scanRecordExec", func(_ failpoint.Value) {
- OperatorCallBackForTest()
- })
+ failpoint.InjectCall("scanRecordExec")
rs, err := buildTableScan(w.ctx, w.copCtx.GetBase(), startTS, task.Start, task.End)
if err != nil {
return err
@@ -656,7 +649,7 @@ func (w *indexIngestLocalWorker) HandleTask(rs IndexRecordChunk, send func(Index
}()
w.indexIngestBaseWorker.HandleTask(rs, send)
// needs to flush and import to avoid too much use of disk.
- _, _, _, err := ingest.TryFlushAllIndexes(w.backendCtx, ingest.FlushModeAuto, w.indexIDs)
+ _, _, _, err := w.backendCtx.Flush(ingest.FlushModeAuto)
if err != nil {
w.ctx.onError(err)
return
@@ -714,10 +707,7 @@ func (w *indexIngestBaseWorker) initSessCtx() {
return
}
w.restore = restoreSessCtx(sessCtx)
- if err := initSessCtx(sessCtx,
- w.reorgMeta.SQLMode,
- w.reorgMeta.Location,
- w.reorgMeta.ResourceGroupName); err != nil {
+ if err := initSessCtx(sessCtx, w.reorgMeta); err != nil {
w.ctx.onError(err)
return
}
@@ -726,8 +716,14 @@ func (w *indexIngestBaseWorker) initSessCtx() {
}
func (w *indexIngestBaseWorker) Close() {
+ // TODO(lance6716): unify the real write action for engineInfo and external
+ // writer.
for _, writer := range w.writers {
- err := writer.Close(w.ctx)
+ ew, ok := writer.(*external.Writer)
+ if !ok {
+ break
+ }
+ err := ew.Close(w.ctx)
if err != nil {
w.ctx.onError(err)
}
@@ -743,13 +739,12 @@ func (w *indexIngestBaseWorker) WriteChunk(rs *IndexRecordChunk) (count int, nex
failpoint.Inject("mockWriteLocalError", func(_ failpoint.Value) {
failpoint.Return(0, nil, errors.New("mock write local error"))
})
- failpoint.Inject("writeLocalExec", func(_ failpoint.Value) {
- OperatorCallBackForTest()
- })
+ failpoint.InjectCall("writeLocalExec", rs.Done)
oprStartTime := time.Now()
vars := w.se.GetSessionVars()
- cnt, lastHandle, err := writeChunkToLocal(w.ctx, w.writers, w.indexes, w.copCtx, vars, rs.Chunk)
+ sc := vars.StmtCtx
+ cnt, lastHandle, err := writeChunkToLocal(w.ctx, w.writers, w.indexes, w.copCtx, sc.TimeZone(), sc.ErrCtx(), vars.GetWriteStmtBufs(), rs.Chunk)
if err != nil || cnt == 0 {
return 0, nil, err
}
@@ -827,24 +822,36 @@ func (s *indexWriteResultSink) flush() error {
failpoint.Inject("mockFlushError", func(_ failpoint.Value) {
failpoint.Return(errors.New("mock flush error"))
})
- for _, index := range s.indexes {
- idxInfo := index.Meta()
- _, _, err := s.backendCtx.Flush(idxInfo.ID, ingest.FlushModeForceGlobal)
- if err != nil {
- if common.ErrFoundDuplicateKeys.Equal(err) {
- err = convertToKeyExistsErr(err, idxInfo, s.tbl.Meta())
- return err
+ _, _, errIdxID, err := s.backendCtx.Flush(ingest.FlushModeForceFlushAndImport)
+ if err != nil {
+ if common.ErrFoundDuplicateKeys.Equal(err) {
+ var idxInfo table.Index
+ for _, idx := range s.indexes {
+ if idx.Meta().ID == errIdxID {
+ idxInfo = idx
+ break
+ }
}
- logutil.Logger(s.ctx).Error("flush error",
- zap.String("category", "ddl"), zap.Error(err))
- return err
+ if idxInfo == nil {
+ logutil.Logger(s.ctx).Error("index not found", zap.Int64("indexID", errIdxID))
+ return kv.ErrKeyExists
+ }
+ return convertToKeyExistsErr(err, idxInfo.Meta(), s.tbl.Meta())
}
+ logutil.Logger(s.ctx).Error("flush error",
+ zap.String("category", "ddl"), zap.Error(err))
+ return err
}
return nil
}
func (s *indexWriteResultSink) Close() error {
- return s.errGroup.Wait()
+ err := s.errGroup.Wait()
+ // for local pipeline
+ if bc := s.backendCtx; bc != nil {
+ bc.UnregisterEngines()
+ }
+ return err
}
func (*indexWriteResultSink) String() string {
diff --git a/pkg/ddl/backfilling_read_index.go b/pkg/ddl/backfilling_read_index.go
index fddfd946c229d..003e23157b879 100644
--- a/pkg/ddl/backfilling_read_index.go
+++ b/pkg/ddl/backfilling_read_index.go
@@ -32,7 +32,6 @@ import (
"github.com/pingcap/tidb/pkg/lightning/backend/external"
"github.com/pingcap/tidb/pkg/metrics"
"github.com/pingcap/tidb/pkg/parser/model"
- "github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/table"
tidblogutil "github.com/pingcap/tidb/pkg/util/logutil"
"go.uber.org/zap"
@@ -105,21 +104,15 @@ func (r *readIndexExecutor) RunSubtask(ctx context.Context, subtask *proto.Subta
return err
}
- sessCtx, err := newSessCtx(
- r.d.store, r.job.ReorgMeta.SQLMode, r.job.ReorgMeta.Location, r.job.ReorgMeta.ResourceGroupName)
- if err != nil {
- return err
- }
-
opCtx := NewOperatorCtx(ctx, subtask.TaskID, subtask.ID)
defer opCtx.Cancel()
r.curRowCount.Store(0)
var pipe *operator.AsyncPipeline
if len(r.cloudStorageURI) > 0 {
- pipe, err = r.buildExternalStorePipeline(opCtx, subtask.ID, sessCtx, sm, subtask.Concurrency)
+ pipe, err = r.buildExternalStorePipeline(opCtx, subtask.ID, sm, subtask.Concurrency)
} else {
- pipe, err = r.buildLocalStorePipeline(opCtx, sessCtx, sm, subtask.Concurrency)
+ pipe, err = r.buildLocalStorePipeline(opCtx, sm, subtask.Concurrency)
}
if err != nil {
return err
@@ -133,12 +126,7 @@ func (r *readIndexExecutor) RunSubtask(ctx context.Context, subtask *proto.Subta
if opCtx.OperatorErr() != nil {
return opCtx.OperatorErr()
}
- if err != nil {
- return err
- }
-
- r.bc.ResetWorkers(r.job.ID)
- return nil
+ return err
}
func (r *readIndexExecutor) RealtimeSummary() *execute.SubtaskSummary {
@@ -154,16 +142,8 @@ func (r *readIndexExecutor) Cleanup(ctx context.Context) error {
return nil
}
-// MockDMLExecutionAddIndexSubTaskFinish is used to mock DML execution during distributed add index.
-var MockDMLExecutionAddIndexSubTaskFinish func()
-
func (r *readIndexExecutor) OnFinished(ctx context.Context, subtask *proto.Subtask) error {
- failpoint.Inject("mockDMLExecutionAddIndexSubTaskFinish", func(val failpoint.Value) {
- //nolint:forcetypeassert
- if val.(bool) {
- MockDMLExecutionAddIndexSubTaskFinish()
- }
- })
+ failpoint.InjectCall("mockDMLExecutionAddIndexSubTaskFinish")
if len(r.cloudStorageURI) == 0 {
return nil
}
@@ -217,7 +197,6 @@ func (r *readIndexExecutor) getTableStartEndKey(sm *BackfillSubTaskMeta) (
func (r *readIndexExecutor) buildLocalStorePipeline(
opCtx *OperatorCtx,
- sessCtx sessionctx.Context,
sm *BackfillSubTaskMeta,
concurrency int,
) (*operator.AsyncPipeline, error) {
@@ -226,15 +205,17 @@ func (r *readIndexExecutor) buildLocalStorePipeline(
return nil, err
}
d := r.d
- engines := make([]ingest.Engine, 0, len(r.indexes))
+ indexIDs := make([]int64, 0, len(r.indexes))
for _, index := range r.indexes {
- ei, err := r.bc.Register(r.job.ID, index.ID, r.job.SchemaName, r.job.TableName)
- if err != nil {
- tidblogutil.Logger(opCtx).Warn("cannot register new engine", zap.Error(err),
- zap.Int64("job ID", r.job.ID), zap.Int64("index ID", index.ID))
- return nil, err
- }
- engines = append(engines, ei)
+ indexIDs = append(indexIDs, index.ID)
+ }
+ engines, err := r.bc.Register(indexIDs, r.job.TableName)
+ if err != nil {
+ tidblogutil.Logger(opCtx).Error("cannot register new engine",
+ zap.Error(err),
+ zap.Int64("job ID", r.job.ID),
+ zap.Int64s("index IDs", indexIDs))
+ return nil, err
}
counter := metrics.BackfillTotalCounter.WithLabelValues(
metrics.GenerateReorgLabel("add_idx_rate", r.job.SchemaName, tbl.Meta().Name.O))
@@ -244,7 +225,6 @@ func (r *readIndexExecutor) buildLocalStorePipeline(
d.sessPool,
r.bc,
engines,
- sessCtx,
r.job.ID,
tbl,
r.indexes,
@@ -261,7 +241,6 @@ func (r *readIndexExecutor) buildLocalStorePipeline(
func (r *readIndexExecutor) buildExternalStorePipeline(
opCtx *OperatorCtx,
subtaskID int64,
- sessCtx sessionctx.Context,
sm *BackfillSubTaskMeta,
concurrency int,
) (*operator.AsyncPipeline, error) {
@@ -290,7 +269,6 @@ func (r *readIndexExecutor) buildExternalStorePipeline(
d.store,
r.cloudStorageURI,
r.d.sessPool,
- sessCtx,
r.job.ID,
subtaskID,
tbl,
diff --git a/pkg/ddl/backfilling_scheduler.go b/pkg/ddl/backfilling_scheduler.go
index 6e3a0ee76fbda..a5107cabbb3f9 100644
--- a/pkg/ddl/backfilling_scheduler.go
+++ b/pkg/ddl/backfilling_scheduler.go
@@ -26,23 +26,29 @@ import (
"github.com/pingcap/tidb/pkg/ddl/ingest"
sess "github.com/pingcap/tidb/pkg/ddl/internal/session"
ddllogutil "github.com/pingcap/tidb/pkg/ddl/logutil"
+ distsqlctx "github.com/pingcap/tidb/pkg/distsql/context"
"github.com/pingcap/tidb/pkg/errctx"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/metrics"
"github.com/pingcap/tidb/pkg/parser/model"
- "github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/resourcemanager/pool/workerpool"
poolutil "github.com/pingcap/tidb/pkg/resourcemanager/util"
"github.com/pingcap/tidb/pkg/sessionctx"
+ "github.com/pingcap/tidb/pkg/sessionctx/stmtctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/table"
- "github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util"
+ contextutil "github.com/pingcap/tidb/pkg/util/context"
"github.com/pingcap/tidb/pkg/util/dbterror"
+ "github.com/pingcap/tidb/pkg/util/execdetails"
"github.com/pingcap/tidb/pkg/util/intest"
"github.com/pingcap/tidb/pkg/util/logutil"
+ "github.com/pingcap/tidb/pkg/util/memory"
"github.com/pingcap/tidb/pkg/util/mock"
decoder "github.com/pingcap/tidb/pkg/util/rowDecoder"
+ "github.com/pingcap/tidb/pkg/util/sqlkiller"
+ "github.com/pingcap/tidb/pkg/util/tiflash"
+ tikvstore "github.com/tikv/client-go/v2/kv"
"go.uber.org/zap"
)
@@ -88,20 +94,19 @@ func newBackfillScheduler(
sessPool *sess.Pool,
tp backfillerType,
tbl table.PhysicalTable,
- sessCtx sessionctx.Context,
jobCtx *JobContext,
) (backfillScheduler, error) {
if tp == typeAddIndexWorker && info.ReorgMeta.ReorgTp == model.ReorgTypeLitMerge {
ctx = logutil.WithCategory(ctx, "ddl-ingest")
return newIngestBackfillScheduler(ctx, info, sessPool, tbl)
}
- return newTxnBackfillScheduler(ctx, info, sessPool, tp, tbl, sessCtx, jobCtx)
+ return newTxnBackfillScheduler(ctx, info, sessPool, tp, tbl, jobCtx)
}
func newTxnBackfillScheduler(ctx context.Context, info *reorgInfo, sessPool *sess.Pool,
- tp backfillerType, tbl table.PhysicalTable, sessCtx sessionctx.Context,
+ tp backfillerType, tbl table.PhysicalTable,
jobCtx *JobContext) (backfillScheduler, error) {
- decColMap, err := makeupDecodeColMap(sessCtx, info.dbInfo.Name, tbl)
+ decColMap, err := makeupDecodeColMap(info.dbInfo.Name, tbl)
if err != nil {
return nil, err
}
@@ -136,26 +141,63 @@ func (b *txnBackfillScheduler) resultChan() <-chan *backfillResult {
return b.resultCh
}
-func newSessCtx(
+// NewReorgCopContext creates a CopContext for reorg
+func NewReorgCopContext(
store kv.Storage,
- sqlMode mysql.SQLMode,
- tzLocation *model.TimeZoneLocation,
- resourceGroupName string,
-) (sessionctx.Context, error) {
+ reorgMeta *model.DDLReorgMeta,
+ tblInfo *model.TableInfo,
+ allIdxInfo []*model.IndexInfo,
+ requestSource string,
+) (copr.CopContext, error) {
+ sessCtx, err := newSessCtx(store, reorgMeta)
+ if err != nil {
+ return nil, err
+ }
+ return copr.NewCopContext(
+ sessCtx.GetExprCtx(),
+ sessCtx.GetDistSQLCtx(),
+ sessCtx.GetSessionVars().StmtCtx.PushDownFlags(),
+ tblInfo,
+ allIdxInfo,
+ requestSource,
+ )
+}
+
+func newSessCtx(store kv.Storage, reorgMeta *model.DDLReorgMeta) (sessionctx.Context, error) {
sessCtx := newReorgSessCtx(store)
- if err := initSessCtx(sessCtx, sqlMode, tzLocation, resourceGroupName); err != nil {
+ if err := initSessCtx(sessCtx, reorgMeta); err != nil {
return nil, errors.Trace(err)
}
return sessCtx, nil
}
+func newDefaultReorgDistSQLCtx(kvClient kv.Client) *distsqlctx.DistSQLContext {
+ warnHandler := contextutil.NewStaticWarnHandler(0)
+ var sqlKiller sqlkiller.SQLKiller
+ var execDetails execdetails.SyncExecDetails
+ return &distsqlctx.DistSQLContext{
+ WarnHandler: warnHandler,
+ Client: kvClient,
+ EnableChunkRPC: true,
+ EnabledRateLimitAction: variable.DefTiDBEnableRateLimitAction,
+ KVVars: tikvstore.NewVariables(&sqlKiller.Signal),
+ SessionMemTracker: memory.NewTracker(memory.LabelForSession, -1),
+ Location: time.UTC,
+ SQLKiller: &sqlKiller,
+ ErrCtx: errctx.NewContextWithLevels(stmtctx.DefaultStmtErrLevels, warnHandler),
+ TiFlashReplicaRead: tiflash.GetTiFlashReplicaReadByStr(variable.DefTiFlashReplicaRead),
+ TiFlashMaxThreads: variable.DefTiFlashMaxThreads,
+ TiFlashMaxBytesBeforeExternalJoin: variable.DefTiFlashMaxBytesBeforeExternalJoin,
+ TiFlashMaxBytesBeforeExternalGroupBy: variable.DefTiFlashMaxBytesBeforeExternalGroupBy,
+ TiFlashMaxBytesBeforeExternalSort: variable.DefTiFlashMaxBytesBeforeExternalSort,
+ TiFlashMaxQueryMemoryPerNode: variable.DefTiFlashMemQuotaQueryPerNode,
+ TiFlashQuerySpillRatio: variable.DefTiFlashQuerySpillRatio,
+ ExecDetails: &execDetails,
+ }
+}
+
// initSessCtx initializes the session context. Be careful to the timezone.
-func initSessCtx(
- sessCtx sessionctx.Context,
- sqlMode mysql.SQLMode,
- tzLocation *model.TimeZoneLocation,
- resGroupName string,
-) error {
+func initSessCtx(sessCtx sessionctx.Context, reorgMeta *model.DDLReorgMeta) error {
// Correct the initial timezone.
tz := *time.UTC
sessCtx.GetSessionVars().TimeZone = &tz
@@ -165,25 +207,21 @@ func initSessCtx(
rowFormat := variable.GetDDLReorgRowFormat()
sessCtx.GetSessionVars().RowEncoder.Enable = rowFormat != variable.DefTiDBRowFormatV1
// Simulate the sql mode environment in the worker sessionCtx.
+ sqlMode := reorgMeta.SQLMode
sessCtx.GetSessionVars().SQLMode = sqlMode
- if err := setSessCtxLocation(sessCtx, tzLocation); err != nil {
+ loc, err := reorgTimeZoneWithTzLoc(reorgMeta.Location)
+ if err != nil {
return errors.Trace(err)
}
- sessCtx.GetSessionVars().StmtCtx.SetTimeZone(sessCtx.GetSessionVars().Location())
+ sessCtx.GetSessionVars().TimeZone = loc
+ sessCtx.GetSessionVars().StmtCtx.SetTimeZone(loc)
- errLevels := sessCtx.GetSessionVars().StmtCtx.ErrLevels()
- errLevels[errctx.ErrGroupBadNull] = errctx.ResolveErrLevel(false, !sqlMode.HasStrictMode())
- errLevels[errctx.ErrGroupDividedByZero] =
- errctx.ResolveErrLevel(!sqlMode.HasErrorForDivisionByZeroMode(), !sqlMode.HasStrictMode())
+ errLevels := reorgErrLevelsWithSQLMode(sqlMode)
sessCtx.GetSessionVars().StmtCtx.SetErrLevels(errLevels)
- typeFlags := types.StrictFlags.
- WithTruncateAsWarning(!sqlMode.HasStrictMode()).
- WithIgnoreInvalidDateErr(sqlMode.HasAllowInvalidDatesMode()).
- WithIgnoreZeroInDate(!sqlMode.HasStrictMode() || sqlMode.HasAllowInvalidDatesMode()).
- WithCastTimeToYearThroughConcat(true)
+ typeFlags := reorgTypeFlagsWithSQLMode(sqlMode)
sessCtx.GetSessionVars().StmtCtx.SetTypeFlags(typeFlags)
- sessCtx.GetSessionVars().StmtCtx.ResourceGroupName = resGroupName
+ sessCtx.GetSessionVars().StmtCtx.ResourceGroupName = reorgMeta.ResourceGroupName
// Prevent initializing the mock context in the workers concurrently.
// For details, see https://github.com/pingcap/tidb/issues/40879.
@@ -236,17 +274,17 @@ func (b *txnBackfillScheduler) adjustWorkerSize() error {
workerCnt := b.expectedWorkerSize()
// Increase the worker.
for i := len(b.workers); i < workerCnt; i++ {
- sessCtx, err := newSessCtx(reorgInfo.d.store, reorgInfo.ReorgMeta.SQLMode, reorgInfo.ReorgMeta.Location, reorgInfo.ReorgMeta.ResourceGroupName)
- if err != nil {
- return err
- }
var (
runner *backfillWorker
worker backfiller
)
switch b.tp {
case typeAddIndexWorker:
- backfillCtx := newBackfillCtx(reorgInfo.d, i, sessCtx, job.SchemaName, b.tbl, jc, "add_idx_rate", false)
+ backfillCtx, err := newBackfillCtx(i, reorgInfo, job.SchemaName, b.tbl, jc, "add_idx_rate", false)
+ if err != nil {
+ return err
+ }
+
idxWorker, err := newAddIndexTxnWorker(b.decodeColMap, b.tbl, backfillCtx,
job.ID, reorgInfo.elements, reorgInfo.currElement.TypeKey)
if err != nil {
@@ -255,25 +293,29 @@ func (b *txnBackfillScheduler) adjustWorkerSize() error {
runner = newBackfillWorker(b.ctx, idxWorker)
worker = idxWorker
case typeAddIndexMergeTmpWorker:
- backfillCtx := newBackfillCtx(reorgInfo.d, i, sessCtx, job.SchemaName, b.tbl, jc, "merge_tmp_idx_rate", false)
+ backfillCtx, err := newBackfillCtx(i, reorgInfo, job.SchemaName, b.tbl, jc, "merge_tmp_idx_rate", false)
+ if err != nil {
+ return err
+ }
tmpIdxWorker := newMergeTempIndexWorker(backfillCtx, b.tbl, reorgInfo.elements)
runner = newBackfillWorker(b.ctx, tmpIdxWorker)
worker = tmpIdxWorker
case typeUpdateColumnWorker:
- // Setting InCreateOrAlterStmt tells the difference between SELECT casting and ALTER COLUMN casting.
- sessCtx.GetSessionVars().StmtCtx.InCreateOrAlterStmt = true
- sessCtx.GetSessionVars().StmtCtx.SetTypeFlags(
- sessCtx.GetSessionVars().StmtCtx.TypeFlags().
- WithIgnoreZeroDateErr(!reorgInfo.ReorgMeta.SQLMode.HasStrictMode()))
- updateWorker := newUpdateColumnWorker(sessCtx, i, b.tbl, b.decodeColMap, reorgInfo, jc)
+ updateWorker, err := newUpdateColumnWorker(i, b.tbl, b.decodeColMap, reorgInfo, jc)
+ if err != nil {
+ return err
+ }
runner = newBackfillWorker(b.ctx, updateWorker)
worker = updateWorker
case typeCleanUpIndexWorker:
- idxWorker := newCleanUpIndexWorker(sessCtx, i, b.tbl, b.decodeColMap, reorgInfo, jc)
+ idxWorker, err := newCleanUpIndexWorker(i, b.tbl, b.decodeColMap, reorgInfo, jc)
+ if err != nil {
+ return err
+ }
runner = newBackfillWorker(b.ctx, idxWorker)
worker = idxWorker
case typeReorgPartitionWorker:
- partWorker, err := newReorgPartitionWorker(sessCtx, i, b.tbl, b.decodeColMap, reorgInfo, jc)
+ partWorker, err := newReorgPartitionWorker(i, b.tbl, b.decodeColMap, reorgInfo, jc)
if err != nil {
return err
}
@@ -354,6 +396,15 @@ func newIngestBackfillScheduler(
}, nil
}
+func (b *ingestBackfillScheduler) importStarted() bool {
+ job := b.reorgInfo.Job
+ bc, ok := ingest.LitBackCtxMgr.Load(job.ID)
+ if !ok {
+ return false
+ }
+ return bc.ImportStarted()
+}
+
func (b *ingestBackfillScheduler) setupWorkers() error {
job := b.reorgInfo.Job
bc, ok := ingest.LitBackCtxMgr.Load(job.ID)
@@ -371,10 +422,26 @@ func (b *ingestBackfillScheduler) setupWorkers() error {
if err != nil {
return errors.Trace(err)
}
+
+ indexIDs := make([]int64, 0, len(b.reorgInfo.elements))
+ for _, e := range b.reorgInfo.elements {
+ indexIDs = append(indexIDs, e.ID)
+ }
+ engines, err := b.backendCtx.Register(indexIDs, job.TableName)
+ if err != nil {
+ return errors.Trace(err)
+ }
+
b.copReqSenderPool = copReqSenderPool
readerCnt, writerCnt := b.expectedWorkerSize()
- writerPool := workerpool.NewWorkerPool[IndexRecordChunk]("ingest_writer",
- poolutil.DDL, writerCnt, b.createWorker)
+ writerPool := workerpool.NewWorkerPool[IndexRecordChunk](
+ "ingest_writer",
+ poolutil.DDL,
+ writerCnt,
+ func() workerpool.Worker[IndexRecordChunk, workerpool.None] {
+ return b.createWorker(indexIDs, engines)
+ },
+ )
writerPool.Start(b.ctx)
b.writerPool = writerPool
b.copReqSenderPool.chunkSender = writerPool
@@ -397,7 +464,7 @@ func (b *ingestBackfillScheduler) close(force bool) {
b.writerPool.ReleaseAndWait()
}
if b.checkpointMgr != nil {
- b.checkpointMgr.Sync()
+ b.checkpointMgr.Flush()
// Get the latest status after all workers are closed so that the result is more accurate.
cnt, nextKey := b.checkpointMgr.Status()
b.sendResult(&backfillResult{
@@ -406,13 +473,9 @@ func (b *ingestBackfillScheduler) close(force bool) {
})
}
close(b.resultCh)
- if intest.InTest && len(b.copReqSenderPool.srcChkPool) != copReadChunkPoolSize() {
+ if intest.InTest && b.copReqSenderPool != nil && len(b.copReqSenderPool.srcChkPool) != copReadChunkPoolSize() {
panic(fmt.Sprintf("unexpected chunk size %d", len(b.copReqSenderPool.srcChkPool)))
}
- if !force {
- jobID := b.reorgInfo.ID
- b.backendCtx.ResetWorkers(jobID)
- }
}
func (b *ingestBackfillScheduler) sendTask(task *reorgBackfillTask) error {
@@ -446,37 +509,16 @@ func (b *ingestBackfillScheduler) adjustWorkerSize() error {
return nil
}
-func (b *ingestBackfillScheduler) createWorker() workerpool.Worker[IndexRecordChunk, workerpool.None] {
+func (b *ingestBackfillScheduler) createWorker(
+ indexIDs []int64,
+ engines []ingest.Engine,
+) workerpool.Worker[IndexRecordChunk, workerpool.None] {
reorgInfo := b.reorgInfo
job := reorgInfo.Job
- sessCtx, err := newSessCtx(reorgInfo.d.store, reorgInfo.ReorgMeta.SQLMode, reorgInfo.ReorgMeta.Location, reorgInfo.ReorgMeta.ResourceGroupName)
- if err != nil {
- b.sendResult(&backfillResult{err: err})
- return nil
- }
- bcCtx := b.backendCtx
- indexIDs := make([]int64, 0, len(reorgInfo.elements))
- engines := make([]ingest.Engine, 0, len(reorgInfo.elements))
- for _, elem := range reorgInfo.elements {
- ei, err := bcCtx.Register(job.ID, elem.ID, job.SchemaName, job.TableName)
- if err != nil {
- // Return an error only if it is the first worker.
- if b.writerMaxID == 0 {
- b.sendResult(&backfillResult{err: err})
- return nil
- }
- logutil.Logger(b.ctx).Warn("cannot create new writer", zap.Error(err),
- zap.Int64("job ID", reorgInfo.ID), zap.Int64("index ID", elem.ID))
- return nil
- }
- indexIDs = append(indexIDs, elem.ID)
- engines = append(engines, ei)
- }
-
worker, err := newAddIndexIngestWorker(
- b.ctx, b.tbl, reorgInfo.d, engines, b.resultCh, job.ID,
- reorgInfo.SchemaName, indexIDs, b.writerMaxID,
- b.copReqSenderPool, sessCtx, b.checkpointMgr)
+ b.ctx, b.tbl, reorgInfo, engines, b.resultCh, job.ID,
+ indexIDs, b.writerMaxID,
+ b.copReqSenderPool, b.checkpointMgr)
if err != nil {
// Return an error only if it is the first worker.
if b.writerMaxID == 0 {
@@ -503,18 +545,13 @@ func (b *ingestBackfillScheduler) createCopReqSenderPool() (*copReqSenderPool, e
}
allIndexInfos = append(allIndexInfos, indexInfo)
}
- sessCtx, err := newSessCtx(ri.d.store, ri.ReorgMeta.SQLMode, ri.ReorgMeta.Location, ri.ReorgMeta.ResourceGroupName)
- if err != nil {
- logutil.Logger(b.ctx).Warn("cannot init cop request sender", zap.Error(err))
- return nil, err
- }
reqSrc := getDDLRequestSource(model.ActionAddIndex)
- copCtx, err := copr.NewCopContext(b.tbl.Meta(), allIndexInfos, sessCtx, reqSrc)
+ copCtx, err := NewReorgCopContext(ri.d.store, ri.ReorgMeta, b.tbl.Meta(), allIndexInfos, reqSrc)
if err != nil {
logutil.Logger(b.ctx).Warn("cannot init cop request sender", zap.Error(err))
return nil, err
}
- return newCopReqSenderPool(b.ctx, copCtx, sessCtx.GetStore(), b.taskCh, b.sessPool, b.checkpointMgr), nil
+ return newCopReqSenderPool(b.ctx, copCtx, ri.d.store, b.taskCh, b.sessPool, b.checkpointMgr), nil
}
func (b *ingestBackfillScheduler) expectedWorkerSize() (readerSize int, writerSize int) {
@@ -585,7 +622,7 @@ func (w *addIndexIngestWorker) HandleTask(rs IndexRecordChunk, _ func(workerpool
cnt, nextKey := w.checkpointMgr.Status()
result.totalCount = cnt
result.nextKey = nextKey
- result.err = w.checkpointMgr.UpdateCurrent(rs.ID, count)
+ result.err = w.checkpointMgr.UpdateWrittenKeys(rs.ID, count)
} else {
result.addedCount = count
result.scanCount = count
diff --git a/pkg/ddl/backfilling_test.go b/pkg/ddl/backfilling_test.go
index 736c08eb0bde8..21b8255b12e95 100644
--- a/pkg/ddl/backfilling_test.go
+++ b/pkg/ddl/backfilling_test.go
@@ -18,11 +18,15 @@ import (
"bytes"
"context"
"testing"
+ "time"
+ "github.com/pingcap/errors"
"github.com/pingcap/tidb/pkg/ddl/ingest"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/sessionctx"
+ contextutil "github.com/pingcap/tidb/pkg/util/context"
+ "github.com/pingcap/tidb/pkg/util/mock"
"github.com/stretchr/testify/require"
)
@@ -84,3 +88,107 @@ func TestPickBackfillType(t *testing.T) {
require.NoError(t, err)
require.Equal(t, tp, model.ReorgTypeLitMerge)
}
+
+// TestReorgExprContext is used in refactor stage to make sure the newReorgExprCtx() is
+// compatible with newReorgSessCtx(nil).GetExprCtx() to make it safe to replace `mock.Context` usage.
+// After refactor, the TestReorgExprContext can be removed.
+func TestReorgExprContext(t *testing.T) {
+ sctx := newReorgSessCtx(nil)
+ sessCtx := sctx.GetExprCtx()
+ exprCtx := newReorgExprCtx()
+ cs1, col1 := sessCtx.GetCharsetInfo()
+ cs2, col2 := exprCtx.GetCharsetInfo()
+ require.Equal(t, cs1, cs2)
+ require.Equal(t, col1, col2)
+ require.Equal(t, sessCtx.GetDefaultCollationForUTF8MB4(), exprCtx.GetDefaultCollationForUTF8MB4())
+ if sessCtx.GetBlockEncryptionMode() == "" {
+ // The newReorgSessCtx returns a block encryption mode as an empty string.
+ // Though it is not a valid value, it does not matter because `GetBlockEncryptionMode` is never used in DDL.
+ // So we do not want to modify the behavior of `newReorgSessCtx` or `newReorgExprCtx`, and just to
+ // place the test code here to check:
+ // If `GetBlockEncryptionMode` still returns empty string in `newReorgSessCtx`, that means the behavior is
+ // not changed, and we just need to return a default value for `newReorgExprCtx`.
+ // If `GetBlockEncryptionMode` returns some other values, that means `GetBlockEncryptionMode` may have been
+ // used in somewhere and two return values should be the same.
+ require.Equal(t, "aes-128-ecb", exprCtx.GetBlockEncryptionMode())
+ } else {
+ require.Equal(t, sessCtx.GetBlockEncryptionMode(), exprCtx.GetBlockEncryptionMode())
+ }
+ require.Equal(t, sessCtx.GetSysdateIsNow(), exprCtx.GetSysdateIsNow())
+ require.Equal(t, sessCtx.GetNoopFuncsMode(), exprCtx.GetNoopFuncsMode())
+ require.Equal(t, sessCtx.IsUseCache(), exprCtx.IsUseCache())
+ require.Equal(t, sessCtx.IsInNullRejectCheck(), exprCtx.IsInNullRejectCheck())
+ require.Equal(t, sessCtx.ConnectionID(), exprCtx.ConnectionID())
+ require.Equal(t, sessCtx.AllocPlanColumnID(), exprCtx.AllocPlanColumnID())
+ require.Equal(t, sessCtx.GetWindowingUseHighPrecision(), exprCtx.GetWindowingUseHighPrecision())
+ require.Equal(t, sessCtx.GetGroupConcatMaxLen(), exprCtx.GetGroupConcatMaxLen())
+
+ evalCtx1 := sessCtx.GetEvalCtx()
+ evalCtx := exprCtx.GetEvalCtx()
+ require.Equal(t, evalCtx1.SQLMode(), evalCtx.SQLMode())
+ tc1 := evalCtx1.TypeCtx()
+ tc2 := evalCtx.TypeCtx()
+ require.Equal(t, tc1.Flags(), tc2.Flags())
+ require.Equal(t, tc1.Location().String(), tc2.Location().String())
+ ec1 := evalCtx1.ErrCtx()
+ ec2 := evalCtx.ErrCtx()
+ require.Equal(t, ec1.LevelMap(), ec2.LevelMap())
+ require.Equal(t, time.UTC, sctx.GetSessionVars().Location())
+ require.Equal(t, time.UTC, sctx.GetSessionVars().StmtCtx.TimeZone())
+ require.Equal(t, time.UTC, evalCtx1.Location())
+ require.Equal(t, time.UTC, evalCtx.Location())
+ require.Equal(t, evalCtx1.CurrentDB(), evalCtx.CurrentDB())
+ tm1, err := evalCtx1.CurrentTime()
+ require.NoError(t, err)
+ tm2, err := evalCtx.CurrentTime()
+ require.NoError(t, err)
+ require.InDelta(t, tm1.Unix(), tm2.Unix(), 2)
+ require.Equal(t, evalCtx1.GetMaxAllowedPacket(), evalCtx.GetMaxAllowedPacket())
+ require.Equal(t, evalCtx1.GetDefaultWeekFormatMode(), evalCtx.GetDefaultWeekFormatMode())
+ require.Equal(t, evalCtx1.GetDivPrecisionIncrement(), evalCtx.GetDivPrecisionIncrement())
+}
+
+type mockStorage struct {
+ kv.Storage
+ client kv.Client
+}
+
+func (s *mockStorage) GetClient() kv.Client {
+ return s.client
+}
+
+// TestReorgExprContext is used in refactor stage to make sure the newDefaultReorgDistSQLCtx() is
+// compatible with newReorgSessCtx(nil).GetDistSQLCtx() to make it safe to replace `mock.Context` usage.
+// After refactor, the TestReorgExprContext can be removed.
+func TestReorgDistSQLCtx(t *testing.T) {
+ store := &mockStorage{client: &mock.Client{}}
+ ctx1 := newReorgSessCtx(store).GetDistSQLCtx()
+ ctx2 := newDefaultReorgDistSQLCtx(store.client)
+
+ // set the same warnHandler to make two contexts equal
+ ctx1.WarnHandler = ctx2.WarnHandler
+
+ // set the same KVVars to make two contexts equal
+ require.Equal(t, uint32(0), *ctx1.KVVars.Killed)
+ require.Equal(t, uint32(0), *ctx2.KVVars.Killed)
+ ctx1.KVVars.Killed = ctx2.KVVars.Killed
+
+ // set the same SessionMemTracker to make two contexts equal
+ require.Equal(t, ctx1.SessionMemTracker.Label(), ctx2.SessionMemTracker.Label())
+ require.Equal(t, ctx1.SessionMemTracker.GetBytesLimit(), ctx2.SessionMemTracker.GetBytesLimit())
+ ctx1.SessionMemTracker = ctx2.SessionMemTracker
+
+ // set the same ErrCtx to make two contexts equal
+ require.Equal(t, ctx1.ErrCtx.LevelMap(), ctx2.ErrCtx.LevelMap())
+ require.Equal(t, 0, ctx2.WarnHandler.(contextutil.WarnHandler).WarningCount())
+ ctx2.ErrCtx.AppendWarning(errors.New("warn"))
+ require.Equal(t, 1, ctx2.WarnHandler.(contextutil.WarnHandler).WarningCount())
+ ctx1.ErrCtx = ctx2.ErrCtx
+
+ // set the same ExecDetails to make two contexts equal
+ require.NotNil(t, ctx1.ExecDetails)
+ require.NotNil(t, ctx2.ExecDetails)
+ ctx1.ExecDetails = ctx2.ExecDetails
+
+ require.Equal(t, ctx1, ctx2)
+}
diff --git a/pkg/ddl/bench_test.go b/pkg/ddl/bench_test.go
index e26f751618a2a..fc0cd7ed44678 100644
--- a/pkg/ddl/bench_test.go
+++ b/pkg/ddl/bench_test.go
@@ -43,7 +43,10 @@ func BenchmarkExtractDatumByOffsets(b *testing.B) {
require.NoError(b, err)
tblInfo := tbl.Meta()
idxInfo := tblInfo.FindIndexByName("idx")
- copCtx, err := copr.NewCopContextSingleIndex(tblInfo, idxInfo, tk.Session(), "")
+ sctx := tk.Session()
+ copCtx, err := ddl.NewReorgCopContext(store, ddl.NewDDLReorgMeta(sctx), tblInfo, []*model.IndexInfo{idxInfo}, "")
+ require.NoError(b, err)
+ require.IsType(b, copCtx, &copr.CopContextSingleIndex{})
require.NoError(b, err)
startKey := tbl.RecordPrefix()
endKey := startKey.PrefixNext()
@@ -62,7 +65,7 @@ func BenchmarkExtractDatumByOffsets(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
- ddl.ExtractDatumByOffsetsForTest(row, offsets, c.ExprColumnInfos, handleDataBuf)
+ ddl.ExtractDatumByOffsetsForTest(tk.Session().GetExprCtx().GetEvalCtx(), row, offsets, c.ExprColumnInfos, handleDataBuf)
}
}
diff --git a/pkg/ddl/cluster.go b/pkg/ddl/cluster.go
index 5a6aabd5b65f0..863e8d3ebb526 100644
--- a/pkg/ddl/cluster.go
+++ b/pkg/ddl/cluster.go
@@ -220,8 +220,8 @@ func checkSystemSchemaID(t *meta.Meta, schemaID int64, flashbackTSString string)
return nil
}
-func checkAndSetFlashbackClusterInfo(se sessionctx.Context, d *ddlCtx, t *meta.Meta, job *model.Job, flashbackTS uint64) (err error) {
- if err = ValidateFlashbackTS(d.ctx, se, flashbackTS); err != nil {
+func checkAndSetFlashbackClusterInfo(ctx context.Context, se sessionctx.Context, d *ddlCtx, t *meta.Meta, job *model.Job, flashbackTS uint64) (err error) {
+ if err = ValidateFlashbackTS(ctx, se, flashbackTS); err != nil {
return err
}
@@ -231,13 +231,13 @@ func checkAndSetFlashbackClusterInfo(se sessionctx.Context, d *ddlCtx, t *meta.M
if err = closePDSchedule(); err != nil {
return err
}
- if err = setTiDBEnableAutoAnalyze(d.ctx, se, variable.Off); err != nil {
+ if err = setTiDBEnableAutoAnalyze(ctx, se, variable.Off); err != nil {
return err
}
- if err = setTiDBSuperReadOnly(d.ctx, se, variable.On); err != nil {
+ if err = setTiDBSuperReadOnly(ctx, se, variable.On); err != nil {
return err
}
- if err = setTiDBTTLJobEnable(d.ctx, se, variable.Off); err != nil {
+ if err = setTiDBTTLJobEnable(ctx, se, variable.Off); err != nil {
return err
}
@@ -256,12 +256,12 @@ func checkAndSetFlashbackClusterInfo(se sessionctx.Context, d *ddlCtx, t *meta.M
// Check if there is an upgrade during [flashbackTS, now)
sql := fmt.Sprintf("select VARIABLE_VALUE from mysql.tidb as of timestamp '%s' where VARIABLE_NAME='tidb_server_version'", flashbackTSString)
- rows, err := sess.NewSession(se).Execute(d.ctx, sql, "check_tidb_server_version")
+ rows, err := sess.NewSession(se).Execute(ctx, sql, "check_tidb_server_version")
if err != nil || len(rows) == 0 {
return errors.Errorf("Get history `tidb_server_version` failed, can't do flashback")
}
sql = fmt.Sprintf("select 1 from mysql.tidb where VARIABLE_NAME='tidb_server_version' and VARIABLE_VALUE=%s", rows[0].GetString(0))
- rows, err = sess.NewSession(se).Execute(d.ctx, sql, "check_tidb_server_version")
+ rows, err = sess.NewSession(se).Execute(ctx, sql, "check_tidb_server_version")
if err != nil {
return errors.Trace(err)
}
@@ -271,7 +271,7 @@ func checkAndSetFlashbackClusterInfo(se sessionctx.Context, d *ddlCtx, t *meta.M
// Check is there a DDL task at flashbackTS.
sql = fmt.Sprintf("select count(*) from mysql.%s as of timestamp '%s'", JobTable, flashbackTSString)
- rows, err = sess.NewSession(se).Execute(d.ctx, sql, "check_history_job")
+ rows, err = sess.NewSession(se).Execute(ctx, sql, "check_history_job")
if err != nil || len(rows) == 0 {
return errors.Errorf("Get history ddl jobs failed, can't do flashback")
}
@@ -609,12 +609,12 @@ func flashbackToVersion(
).RunOnRange(ctx, startKey, endKey)
}
-func splitRegionsByKeyRanges(d *ddlCtx, keyRanges []kv.KeyRange) {
+func splitRegionsByKeyRanges(ctx context.Context, d *ddlCtx, keyRanges []kv.KeyRange) {
if s, ok := d.store.(kv.SplittableStore); ok {
for _, keys := range keyRanges {
for {
// tableID is useless when scatter == false
- _, err := s.SplitRegions(d.ctx, [][]byte{keys.StartKey, keys.EndKey}, false, nil)
+ _, err := s.SplitRegions(ctx, [][]byte{keys.StartKey, keys.EndKey}, false, nil)
if err == nil {
break
}
@@ -696,12 +696,12 @@ func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
return ver, nil
// Stage 2, check flashbackTS, close GC and PD schedule, get flashback key ranges.
case model.StateDeleteOnly:
- if err = checkAndSetFlashbackClusterInfo(sess, d, t, job, flashbackTS); err != nil {
+ if err = checkAndSetFlashbackClusterInfo(w.ctx, sess, d, t, job, flashbackTS); err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
// We should get startTS here to avoid lost startTS when TiDB crashed during send prepare flashback RPC.
- startTS, err = d.store.GetOracle().GetTimestamp(d.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope})
+ startTS, err = d.store.GetOracle().GetTimestamp(w.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope})
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
@@ -722,10 +722,10 @@ func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
return updateSchemaVersion(d, t, job)
}
// Split region by keyRanges, make sure no unrelated key ranges be locked.
- splitRegionsByKeyRanges(d, keyRanges)
+ splitRegionsByKeyRanges(w.ctx, d, keyRanges)
totalRegions.Store(0)
for _, r := range keyRanges {
- if err = flashbackToVersion(d.ctx, d,
+ if err = flashbackToVersion(w.ctx, d,
func(ctx context.Context, r tikvstore.KeyRange) (rangetask.TaskStat, error) {
stats, err := SendPrepareFlashbackToVersionRPC(ctx, d.store.(tikv.Storage), flashbackTS, startTS, r)
totalRegions.Add(uint64(stats.CompletedRegions))
@@ -738,7 +738,7 @@ func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
job.Args[totalLockedRegionsOffset] = totalRegions.Load()
// We should get commitTS here to avoid lost commitTS when TiDB crashed during send flashback RPC.
- commitTS, err = d.store.GetOracle().GetTimestamp(d.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope})
+ commitTS, err = d.store.GetOracle().GetTimestamp(w.ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope})
if err != nil {
return ver, errors.Trace(err)
}
@@ -756,7 +756,7 @@ func (w *worker) onFlashbackCluster(d *ddlCtx, t *meta.Meta, job *model.Job) (ve
}
for _, r := range keyRanges {
- if err = flashbackToVersion(d.ctx, d,
+ if err = flashbackToVersion(w.ctx, d,
func(ctx context.Context, r tikvstore.KeyRange) (rangetask.TaskStat, error) {
// Use same startTS as prepare phase to simulate 1PC txn.
stats, err := SendFlashbackToVersionRPC(ctx, d.store.(tikv.Storage), flashbackTS, startTS, commitTS, r)
diff --git a/pkg/ddl/column.go b/pkg/ddl/column.go
index 16c4a1664b81b..f263fcf8a2619 100644
--- a/pkg/ddl/column.go
+++ b/pkg/ddl/column.go
@@ -31,6 +31,7 @@ import (
sess "github.com/pingcap/tidb/pkg/ddl/internal/session"
"github.com/pingcap/tidb/pkg/ddl/logutil"
"github.com/pingcap/tidb/pkg/expression"
+ exprctx "github.com/pingcap/tidb/pkg/expression/context"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/meta"
@@ -171,6 +172,7 @@ func onAddColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error)
case model.StateWriteReorganization:
// reorganization -> public
// Adjust table column offset.
+ failpoint.InjectCall("onAddColumnStateWriteReorg")
offset, err := LocateOffsetToMove(columnInfo.Offset, pos, tblInfo)
if err != nil {
return ver, errors.Trace(err)
@@ -270,6 +272,7 @@ func onDropColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
}
case model.StateWriteOnly:
// write only -> delete only
+ failpoint.InjectCall("onDropColumnStateWriteOnly")
colInfo.State = model.StateDeleteOnly
tblInfo.MoveColumnInfo(colInfo.Offset, len(tblInfo.Columns)-1)
if len(idxInfos) > 0 {
@@ -491,11 +494,11 @@ func getModifyColumnInfo(t *meta.Meta, job *model.Job) (*model.DBInfo, *model.Ta
// Otherwise we set the zero value as original default value.
// Besides, in insert & update records, we have already implement using the casted value of relative column to insert
// rather than the original default value.
-func GetOriginDefaultValueForModifyColumn(sessCtx sessionctx.Context, changingCol, oldCol *model.ColumnInfo) (any, error) {
+func GetOriginDefaultValueForModifyColumn(ctx exprctx.BuildContext, changingCol, oldCol *model.ColumnInfo) (any, error) {
var err error
originDefVal := oldCol.GetOriginDefaultValue()
if originDefVal != nil {
- odv, err := table.CastValue(sessCtx, types.NewDatum(originDefVal), changingCol, false, false)
+ odv, err := table.CastColumnValue(ctx, types.NewDatum(originDefVal), changingCol, false, false)
if err != nil {
logutil.DDLLogger().Info("cast origin default value failed", zap.Error(err))
}
@@ -580,7 +583,7 @@ func (w *worker) onModifyColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver in
changingCol.Name = newColName
changingCol.ChangeStateInfo = &model.ChangeStateInfo{DependencyColumnOffset: oldCol.Offset}
- originDefVal, err := GetOriginDefaultValueForModifyColumn(newReorgSessCtx(d.store), changingCol, oldCol)
+ originDefVal, err := GetOriginDefaultValueForModifyColumn(newReorgExprCtx(), changingCol, oldCol)
if err != nil {
return ver, errors.Trace(err)
}
@@ -1093,7 +1096,7 @@ func (w *worker) updatePhysicalTableRow(t table.Table, reorgInfo *reorgInfo) err
// https://github.com/pingcap/tidb/issues/38297
return dbterror.ErrCancelledDDLJob.GenWithStack("Modify Column on partitioned table / typeUpdateColumnWorker not yet supported.")
}
- err := w.writePhysicalTableRecord(w.sessPool, p, workType, reorgInfo)
+ err := w.writePhysicalTableRecord(w.ctx, w.sessPool, p, workType, reorgInfo)
if err != nil {
return err
}
@@ -1105,7 +1108,7 @@ func (w *worker) updatePhysicalTableRow(t table.Table, reorgInfo *reorgInfo) err
return nil
}
if tbl, ok := t.(table.PhysicalTable); ok {
- return w.writePhysicalTableRecord(w.sessPool, tbl, typeUpdateColumnWorker, reorgInfo)
+ return w.writePhysicalTableRecord(w.ctx, w.sessPool, tbl, typeUpdateColumnWorker, reorgInfo)
}
return dbterror.ErrCancelledDDLJob.GenWithStack("internal error for phys tbl id: %d tbl id: %d", reorgInfo.PhysicalTableID, t.Meta().ID)
}
@@ -1209,11 +1212,23 @@ type updateColumnWorker struct {
checksumNeeded bool
}
-func newUpdateColumnWorker(sessCtx sessionctx.Context, id int, t table.PhysicalTable, decodeColMap map[int64]decoder.Column, reorgInfo *reorgInfo, jc *JobContext) *updateColumnWorker {
+func newUpdateColumnWorker(id int, t table.PhysicalTable, decodeColMap map[int64]decoder.Column, reorgInfo *reorgInfo, jc *JobContext) (*updateColumnWorker, error) {
+ bCtx, err := newBackfillCtx(id, reorgInfo, reorgInfo.SchemaName, t, jc, "update_col_rate", false)
+ if err != nil {
+ return nil, err
+ }
+
+ sessCtx := bCtx.sessCtx
+ sessCtx.GetSessionVars().StmtCtx.SetTypeFlags(
+ sessCtx.GetSessionVars().StmtCtx.TypeFlags().
+ WithIgnoreZeroDateErr(!reorgInfo.ReorgMeta.SQLMode.HasStrictMode()))
+ bCtx.exprCtx = bCtx.sessCtx.GetExprCtx()
+ bCtx.tblCtx = bCtx.sessCtx.GetTableCtx()
+
if !bytes.Equal(reorgInfo.currElement.TypeKey, meta.ColumnElementKey) {
logutil.DDLLogger().Error("Element type for updateColumnWorker incorrect", zap.String("jobQuery", reorgInfo.Query),
zap.Stringer("reorgInfo", reorgInfo))
- return nil
+ return nil, nil
}
var oldCol, newCol *model.ColumnInfo
for _, col := range t.WritableCols() {
@@ -1245,13 +1260,13 @@ func newUpdateColumnWorker(sessCtx sessionctx.Context, id int, t table.PhysicalT
}
}
return &updateColumnWorker{
- backfillCtx: newBackfillCtx(reorgInfo.d, id, sessCtx, reorgInfo.SchemaName, t, jc, "update_col_rate", false),
+ backfillCtx: bCtx,
oldColInfo: oldCol,
newColInfo: newCol,
rowDecoder: rowDecoder,
rowMap: make(map[int64]types.Datum, len(decodeColMap)),
checksumNeeded: checksumNeeded,
- }
+ }, nil
}
func (w *updateColumnWorker) AddMetricInfo(cnt float64) {
@@ -1291,7 +1306,7 @@ func (w *updateColumnWorker) fetchRowColVals(txn kv.Transaction, taskRange reorg
taskDone := false
var lastAccessedHandle kv.Key
oprStartTime := startTime
- err := iterateSnapshotKeys(w.jobContext, w.sessCtx.GetStore(), taskRange.priority, taskRange.physicalTable.RecordPrefix(),
+ err := iterateSnapshotKeys(w.jobContext, w.ddlCtx.store, taskRange.priority, taskRange.physicalTable.RecordPrefix(),
txn.StartTS(), taskRange.startKey, taskRange.endKey, func(handle kv.Handle, recordKey kv.Key, rawRow []byte) (bool, error) {
oprEndTime := time.Now()
logSlowOperations(oprEndTime.Sub(oprStartTime), "iterateSnapshotKeys in updateColumnWorker fetchRowColVals", 0)
@@ -1326,8 +1341,8 @@ func (w *updateColumnWorker) fetchRowColVals(txn kv.Transaction, taskRange reorg
}
func (w *updateColumnWorker) getRowRecord(handle kv.Handle, recordKey []byte, rawRow []byte) error {
- sysTZ := w.sessCtx.GetSessionVars().StmtCtx.TimeZone()
- _, err := w.rowDecoder.DecodeTheExistedColumnMap(w.sessCtx, handle, rawRow, sysTZ, w.rowMap)
+ sysTZ := w.loc
+ _, err := w.rowDecoder.DecodeTheExistedColumnMap(w.exprCtx, handle, rawRow, sysTZ, w.rowMap)
if err != nil {
return errors.Trace(dbterror.ErrCantDecodeRecord.GenWithStackByArgs("column", err))
}
@@ -1340,26 +1355,26 @@ func (w *updateColumnWorker) getRowRecord(handle kv.Handle, recordKey []byte, ra
var recordWarning *terror.Error
// Since every updateColumnWorker handle their own work individually, we can cache warning in statement context when casting datum.
- oldWarn := w.sessCtx.GetSessionVars().StmtCtx.GetWarnings()
+ oldWarn := w.warnings.GetWarnings()
if oldWarn == nil {
oldWarn = []contextutil.SQLWarn{}
} else {
oldWarn = oldWarn[:0]
}
- w.sessCtx.GetSessionVars().StmtCtx.SetWarnings(oldWarn)
+ w.warnings.SetWarnings(oldWarn)
val := w.rowMap[w.oldColInfo.ID]
col := w.newColInfo
if val.Kind() == types.KindNull && col.FieldType.GetType() == mysql.TypeTimestamp && mysql.HasNotNullFlag(col.GetFlag()) {
- if v, err := expression.GetTimeCurrentTimestamp(w.sessCtx.GetExprCtx().GetEvalCtx(), col.GetType(), col.GetDecimal()); err == nil {
+ if v, err := expression.GetTimeCurrentTimestamp(w.exprCtx.GetEvalCtx(), col.GetType(), col.GetDecimal()); err == nil {
// convert null value to timestamp should be substituted with current timestamp if NOT_NULL flag is set.
w.rowMap[w.oldColInfo.ID] = v
}
}
- newColVal, err := table.CastValue(w.sessCtx, w.rowMap[w.oldColInfo.ID], w.newColInfo, false, false)
+ newColVal, err := table.CastColumnValue(w.exprCtx, w.rowMap[w.oldColInfo.ID], w.newColInfo, false, false)
if err != nil {
return w.reformatErrors(err)
}
- warn := w.sessCtx.GetSessionVars().StmtCtx.GetWarnings()
+ warn := w.warnings.GetWarnings()
if len(warn) != 0 {
//nolint:forcetypeassert
recordWarning = errors.Cause(w.reformatErrors(warn[0].Err)).(*terror.Error)
@@ -1375,7 +1390,7 @@ func (w *updateColumnWorker) getRowRecord(handle kv.Handle, recordKey []byte, ra
})
w.rowMap[w.newColInfo.ID] = newColVal
- _, err = w.rowDecoder.EvalRemainedExprColumnMap(w.sessCtx, w.rowMap)
+ _, err = w.rowDecoder.EvalRemainedExprColumnMap(w.exprCtx, w.rowMap)
if err != nil {
return errors.Trace(err)
}
@@ -1386,9 +1401,10 @@ func (w *updateColumnWorker) getRowRecord(handle kv.Handle, recordKey []byte, ra
newRow = append(newRow, val)
}
checksums := w.calcChecksums()
- sctx, rd := w.sessCtx.GetSessionVars().StmtCtx, &w.sessCtx.GetSessionVars().RowEncoder
- newRowVal, err := tablecodec.EncodeRow(sctx.TimeZone(), newRow, newColumnIDs, nil, nil, rd, checksums...)
- err = sctx.HandleError(err)
+ rd := &w.tblCtx.GetSessionVars().RowEncoder
+ ec := w.exprCtx.GetEvalCtx().ErrCtx()
+ newRowVal, err := tablecodec.EncodeRow(w.loc, newRow, newColumnIDs, nil, nil, rd, checksums...)
+ err = ec.HandleError(err)
if err != nil {
return errors.Trace(err)
}
@@ -1420,7 +1436,7 @@ func (w *updateColumnWorker) calcChecksums() []uint32 {
if !sort.IsSorted(w.checksumBuffer) {
sort.Sort(w.checksumBuffer)
}
- checksum, err := w.checksumBuffer.Checksum(w.sessCtx.GetSessionVars().StmtCtx.TimeZone())
+ checksum, err := w.checksumBuffer.Checksum(w.loc)
if err != nil {
logutil.DDLLogger().Warn("skip checksum in update-column backfill due to encode error", zap.Error(err))
return nil
@@ -1462,7 +1478,7 @@ func (w *updateColumnWorker) cleanRowMap() {
func (w *updateColumnWorker) BackfillData(handleRange reorgBackfillTask) (taskCtx backfillTaskContext, errInTxn error) {
oprStartTime := time.Now()
ctx := kv.WithInternalSourceAndTaskType(context.Background(), w.jobContext.ddlJobSourceType(), kvutil.ExplicitTypeDDL)
- errInTxn = kv.RunInNewTxn(ctx, w.sessCtx.GetStore(), true, func(_ context.Context, txn kv.Transaction) error {
+ errInTxn = kv.RunInNewTxn(ctx, w.ddlCtx.store, true, func(_ context.Context, txn kv.Transaction) error {
taskCtx.addedCount = 0
taskCtx.scanCount = 0
updateTxnEntrySizeLimitIfNeeded(txn)
@@ -1803,7 +1819,7 @@ func updateColumnDefaultValue(d *ddlCtx, t *meta.Meta, job *model.Job, newCol *m
return ver, infoschema.ErrColumnNotExists.GenWithStackByArgs(newCol.Name, tblInfo.Name)
}
- if hasDefaultValue, _, err := checkColumnDefaultValue(newReorgSessCtx(d.store), table.ToColumn(oldCol.Clone()), newCol.DefaultValue); err != nil {
+ if hasDefaultValue, _, err := checkColumnDefaultValue(newReorgExprCtx(), table.ToColumn(oldCol.Clone()), newCol.DefaultValue); err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
} else if !hasDefaultValue {
@@ -1819,8 +1835,7 @@ func updateColumnDefaultValue(d *ddlCtx, t *meta.Meta, job *model.Job, newCol *m
oldCol.AddFlag(mysql.NoDefaultValueFlag)
} else {
oldCol.DelFlag(mysql.NoDefaultValueFlag)
- sctx := newReorgSessCtx(d.store)
- err = checkDefaultValue(sctx, table.ToColumn(oldCol), true)
+ err = checkDefaultValue(newReorgExprCtx(), table.ToColumn(oldCol), true)
if err != nil {
job.State = model.JobStateCancelled
return ver, err
diff --git a/pkg/ddl/constraint_test.go b/pkg/ddl/constraint_test.go
index e8ad736f21eb8..bfc612c720df6 100644
--- a/pkg/ddl/constraint_test.go
+++ b/pkg/ddl/constraint_test.go
@@ -216,7 +216,7 @@ func TestAlterAddConstraintStateChange3(t *testing.T) {
return
}
originalCallback.OnChanged(nil)
- if job.SchemaState == model.StatePublic {
+ if job.SchemaState == model.StatePublic && job.IsDone() {
// set constraint state
constraintTable := external.GetTableByName(t, tk1, "test", "t")
tableCommon, ok := constraintTable.(*tables.TableCommon)
diff --git a/pkg/ddl/copr/BUILD.bazel b/pkg/ddl/copr/BUILD.bazel
index 3cb9d8a12584e..b9a4357ed53f9 100644
--- a/pkg/ddl/copr/BUILD.bazel
+++ b/pkg/ddl/copr/BUILD.bazel
@@ -6,9 +6,11 @@ go_library(
importpath = "github.com/pingcap/tidb/pkg/ddl/copr",
visibility = ["//visibility:public"],
deps = [
+ "//pkg/distsql/context",
"//pkg/expression",
+ "//pkg/expression/context",
+ "//pkg/infoschema",
"//pkg/parser/model",
- "//pkg/sessionctx",
"//pkg/table/tables",
"//pkg/types",
"@com_github_pingcap_errors//:errors",
@@ -24,6 +26,7 @@ go_test(
shard_count = 3,
deps = [
"//pkg/expression",
+ "//pkg/expression/contextstatic",
"//pkg/parser/model",
"//pkg/parser/mysql",
"//pkg/types",
diff --git a/pkg/ddl/copr/copr_ctx.go b/pkg/ddl/copr/copr_ctx.go
index 87f1b43adea6f..023f7bdbca034 100644
--- a/pkg/ddl/copr/copr_ctx.go
+++ b/pkg/ddl/copr/copr_ctx.go
@@ -16,9 +16,12 @@ package copr
import (
"github.com/pingcap/errors"
+ distsqlctx "github.com/pingcap/tidb/pkg/distsql/context"
"github.com/pingcap/tidb/pkg/expression"
+ exprctx "github.com/pingcap/tidb/pkg/expression/context"
+ // make sure mock.MockInfoschema is initialized to make sure the test pass
+ _ "github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/parser/model"
- "github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/table/tables"
"github.com/pingcap/tidb/pkg/types"
)
@@ -35,9 +38,10 @@ type CopContext interface {
type CopContextBase struct {
TableInfo *model.TableInfo
PrimaryKeyInfo *model.IndexInfo
- SessionContext sessionctx.Context
-
- RequestSource string
+ ExprCtx exprctx.BuildContext
+ DistSQLCtx *distsqlctx.DistSQLContext
+ PushDownFlags uint64
+ RequestSource string
ColumnInfos []*model.ColumnInfo
FieldTypes []*types.FieldType
@@ -66,9 +70,11 @@ type CopContextMultiIndex struct {
// NewCopContextBase creates a CopContextBase.
func NewCopContextBase(
+ exprCtx exprctx.BuildContext,
+ distSQLCtx *distsqlctx.DistSQLContext,
+ pushDownFlags uint64,
tblInfo *model.TableInfo,
idxCols []*model.IndexColumn,
- sessCtx sessionctx.Context,
requestSource string,
) (*CopContextBase, error) {
var err error
@@ -115,18 +121,20 @@ func NewCopContextBase(
handleIDs = []int64{extra.ID}
}
- expColInfos, _, err := expression.ColumnInfos2ColumnsAndNames(sessCtx.GetExprCtx(),
+ expColInfos, _, err := expression.ColumnInfos2ColumnsAndNames(exprCtx,
model.CIStr{} /* unused */, tblInfo.Name, colInfos, tblInfo)
if err != nil {
return nil, err
}
hdColOffsets := resolveIndicesForHandle(expColInfos, handleIDs)
- vColOffsets, vColFts := collectVirtualColumnOffsetsAndTypes(expColInfos)
+ vColOffsets, vColFts := collectVirtualColumnOffsetsAndTypes(exprCtx.GetEvalCtx(), expColInfos)
return &CopContextBase{
TableInfo: tblInfo,
PrimaryKeyInfo: primaryIdx,
- SessionContext: sessCtx,
+ ExprCtx: exprCtx,
+ DistSQLCtx: distSQLCtx,
+ PushDownFlags: pushDownFlags,
RequestSource: requestSource,
ColumnInfos: colInfos,
FieldTypes: fieldTps,
@@ -139,25 +147,29 @@ func NewCopContextBase(
// NewCopContext creates a CopContext.
func NewCopContext(
+ exprCtx exprctx.BuildContext,
+ distSQLCtx *distsqlctx.DistSQLContext,
+ pushDownFlags uint64,
tblInfo *model.TableInfo,
allIdxInfo []*model.IndexInfo,
- sessCtx sessionctx.Context,
requestSource string,
) (CopContext, error) {
if len(allIdxInfo) == 1 {
- return NewCopContextSingleIndex(tblInfo, allIdxInfo[0], sessCtx, requestSource)
+ return NewCopContextSingleIndex(exprCtx, distSQLCtx, pushDownFlags, tblInfo, allIdxInfo[0], requestSource)
}
- return NewCopContextMultiIndex(tblInfo, allIdxInfo, sessCtx, requestSource)
+ return NewCopContextMultiIndex(exprCtx, distSQLCtx, pushDownFlags, tblInfo, allIdxInfo, requestSource)
}
// NewCopContextSingleIndex creates a CopContextSingleIndex.
func NewCopContextSingleIndex(
+ exprCtx exprctx.BuildContext,
+ distSQLCtx *distsqlctx.DistSQLContext,
+ pushDownFlags uint64,
tblInfo *model.TableInfo,
idxInfo *model.IndexInfo,
- sessCtx sessionctx.Context,
requestSource string,
) (*CopContextSingleIndex, error) {
- base, err := NewCopContextBase(tblInfo, idxInfo.Columns, sessCtx, requestSource)
+ base, err := NewCopContextBase(exprCtx, distSQLCtx, pushDownFlags, tblInfo, idxInfo.Columns, requestSource)
if err != nil {
return nil, err
}
@@ -186,9 +198,11 @@ func (c *CopContextSingleIndex) IndexInfo(_ int64) *model.IndexInfo {
// NewCopContextMultiIndex creates a CopContextMultiIndex.
func NewCopContextMultiIndex(
+ exprCtx exprctx.BuildContext,
+ distSQLCtx *distsqlctx.DistSQLContext,
+ pushDownFlags uint64,
tblInfo *model.TableInfo,
allIdxInfo []*model.IndexInfo,
- sessCtx sessionctx.Context,
requestSource string,
) (*CopContextMultiIndex, error) {
approxColLen := 0
@@ -206,7 +220,7 @@ func NewCopContextMultiIndex(
}
}
- base, err := NewCopContextBase(tblInfo, allIdxCols, sessCtx, requestSource)
+ base, err := NewCopContextBase(exprCtx, distSQLCtx, pushDownFlags, tblInfo, allIdxCols, requestSource)
if err != nil {
return nil, err
}
@@ -305,13 +319,13 @@ func resolveIndicesForHandle(cols []*expression.Column, handleIDs []int64) []int
return offsets
}
-func collectVirtualColumnOffsetsAndTypes(cols []*expression.Column) ([]int, []*types.FieldType) {
+func collectVirtualColumnOffsetsAndTypes(ctx expression.EvalContext, cols []*expression.Column) ([]int, []*types.FieldType) {
var offsets []int
var fts []*types.FieldType
for i, col := range cols {
if col.VirtualExpr != nil {
offsets = append(offsets, i)
- fts = append(fts, col.GetType())
+ fts = append(fts, col.GetType(ctx))
}
}
return offsets, fts
diff --git a/pkg/ddl/copr/copr_ctx_test.go b/pkg/ddl/copr/copr_ctx_test.go
index 4d43f14f6de00..a4d84d1130562 100644
--- a/pkg/ddl/copr/copr_ctx_test.go
+++ b/pkg/ddl/copr/copr_ctx_test.go
@@ -19,6 +19,7 @@ import (
"testing"
"github.com/pingcap/tidb/pkg/expression"
+ "github.com/pingcap/tidb/pkg/expression/contextstatic"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/types"
@@ -104,7 +105,13 @@ func TestNewCopContextSingleIndex(t *testing.T) {
})
}
- copCtx, err := NewCopContextSingleIndex(mockTableInfo, mockIdxInfo, mock.NewContext(), "")
+ sctx := mock.NewContext()
+ copCtx, err := NewCopContextSingleIndex(
+ sctx.GetExprCtx(),
+ sctx.GetDistSQLCtx(),
+ sctx.GetSessionVars().StmtCtx.PushDownFlags(),
+ mockTableInfo, mockIdxInfo, "",
+ )
require.NoError(t, err)
base := copCtx.GetBase()
require.Equal(t, "t", base.TableInfo.Name.L)
@@ -194,7 +201,8 @@ func TestCollectVirtualColumnOffsetsAndTypes(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- gotOffsets, gotFt := collectVirtualColumnOffsetsAndTypes(tt.cols)
+ ctx := contextstatic.NewStaticEvalContext()
+ gotOffsets, gotFt := collectVirtualColumnOffsetsAndTypes(ctx, tt.cols)
require.Equal(t, gotOffsets, tt.offsets)
require.Equal(t, len(gotFt), len(tt.fieldTp))
for i, ft := range gotFt {
diff --git a/pkg/ddl/db_integration_test.go b/pkg/ddl/db_integration_test.go
index 7e86704dc6eba..2f1c775975df4 100644
--- a/pkg/ddl/db_integration_test.go
+++ b/pkg/ddl/db_integration_test.go
@@ -3082,3 +3082,31 @@ func TestIssue52680(t *testing.T) {
tk.MustExec("insert into issue52680 values(default);")
tk.MustQuery("select * from issue52680").Check(testkit.Rows("1", "2", "3"))
}
+
+func TestCreateIndexWithChangeMaxIndexLength(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ originCfg := config.GetGlobalConfig()
+ defer func() {
+ config.StoreGlobalConfig(originCfg)
+ }()
+
+ originHook := dom.DDL().GetHook()
+ defer dom.DDL().SetHook(originHook)
+ hook := &callback.TestDDLCallback{Do: dom}
+ hook.OnJobRunBeforeExported = func(job *model.Job) {
+ if job.Type != model.ActionAddIndex {
+ return
+ }
+ if job.SchemaState == model.StateNone {
+ newCfg := *originCfg
+ newCfg.MaxIndexLength = 1000
+ config.StoreGlobalConfig(&newCfg)
+ }
+ }
+ dom.DDL().SetHook(hook)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test;")
+ tk.MustExec("create table t(id int, a json DEFAULT NULL, b varchar(2) DEFAULT NULL);")
+ tk.MustGetErrMsg("CREATE INDEX idx_test on t ((cast(a as char(2000) array)),b);", "[ddl:1071]Specified key was too long (2000 bytes); max key length is 1000 bytes")
+}
diff --git a/pkg/ddl/db_test.go b/pkg/ddl/db_test.go
index 72dcc7f7eeb5a..1c3f64c8d9c8e 100644
--- a/pkg/ddl/db_test.go
+++ b/pkg/ddl/db_test.go
@@ -1076,18 +1076,17 @@ func TestTruncateTableAndSchemaDependence(t *testing.T) {
tk.MustExec("create table t(a int);")
var wg sync.WaitGroup
-
- hook := &callback.TestDDLCallback{Do: dom}
wg.Add(2)
+
var timetk2 time.Time
var timetk3 time.Time
- one := false
+ first := false
f := func(job *model.Job) {
- if one {
+ if first || job.Type != model.ActionTruncateTable {
return
}
- one = true
+ first = true
go func() {
tk3.MustExec("drop database test")
timetk3 = time.Now()
@@ -1096,6 +1095,7 @@ func TestTruncateTableAndSchemaDependence(t *testing.T) {
time.Sleep(3 * time.Second)
}
+ hook := &callback.TestDDLCallback{Do: dom}
hook.OnJobUpdatedExported.Store(&f)
dom.DDL().SetHook(hook)
diff --git a/pkg/ddl/ddl.go b/pkg/ddl/ddl.go
index 8e643fa99d046..c3e283ecbc1da 100644
--- a/pkg/ddl/ddl.go
+++ b/pkg/ddl/ddl.go
@@ -64,7 +64,7 @@ import (
"github.com/pingcap/tidb/pkg/util/dbterror"
"github.com/pingcap/tidb/pkg/util/dbterror/exeerrors"
"github.com/pingcap/tidb/pkg/util/gcutil"
- "github.com/pingcap/tidb/pkg/util/syncutil"
+ "github.com/pingcap/tidb/pkg/util/generic"
"github.com/tikv/client-go/v2/tikvrpc"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/concurrency"
@@ -291,11 +291,9 @@ type ddl struct {
delRangeMgr delRangeManager
enableTiFlashPoll *atomicutil.Bool
// used in the concurrency ddl.
- reorgWorkerPool *workerPool
- generalDDLWorkerPool *workerPool
- localWorkerPool *workerPool
- // get notification if any DDL coming.
- ddlJobCh chan struct{}
+ localWorkerPool *workerPool
+ // get notification if any DDL job submitted or finished.
+ ddlJobNotifyCh chan struct{}
// localJobCh is used to delivery job in local TiDB nodes.
localJobCh chan *limitJobTask
@@ -357,6 +355,12 @@ func (w *waitSchemaSyncedController) setAlreadyRunOnce(id int64) {
w.onceMap[id] = struct{}{}
}
+func (w *waitSchemaSyncedController) clearOnceMap() {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ w.onceMap = make(map[int64]struct{}, jobOnceCapacity)
+}
+
// ddlCtx is the context when we use worker to handle DDL jobs.
type ddlCtx struct {
ctx context.Context
@@ -366,28 +370,23 @@ type ddlCtx struct {
ownerManager owner.Manager
schemaSyncer syncer.SchemaSyncer
stateSyncer syncer.StateSyncer
- ddlJobDoneCh chan struct{}
- ddlEventCh chan<- *statsutil.DDLEvent
- lease time.Duration // lease is schema lease, default 45s, see config.Lease.
- binlogCli *pumpcli.PumpsClient // binlogCli is used for Binlog.
- infoCache *infoschema.InfoCache
- statsHandle *handle.Handle
- tableLockCkr util.DeadTableLockChecker
- etcdCli *clientv3.Client
- autoidCli *autoid.ClientDiscover
+ // ddlJobDoneChMap is used to notify the session that the DDL job is finished.
+ // jobID -> chan struct{}
+ ddlJobDoneChMap generic.SyncMap[int64, chan struct{}]
+ ddlEventCh chan<- *statsutil.DDLEvent
+ lease time.Duration // lease is schema lease, default 45s, see config.Lease.
+ binlogCli *pumpcli.PumpsClient // binlogCli is used for Binlog.
+ infoCache *infoschema.InfoCache
+ statsHandle *handle.Handle
+ tableLockCkr util.DeadTableLockChecker
+ etcdCli *clientv3.Client
+ autoidCli *autoid.ClientDiscover
*waitSchemaSyncedController
*schemaVersionManager
- runningJobs *runningJobs
-
// reorgCtx is used for reorganization.
reorgCtx reorgContexts
- // backfillCtx is used for backfill workers.
- backfillCtx struct {
- syncutil.RWMutex
- jobCtxMap map[int64]*JobContext
- }
jobCtx struct {
sync.RWMutex
@@ -403,6 +402,7 @@ type ddlCtx struct {
interceptor Interceptor
}
+ // TODO merge with *waitSchemaSyncedController into another new struct.
ddlSeqNumMu struct {
sync.Mutex
seqNum uint64
@@ -618,6 +618,27 @@ func (dc *ddlCtx) notifyReorgWorkerJobStateChange(job *model.Job) {
rc.notifyJobState(job.State)
}
+func (dc *ddlCtx) initJobDoneCh(jobID int64) {
+ dc.ddlJobDoneChMap.Store(jobID, make(chan struct{}, 1))
+}
+
+func (dc *ddlCtx) getJobDoneCh(jobID int64) (chan struct{}, bool) {
+ return dc.ddlJobDoneChMap.Load(jobID)
+}
+
+func (dc *ddlCtx) delJobDoneCh(jobID int64) {
+ dc.ddlJobDoneChMap.Delete(jobID)
+}
+
+func (dc *ddlCtx) notifyJobDone(jobID int64) {
+ if ch, ok := dc.ddlJobDoneChMap.Load(jobID); ok {
+ select {
+ case ch <- struct{}{}:
+ default:
+ }
+ }
+}
+
// EnableTiFlashPoll enables TiFlash poll loop aka PollTiFlashReplicaStatus.
func EnableTiFlashPoll(d any) {
if dd, ok := d.(*ddl); ok {
@@ -711,7 +732,7 @@ func newDDL(ctx context.Context, options ...Option) *ddl {
uuid: id,
store: opt.Store,
lease: opt.Lease,
- ddlJobDoneCh: make(chan struct{}, 1),
+ ddlJobDoneChMap: generic.NewSyncMap[int64, chan struct{}](10),
ownerManager: manager,
schemaSyncer: schemaSyncer,
stateSyncer: stateSyncer,
@@ -721,7 +742,6 @@ func newDDL(ctx context.Context, options ...Option) *ddl {
etcdCli: opt.EtcdCli,
autoidCli: opt.AutoIDClient,
waitSchemaSyncedController: newWaitSchemaSyncedController(),
- runningJobs: newRunningJobs(),
}
ddlCtx.reorgCtx.reorgCtxMap = make(map[int64]*reorgCtx)
ddlCtx.jobCtx.jobCtxMap = make(map[int64]*JobContext)
@@ -736,7 +756,7 @@ func newDDL(ctx context.Context, options ...Option) *ddl {
limitJobCh: make(chan *limitJobTask, batchAddingJobs),
limitJobChV2: make(chan *limitJobTask, batchAddingJobs),
enableTiFlashPoll: atomicutil.NewBool(true),
- ddlJobCh: make(chan struct{}, 100),
+ ddlJobNotifyCh: make(chan struct{}, 100),
localJobCh: make(chan *limitJobTask, 1),
}
@@ -783,7 +803,7 @@ func (d *ddl) newDeleteRangeManager(mock bool) delRangeManager {
return delRangeMgr
}
-func (d *ddl) prepareWorkers4ConcurrencyDDL() {
+func (d *ddl) prepareLocalModeWorkers() {
workerFactory := func(tp workerType) func() (pools.Resource, error) {
return func() (pools.Resource, error) {
wk := newWorker(d.ctx, tp, d.sessPool, d.delRangeMgr, d.ddlCtx)
@@ -797,19 +817,14 @@ func (d *ddl) prepareWorkers4ConcurrencyDDL() {
return wk, nil
}
}
- // reorg worker count at least 1 at most 10.
- reorgCnt := min(max(runtime.GOMAXPROCS(0)/4, 1), reorgWorkerCnt)
// local worker count at least 2 at most 10.
localCnt := min(max(runtime.GOMAXPROCS(0)/4, 2), localWorkerCnt)
- d.reorgWorkerPool = newDDLWorkerPool(pools.NewResourcePool(workerFactory(addIdxWorker), reorgCnt, reorgCnt, 0), jobTypeReorg)
- d.generalDDLWorkerPool = newDDLWorkerPool(pools.NewResourcePool(workerFactory(generalWorker), generalWorkerCnt, generalWorkerCnt, 0), jobTypeGeneral)
d.localWorkerPool = newDDLWorkerPool(pools.NewResourcePool(workerFactory(localWorker), localCnt, localCnt, 0), jobTypeLocal)
failpoint.Inject("NoDDLDispatchLoop", func(val failpoint.Value) {
if val.(bool) {
failpoint.Return()
}
})
- d.wg.Run(d.startDispatchLoop)
d.wg.Run(d.startLocalWorkerLoop)
}
@@ -824,16 +839,6 @@ func (d *ddl) Start(ctxPool *pools.ResourcePool) error {
d.limitDDLJobs(d.limitJobChV2, d.addBatchLocalDDLJobs)
})
d.sessPool = sess.NewSessionPool(ctxPool, d.store)
- d.ownerManager.SetBeOwnerHook(func() {
- var err error
- d.ddlSeqNumMu.Lock()
- defer d.ddlSeqNumMu.Unlock()
- d.ddlSeqNumMu.seqNum, err = d.GetNextDDLSeqNum()
- if err != nil {
- logutil.DDLLogger().Error("error when getting the ddl history count", zap.Error(err))
- }
- d.runningJobs.clear()
- })
d.delRangeMgr = d.newDeleteRangeManager(ctxPool == nil)
@@ -841,8 +846,11 @@ func (d *ddl) Start(ctxPool *pools.ResourcePool) error {
logutil.DDLLogger().Warn("start DDL init state syncer failed", zap.Error(err))
return errors.Trace(err)
}
+ d.ownerManager.SetListener(&ownerListener{
+ ddl: d,
+ })
- d.prepareWorkers4ConcurrencyDDL()
+ d.prepareLocalModeWorkers()
if config.TableLockEnabled() {
d.wg.Add(1)
@@ -906,10 +914,10 @@ func (d *ddl) DisableDDL() error {
}
// GetNextDDLSeqNum return the next DDL seq num.
-func (d *ddl) GetNextDDLSeqNum() (uint64, error) {
+func (s *jobScheduler) GetNextDDLSeqNum() (uint64, error) {
var count uint64
- ctx := kv.WithInternalSourceType(d.ctx, kv.InternalTxnDDL)
- err := kv.RunInNewTxn(ctx, d.store, true, func(_ context.Context, txn kv.Transaction) error {
+ ctx := kv.WithInternalSourceType(s.schCtx, kv.InternalTxnDDL)
+ err := kv.RunInNewTxn(ctx, s.store, true, func(_ context.Context, txn kv.Transaction) error {
t := meta.NewMeta(txn)
var err error
count, err = t.GetHistoryDDLCount()
@@ -928,12 +936,6 @@ func (d *ddl) close() {
d.wg.Wait()
d.ownerManager.Cancel()
d.schemaSyncer.Close()
- if d.reorgWorkerPool != nil {
- d.reorgWorkerPool.close()
- }
- if d.generalDDLWorkerPool != nil {
- d.generalDDLWorkerPool.close()
- }
if d.localWorkerPool != nil {
d.localWorkerPool.close()
}
@@ -1063,7 +1065,7 @@ func getJobCheckInterval(job *model.Job, i int) (time.Duration, bool) {
}
}
-func (dc *ddlCtx) asyncNotifyWorker(ch chan struct{}, etcdPath string, jobID int64, jobType string) {
+func (dc *ddlCtx) notifyNewJobSubmitted(ch chan struct{}, etcdPath string, jobID int64, jobType string) {
// If the workers don't run, we needn't notify workers.
// TODO: It does not affect informing the backfill worker.
if !config.GetGlobalConfig().Instance.TiDBEnableDDL.Load() {
@@ -1072,7 +1074,7 @@ func (dc *ddlCtx) asyncNotifyWorker(ch chan struct{}, etcdPath string, jobID int
if dc.isOwner() {
asyncNotify(ch)
} else {
- dc.asyncNotifyByEtcd(etcdPath, jobID, jobType)
+ dc.notifyNewJobByEtcd(etcdPath, jobID, jobType)
}
}
@@ -1169,16 +1171,18 @@ func (d *ddl) DoDDLJob(ctx sessionctx.Context, job *model.Job) error {
// worker should restart to continue handling tasks in limitJobCh, and send back through task.err
err := <-task.errChs[0]
+ defer d.delJobDoneCh(job.ID)
if err != nil {
// The transaction of enqueuing job is failed.
return errors.Trace(err)
}
+ failpoint.InjectCall("waitJobSubmitted")
sessVars := ctx.GetSessionVars()
sessVars.StmtCtx.IsDDLJobInQueue = true
// Notice worker that we push a new job and wait the job done.
- d.asyncNotifyWorker(d.ddlJobCh, addingDDLJobConcurrent, job.ID, job.Type.String())
+ d.notifyNewJobSubmitted(d.ddlJobNotifyCh, addingDDLJobNotifyKey, job.ID, job.Type.String())
logutil.DDLLogger().Info("start DDL job", zap.Stringer("job", job), zap.String("query", job.Query))
if !d.shouldCheckHistoryJob(job) {
return nil
@@ -1205,13 +1209,14 @@ func (d *ddl) DoDDLJob(ctx sessionctx.Context, job *model.Job) error {
recordLastDDLInfo(ctx, historyJob)
}()
i := 0
+ notifyCh, _ := d.getJobDoneCh(job.ID)
for {
failpoint.Inject("storeCloseInLoop", func(_ failpoint.Value) {
_ = d.Stop()
})
select {
- case <-d.ddlJobDoneCh:
+ case <-notifyCh:
case <-ticker.C:
i++
ticker = updateTickerInterval(ticker, 10*d.lease, job, i)
diff --git a/pkg/ddl/ddl_api.go b/pkg/ddl/ddl_api.go
index 9262f417ed8ed..b34923abb4f83 100644
--- a/pkg/ddl/ddl_api.go
+++ b/pkg/ddl/ddl_api.go
@@ -40,6 +40,7 @@ import (
rg "github.com/pingcap/tidb/pkg/domain/resourcegroup"
"github.com/pingcap/tidb/pkg/errctx"
"github.com/pingcap/tidb/pkg/expression"
+ exprctx "github.com/pingcap/tidb/pkg/expression/context"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/meta"
@@ -96,6 +97,8 @@ const (
tiflashCheckPendingTablesRetry = 7
)
+var errCheckConstraintIsOff = errors.NewNoStackError(variable.TiDBEnableCheckConstraint + " is off")
+
func (d *ddl) CreateSchema(ctx sessionctx.Context, stmt *ast.CreateDatabaseStmt) (err error) {
var placementPolicyRef *model.PolicyRefInfo
sessionVars := ctx.GetSessionVars()
@@ -1021,13 +1024,13 @@ func buildColumnAndConstraint(
// In non-strict SQL mode, if the default value of the column is an empty string, the default value can be ignored.
// In strict SQL mode, TEXT/BLOB/JSON can't have not null default values.
// In NO_ZERO_DATE SQL mode, TIMESTAMP/DATE/DATETIME type can't have zero date like '0000-00-00' or '0000-00-00 00:00:00'.
-func checkColumnDefaultValue(ctx sessionctx.Context, col *table.Column, value any) (bool, any, error) {
+func checkColumnDefaultValue(ctx exprctx.BuildContext, col *table.Column, value any) (bool, any, error) {
hasDefaultValue := true
if value != nil && (col.GetType() == mysql.TypeJSON ||
col.GetType() == mysql.TypeTinyBlob || col.GetType() == mysql.TypeMediumBlob ||
col.GetType() == mysql.TypeLongBlob || col.GetType() == mysql.TypeBlob) {
// In non-strict SQL mode.
- if !ctx.GetSessionVars().SQLMode.HasStrictMode() && value == "" {
+ if !ctx.GetEvalCtx().SQLMode().HasStrictMode() && value == "" {
if col.GetType() == mysql.TypeBlob || col.GetType() == mysql.TypeLongBlob {
// The TEXT/BLOB default value can be ignored.
hasDefaultValue = false
@@ -1036,17 +1039,16 @@ func checkColumnDefaultValue(ctx sessionctx.Context, col *table.Column, value an
if col.GetType() == mysql.TypeJSON {
value = `null`
}
- sc := ctx.GetSessionVars().StmtCtx
- sc.AppendWarning(dbterror.ErrBlobCantHaveDefault.FastGenByArgs(col.Name.O))
+ ctx.GetEvalCtx().AppendWarning(dbterror.ErrBlobCantHaveDefault.FastGenByArgs(col.Name.O))
return hasDefaultValue, value, nil
}
// In strict SQL mode or default value is not an empty string.
return hasDefaultValue, value, dbterror.ErrBlobCantHaveDefault.GenWithStackByArgs(col.Name.O)
}
- if value != nil && ctx.GetSessionVars().SQLMode.HasNoZeroDateMode() &&
- ctx.GetSessionVars().SQLMode.HasStrictMode() && types.IsTypeTime(col.GetType()) {
+ if value != nil && ctx.GetEvalCtx().SQLMode().HasNoZeroDateMode() &&
+ ctx.GetEvalCtx().SQLMode().HasStrictMode() && types.IsTypeTime(col.GetType()) {
if vv, ok := value.(string); ok {
- timeValue, err := expression.GetTimeValue(ctx.GetExprCtx(), vv, col.GetType(), col.GetDecimal(), nil)
+ timeValue, err := expression.GetTimeValue(ctx, vv, col.GetType(), col.GetDecimal(), nil)
if err != nil {
return hasDefaultValue, value, errors.Trace(err)
}
@@ -1254,7 +1256,7 @@ func columnDefToCol(ctx sessionctx.Context, offset int, colDef *ast.ColumnDef, o
ctx.GetSessionVars().StmtCtx.AppendWarning(dbterror.ErrTableCantHandleFt.FastGenByArgs())
case ast.ColumnOptionCheck:
if !variable.EnableCheckConstraint.Load() {
- ctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackError("the switch of check constraint is off"))
+ ctx.GetSessionVars().StmtCtx.AppendWarning(errCheckConstraintIsOff)
} else {
// Check the column CHECK constraint dependency lazily, after fill all the name.
// Extract column constraint from column option.
@@ -1431,7 +1433,7 @@ func getFuncCallDefaultValue(col *table.Column, option *ast.ColumnOption, expr *
// getDefaultValue will get the default value for column.
// 1: get the expr restored string for the column which uses sequence next value as default value.
// 2: get specific default value for the other column.
-func getDefaultValue(ctx sessionctx.Context, col *table.Column, option *ast.ColumnOption) (any, bool, error) {
+func getDefaultValue(ctx exprctx.BuildContext, col *table.Column, option *ast.ColumnOption) (any, bool, error) {
// handle default value with function call
tp, fsp := col.FieldType.GetType(), col.FieldType.GetDecimal()
if x, ok := option.Expr.(*ast.FuncCallExpr); ok {
@@ -1443,7 +1445,7 @@ func getDefaultValue(ctx sessionctx.Context, col *table.Column, option *ast.Colu
}
if tp == mysql.TypeTimestamp || tp == mysql.TypeDatetime || tp == mysql.TypeDate {
- vd, err := expression.GetTimeValue(ctx.GetExprCtx(), option.Expr, tp, fsp, nil)
+ vd, err := expression.GetTimeValue(ctx, option.Expr, tp, fsp, nil)
value := vd.GetValue()
if err != nil {
return nil, false, dbterror.ErrInvalidDefaultValue.GenWithStackByArgs(col.Name.O)
@@ -1463,7 +1465,7 @@ func getDefaultValue(ctx sessionctx.Context, col *table.Column, option *ast.Colu
}
// evaluate the non-function-call expr to a certain value.
- v, err := expression.EvalSimpleAst(ctx.GetExprCtx(), option.Expr)
+ v, err := expression.EvalSimpleAst(ctx, option.Expr)
if err != nil {
return nil, false, errors.Trace(err)
}
@@ -1489,7 +1491,7 @@ func getDefaultValue(ctx sessionctx.Context, col *table.Column, option *ast.Colu
return str, false, err
}
// For other kind of fields (e.g. INT), we supply its integer as string value.
- value, err := v.GetBinaryLiteral().ToInt(ctx.GetSessionVars().StmtCtx.TypeCtx())
+ value, err := v.GetBinaryLiteral().ToInt(ctx.GetEvalCtx().TypeCtx())
if err != nil {
return nil, false, err
}
@@ -1504,7 +1506,7 @@ func getDefaultValue(ctx sessionctx.Context, col *table.Column, option *ast.Colu
val, err := getEnumDefaultValue(v, col)
return val, false, err
case mysql.TypeDuration, mysql.TypeDate:
- if v, err = v.ConvertTo(ctx.GetSessionVars().StmtCtx.TypeCtx(), &col.FieldType); err != nil {
+ if v, err = v.ConvertTo(ctx.GetEvalCtx().TypeCtx(), &col.FieldType); err != nil {
return "", false, errors.Trace(err)
}
case mysql.TypeBit:
@@ -1516,7 +1518,7 @@ func getDefaultValue(ctx sessionctx.Context, col *table.Column, option *ast.Colu
// For these types, convert it to standard format firstly.
// like integer fields, convert it into integer string literals. like convert "1.25" into "1" and "2.8" into "3".
// if raise a error, we will use original expression. We will handle it in check phase
- if temp, err := v.ConvertTo(ctx.GetSessionVars().StmtCtx.TypeCtx(), &col.FieldType); err == nil {
+ if temp, err := v.ConvertTo(ctx.GetEvalCtx().TypeCtx(), &col.FieldType); err == nil {
v = temp
}
}
@@ -1663,7 +1665,7 @@ func setNoDefaultValueFlag(c *table.Column, hasDefaultValue bool) {
}
}
-func checkDefaultValue(ctx sessionctx.Context, c *table.Column, hasDefaultValue bool) (err error) {
+func checkDefaultValue(ctx exprctx.BuildContext, c *table.Column, hasDefaultValue bool) (err error) {
if !hasDefaultValue {
return nil
}
@@ -1675,9 +1677,10 @@ func checkDefaultValue(ctx sessionctx.Context, c *table.Column, hasDefaultValue
}
return nil
}
- handleWithTruncateErr(ctx, func() {
- _, err = table.GetColDefaultValue(ctx.GetExprCtx(), c.ToInfo())
- })
+ _, err = table.GetColDefaultValue(
+ exprctx.CtxWithHandleTruncateErrLevel(ctx, errctx.LevelError),
+ c.ToInfo(),
+ )
if err != nil {
return types.ErrInvalidDefault.GenWithStackByArgs(c.Name)
}
@@ -2197,7 +2200,7 @@ func BuildTableInfo(
// check constraint
if constr.Tp == ast.ConstraintCheck {
if !variable.EnableCheckConstraint.Load() {
- ctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackError("the switch of check constraint is off"))
+ ctx.GetSessionVars().StmtCtx.AppendWarning(errCheckConstraintIsOff)
continue
}
// Since column check constraint dependency has been done in columnDefToCol.
@@ -2249,7 +2252,7 @@ func BuildTableInfo(
return nil, errors.Trace(err)
}
// check if the expression is bool type
- if err := table.IfCheckConstraintExprBoolType(constraintInfo, tbInfo); err != nil {
+ if err := table.IfCheckConstraintExprBoolType(ctx.GetExprCtx().GetEvalCtx(), constraintInfo, tbInfo); err != nil {
return nil, err
}
constraintInfo.ID = allocateConstraintID(tbInfo)
@@ -3004,11 +3007,32 @@ func (d *ddl) BatchCreateTableWithInfo(ctx sessionctx.Context,
}
}
- return nil
+ return d.callHookOnChanged(jobs, err)
+}
+
+// BuildQueryStringFromJobs takes a slice of Jobs and concatenates their
+// queries into a single query string.
+// Each query is separated by a semicolon and a space.
+// Trailing spaces are removed from each query, and a semicolon is appended
+// if it's not already present.
+func BuildQueryStringFromJobs(jobs []*model.Job) string {
+ var queryBuilder strings.Builder
+ for i, job := range jobs {
+ q := strings.TrimSpace(job.Query)
+ if !strings.HasSuffix(q, ";") {
+ q += ";"
+ }
+ queryBuilder.WriteString(q)
+
+ if i < len(jobs)-1 {
+ queryBuilder.WriteString(" ")
+ }
+ }
+ return queryBuilder.String()
}
// BatchCreateTableWithJobs combine CreateTableJobs to BatchCreateTableJob.
-func (*ddl) BatchCreateTableWithJobs(jobs []*model.Job) (*model.Job, error) {
+func BatchCreateTableWithJobs(jobs []*model.Job) (*model.Job, error) {
if len(jobs) == 0 {
return nil, errors.Trace(fmt.Errorf("expect non-empty jobs"))
}
@@ -3052,6 +3076,7 @@ func (*ddl) BatchCreateTableWithJobs(jobs []*model.Job) (*model.Job, error) {
combinedJob.Args = append(combinedJob.Args, args)
combinedJob.Args = append(combinedJob.Args, foreignKeyChecks)
combinedJob.InvolvingSchemaInfo = involvingSchemaInfo
+ combinedJob.Query = BuildQueryStringFromJobs(jobs)
return combinedJob, nil
}
@@ -3947,7 +3972,7 @@ func (d *ddl) AlterTable(ctx context.Context, sctx sessionctx.Context, stmt *ast
sctx.GetSessionVars().StmtCtx.AppendWarning(dbterror.ErrTableCantHandleFt)
case ast.ConstraintCheck:
if !variable.EnableCheckConstraint.Load() {
- sctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackError("the switch of check constraint is off"))
+ sctx.GetSessionVars().StmtCtx.AppendWarning(errCheckConstraintIsOff)
} else {
err = d.CreateCheckConstraint(sctx, ident, model.NewCIStr(constr.Name), spec.Constraint)
}
@@ -4048,13 +4073,13 @@ func (d *ddl) AlterTable(ctx context.Context, sctx sessionctx.Context, stmt *ast
err = d.AlterIndexVisibility(sctx, ident, spec.IndexName, spec.Visibility)
case ast.AlterTableAlterCheck:
if !variable.EnableCheckConstraint.Load() {
- sctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackError("the switch of check constraint is off"))
+ sctx.GetSessionVars().StmtCtx.AppendWarning(errCheckConstraintIsOff)
} else {
err = d.AlterCheckConstraint(sctx, ident, model.NewCIStr(spec.Constraint.Name), spec.Constraint.Enforced)
}
case ast.AlterTableDropCheck:
if !variable.EnableCheckConstraint.Load() {
- sctx.GetSessionVars().StmtCtx.AppendWarning(errors.NewNoStackError("the switch of check constraint is off"))
+ sctx.GetSessionVars().StmtCtx.AppendWarning(errCheckConstraintIsOff)
} else {
err = d.DropCheckConstraint(sctx, ident, model.NewCIStr(spec.Constraint.Name))
}
@@ -4865,7 +4890,7 @@ func checkReorgPartitionDefs(ctx sessionctx.Context, action model.ActionType, tb
return nil
}
- isUnsigned := isPartExprUnsigned(tblInfo)
+ isUnsigned := isPartExprUnsigned(ctx.GetExprCtx().GetEvalCtx(), tblInfo)
currentRangeValue, _, err := getRangeValue(ctx.GetExprCtx(), pi.Definitions[lastPartIdx].LessThan[0], isUnsigned)
if err != nil {
return errors.Trace(err)
@@ -5516,23 +5541,14 @@ func checkModifyTypes(origin *types.FieldType, to *types.FieldType, needRewriteC
return errors.Trace(err)
}
-// handleWithTruncateErr handles the doFunc with FlagTruncateAsWarning and FlagIgnoreTruncateErr flags, both of which are false.
-func handleWithTruncateErr(ctx sessionctx.Context, doFunc func()) {
- sv := ctx.GetSessionVars().StmtCtx
- oldTypeFlags := sv.TypeFlags()
- newTypeFlags := oldTypeFlags.WithTruncateAsWarning(false).WithIgnoreTruncateErr(false)
- sv.SetTypeFlags(newTypeFlags)
- doFunc()
- sv.SetTypeFlags(oldTypeFlags)
-}
-
// SetDefaultValue sets the default value of the column.
func SetDefaultValue(ctx sessionctx.Context, col *table.Column, option *ast.ColumnOption) (hasDefaultValue bool, err error) {
var value any
var isSeqExpr bool
- handleWithTruncateErr(ctx, func() {
- value, isSeqExpr, err = getDefaultValue(ctx, col, option)
- })
+ value, isSeqExpr, err = getDefaultValue(
+ exprctx.CtxWithHandleTruncateErrLevel(ctx.GetExprCtx(), errctx.LevelError),
+ col, option,
+ )
if err != nil {
return false, errors.Trace(err)
}
@@ -5545,7 +5561,7 @@ func SetDefaultValue(ctx sessionctx.Context, col *table.Column, option *ast.Colu
// When the default value is expression, we skip check and convert.
if !col.DefaultIsExpr {
- if hasDefaultValue, value, err = checkColumnDefaultValue(ctx, col, value); err != nil {
+ if hasDefaultValue, value, err = checkColumnDefaultValue(ctx.GetExprCtx(), col, value); err != nil {
return hasDefaultValue, errors.Trace(err)
}
value, err = convertTimestampDefaultValToUTC(ctx, value, col)
@@ -5683,7 +5699,7 @@ func processAndCheckDefaultValueAndColumn(ctx sessionctx.Context, col *table.Col
if err = checkColumnValueConstraint(col, col.GetCollate()); err != nil {
return errors.Trace(err)
}
- if err = checkDefaultValue(ctx, col, hasDefaultValue); err != nil {
+ if err = checkDefaultValue(ctx.GetExprCtx(), col, hasDefaultValue); err != nil {
return errors.Trace(err)
}
if err = checkColumnFieldLength(col); err != nil {
@@ -5939,9 +5955,10 @@ func GetModifiableColumnJob(
return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStack("cannot parse generated PartitionInfo")
}
pAst := at.Specs[0].Partition
- handleWithTruncateErr(sctx, func() {
- _, err = buildPartitionDefinitionsInfo(sctx.GetExprCtx(), pAst.Definitions, &newTblInfo, uint64(len(newTblInfo.Partition.Definitions)))
- })
+ _, err = buildPartitionDefinitionsInfo(
+ exprctx.CtxWithHandleTruncateErrLevel(sctx.GetExprCtx(), errctx.LevelError),
+ pAst.Definitions, &newTblInfo, uint64(len(newTblInfo.Partition.Definitions)),
+ )
if err != nil {
return nil, dbterror.ErrUnsupportedModifyColumn.GenWithStack("New column does not match partition definitions: %s", err.Error())
}
@@ -6359,7 +6376,7 @@ func (d *ddl) AlterColumn(ctx sessionctx.Context, ident ast.Ident, spec *ast.Alt
if err != nil {
return errors.Trace(err)
}
- if err = checkDefaultValue(ctx, col, hasDefaultValue); err != nil {
+ if err = checkDefaultValue(ctx.GetExprCtx(), col, hasDefaultValue); err != nil {
return errors.Trace(err)
}
}
@@ -7544,7 +7561,7 @@ func BuildHiddenColumnInfo(ctx sessionctx.Context, indexPartSpecifications []*as
Version: model.CurrLatestColumnInfoVersion,
Dependences: make(map[string]struct{}),
Hidden: true,
- FieldType: *expr.GetType(),
+ FieldType: *expr.GetType(ctx.GetExprCtx().GetEvalCtx()),
}
// Reset some flag, it may be caused by wrong type infer. But it's not easy to fix them all, so reset them here for safety.
colInfo.DelFlag(mysql.PriKeyFlag | mysql.UniqueKeyFlag | mysql.AutoIncrementFlag)
@@ -9388,7 +9405,7 @@ func (d *ddl) CreateCheckConstraint(ctx sessionctx.Context, ti ast.Ident, constr
return errors.Trace(err)
}
// check if the expression is bool type
- if err := table.IfCheckConstraintExprBoolType(constraintInfo, tblInfo); err != nil {
+ if err := table.IfCheckConstraintExprBoolType(ctx.GetExprCtx().GetEvalCtx(), constraintInfo, tblInfo); err != nil {
return err
}
job := &model.Job{
diff --git a/pkg/ddl/ddl_api_test.go b/pkg/ddl/ddl_api_test.go
index e849b43b3ff17..af431dc3d88c2 100644
--- a/pkg/ddl/ddl_api_test.go
+++ b/pkg/ddl/ddl_api_test.go
@@ -31,6 +31,7 @@ import (
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/util/chunk"
"github.com/stretchr/testify/require"
+ "golang.org/x/sync/errgroup"
)
func TestGetDDLJobs(t *testing.T) {
@@ -151,6 +152,46 @@ func enQueueDDLJobs(t *testing.T, sess sessiontypes.Session, txn kv.Transaction,
}
}
+func TestCreateViewConcurrently(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+
+ tk.MustExec("create table t (a int);")
+ tk.MustExec("create view v as select * from t;")
+ var (
+ counterErr error
+ counter int
+ )
+ failpoint.EnableCall("github.com/pingcap/tidb/pkg/ddl/onDDLCreateView", func(job *model.Job) {
+ counter++
+ if counter > 1 {
+ counterErr = fmt.Errorf("create view job should not run concurrently")
+ return
+ }
+ })
+ failpoint.EnableCall("github.com/pingcap/tidb/pkg/ddl/afterDelivery2Worker", func(job *model.Job) {
+ if job.Type == model.ActionCreateView {
+ counter--
+ }
+ })
+ var eg errgroup.Group
+ for i := 0; i < 5; i++ {
+ eg.Go(func() error {
+ newTk := testkit.NewTestKit(t, store)
+ _, err := newTk.Exec("use test")
+ if err != nil {
+ return err
+ }
+ _, err = newTk.Exec("create or replace view v as select * from t;")
+ return err
+ })
+ }
+ err := eg.Wait()
+ require.NoError(t, err)
+ require.NoError(t, counterErr)
+}
+
func TestCreateDropCreateTable(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
tk := testkit.NewTestKit(t, store)
@@ -213,3 +254,66 @@ func TestCreateDropCreateTable(t *testing.T) {
require.Less(t, create0TS, dropTS, "first create should finish before drop")
require.Less(t, dropTS, create1TS, "second create should finish after drop")
}
+
+func TestBuildQueryStringFromJobs(t *testing.T) {
+ testCases := []struct {
+ name string
+ jobs []*model.Job
+ expected string
+ }{
+ {
+ name: "Empty jobs",
+ jobs: []*model.Job{},
+ expected: "",
+ },
+ {
+ name: "Single create table job",
+ jobs: []*model.Job{{Query: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255));"}},
+ expected: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255));",
+ },
+ {
+ name: "Multiple create table jobs with trailing semicolons",
+ jobs: []*model.Job{
+ {Query: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255));"},
+ {Query: "CREATE TABLE products (id INT PRIMARY KEY, description TEXT);"},
+ },
+ expected: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255)); CREATE TABLE products (id INT PRIMARY KEY, description TEXT);",
+ },
+ {
+ name: "Multiple create table jobs with and without trailing semicolons",
+ jobs: []*model.Job{
+ {Query: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255))"},
+ {Query: "CREATE TABLE products (id INT PRIMARY KEY, description TEXT);"},
+ {Query: " CREATE TABLE orders (id INT PRIMARY KEY, user_id INT, product_id INT) "},
+ },
+ expected: "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(255)); CREATE TABLE products (id INT PRIMARY KEY, description TEXT); CREATE TABLE orders (id INT PRIMARY KEY, user_id INT, product_id INT);",
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ actual := ddl.BuildQueryStringFromJobs(tc.jobs)
+ require.Equal(t, tc.expected, actual, "Query strings do not match")
+ })
+ }
+}
+
+func TestBatchCreateTableWithJobs(t *testing.T) {
+ job1 := &model.Job{
+ SchemaID: 1,
+ Type: model.ActionCreateTable,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []any{&model.TableInfo{Name: model.CIStr{O: "t1", L: "t1"}}, false},
+ Query: "create table db1.t1 (c1 int, c2 int)",
+ }
+ job2 := &model.Job{
+ SchemaID: 1,
+ Type: model.ActionCreateTable,
+ BinlogInfo: &model.HistoryInfo{},
+ Args: []any{&model.TableInfo{Name: model.CIStr{O: "t2", L: "t2"}}, &model.TableInfo{}},
+ Query: "create table db1.t2 (c1 int, c2 int);",
+ }
+ job, err := ddl.BatchCreateTableWithJobs([]*model.Job{job1, job2})
+ require.NoError(t, err)
+ require.Equal(t, "create table db1.t1 (c1 int, c2 int); create table db1.t2 (c1 int, c2 int);", job.Query)
+}
diff --git a/pkg/ddl/ddl_tiflash_api.go b/pkg/ddl/ddl_tiflash_api.go
index 3281beef38de6..6eacfcdecf421 100644
--- a/pkg/ddl/ddl_tiflash_api.go
+++ b/pkg/ddl/ddl_tiflash_api.go
@@ -451,10 +451,9 @@ func (d *ddl) refreshTiFlashTicker(ctx sessionctx.Context, pollTiFlashContext *T
var tableList = make([]TiFlashReplicaStatus, 0)
// Collect TiFlash Replica info, for every table.
- for _, db := range schema.AllSchemaNames() {
- tbls := schema.SchemaTables(db)
- for _, tbl := range tbls {
- tblInfo := tbl.Meta()
+ ch := schema.ListTablesWithSpecialAttribute(infoschema.TiFlashAttribute)
+ for _, v := range ch {
+ for _, tblInfo := range v.TableInfos {
LoadTiFlashReplicaInfo(tblInfo, &tableList)
}
}
diff --git a/pkg/ddl/ddl_worker.go b/pkg/ddl/ddl_worker.go
index 4fb7d5a716a96..8477829c69269 100644
--- a/pkg/ddl/ddl_worker.go
+++ b/pkg/ddl/ddl_worker.go
@@ -20,6 +20,7 @@ import (
"math/rand"
"os"
"strconv"
+ "strings"
"sync"
"sync/atomic"
"time"
@@ -96,8 +97,9 @@ type worker struct {
tp workerType
addingDDLJobKey string
ddlJobCh chan struct{}
- ctx context.Context
- wg sync.WaitGroup
+ // for local mode worker, it's ctx of 'ddl', else it's the ctx of 'job scheduler'.
+ ctx context.Context
+ wg sync.WaitGroup
sessPool *sess.Pool // sessPool is used to new sessions to execute SQL in ddl package.
sess *sess.Session // sess is used and only used in running DDL job.
@@ -186,7 +188,7 @@ func (w *worker) Close() {
tidblogutil.Logger(w.logCtx).Info("DDL worker closed", zap.Duration("take time", time.Since(startTime)))
}
-func (dc *ddlCtx) asyncNotifyByEtcd(etcdPath string, jobID int64, jobType string) {
+func (dc *ddlCtx) notifyNewJobByEtcd(etcdPath string, jobID int64, jobType string) {
if dc.etcdCli == nil {
return
}
@@ -418,7 +420,7 @@ func (d *ddl) addBatchDDLJobs(tasks []*limitJobTask) error {
bdrRole = string(ast.BDRRoleNone)
)
- if newTasks, err := d.combineBatchCreateTableJobs(tasks); err == nil {
+ if newTasks, err := combineBatchCreateTableJobs(tasks); err == nil {
tasks = newTasks
}
@@ -492,6 +494,9 @@ func (d *ddl) addBatchDDLJobs(tasks []*limitJobTask) error {
jobTasks = append(jobTasks, job)
injectModifyJobArgFailPoint(job)
+ if !job.LocalMode {
+ d.initJobDoneCh(job.ID)
+ }
}
se.GetSessionVars().SetDiskFullOpt(kvrpcpb.DiskFullOpt_AllowedOnAlmostFull)
@@ -507,7 +512,7 @@ func (d *ddl) addBatchDDLJobs(tasks []*limitJobTask) error {
// combineBatchCreateTableJobs combine batch jobs to another batch jobs.
// currently it only support combine CreateTable to CreateTables.
-func (d *ddl) combineBatchCreateTableJobs(tasks []*limitJobTask) ([]*limitJobTask, error) {
+func combineBatchCreateTableJobs(tasks []*limitJobTask) ([]*limitJobTask, error) {
if len(tasks) <= 1 || !tasks[0].job.LocalMode {
return tasks, nil
}
@@ -525,7 +530,7 @@ func (d *ddl) combineBatchCreateTableJobs(tasks []*limitJobTask) ([]*limitJobTas
jobs = append(jobs, task.job)
}
- job, err := d.BatchCreateTableWithJobs(jobs)
+ job, err := BatchCreateTableWithJobs(jobs)
if err != nil {
return tasks, err
}
@@ -605,32 +610,47 @@ func (w *worker) registerMDLInfo(job *model.Job, ver int64) error {
if len(rows) == 0 {
return errors.Errorf("can't find ddl job %d", job.ID)
}
+ ownerID := w.ownerManager.ID()
ids := rows[0].GetString(0)
- sql := fmt.Sprintf("replace into mysql.tidb_mdl_info (job_id, version, table_ids) values (%d, %d, '%s')", job.ID, ver, ids)
+ var sql string
+ if tidbutil.IsSysDB(strings.ToLower(job.SchemaName)) {
+ // DDLs that modify system tables could only happen in upgrade process,
+ // we should not reference 'owner_id'. Otherwise, there is a circular blocking problem.
+ sql = fmt.Sprintf("replace into mysql.tidb_mdl_info (job_id, version, table_ids) values (%d, %d, '%s')", job.ID, ver, ids)
+ } else {
+ sql = fmt.Sprintf("replace into mysql.tidb_mdl_info (job_id, version, table_ids, owner_id) values (%d, %d, '%s', '%s')", job.ID, ver, ids, ownerID)
+ }
_, err = w.sess.Execute(context.Background(), sql, "register-mdl-info")
return err
}
// cleanMDLInfo cleans metadata lock info.
-func cleanMDLInfo(pool *sess.Pool, jobID int64, ec *clientv3.Client, cleanETCD bool) {
+func cleanMDLInfo(pool *sess.Pool, job *model.Job, ec *clientv3.Client, ownerID string, cleanETCD bool) {
if !variable.EnableMDL.Load() {
return
}
- sql := fmt.Sprintf("delete from mysql.tidb_mdl_info where job_id = %d", jobID)
+ var sql string
+ if tidbutil.IsSysDB(strings.ToLower(job.SchemaName)) {
+ // DDLs that modify system tables could only happen in upgrade process,
+ // we should not reference 'owner_id'. Otherwise, there is a circular blocking problem.
+ sql = fmt.Sprintf("delete from mysql.tidb_mdl_info where job_id = %d", job.ID)
+ } else {
+ sql = fmt.Sprintf("delete from mysql.tidb_mdl_info where job_id = %d and owner_id = '%s'", job.ID, ownerID)
+ }
sctx, _ := pool.Get()
defer pool.Put(sctx)
se := sess.NewSession(sctx)
se.GetSessionVars().SetDiskFullOpt(kvrpcpb.DiskFullOpt_AllowedOnAlmostFull)
_, err := se.Execute(context.Background(), sql, "delete-mdl-info")
if err != nil {
- logutil.DDLLogger().Warn("unexpected error when clean mdl info", zap.Int64("job ID", jobID), zap.Error(err))
+ logutil.DDLLogger().Warn("unexpected error when clean mdl info", zap.Int64("job ID", job.ID), zap.Error(err))
return
}
if cleanETCD && ec != nil {
- path := fmt.Sprintf("%s/%d/", util.DDLAllSchemaVersionsByJob, jobID)
+ path := fmt.Sprintf("%s/%d/", util.DDLAllSchemaVersionsByJob, job.ID)
_, err = ec.Delete(context.Background(), path, clientv3.WithPrefix())
if err != nil {
- logutil.DDLLogger().Warn("delete versions failed", zap.Int64("job ID", jobID), zap.Error(err))
+ logutil.DDLLogger().Warn("delete versions failed", zap.Int64("job ID", job.ID), zap.Error(err))
}
}
}
@@ -845,7 +865,7 @@ func (w *JobContext) setDDLLabelForDiagnosis(jobType model.ActionType) {
}
func (w *worker) HandleJobDone(d *ddlCtx, job *model.Job, t *meta.Meta) error {
- if err := w.checkOwnerBeforeCommit(); err != nil {
+ if err := w.checkBeforeCommit(); err != nil {
return err
}
err := w.finishDDLJob(t, job)
@@ -859,7 +879,7 @@ func (w *worker) HandleJobDone(d *ddlCtx, job *model.Job, t *meta.Meta) error {
return err
}
CleanupDDLReorgHandles(job, w.sess)
- asyncNotify(d.ddlJobDoneCh)
+ d.notifyJobDone(job.ID)
return nil
}
@@ -956,7 +976,7 @@ func (w *worker) HandleDDLJobTable(d *ddlCtx, job *model.Job) (int64, error) {
return 0, err
}
- if err = w.checkOwnerBeforeCommit(); err != nil {
+ if err = w.checkBeforeCommit(); err != nil {
d.unlockSchemaVersion(job.ID)
return 0, err
}
@@ -1008,13 +1028,18 @@ func (w *worker) HandleDDLJobTable(d *ddlCtx, job *model.Job) (int64, error) {
return schemaVer, nil
}
-func (w *worker) checkOwnerBeforeCommit() error {
+func (w *worker) checkBeforeCommit() error {
if !w.ddlCtx.isOwner() && w.tp != localWorker {
// Since this TiDB instance is not a DDL owner anymore,
// it should not commit any transaction.
w.sess.Rollback()
return dbterror.ErrNotOwner
}
+
+ if err := w.ctx.Err(); err != nil {
+ // The worker context is canceled, it should not commit any transaction.
+ return err
+ }
return nil
}
@@ -1315,7 +1340,7 @@ func (w *worker) runDDLJob(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64,
case model.ActionAlterTablePlacement:
ver, err = onAlterTablePlacement(d, t, job)
case model.ActionCreateResourceGroup:
- ver, err = onCreateResourceGroup(d, t, job)
+ ver, err = onCreateResourceGroup(w.ctx, d, t, job)
case model.ActionAlterResourceGroup:
ver, err = onAlterResourceGroup(d, t, job)
case model.ActionDropResourceGroup:
@@ -1378,7 +1403,7 @@ func toTError(err error) *terror.Error {
// waitSchemaChanged waits for the completion of updating all servers' schema or MDL synced. In order to make sure that happens,
// we wait at most 2 * lease time(sessionTTL, 90 seconds).
-func waitSchemaChanged(d *ddlCtx, waitTime time.Duration, latestSchemaVersion int64, job *model.Job) error {
+func waitSchemaChanged(ctx context.Context, d *ddlCtx, waitTime time.Duration, latestSchemaVersion int64, job *model.Job) error {
if !job.IsRunning() && !job.IsRollingback() && !job.IsDone() && !job.IsRollbackDone() {
return nil
}
@@ -1393,13 +1418,13 @@ func waitSchemaChanged(d *ddlCtx, waitTime time.Duration, latestSchemaVersion in
}()
if latestSchemaVersion == 0 {
- tidblogutil.Logger(d.ctx).Info("schema version doesn't change", zap.String("category", "ddl"), zap.Int64("jobID", job.ID))
+ logutil.DDLLogger().Info("schema version doesn't change", zap.Int64("jobID", job.ID))
return nil
}
- err = d.schemaSyncer.OwnerUpdateGlobalVersion(d.ctx, latestSchemaVersion)
+ err = d.schemaSyncer.OwnerUpdateGlobalVersion(ctx, latestSchemaVersion)
if err != nil {
- tidblogutil.Logger(d.ctx).Info("update latest schema version failed", zap.String("category", "ddl"), zap.Int64("ver", latestSchemaVersion), zap.Error(err))
+ logutil.DDLLogger().Info("update latest schema version failed", zap.Int64("ver", latestSchemaVersion), zap.Error(err))
if variable.EnableMDL.Load() {
return err
}
@@ -1410,13 +1435,13 @@ func waitSchemaChanged(d *ddlCtx, waitTime time.Duration, latestSchemaVersion in
}
}
- return checkAllVersions(d, job, latestSchemaVersion, timeStart)
+ return checkAllVersions(ctx, d, job, latestSchemaVersion, timeStart)
}
// waitSchemaSyncedForMDL likes waitSchemaSynced, but it waits for getting the metadata lock of the latest version of this DDL.
-func waitSchemaSyncedForMDL(d *ddlCtx, job *model.Job, latestSchemaVersion int64) error {
+func waitSchemaSyncedForMDL(ctx context.Context, d *ddlCtx, job *model.Job, latestSchemaVersion int64) error {
timeStart := time.Now()
- return checkAllVersions(d, job, latestSchemaVersion, timeStart)
+ return checkAllVersions(ctx, d, job, latestSchemaVersion, timeStart)
}
func buildPlacementAffects(oldIDs []int64, newIDs []int64) []*model.AffectedOption {
diff --git a/pkg/ddl/ddl_worker_test.go b/pkg/ddl/ddl_worker_test.go
index bfcbdf866d2e8..9ea7e2bb8d6c1 100644
--- a/pkg/ddl/ddl_worker_test.go
+++ b/pkg/ddl/ddl_worker_test.go
@@ -17,6 +17,7 @@ package ddl_test
import (
"strconv"
"sync"
+ "sync/atomic"
"testing"
"time"
@@ -27,6 +28,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/util"
"github.com/stretchr/testify/require"
)
@@ -168,76 +170,38 @@ func TestParallelDDL(t *testing.T) {
seqIDs := make([]int, 11)
- wg.Run(func() {
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("alter table test_parallel_ddl_1.t1 add index db1_idx1(c1)")
- rs := tk.MustQuery("select json_extract(@@tidb_last_ddl_info, '$.seq_num')")
- seqIDs[0], _ = strconv.Atoi(rs.Rows()[0][0].(string))
- })
- time.Sleep(5 * time.Millisecond)
- wg.Run(func() {
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("alter table test_parallel_ddl_1.t1 add column c3 int")
- rs := tk.MustQuery("select json_extract(@@tidb_last_ddl_info, '$.seq_num')")
- seqIDs[1], _ = strconv.Atoi(rs.Rows()[0][0].(string))
- })
- time.Sleep(5 * time.Millisecond)
- wg.Run(func() {
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("alter table test_parallel_ddl_1.t1 add index db1_idxx(c1)")
- rs := tk.MustQuery("select json_extract(@@tidb_last_ddl_info, '$.seq_num')")
- seqIDs[2], _ = strconv.Atoi(rs.Rows()[0][0].(string))
- })
- time.Sleep(5 * time.Millisecond)
- wg.Run(func() {
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("alter table test_parallel_ddl_1.t2 drop column c3")
- rs := tk.MustQuery("select json_extract(@@tidb_last_ddl_info, '$.seq_num')")
- seqIDs[3], _ = strconv.Atoi(rs.Rows()[0][0].(string))
- })
- time.Sleep(5 * time.Millisecond)
- wg.Run(func() {
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("alter table test_parallel_ddl_1.t1 drop index db1_idx2")
- rs := tk.MustQuery("select json_extract(@@tidb_last_ddl_info, '$.seq_num')")
- seqIDs[4], _ = strconv.Atoi(rs.Rows()[0][0].(string))
- })
- time.Sleep(5 * time.Millisecond)
- wg.Run(func() {
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("alter table test_parallel_ddl_1.t2 add index db1_idx2(c2)")
- rs := tk.MustQuery("select json_extract(@@tidb_last_ddl_info, '$.seq_num')")
- seqIDs[5], _ = strconv.Atoi(rs.Rows()[0][0].(string))
- })
- time.Sleep(5 * time.Millisecond)
- wg.Run(func() {
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("alter table test_parallel_ddl_2.t3 drop column c4")
- rs := tk.MustQuery("select json_extract(@@tidb_last_ddl_info, '$.seq_num')")
- seqIDs[6], _ = strconv.Atoi(rs.Rows()[0][0].(string))
- })
- time.Sleep(5 * time.Millisecond)
- wg.Run(func() {
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("alter table test_parallel_ddl_2.t3 auto_id_cache 1024")
- rs := tk.MustQuery("select json_extract(@@tidb_last_ddl_info, '$.seq_num')")
- seqIDs[7], _ = strconv.Atoi(rs.Rows()[0][0].(string))
- })
- time.Sleep(5 * time.Millisecond)
- wg.Run(func() {
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("alter table test_parallel_ddl_1.t1 add index db1_idx3(c2)")
- rs := tk.MustQuery("select json_extract(@@tidb_last_ddl_info, '$.seq_num')")
- seqIDs[8], _ = strconv.Atoi(rs.Rows()[0][0].(string))
- })
- time.Sleep(5 * time.Millisecond)
- wg.Run(func() {
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("drop database test_parallel_ddl_2")
- rs := tk.MustQuery("select json_extract(@@tidb_last_ddl_info, '$.seq_num')")
- seqIDs[9], _ = strconv.Atoi(rs.Rows()[0][0].(string))
- })
- time.Sleep(5 * time.Millisecond)
+ var enable atomic.Bool
+ ch := make(chan struct{})
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/waitJobSubmitted",
+ func() {
+ if enable.Load() {
+ <-ch
+ }
+ },
+ )
+ enable.Store(true)
+ for i, sql := range []string{
+ "alter table test_parallel_ddl_1.t1 add index db1_idx1(c1)",
+ "alter table test_parallel_ddl_1.t1 add column c3 int",
+ "alter table test_parallel_ddl_1.t1 add index db1_idxx(c1)",
+ "alter table test_parallel_ddl_1.t2 drop column c3",
+ "alter table test_parallel_ddl_1.t1 drop index db1_idx2",
+ "alter table test_parallel_ddl_1.t2 add index db1_idx2(c2)",
+ "alter table test_parallel_ddl_2.t3 drop column c4",
+ "alter table test_parallel_ddl_2.t3 auto_id_cache 1024",
+ "alter table test_parallel_ddl_1.t1 add index db1_idx3(c2)",
+ "drop database test_parallel_ddl_2",
+ } {
+ idx := i
+ wg.Run(func() {
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec(sql)
+ rs := tk2.MustQuery("select json_extract(@@tidb_last_ddl_info, '$.seq_num')")
+ seqIDs[idx], _ = strconv.Atoi(rs.Rows()[0][0].(string))
+ })
+ ch <- struct{}{}
+ }
+ enable.Store(false)
wg.Run(func() {
tk := testkit.NewTestKit(t, store)
_ = tk.ExecToErr("alter table test_parallel_ddl_2.t3 add index db3_idx1(c2)")
diff --git a/pkg/ddl/export_test.go b/pkg/ddl/export_test.go
index 87d8920486c45..4a92aeb66883d 100644
--- a/pkg/ddl/export_test.go
+++ b/pkg/ddl/export_test.go
@@ -18,14 +18,17 @@ import (
"context"
"time"
+ "github.com/ngaut/pools"
"github.com/pingcap/tidb/pkg/ddl/copr"
"github.com/pingcap/tidb/pkg/ddl/internal/session"
+ "github.com/pingcap/tidb/pkg/errctx"
+ "github.com/pingcap/tidb/pkg/expression"
"github.com/pingcap/tidb/pkg/kv"
- "github.com/pingcap/tidb/pkg/sessionctx/stmtctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/chunk"
+ "github.com/pingcap/tidb/pkg/util/mock"
)
type resultChanForTest struct {
@@ -47,7 +50,12 @@ func FetchChunk4Test(copCtx copr.CopContext, tbl table.PhysicalTable, startKey,
}
taskCh := make(chan *reorgBackfillTask, 5)
resultCh := make(chan IndexRecordChunk, 5)
- sessPool := session.NewSessionPool(nil, store)
+ resPool := pools.NewResourcePool(func() (pools.Resource, error) {
+ ctx := mock.NewContext()
+ ctx.Store = store
+ return ctx, nil
+ }, 8, 8, 0)
+ sessPool := session.NewSessionPool(resPool, store)
pool := newCopReqSenderPool(context.Background(), copCtx, store, taskCh, sessPool, nil)
pool.chunkSender = &resultChanForTest{ch: resultCh}
pool.adjustSize(1)
@@ -55,16 +63,18 @@ func FetchChunk4Test(copCtx copr.CopContext, tbl table.PhysicalTable, startKey,
rs := <-resultCh
close(taskCh)
pool.close(false)
+ sessPool.Close()
return rs.Chunk
}
func ConvertRowToHandleAndIndexDatum(
+ ctx expression.EvalContext,
handleDataBuf, idxDataBuf []types.Datum,
row chunk.Row, copCtx copr.CopContext, idxID int64) (kv.Handle, []types.Datum, error) {
c := copCtx.GetBase()
- idxData := extractDatumByOffsets(row, copCtx.IndexColumnOutputOffsets(idxID), c.ExprColumnInfos, idxDataBuf)
- handleData := extractDatumByOffsets(row, c.HandleOutputOffsets, c.ExprColumnInfos, handleDataBuf)
- handle, err := buildHandle(handleData, c.TableInfo, c.PrimaryKeyInfo, stmtctx.NewStmtCtxWithTimeZone(time.Local))
+ idxData := extractDatumByOffsets(ctx, row, copCtx.IndexColumnOutputOffsets(idxID), c.ExprColumnInfos, idxDataBuf)
+ handleData := extractDatumByOffsets(ctx, row, c.HandleOutputOffsets, c.ExprColumnInfos, handleDataBuf)
+ handle, err := buildHandle(handleData, c.TableInfo, c.PrimaryKeyInfo, time.Local, errctx.StrictNoWarningContext)
return handle, idxData, err
}
diff --git a/pkg/ddl/index.go b/pkg/ddl/index.go
index f0468554be25b..ca96a9f30b3bc 100644
--- a/pkg/ddl/index.go
+++ b/pkg/ddl/index.go
@@ -39,6 +39,7 @@ import (
"github.com/pingcap/tidb/pkg/disttask/framework/proto"
"github.com/pingcap/tidb/pkg/disttask/framework/scheduler"
"github.com/pingcap/tidb/pkg/disttask/framework/storage"
+ "github.com/pingcap/tidb/pkg/errctx"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/lightning/common"
@@ -50,7 +51,6 @@ import (
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/parser/terror"
"github.com/pingcap/tidb/pkg/sessionctx"
- "github.com/pingcap/tidb/pkg/sessionctx/stmtctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/store/helper"
"github.com/pingcap/tidb/pkg/table"
@@ -61,6 +61,7 @@ import (
"github.com/pingcap/tidb/pkg/util/backoff"
"github.com/pingcap/tidb/pkg/util/chunk"
"github.com/pingcap/tidb/pkg/util/codec"
+ contextutil "github.com/pingcap/tidb/pkg/util/context"
"github.com/pingcap/tidb/pkg/util/dbterror"
tidblogutil "github.com/pingcap/tidb/pkg/util/logutil"
decoder "github.com/pingcap/tidb/pkg/util/rowDecoder"
@@ -133,8 +134,7 @@ func buildIndexColumns(ctx sessionctx.Context, columns []*model.ColumnInfo, inde
if sumLength > maxIndexLength {
// The multiple column index and the unique index in which the length sum exceeds the maximum size
// will return an error instead produce a warning.
- suppress := suppressErrorTooLongKeyKey(ctx)
- if ctx == nil || (ctx.GetSessionVars().SQLMode.HasStrictMode() && !suppress) || mysql.HasUniKeyFlag(col.GetFlag()) || len(indexPartSpecifications) > 1 {
+ if ctx == nil || (ctx.GetSessionVars().SQLMode.HasStrictMode() && !suppressErrorTooLongKeyKey(ctx)) || mysql.HasUniKeyFlag(col.GetFlag()) || len(indexPartSpecifications) > 1 {
return nil, false, dbterror.ErrTooLongKey.GenWithStackByArgs(sumLength, maxIndexLength)
}
// truncate index length and produce warning message in non-restrict sql mode.
@@ -244,8 +244,7 @@ func checkIndexColumn(ctx sessionctx.Context, col *model.ColumnInfo, indexColumn
// Specified length must be shorter than the max length for prefix.
maxIndexLength := config.GetGlobalConfig().MaxIndexLength
if indexColumnLen > maxIndexLength {
- suppress := suppressErrorTooLongKeyKey(ctx)
- if ctx == nil || (ctx.GetSessionVars().SQLMode.HasStrictMode() && !suppress) {
+ if ctx == nil || (ctx.GetSessionVars().SQLMode.HasStrictMode() && !suppressErrorTooLongKeyKey(ctx)) {
// return error in strict sql mode
return dbterror.ErrTooLongKey.GenWithStackByArgs(indexColumnLen, maxIndexLength)
}
@@ -877,6 +876,11 @@ func doReorgWorkForCreateIndex(w *worker, d *ddlCtx, t *meta.Meta, job *model.Jo
ver, err = updateVersionAndTableInfo(d, t, job, tbl.Meta(), true)
return false, ver, errors.Trace(err)
case model.BackfillStateReadyToMerge:
+ failpoint.Inject("mockDMLExecutionStateBeforeMerge", func(_ failpoint.Value) {
+ if MockDMLExecutionStateBeforeMerge != nil {
+ MockDMLExecutionStateBeforeMerge()
+ }
+ })
logutil.DDLLogger().Info("index backfill state ready to merge",
zap.Int64("job ID", job.ID),
zap.String("table", tbl.Meta().Name.O),
@@ -1434,7 +1438,7 @@ func (w *baseIndexWorker) getIndexRecord(idxInfo *model.IndexInfo, handle kv.Han
idxVal[j] = idxColumnVal
continue
}
- idxColumnVal, err = tables.GetColDefaultValue(w.sessCtx, col, w.defaultVals)
+ idxColumnVal, err = tables.GetColDefaultValue(w.exprCtx, col, w.defaultVals)
if err != nil {
return nil, errors.Trace(err)
}
@@ -1465,8 +1469,8 @@ func (w *baseIndexWorker) getNextKey(taskRange reorgBackfillTask, taskDone bool)
}
func (w *baseIndexWorker) updateRowDecoder(handle kv.Handle, rawRecord []byte) error {
- sysZone := w.sessCtx.GetSessionVars().StmtCtx.TimeZone()
- _, err := w.rowDecoder.DecodeAndEvalRowWithMap(w.sessCtx, handle, rawRecord, sysZone, w.rowMap)
+ sysZone := w.loc
+ _, err := w.rowDecoder.DecodeAndEvalRowWithMap(w.exprCtx, handle, rawRecord, sysZone, w.rowMap)
return errors.Trace(err)
}
@@ -1484,7 +1488,7 @@ func (w *baseIndexWorker) fetchRowColVals(txn kv.Transaction, taskRange reorgBac
// taskDone means that the reorged handle is out of taskRange.endHandle.
taskDone := false
oprStartTime := startTime
- err := iterateSnapshotKeys(w.jobContext, w.sessCtx.GetStore(), taskRange.priority, taskRange.physicalTable.RecordPrefix(), txn.StartTS(),
+ err := iterateSnapshotKeys(w.jobContext, w.ddlCtx.store, taskRange.priority, taskRange.physicalTable.RecordPrefix(), txn.StartTS(),
taskRange.startKey, taskRange.endKey, func(handle kv.Handle, recordKey kv.Key, rawRow []byte) (bool, error) {
oprEndTime := time.Now()
logSlowOperations(oprEndTime.Sub(oprStartTime), "iterateSnapshotKeys in baseIndexWorker fetchRowColVals", 0)
@@ -1568,7 +1572,8 @@ func genKeyExistsErr(key, value []byte, idxInfo *model.IndexInfo, tblInfo *model
// Note that `idxRecords` may belong to multiple indexes.
func (w *addIndexTxnWorker) batchCheckUniqueKey(txn kv.Transaction, idxRecords []*indexRecord) error {
w.initBatchCheckBufs(len(idxRecords))
- stmtCtx := w.sessCtx.GetSessionVars().StmtCtx
+ evalCtx := w.exprCtx.GetEvalCtx()
+ ec := evalCtx.ErrCtx()
uniqueBatchKeys := make([]kv.Key, 0, len(idxRecords))
cnt := 0
for i, record := range idxRecords {
@@ -1584,7 +1589,7 @@ func (w *addIndexTxnWorker) batchCheckUniqueKey(txn kv.Transaction, idxRecords [
}
// skip by default.
idxRecords[i].skip = true
- iter := idx.GenIndexKVIter(stmtCtx.ErrCtx(), stmtCtx.TimeZone(), record.vals, record.handle, idxRecords[i].rsData)
+ iter := idx.GenIndexKVIter(ec, w.loc, record.vals, record.handle, idxRecords[i].rsData)
for iter.Valid() {
var buf []byte
if cnt < len(w.idxKeyBufs) {
@@ -1640,7 +1645,7 @@ func (w *addIndexTxnWorker) batchCheckUniqueKey(txn kv.Transaction, idxRecords [
idxRecords[w.recordIdx[i]].skip = found && idxRecords[w.recordIdx[i]].skip
}
// Constrains is already checked.
- stmtCtx.BatchCheck = true
+ w.tblCtx.GetSessionVars().StmtCtx.BatchCheck = true
return nil
}
@@ -1648,7 +1653,9 @@ type addIndexIngestWorker struct {
ctx context.Context
d *ddlCtx
metricCounter prometheus.Counter
- sessCtx sessionctx.Context
+ writeLoc *time.Location
+ writeErrCtx errctx.Context
+ writeStmtBufs variable.WriteStmtBufs
tbl table.PhysicalTable
indexes []table.Index
@@ -1663,17 +1670,20 @@ type addIndexIngestWorker struct {
func newAddIndexIngestWorker(
ctx context.Context,
t table.PhysicalTable,
- d *ddlCtx,
+ info *reorgInfo,
engines []ingest.Engine,
resultCh chan *backfillResult,
jobID int64,
- schemaName string,
indexIDs []int64,
writerID int,
copReqSenderPool *copReqSenderPool,
- sessCtx sessionctx.Context,
checkpointMgr *ingest.CheckpointManager,
) (*addIndexIngestWorker, error) {
+ writeLoc, err := reorgTimeZoneWithTzLoc(info.ReorgMeta.Location)
+ if err != nil {
+ return nil, err
+ }
+
indexes := make([]table.Index, 0, len(indexIDs))
writers := make([]ingest.Writer, 0, len(indexIDs))
for i, indexID := range indexIDs {
@@ -1687,12 +1697,18 @@ func newAddIndexIngestWorker(
writers = append(writers, lw)
}
+ writeErrCtx := errctx.NewContextWithLevels(
+ reorgErrLevelsWithSQLMode(info.ReorgMeta.SQLMode),
+ contextutil.IgnoreWarn,
+ )
+
return &addIndexIngestWorker{
- ctx: ctx,
- d: d,
- sessCtx: sessCtx,
+ ctx: ctx,
+ d: info.d,
+ writeLoc: writeLoc,
+ writeErrCtx: writeErrCtx,
metricCounter: metrics.BackfillTotalCounter.WithLabelValues(
- metrics.GenerateReorgLabel("add_idx_rate", schemaName, t.Meta().Name.O)),
+ metrics.GenerateReorgLabel("add_idx_rate", info.SchemaName, t.Meta().Name.O)),
tbl: t,
indexes: indexes,
writers: writers,
@@ -1707,9 +1723,8 @@ func newAddIndexIngestWorker(
func (w *addIndexIngestWorker) WriteLocal(rs *IndexRecordChunk) (count int, nextKey kv.Key, err error) {
oprStartTime := time.Now()
copCtx := w.copReqSenderPool.copCtx
- vars := w.sessCtx.GetSessionVars()
cnt, lastHandle, err := writeChunkToLocal(
- w.ctx, w.writers, w.indexes, copCtx, vars, rs.Chunk)
+ w.ctx, w.writers, w.indexes, copCtx, w.writeLoc, w.writeErrCtx, &w.writeStmtBufs, rs.Chunk)
if err != nil || cnt == 0 {
return 0, nil, err
}
@@ -1724,12 +1739,14 @@ func writeChunkToLocal(
writers []ingest.Writer,
indexes []table.Index,
copCtx copr.CopContext,
- vars *variable.SessionVars,
+ loc *time.Location,
+ errCtx errctx.Context,
+ writeStmtBufs *variable.WriteStmtBufs,
copChunk *chunk.Chunk,
) (int, kv.Handle, error) {
- sCtx, writeBufs := vars.StmtCtx, vars.GetWriteStmtBufs()
iter := chunk.NewIterator4Chunk(copChunk)
c := copCtx.GetBase()
+ ectx := c.ExprCtx.GetEvalCtx()
maxIdxColCnt := maxIndexColumnCount(indexes)
idxDataBuf := make([]types.Datum, maxIdxColCnt)
@@ -1749,9 +1766,12 @@ func writeChunkToLocal(
}
}()
needRestoreForIndexes := make([]bool, len(indexes))
- restore := false
+ restore, pkNeedRestore := false, false
+ if c.PrimaryKeyInfo != nil && c.TableInfo.IsCommonHandle && c.TableInfo.CommonHandleVersion != 0 {
+ pkNeedRestore = tables.NeedRestoredData(c.PrimaryKeyInfo.Columns, c.TableInfo.Columns)
+ }
for i, index := range indexes {
- needRestore := tables.NeedRestoredData(index.Meta().Columns, c.TableInfo.Columns)
+ needRestore := pkNeedRestore || tables.NeedRestoredData(index.Meta().Columns, c.TableInfo.Columns)
needRestoreForIndexes[i] = needRestore
restore = restore || needRestore
}
@@ -1759,27 +1779,27 @@ func writeChunkToLocal(
restoreDataBuf = make([]types.Datum, len(c.HandleOutputOffsets))
}
for row := iter.Begin(); row != iter.End(); row = iter.Next() {
- handleDataBuf := extractDatumByOffsets(row, c.HandleOutputOffsets, c.ExprColumnInfos, handleDataBuf)
+ handleDataBuf := extractDatumByOffsets(ectx, row, c.HandleOutputOffsets, c.ExprColumnInfos, handleDataBuf)
if restore {
// restoreDataBuf should not truncate index values.
for i, datum := range handleDataBuf {
restoreDataBuf[i] = *datum.Clone()
}
}
- h, err := buildHandle(handleDataBuf, c.TableInfo, c.PrimaryKeyInfo, sCtx)
+ h, err := buildHandle(handleDataBuf, c.TableInfo, c.PrimaryKeyInfo, loc, errCtx)
if err != nil {
return 0, nil, errors.Trace(err)
}
for i, index := range indexes {
idxID := index.Meta().ID
- idxDataBuf = extractDatumByOffsets(
+ idxDataBuf = extractDatumByOffsets(ectx,
row, copCtx.IndexColumnOutputOffsets(idxID), c.ExprColumnInfos, idxDataBuf)
idxData := idxDataBuf[:len(index.Meta().Columns)]
var rsData []types.Datum
if needRestoreForIndexes[i] {
rsData = getRestoreData(c.TableInfo, copCtx.IndexInfo(idxID), c.PrimaryKeyInfo, restoreDataBuf)
}
- err = writeOneKVToLocal(ctx, writers[i], index, sCtx, writeBufs, idxData, rsData, h)
+ err = writeOneKVToLocal(ctx, writers[i], index, loc, errCtx, writeStmtBufs, idxData, rsData, h)
if err != nil {
return 0, nil, errors.Trace(err)
}
@@ -1805,12 +1825,13 @@ func writeOneKVToLocal(
ctx context.Context,
writer ingest.Writer,
index table.Index,
- sCtx *stmtctx.StatementContext,
+ loc *time.Location,
+ errCtx errctx.Context,
writeBufs *variable.WriteStmtBufs,
idxDt, rsData []types.Datum,
handle kv.Handle,
) error {
- iter := index.GenIndexKVIter(sCtx.ErrCtx(), sCtx.TimeZone(), idxDt, handle, rsData)
+ iter := index.GenIndexKVIter(errCtx, loc, idxDt, handle, rsData)
for iter.Valid() {
key, idxVal, _, err := iter.Next(writeBufs.IndexKeyBuf, writeBufs.RowValBuf)
if err != nil {
@@ -1849,7 +1870,7 @@ func (w *addIndexTxnWorker) BackfillData(handleRange reorgBackfillTask) (taskCtx
oprStartTime := time.Now()
jobID := handleRange.getJobID()
ctx := kv.WithInternalSourceAndTaskType(context.Background(), w.jobContext.ddlJobSourceType(), kvutil.ExplicitTypeDDL)
- errInTxn = kv.RunInNewTxn(ctx, w.sessCtx.GetStore(), true, func(_ context.Context, txn kv.Transaction) (err error) {
+ errInTxn = kv.RunInNewTxn(ctx, w.ddlCtx.store, true, func(_ context.Context, txn kv.Transaction) (err error) {
taskCtx.finishTS = txn.StartTS()
taskCtx.addedCount = 0
taskCtx.scanCount = 0
@@ -1888,7 +1909,7 @@ func (w *addIndexTxnWorker) BackfillData(handleRange reorgBackfillTask) (taskCtx
}
handle, err := w.indexes[i%len(w.indexes)].Create(
- w.sessCtx.GetTableCtx(), txn, idxRecord.vals, idxRecord.handle, idxRecord.rsData, table.WithIgnoreAssertion, table.FromBackfill)
+ w.tblCtx, txn, idxRecord.vals, idxRecord.handle, idxRecord.rsData, table.WithIgnoreAssertion, table.FromBackfill)
if err != nil {
if kv.ErrKeyExists.Equal(err) && idxRecord.handle.Equal(handle) {
// Index already exists, skip it.
@@ -1923,13 +1944,16 @@ var MockDMLExecutionStateMerging func()
// MockDMLExecutionStateBeforeImport is only used for test.
var MockDMLExecutionStateBeforeImport func()
+// MockDMLExecutionStateBeforeMerge is only used for test.
+var MockDMLExecutionStateBeforeMerge func()
+
func (w *worker) addPhysicalTableIndex(t table.PhysicalTable, reorgInfo *reorgInfo) error {
if reorgInfo.mergingTmpIdx {
logutil.DDLLogger().Info("start to merge temp index", zap.Stringer("job", reorgInfo.Job), zap.Stringer("reorgInfo", reorgInfo))
- return w.writePhysicalTableRecord(w.sessPool, t, typeAddIndexMergeTmpWorker, reorgInfo)
+ return w.writePhysicalTableRecord(w.ctx, w.sessPool, t, typeAddIndexMergeTmpWorker, reorgInfo)
}
logutil.DDLLogger().Info("start to add table index", zap.Stringer("job", reorgInfo.Job), zap.Stringer("reorgInfo", reorgInfo))
- return w.writePhysicalTableRecord(w.sessPool, t, typeAddIndexWorker, reorgInfo)
+ return w.writePhysicalTableRecord(w.ctx, w.sessPool, t, typeAddIndexWorker, reorgInfo)
}
// addTableIndex handles the add index reorganization state for a table.
@@ -1962,6 +1986,7 @@ func (w *worker) addTableIndex(t table.Table, reorgInfo *reorgInfo) error {
w.ddlCtx.mu.RLock()
w.ddlCtx.mu.hook.OnUpdateReorgInfo(reorgInfo.Job, reorgInfo.PhysicalTableID)
w.ddlCtx.mu.RUnlock()
+ failpoint.InjectCall("beforeUpdateReorgInfo-addTableIndex", reorgInfo.Job)
finish, err = updateReorgInfo(w.sessPool, tbl, reorgInfo)
if err != nil {
@@ -2010,15 +2035,6 @@ func checkDuplicateForUniqueIndex(ctx context.Context, t table.Table, reorgInfo
return nil
}
-// MockDMLExecutionOnTaskFinished is used to mock DML execution when tasks finished.
-var MockDMLExecutionOnTaskFinished func()
-
-// MockDMLExecutionOnDDLPaused is used to mock DML execution when ddl job paused.
-var MockDMLExecutionOnDDLPaused func()
-
-// TestSyncChan is used to sync the test.
-var TestSyncChan = make(chan struct{})
-
func (w *worker) executeDistTask(t table.Table, reorgInfo *reorgInfo) error {
if reorgInfo.mergingTmpIdx {
return errors.New("do not support merge index")
@@ -2026,7 +2042,7 @@ func (w *worker) executeDistTask(t table.Table, reorgInfo *reorgInfo) error {
taskType := proto.Backfill
taskKey := fmt.Sprintf("ddl/%s/%d", taskType, reorgInfo.Job.ID)
- g, ctx := errgroup.WithContext(context.Background())
+ g, ctx := errgroup.WithContext(w.ctx)
ctx = kv.WithInternalSourceType(ctx, kv.InternalDistTask)
done := make(chan struct{})
@@ -2106,9 +2122,7 @@ func (w *worker) executeDistTask(t table.Table, reorgInfo *reorgInfo) error {
g.Go(func() error {
defer close(done)
err := submitAndWaitTask(ctx, taskKey, taskType, concurrency, reorgInfo.ReorgMeta.TargetScope, metaData)
- failpoint.Inject("pauseAfterDistTaskFinished", func() {
- MockDMLExecutionOnTaskFinished()
- })
+ failpoint.InjectCall("pauseAfterDistTaskFinished")
if err := w.isReorgRunnable(reorgInfo.Job.ID, true); err != nil {
if dbterror.ErrPausedDDLJob.Equal(err) {
logutil.DDLLogger().Warn("job paused by user", zap.Error(err))
@@ -2136,10 +2150,7 @@ func (w *worker) executeDistTask(t table.Table, reorgInfo *reorgInfo) error {
logutil.DDLLogger().Error("pause task error", zap.String("task_key", taskKey), zap.Error(err))
continue
}
- failpoint.Inject("syncDDLTaskPause", func() {
- // make sure the task is paused.
- TestSyncChan <- struct{}{}
- })
+ failpoint.InjectCall("syncDDLTaskPause")
}
if !dbterror.ErrCancelledDDLJob.Equal(err) {
return errors.Trace(err)
@@ -2408,7 +2419,12 @@ type cleanUpIndexWorker struct {
baseIndexWorker
}
-func newCleanUpIndexWorker(sessCtx sessionctx.Context, id int, t table.PhysicalTable, decodeColMap map[int64]decoder.Column, reorgInfo *reorgInfo, jc *JobContext) *cleanUpIndexWorker {
+func newCleanUpIndexWorker(id int, t table.PhysicalTable, decodeColMap map[int64]decoder.Column, reorgInfo *reorgInfo, jc *JobContext) (*cleanUpIndexWorker, error) {
+ bCtx, err := newBackfillCtx(id, reorgInfo, reorgInfo.SchemaName, t, jc, "cleanup_idx_rate", false)
+ if err != nil {
+ return nil, err
+ }
+
indexes := make([]table.Index, 0, len(t.Indices()))
rowDecoder := decoder.NewRowDecoder(t, t.WritableCols(), decodeColMap)
for _, index := range t.Indices() {
@@ -2418,13 +2434,13 @@ func newCleanUpIndexWorker(sessCtx sessionctx.Context, id int, t table.PhysicalT
}
return &cleanUpIndexWorker{
baseIndexWorker: baseIndexWorker{
- backfillCtx: newBackfillCtx(reorgInfo.d, id, sessCtx, reorgInfo.SchemaName, t, jc, "cleanup_idx_rate", false),
+ backfillCtx: bCtx,
indexes: indexes,
rowDecoder: rowDecoder,
defaultVals: make([]types.Datum, len(t.WritableCols())),
rowMap: make(map[int64]types.Datum, len(decodeColMap)),
},
- }
+ }, nil
}
func (w *cleanUpIndexWorker) BackfillData(handleRange reorgBackfillTask) (taskCtx backfillTaskContext, errInTxn error) {
@@ -2437,7 +2453,7 @@ func (w *cleanUpIndexWorker) BackfillData(handleRange reorgBackfillTask) (taskCt
oprStartTime := time.Now()
ctx := kv.WithInternalSourceAndTaskType(context.Background(), w.jobContext.ddlJobSourceType(), kvutil.ExplicitTypeDDL)
- errInTxn = kv.RunInNewTxn(ctx, w.sessCtx.GetStore(), true, func(_ context.Context, txn kv.Transaction) error {
+ errInTxn = kv.RunInNewTxn(ctx, w.ddlCtx.store, true, func(_ context.Context, txn kv.Transaction) error {
taskCtx.addedCount = 0
taskCtx.scanCount = 0
updateTxnEntrySizeLimitIfNeeded(txn)
@@ -2462,7 +2478,7 @@ func (w *cleanUpIndexWorker) BackfillData(handleRange reorgBackfillTask) (taskCt
// we fetch records row by row, so records will belong to
// index[0], index[1] ... index[n-1], index[0], index[1] ...
// respectively. So indexes[i%n] is the index of idxRecords[i].
- err := w.indexes[i%n].Delete(w.sessCtx.GetTableCtx(), txn, idxRecord.vals, idxRecord.handle)
+ err := w.indexes[i%n].Delete(w.tblCtx, txn, idxRecord.vals, idxRecord.handle)
if err != nil {
return errors.Trace(err)
}
@@ -2471,6 +2487,12 @@ func (w *cleanUpIndexWorker) BackfillData(handleRange reorgBackfillTask) (taskCt
return nil
})
logSlowOperations(time.Since(oprStartTime), "cleanUpIndexBackfillDataInTxn", 3000)
+ failpoint.Inject("mockDMLExecution", func(val failpoint.Value) {
+ //nolint:forcetypeassert
+ if val.(bool) && MockDMLExecution != nil {
+ MockDMLExecution()
+ }
+ })
return
}
@@ -2478,7 +2500,7 @@ func (w *cleanUpIndexWorker) BackfillData(handleRange reorgBackfillTask) (taskCt
// cleanupPhysicalTableIndex handles the drop partition reorganization state for a non-partitioned table or a partition.
func (w *worker) cleanupPhysicalTableIndex(t table.PhysicalTable, reorgInfo *reorgInfo) error {
logutil.DDLLogger().Info("start to clean up index", zap.Stringer("job", reorgInfo.Job), zap.Stringer("reorgInfo", reorgInfo))
- return w.writePhysicalTableRecord(w.sessPool, t, typeCleanUpIndexWorker, reorgInfo)
+ return w.writePhysicalTableRecord(w.ctx, w.sessPool, t, typeCleanUpIndexWorker, reorgInfo)
}
// cleanupGlobalIndex handles the drop partition reorganization state to clean up index entries of partitions.
diff --git a/pkg/ddl/index_cop.go b/pkg/ddl/index_cop.go
index 58d6980ad9788..db599d562779e 100644
--- a/pkg/ddl/index_cop.go
+++ b/pkg/ddl/index_cop.go
@@ -26,13 +26,14 @@ import (
"github.com/pingcap/tidb/pkg/ddl/ingest"
sess "github.com/pingcap/tidb/pkg/ddl/internal/session"
"github.com/pingcap/tidb/pkg/distsql"
+ distsqlctx "github.com/pingcap/tidb/pkg/distsql/context"
+ "github.com/pingcap/tidb/pkg/errctx"
"github.com/pingcap/tidb/pkg/expression"
+ exprctx "github.com/pingcap/tidb/pkg/expression/context"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/metrics"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/terror"
- "github.com/pingcap/tidb/pkg/sessionctx"
- "github.com/pingcap/tidb/pkg/sessionctx/stmtctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/table/tables"
@@ -121,7 +122,7 @@ func (c *copReqSender) run() {
if !ok {
return
}
- if p.checkpointMgr != nil && p.checkpointMgr.IsComplete(task.endKey) {
+ if p.checkpointMgr != nil && p.checkpointMgr.IsKeyProcessed(task.endKey) {
logutil.Logger(p.ctx).Info("checkpoint detected, skip a cop-request task",
zap.Int("task ID", task.id),
zap.String("task end key", hex.EncodeToString(task.endKey)))
@@ -163,7 +164,7 @@ func scanRecords(p *copReqSenderPool, task *reorgBackfillTask, se *sess.Session)
return err
}
if p.checkpointMgr != nil {
- p.checkpointMgr.UpdateTotal(task.id, srcChk.NumRows(), done)
+ p.checkpointMgr.UpdateTotalKeys(task.id, srcChk.NumRows(), done)
}
idxRs := IndexRecordChunk{ID: task.id, Chunk: srcChk, Done: done}
rate := float64(srcChk.MemoryUsage()) / 1024.0 / 1024.0 / time.Since(startTime).Seconds()
@@ -269,7 +270,7 @@ func (c *copReqSenderPool) recycleChunk(chk *chunk.Chunk) {
}
func buildTableScan(ctx context.Context, c *copr.CopContextBase, startTS uint64, start, end kv.Key) (distsql.SelectResult, error) {
- dagPB, err := buildDAGPB(c.SessionContext, c.TableInfo, c.ColumnInfos)
+ dagPB, err := buildDAGPB(c.ExprCtx, c.DistSQLCtx, c.PushDownFlags, c.TableInfo, c.ColumnInfos)
if err != nil {
return nil, err
}
@@ -280,8 +281,7 @@ func buildTableScan(ctx context.Context, c *copr.CopContextBase, startTS uint64,
SetStartTS(startTS).
SetKeyRanges([]kv.KeyRange{{StartKey: start, EndKey: end}}).
SetKeepOrder(true).
- SetFromSessionVars(c.SessionContext.GetDistSQLCtx()).
- SetFromInfoSchema(c.SessionContext.GetDomainInfoSchema()).
+ SetFromSessionVars(c.DistSQLCtx).
SetConcurrency(1).
Build()
kvReq.RequestSource.RequestSourceInternal = true
@@ -290,7 +290,7 @@ func buildTableScan(ctx context.Context, c *copr.CopContextBase, startTS uint64,
if err != nil {
return nil, err
}
- return distsql.Select(ctx, c.SessionContext.GetDistSQLCtx(), kvReq, c.FieldTypes)
+ return distsql.Select(ctx, c.DistSQLCtx, kvReq, c.FieldTypes)
}
func fetchTableScanResult(
@@ -308,7 +308,7 @@ func fetchTableScanResult(
}
err = table.FillVirtualColumnValue(
copCtx.VirtualColumnsFieldTypes, copCtx.VirtualColumnsOutputOffsets,
- copCtx.ExprColumnInfos, copCtx.ColumnInfos, copCtx.SessionContext.GetExprCtx(), chk)
+ copCtx.ExprColumnInfos, copCtx.ColumnInfos, copCtx.ExprCtx, chk)
return false, err
}
@@ -346,44 +346,43 @@ func getRestoreData(tblInfo *model.TableInfo, targetIdx, pkIdx *model.IndexInfo,
return dtToRestored
}
-func buildDAGPB(sCtx sessionctx.Context, tblInfo *model.TableInfo, colInfos []*model.ColumnInfo) (*tipb.DAGRequest, error) {
+func buildDAGPB(exprCtx exprctx.BuildContext, distSQLCtx *distsqlctx.DistSQLContext, pushDownFlags uint64, tblInfo *model.TableInfo, colInfos []*model.ColumnInfo) (*tipb.DAGRequest, error) {
dagReq := &tipb.DAGRequest{}
- dagReq.TimeZoneName, dagReq.TimeZoneOffset = timeutil.Zone(sCtx.GetSessionVars().Location())
- sc := sCtx.GetSessionVars().StmtCtx
- dagReq.Flags = sc.PushDownFlags()
+ dagReq.TimeZoneName, dagReq.TimeZoneOffset = timeutil.Zone(exprCtx.GetEvalCtx().Location())
+ dagReq.Flags = pushDownFlags
for i := range colInfos {
dagReq.OutputOffsets = append(dagReq.OutputOffsets, uint32(i))
}
- execPB, err := constructTableScanPB(sCtx, tblInfo, colInfos)
+ execPB, err := constructTableScanPB(exprCtx, tblInfo, colInfos)
if err != nil {
return nil, err
}
dagReq.Executors = append(dagReq.Executors, execPB)
- distsql.SetEncodeType(sCtx.GetDistSQLCtx(), dagReq)
+ distsql.SetEncodeType(distSQLCtx, dagReq)
return dagReq, nil
}
-func constructTableScanPB(sCtx sessionctx.Context, tblInfo *model.TableInfo, colInfos []*model.ColumnInfo) (*tipb.Executor, error) {
+func constructTableScanPB(ctx exprctx.BuildContext, tblInfo *model.TableInfo, colInfos []*model.ColumnInfo) (*tipb.Executor, error) {
tblScan := tables.BuildTableScanFromInfos(tblInfo, colInfos)
tblScan.TableId = tblInfo.ID
- err := tables.SetPBColumnsDefaultValue(sCtx.GetExprCtx(), tblScan.Columns, colInfos)
+ err := tables.SetPBColumnsDefaultValue(ctx, tblScan.Columns, colInfos)
return &tipb.Executor{Tp: tipb.ExecType_TypeTableScan, TblScan: tblScan}, err
}
-func extractDatumByOffsets(row chunk.Row, offsets []int, expCols []*expression.Column, buf []types.Datum) []types.Datum {
+func extractDatumByOffsets(ctx expression.EvalContext, row chunk.Row, offsets []int, expCols []*expression.Column, buf []types.Datum) []types.Datum {
for i, offset := range offsets {
c := expCols[offset]
- row.DatumWithBuffer(offset, c.GetType(), &buf[i])
+ row.DatumWithBuffer(offset, c.GetType(ctx), &buf[i])
}
return buf
}
func buildHandle(pkDts []types.Datum, tblInfo *model.TableInfo,
- pkInfo *model.IndexInfo, stmtCtx *stmtctx.StatementContext) (kv.Handle, error) {
+ pkInfo *model.IndexInfo, loc *time.Location, errCtx errctx.Context) (kv.Handle, error) {
if tblInfo.IsCommonHandle {
tablecodec.TruncateIndexValues(tblInfo, pkInfo, pkDts)
- handleBytes, err := codec.EncodeKey(stmtCtx.TimeZone(), nil, pkDts...)
- err = stmtCtx.HandleError(err)
+ handleBytes, err := codec.EncodeKey(loc, nil, pkDts...)
+ err = errCtx.HandleError(err)
if err != nil {
return nil, err
}
diff --git a/pkg/ddl/index_cop_test.go b/pkg/ddl/index_cop_test.go
index 6b33e3bb5037e..32817f7e54cef 100644
--- a/pkg/ddl/index_cop_test.go
+++ b/pkg/ddl/index_cop_test.go
@@ -40,8 +40,11 @@ func TestAddIndexFetchRowsFromCoprocessor(t *testing.T) {
require.NoError(t, err)
tblInfo := tbl.Meta()
idxInfo := tblInfo.FindIndexByName(idx)
- copCtx, err := copr.NewCopContextSingleIndex(tblInfo, idxInfo, tk.Session(), "")
+
+ sctx := tk.Session()
+ copCtx, err := ddl.NewReorgCopContext(store, ddl.NewDDLReorgMeta(sctx), tblInfo, []*model.IndexInfo{idxInfo}, "")
require.NoError(t, err)
+ require.IsType(t, copCtx, &copr.CopContextSingleIndex{})
startKey := tbl.RecordPrefix()
endKey := startKey.PrefixNext()
txn, err := store.Begin()
@@ -57,7 +60,7 @@ func TestAddIndexFetchRowsFromCoprocessor(t *testing.T) {
idxDataBuf := make([]types.Datum, len(idxInfo.Columns))
for row := iter.Begin(); row != iter.End(); row = iter.Next() {
- handle, idxDatum, err := ddl.ConvertRowToHandleAndIndexDatum(handleDataBuf, idxDataBuf, row, copCtx, idxInfo.ID)
+ handle, idxDatum, err := ddl.ConvertRowToHandleAndIndexDatum(tk.Session().GetExprCtx().GetEvalCtx(), handleDataBuf, idxDataBuf, row, copCtx, idxInfo.ID)
require.NoError(t, err)
handles = append(handles, handle)
copiedIdxDatum := make([]types.Datum, len(idxDatum))
diff --git a/pkg/ddl/index_merge_tmp.go b/pkg/ddl/index_merge_tmp.go
index ba4619f3c8d33..c0001250b6c22 100644
--- a/pkg/ddl/index_merge_tmp.go
+++ b/pkg/ddl/index_merge_tmp.go
@@ -84,7 +84,7 @@ func checkTempIndexKey(txn kv.Transaction, tmpRec *temporaryIndexRecord, originI
return nil
}
// For distinct index key values, prevent deleting an unexpected index KV in original index.
- hdInVal, err := tablecodec.DecodeHandleInUniqueIndexValue(originIdxVal, tblInfo.Meta().IsCommonHandle)
+ hdInVal, err := tablecodec.DecodeHandleInIndexValue(originIdxVal)
if err != nil {
return errors.Trace(err)
}
@@ -185,7 +185,7 @@ func (w *mergeIndexWorker) BackfillData(taskRange reorgBackfillTask) (taskCtx ba
oprStartTime := time.Now()
ctx := kv.WithInternalSourceAndTaskType(context.Background(), w.jobContext.ddlJobSourceType(), kvutil.ExplicitTypeDDL)
- errInTxn = kv.RunInNewTxn(ctx, w.sessCtx.GetStore(), true, func(_ context.Context, txn kv.Transaction) error {
+ errInTxn = kv.RunInNewTxn(ctx, w.ddlCtx.store, true, func(_ context.Context, txn kv.Transaction) error {
taskCtx.addedCount = 0
taskCtx.scanCount = 0
updateTxnEntrySizeLimitIfNeeded(txn)
@@ -308,7 +308,7 @@ func (w *mergeIndexWorker) fetchTempIndexVals(
oprStartTime := startTime
idxPrefix := w.table.IndexPrefix()
var lastKey kv.Key
- err := iterateSnapshotKeys(w.jobContext, w.sessCtx.GetStore(), taskRange.priority, idxPrefix, txn.StartTS(),
+ err := iterateSnapshotKeys(w.jobContext, w.ddlCtx.store, taskRange.priority, idxPrefix, txn.StartTS(),
taskRange.startKey, taskRange.endKey, func(_ kv.Handle, indexKey kv.Key, rawValue []byte) (more bool, err error) {
oprEndTime := time.Now()
logSlowOperations(oprEndTime.Sub(oprStartTime), "iterate temporary index in merge process", 0)
diff --git a/pkg/ddl/ingest/BUILD.bazel b/pkg/ddl/ingest/BUILD.bazel
index 570fafbecac3b..5f772cfb5c1ae 100644
--- a/pkg/ddl/ingest/BUILD.bazel
+++ b/pkg/ddl/ingest/BUILD.bazel
@@ -11,7 +11,6 @@ go_library(
"engine.go",
"engine_mgr.go",
"env.go",
- "flush.go",
"mem_root.go",
"message.go",
"mock.go",
@@ -48,6 +47,7 @@ go_library(
"@com_github_google_uuid//:uuid",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
+ "@com_github_tikv_client_go_v2//oracle",
"@com_github_tikv_client_go_v2//util",
"@com_github_tikv_pd_client//:client",
"@io_etcd_go_etcd_client_v3//:client",
@@ -88,6 +88,8 @@ go_test(
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",
+ "@com_github_tikv_client_go_v2//oracle",
+ "@com_github_tikv_pd_client//:client",
"@org_uber_go_goleak//:goleak",
],
)
diff --git a/pkg/ddl/ingest/backend.go b/pkg/ddl/ingest/backend.go
index 4a382af699e18..133c4c1aec6de 100644
--- a/pkg/ddl/ingest/backend.go
+++ b/pkg/ddl/ingest/backend.go
@@ -17,6 +17,7 @@ package ingest
import (
"context"
"fmt"
+ "sync/atomic"
"time"
"github.com/pingcap/errors"
@@ -33,7 +34,6 @@ import (
"github.com/pingcap/tidb/pkg/parser/terror"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/util/dbterror"
- "github.com/pingcap/tidb/pkg/util/generic"
"github.com/pingcap/tidb/pkg/util/logutil"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/concurrency"
@@ -41,15 +41,32 @@ import (
"go.uber.org/zap"
)
-// BackendCtx is the backend context for add index reorg task.
+// MockDMLExecutionStateBeforeImport is a failpoint to mock the DML execution state before import.
+var MockDMLExecutionStateBeforeImport func()
+
+// BackendCtx is the backend context for one add index reorg task.
type BackendCtx interface {
- Register(jobID, indexID int64, schemaName, tableName string) (Engine, error)
- Unregister(jobID, indexID int64)
+ // Register create a new engineInfo for each index ID and register it to the
+ // backend context. If the index ID is already registered, it will return the
+ // associated engines. Only one group of index ID is allowed to register for a
+ // BackendCtx.
+ Register(indexIDs []int64, tableName string) ([]Engine, error)
+ UnregisterEngines()
+ // FinishImport imports the engine of given index ID into the storage, collects
+ // the duplicate errors if the `unique` is true. Caller should make sure the
+ // first call of FinishImport means no further data will be wrote to all engines.
+ //
+ // TODO(lance6716): refine the interface to let caller don't need to pass the
+ // indexID, and unify with CollectRemoteDuplicateRows.
+ FinishImport(indexID int64, unique bool, tbl table.Table) error
+ // ImportStarted returns true only when all the engines are finished writing and
+ // import is started by FinishImport. Considering the calling usage of
+ // FinishImport, it will return true after a successful call of FinishImport and
+ // may return true after a failed call of FinishImport.
+ ImportStarted() bool
CollectRemoteDuplicateRows(indexID int64, tbl table.Table) error
- FinishImport(indexID int64, unique bool, tbl table.Table) error
- ResetWorkers(jobID int64)
- Flush(indexID int64, mode FlushMode) (flushed, imported bool, err error)
+ FlushController
Done() bool
SetDone()
@@ -63,29 +80,29 @@ type BackendCtx interface {
type FlushMode byte
const (
- // FlushModeAuto means flush when the memory table size reaches the threshold.
+ // FlushModeAuto means caller does not enforce any flush, the implementation can
+ // decide it.
FlushModeAuto FlushMode = iota
- // FlushModeForceLocal means flush all data to local storage.
- FlushModeForceLocal
- // FlushModeForceLocalAndCheckDiskQuota means flush all data to local storage and check disk quota.
- FlushModeForceLocalAndCheckDiskQuota
- // FlushModeForceGlobal means import all data in local storage to global storage.
- FlushModeForceGlobal
+ // FlushModeForceFlushNoImport means flush all data to local storage, but don't
+ // import the data to TiKV.
+ FlushModeForceFlushNoImport
+ // FlushModeForceFlushAndImport means flush and import all data to TiKV.
+ FlushModeForceFlushAndImport
)
-// litBackendCtx store a backend info for add index reorg task.
+// litBackendCtx implements BackendCtx.
type litBackendCtx struct {
- generic.SyncMap[int64, *engineInfo]
- MemRoot MemRoot
- DiskRoot DiskRoot
+ engines map[int64]*engineInfo
+ memRoot MemRoot
+ diskRoot DiskRoot
jobID int64
backend *local.Backend
ctx context.Context
cfg *lightning.Config
sysVars map[string]string
- diskRoot DiskRoot
done bool
+ flushing atomic.Bool
timeOfLastFlush atomicutil.Time
updateInterval time.Duration
checkpointMgr *CheckpointManager
@@ -135,7 +152,7 @@ func (bc *litBackendCtx) CollectRemoteDuplicateRows(indexID int64, tbl table.Tab
// FinishImport imports all the key-values in engine into the storage, collects the duplicate errors if any, and
// removes the engine from the backend context.
func (bc *litBackendCtx) FinishImport(indexID int64, unique bool, tbl table.Table) error {
- ei, exist := bc.Load(indexID)
+ ei, exist := bc.engines[indexID]
if !exist {
return dbterror.ErrIngestFailed.FastGenByArgs("ingest engine not found")
}
@@ -175,42 +192,39 @@ func acquireLock(ctx context.Context, se *concurrency.Session, key string) (*con
return mu, nil
}
-// Flush checks the disk quota and imports the current key-values in engine to the storage.
-func (bc *litBackendCtx) Flush(indexID int64, mode FlushMode) (flushed, imported bool, err error) {
- ei, exist := bc.Load(indexID)
- if !exist {
- logutil.Logger(bc.ctx).Error(LitErrGetEngineFail, zap.Int64("index ID", indexID))
- return false, false, dbterror.ErrIngestFailed.FastGenByArgs("ingest engine not found")
- }
-
- shouldFlush, shouldImport := bc.ShouldSync(mode)
+// Flush implements FlushController.
+func (bc *litBackendCtx) Flush(mode FlushMode) (flushed, imported bool, errIdxID int64, err error) {
+ shouldFlush, shouldImport := bc.checkFlush(mode)
if !shouldFlush {
- return false, false, nil
+ return false, false, 0, nil
}
- if !ei.flushing.CompareAndSwap(false, true) {
- return false, false, nil
+ if !bc.flushing.CompareAndSwap(false, true) {
+ return false, false, 0, nil
}
- defer ei.flushing.Store(false)
- ei.flushLock.Lock()
- defer ei.flushLock.Unlock()
+ defer bc.flushing.Store(false)
- err = ei.Flush()
- if err != nil {
- return false, false, err
+ for indexID, ei := range bc.engines {
+ ei.flushLock.Lock()
+ //nolint: all_revive,revive
+ defer ei.flushLock.Unlock()
+
+ if err = ei.Flush(); err != nil {
+ return false, false, indexID, err
+ }
}
bc.timeOfLastFlush.Store(time.Now())
if !shouldImport {
- return true, false, nil
+ return true, false, 0, nil
}
// Use distributed lock if run in distributed mode).
if bc.etcdClient != nil {
- distLockKey := fmt.Sprintf("/tidb/distributeLock/%d/%d", bc.jobID, indexID)
+ distLockKey := fmt.Sprintf("/tidb/distributeLock/%d", bc.jobID)
se, _ := concurrency.NewSession(bc.etcdClient)
mu, err := acquireLock(bc.ctx, se, distLockKey)
if err != nil {
- return true, false, errors.Trace(err)
+ return true, false, 0, errors.Trace(err)
}
logutil.Logger(bc.ctx).Info("acquire distributed flush lock success", zap.Int64("jobID", bc.jobID))
defer func() {
@@ -226,34 +240,56 @@ func (bc *litBackendCtx) Flush(indexID int64, mode FlushMode) (flushed, imported
}
}()
}
- err = bc.unsafeImportAndReset(ei)
- if err != nil {
- return true, false, err
+ failpoint.Inject("mockDMLExecutionStateBeforeImport", func(_ failpoint.Value) {
+ if MockDMLExecutionStateBeforeImport != nil {
+ MockDMLExecutionStateBeforeImport()
+ }
+ })
+
+ for indexID, ei := range bc.engines {
+ if err = bc.unsafeImportAndReset(ei); err != nil {
+ return true, false, indexID, err
+ }
}
- return true, true, nil
+
+ return true, true, 0, nil
}
func (bc *litBackendCtx) unsafeImportAndReset(ei *engineInfo) error {
- logutil.Logger(bc.ctx).Info(LitInfoUnsafeImport, zap.Int64("index ID", ei.indexID),
- zap.String("usage info", bc.diskRoot.UsageInfo()))
logger := log.FromContext(bc.ctx).With(
zap.Stringer("engineUUID", ei.uuid),
)
+ logger.Info(LitInfoUnsafeImport,
+ zap.Int64("index ID", ei.indexID),
+ zap.String("usage info", bc.diskRoot.UsageInfo()))
- ei.closedEngine = backend.NewClosedEngine(bc.backend, logger, ei.uuid, 0)
+ closedEngine := backend.NewClosedEngine(bc.backend, logger, ei.uuid, 0)
regionSplitSize := int64(lightning.SplitRegionSize) * int64(lightning.MaxSplitRegionSizeRatio)
regionSplitKeys := int64(lightning.SplitRegionKeys)
- if err := ei.closedEngine.Import(bc.ctx, regionSplitSize, regionSplitKeys); err != nil {
+ if err := closedEngine.Import(bc.ctx, regionSplitSize, regionSplitKeys); err != nil {
logutil.Logger(bc.ctx).Error(LitErrIngestDataErr, zap.Int64("index ID", ei.indexID),
zap.String("usage info", bc.diskRoot.UsageInfo()))
return err
}
- err := bc.backend.ResetEngine(bc.ctx, ei.uuid)
+ resetFn := bc.backend.ResetEngineSkipAllocTS
+ mgr := bc.GetCheckpointManager()
+ if mgr == nil {
+ // disttask case, no need to refresh TS.
+ //
+ // TODO(lance6716): for disttask local sort case, we need to use a fixed TS. But
+ // it doesn't have checkpoint, so we need to find a way to save TS.
+ resetFn = bc.backend.ResetEngine
+ }
+
+ err := resetFn(bc.ctx, ei.uuid)
+ failpoint.Inject("mockResetEngineFailed", func() {
+ err = fmt.Errorf("mock reset engine failed")
+ })
if err != nil {
logutil.Logger(bc.ctx).Error(LitErrResetEngineFail, zap.Int64("index ID", ei.indexID))
- err1 := ei.closedEngine.Cleanup(bc.ctx)
+ err1 := closedEngine.Cleanup(bc.ctx)
if err1 != nil {
logutil.Logger(ei.ctx).Error(LitErrCleanEngineErr, zap.Error(err1),
zap.Int64("job ID", ei.jobID), zap.Int64("index ID", ei.indexID))
@@ -262,34 +298,50 @@ func (bc *litBackendCtx) unsafeImportAndReset(ei *engineInfo) error {
ei.closedEngine = nil
return err
}
+
+ if mgr == nil {
+ return nil
+ }
+
+ // for local disk case, we need to refresh TS because duplicate detection
+ // requires each ingest to have a unique TS.
+ //
+ // TODO(lance6716): there's still a chance that data is imported but because of
+ // checkpoint is low-watermark, the data will still be imported again with
+ // another TS after failover. Need to refine the checkpoint mechanism.
+ newTS, err := mgr.refreshTSAndUpdateCP()
+ if err != nil {
+ return errors.Trace(err)
+ }
+ ei.openedEngine.SetTS(newTS)
return nil
}
// ForceSyncFlagForTest is a flag to force sync only for test.
var ForceSyncFlagForTest = false
-func (bc *litBackendCtx) ShouldSync(mode FlushMode) (shouldFlush bool, shouldImport bool) {
- if mode == FlushModeForceGlobal || ForceSyncFlagForTest {
+func (bc *litBackendCtx) checkFlush(mode FlushMode) (shouldFlush bool, shouldImport bool) {
+ failpoint.Inject("forceSyncFlagForTest", func() {
+ // used in a manual test
+ ForceSyncFlagForTest = true
+ })
+ if mode == FlushModeForceFlushAndImport || ForceSyncFlagForTest {
return true, true
}
- if mode == FlushModeForceLocal {
+ if mode == FlushModeForceFlushNoImport {
return true, false
}
bc.diskRoot.UpdateUsage()
shouldImport = bc.diskRoot.ShouldImport()
- if mode == FlushModeForceLocalAndCheckDiskQuota {
- shouldFlush = true
- } else {
- interval := bc.updateInterval
- // This failpoint will be manually set through HTTP status port.
- failpoint.Inject("mockSyncIntervalMs", func(val failpoint.Value) {
- if v, ok := val.(int); ok {
- interval = time.Duration(v) * time.Millisecond
- }
- })
- shouldFlush = shouldImport ||
- time.Since(bc.timeOfLastFlush.Load()) >= interval
- }
+ interval := bc.updateInterval
+ // This failpoint will be manually set through HTTP status port.
+ failpoint.Inject("mockSyncIntervalMs", func(val failpoint.Value) {
+ if v, ok := val.(int); ok {
+ interval = time.Duration(v) * time.Millisecond
+ }
+ })
+ shouldFlush = shouldImport ||
+ time.Since(bc.timeOfLastFlush.Load()) >= interval
return shouldFlush, shouldImport
}
diff --git a/pkg/ddl/ingest/backend_mgr.go b/pkg/ddl/ingest/backend_mgr.go
index 5276184a4913e..33c7a81a05ee2 100644
--- a/pkg/ddl/ingest/backend_mgr.go
+++ b/pkg/ddl/ingest/backend_mgr.go
@@ -28,7 +28,6 @@ import (
ddllogutil "github.com/pingcap/tidb/pkg/ddl/logutil"
"github.com/pingcap/tidb/pkg/lightning/backend/local"
"github.com/pingcap/tidb/pkg/lightning/config"
- "github.com/pingcap/tidb/pkg/util/generic"
"github.com/pingcap/tidb/pkg/util/logutil"
kvutil "github.com/tikv/client-go/v2/util"
pd "github.com/tikv/pd/client"
@@ -130,7 +129,7 @@ func (m *litBackendCtxMgr) Register(
}
m.memRoot.RefreshConsumption()
- ok := m.memRoot.CheckConsume(StructSizeBackendCtx)
+ ok := m.memRoot.CheckConsume(structSizeBackendCtx)
if !ok {
return nil, genBackendAllocMemFailedErr(ctx, m.memRoot, jobID)
}
@@ -155,7 +154,7 @@ func (m *litBackendCtxMgr) Register(
bcCtx := newBackendContext(ctx, jobID, bd, cfg.lightning, defaultImportantVariables, m.memRoot, m.diskRoot, etcdClient)
m.backends.m[jobID] = bcCtx
- m.memRoot.Consume(StructSizeBackendCtx)
+ m.memRoot.Consume(structSizeBackendCtx)
m.backends.mu.Unlock()
logutil.Logger(ctx).Info(LitInfoCreateBackend, zap.Int64("job ID", jobID),
@@ -268,15 +267,14 @@ func newBackendContext(
etcdClient *clientv3.Client,
) *litBackendCtx {
bCtx := &litBackendCtx{
- SyncMap: generic.NewSyncMap[int64, *engineInfo](10),
- MemRoot: memRoot,
- DiskRoot: diskRoot,
+ engines: make(map[int64]*engineInfo, 10),
+ memRoot: memRoot,
+ diskRoot: diskRoot,
jobID: jobID,
backend: be,
ctx: ctx,
cfg: cfg,
sysVars: vars,
- diskRoot: diskRoot,
updateInterval: checkpointUpdateInterval,
etcdClient: etcdClient,
}
@@ -299,12 +297,12 @@ func (m *litBackendCtxMgr) Unregister(jobID int64) {
if !exist {
return
}
- bc.unregisterAll(jobID)
+ bc.UnregisterEngines()
bc.backend.Close()
if bc.checkpointMgr != nil {
bc.checkpointMgr.Close()
}
- m.memRoot.Release(StructSizeBackendCtx)
+ m.memRoot.Release(structSizeBackendCtx)
m.memRoot.ReleaseWithTag(encodeBackendTag(jobID))
logutil.Logger(bc.ctx).Info(LitInfoCloseBackend, zap.Int64("job ID", jobID),
zap.Int64("current memory usage", m.memRoot.CurrentUsage()),
diff --git a/pkg/ddl/ingest/checkpoint.go b/pkg/ddl/ingest/checkpoint.go
index cde9bcbbc20ea..49fe71f2dfa6e 100644
--- a/pkg/ddl/ingest/checkpoint.go
+++ b/pkg/ddl/ingest/checkpoint.go
@@ -25,86 +25,113 @@ import (
"time"
"github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/config"
sess "github.com/pingcap/tidb/pkg/ddl/internal/session"
"github.com/pingcap/tidb/pkg/ddl/logutil"
"github.com/pingcap/tidb/pkg/ddl/util"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/meta"
+ "github.com/tikv/client-go/v2/oracle"
+ pd "github.com/tikv/pd/client"
"go.uber.org/zap"
)
-// CheckpointManager is a checkpoint manager implementation that used by non-distributed reorganization.
+// CheckpointManager is a checkpoint manager implementation that used by
+// non-distributed reorganization. It manages the data as two-level checkpoints:
+// "flush"ed to local storage and "import"ed to TiKV. The checkpoint is saved in
+// a table in the TiDB cluster.
type CheckpointManager struct {
- ctx context.Context
- flushCtrl FlushController
- sessPool *sess.Pool
- jobID int64
- indexIDs []int64
+ ctx context.Context
+ cancel context.CancelFunc
+ flushCtrl FlushController
+ sessPool *sess.Pool
+ jobID int64
+ indexIDs []int64
+ localStoreDir string
+ pdCli pd.Client
+ logger *zap.Logger
// Derived and unchanged after the initialization.
instanceAddr string
localDataIsValid bool
// Live in memory.
- checkpoints map[int]*TaskCheckpoint // task ID -> checkpoint
- mu sync.Mutex
- minTaskIDSynced int
- dirty bool
+ mu sync.Mutex
+ checkpoints map[int]*taskCheckpoint // task ID -> checkpoint
+ // we require each task ID to be continuous and start from 1.
+ minTaskIDFinished int
+ dirty bool
// Local meta.
- pidLocal int64
- startLocal kv.Key
- endLocal kv.Key
+ pidFlushed int64
+ startKeyFlushed kv.Key
+ endKeyFlushed kv.Key
// Persisted to the storage.
- minKeySyncLocal kv.Key
- minKeySyncGlobal kv.Key
- localCnt int
- globalCnt int
+ flushedKeyLowWatermark kv.Key
+ importedKeyLowWatermark kv.Key
+ flushedKeyCnt int
+ importedKeyCnt int
// Global meta.
- pidGlobal int64
- startGlobal kv.Key
- endGlobal kv.Key
+ pidImported int64
+ startKeyImported kv.Key
+ endKeyImported kv.Key
+ ts uint64
// For persisting the checkpoint periodically.
- updating bool
- updaterWg sync.WaitGroup
- updaterCh chan *sync.WaitGroup
- updaterExitCh chan struct{}
+ updaterWg sync.WaitGroup
+ updaterCh chan chan struct{}
}
-// TaskCheckpoint is the checkpoint for a single task.
-type TaskCheckpoint struct {
+// taskCheckpoint is the checkpoint for a single task.
+type taskCheckpoint struct {
totalKeys int
- currentKeys int
+ writtenKeys int
checksum int64
endKey kv.Key
- lastBatchSent bool
+ lastBatchRead bool
}
-// FlushController is an interface to control the flush of the checkpoint.
+// FlushController is an interface to control the flush of data so after it
+// returns caller can save checkpoint.
type FlushController interface {
- Flush(indexID int64, mode FlushMode) (flushed, imported bool, err error)
+ // Flush checks if al engines need to be flushed and imported based on given
+ // FlushMode. It's concurrent safe.
+ Flush(mode FlushMode) (flushed, imported bool, errIdxID int64, err error)
}
// NewCheckpointManager creates a new checkpoint manager.
-func NewCheckpointManager(ctx context.Context, flushCtrl FlushController,
- sessPool *sess.Pool, jobID int64, indexIDs []int64) (*CheckpointManager, error) {
- instanceAddr := InitInstanceAddr()
+func NewCheckpointManager(
+ ctx context.Context,
+ flushCtrl FlushController,
+ sessPool *sess.Pool,
+ jobID int64,
+ indexIDs []int64,
+ localStoreDir string,
+ pdCli pd.Client,
+) (*CheckpointManager, error) {
+ instanceAddr := InstanceAddr()
+ ctx2, cancel := context.WithCancel(ctx)
+ logger := logutil.DDLIngestLogger().With(
+ zap.Int64("jobID", jobID), zap.Int64s("indexIDs", indexIDs))
+
cm := &CheckpointManager{
- ctx: ctx,
+ ctx: ctx2,
+ cancel: cancel,
flushCtrl: flushCtrl,
sessPool: sessPool,
jobID: jobID,
indexIDs: indexIDs,
- checkpoints: make(map[int]*TaskCheckpoint, 16),
+ localStoreDir: localStoreDir,
+ pdCli: pdCli,
+ logger: logger,
+ checkpoints: make(map[int]*taskCheckpoint, 16),
mu: sync.Mutex{},
instanceAddr: instanceAddr,
updaterWg: sync.WaitGroup{},
- updaterExitCh: make(chan struct{}),
- updaterCh: make(chan *sync.WaitGroup),
+ updaterCh: make(chan chan struct{}),
}
- err := cm.resumeCheckpoint()
+ err := cm.resumeOrInitCheckpoint()
if err != nil {
return nil, err
}
@@ -113,141 +140,167 @@ func NewCheckpointManager(ctx context.Context, flushCtrl FlushController,
cm.updateCheckpointLoop()
cm.updaterWg.Done()
}()
- logutil.DDLIngestLogger().Info("create checkpoint manager",
- zap.Int64("jobID", jobID), zap.Int64s("indexIDs", indexIDs))
+ logger.Info("create checkpoint manager")
return cm, nil
}
-// InitInstanceAddr returns the string concat with instance address and temp-dir.
-func InitInstanceAddr() string {
+// InstanceAddr returns the string concat with instance address and temp-dir.
+func InstanceAddr() string {
cfg := config.GetGlobalConfig()
dsn := net.JoinHostPort(cfg.AdvertiseAddress, strconv.Itoa(int(cfg.Port)))
return fmt.Sprintf("%s:%s", dsn, cfg.TempDir)
}
-// IsComplete checks if the task is complete.
-// This is called before the reader reads the data and decides whether to skip the current task.
-func (s *CheckpointManager) IsComplete(end kv.Key) bool {
+// IsKeyProcessed checks if the key is processed. The key may not be imported.
+// This is called before the reader reads the data and decides whether to skip
+// the current task.
+func (s *CheckpointManager) IsKeyProcessed(end kv.Key) bool {
s.mu.Lock()
defer s.mu.Unlock()
- if len(s.minKeySyncGlobal) > 0 && end.Cmp(s.minKeySyncGlobal) <= 0 {
+ if len(s.importedKeyLowWatermark) > 0 && end.Cmp(s.importedKeyLowWatermark) <= 0 {
return true
}
- return s.localDataIsValid && len(s.minKeySyncLocal) > 0 && end.Cmp(s.minKeySyncLocal) <= 0
+ return s.localDataIsValid && len(s.flushedKeyLowWatermark) > 0 && end.Cmp(s.flushedKeyLowWatermark) <= 0
}
// Status returns the status of the checkpoint.
-func (s *CheckpointManager) Status() (int, kv.Key) {
+func (s *CheckpointManager) Status() (keyCnt int, minKeyImported kv.Key) {
s.mu.Lock()
defer s.mu.Unlock()
total := 0
for _, cp := range s.checkpoints {
- total += cp.currentKeys
+ total += cp.writtenKeys
}
- return s.localCnt + total, s.minKeySyncGlobal
+ // TODO(lance6716): ???
+ return s.flushedKeyCnt + total, s.importedKeyLowWatermark
}
-// Register registers a new task.
+// Register registers a new task. taskID MUST be continuous ascending and start
+// from 1.
+//
+// TODO(lance6716): remove this constraint, use endKey as taskID and use
+// ordered map type for checkpoints.
func (s *CheckpointManager) Register(taskID int, end kv.Key) {
s.mu.Lock()
defer s.mu.Unlock()
- s.checkpoints[taskID] = &TaskCheckpoint{
+ s.checkpoints[taskID] = &taskCheckpoint{
endKey: end,
}
}
-// UpdateTotal updates the total keys of the task.
+// UpdateTotalKeys updates the total keys of the task.
// This is called by the reader after reading the data to update the number of rows contained in the current chunk.
-func (s *CheckpointManager) UpdateTotal(taskID int, added int, last bool) {
+func (s *CheckpointManager) UpdateTotalKeys(taskID int, delta int, last bool) {
s.mu.Lock()
defer s.mu.Unlock()
cp := s.checkpoints[taskID]
- cp.totalKeys += added
- cp.lastBatchSent = last
+ cp.totalKeys += delta
+ cp.lastBatchRead = last
}
-// UpdateCurrent updates the current keys of the task.
+// UpdateWrittenKeys updates the written keys of the task.
// This is called by the writer after writing the local engine to update the current number of rows written.
-func (s *CheckpointManager) UpdateCurrent(taskID int, added int) error {
+func (s *CheckpointManager) UpdateWrittenKeys(taskID int, delta int) error {
s.mu.Lock()
cp := s.checkpoints[taskID]
- cp.currentKeys += added
+ cp.writtenKeys += delta
s.mu.Unlock()
- flushed, imported, _, err := TryFlushAllIndexes(s.flushCtrl, FlushModeAuto, s.indexIDs)
+ flushed, imported, _, err := s.flushCtrl.Flush(FlushModeAuto)
if !flushed || err != nil {
return err
}
+ failpoint.Inject("resignAfterFlush", func() {
+ // used in a manual test
+ ResignOwnerForTest.Store(true)
+ // wait until ResignOwnerForTest is processed
+ for ResignOwnerForTest.Load() {
+ time.Sleep(100 * time.Millisecond)
+ }
+ })
+
s.mu.Lock()
defer s.mu.Unlock()
- s.progressLocalSyncMinKey()
- if imported && s.minKeySyncGlobal.Cmp(s.minKeySyncLocal) != 0 {
- s.minKeySyncGlobal = s.minKeySyncLocal
- s.globalCnt = s.localCnt
+ s.afterFlush()
+ if imported && s.importedKeyLowWatermark.Cmp(s.flushedKeyLowWatermark) != 0 {
+ // TODO(lance6716): add warning log if cmp > 0
+ s.importedKeyLowWatermark = s.flushedKeyLowWatermark
+ s.importedKeyCnt = s.flushedKeyCnt
s.dirty = true
- s.pidGlobal = s.pidLocal
- s.startGlobal = s.startLocal
- s.endGlobal = s.endLocal
+ s.pidImported = s.pidFlushed
+ s.startKeyImported = s.startKeyFlushed
+ s.endKeyImported = s.endKeyFlushed
}
return nil
}
-func (s *CheckpointManager) progressLocalSyncMinKey() {
+// afterFlush should be called after all engine is flushed.
+func (s *CheckpointManager) afterFlush() {
for {
- cp := s.checkpoints[s.minTaskIDSynced+1]
- if cp == nil || !cp.lastBatchSent || cp.currentKeys < cp.totalKeys {
+ cp := s.checkpoints[s.minTaskIDFinished+1]
+ if cp == nil || !cp.lastBatchRead || cp.writtenKeys < cp.totalKeys {
break
}
- s.minTaskIDSynced++
- s.minKeySyncLocal = cp.endKey
- s.localCnt += cp.totalKeys
- delete(s.checkpoints, s.minTaskIDSynced)
+ s.minTaskIDFinished++
+ s.flushedKeyLowWatermark = cp.endKey
+ s.flushedKeyCnt += cp.totalKeys
+ delete(s.checkpoints, s.minTaskIDFinished)
s.dirty = true
}
}
// Close closes the checkpoint manager.
func (s *CheckpointManager) Close() {
- s.updaterExitCh <- struct{}{}
+ s.cancel()
s.updaterWg.Wait()
- logutil.DDLIngestLogger().Info("close checkpoint manager",
- zap.Int64("jobID", s.jobID), zap.Int64s("indexIDs", s.indexIDs))
+ s.logger.Info("close checkpoint manager")
}
-// Sync syncs the checkpoint.
-func (s *CheckpointManager) Sync() {
- _, _, _, err := TryFlushAllIndexes(s.flushCtrl, FlushModeForceLocal, s.indexIDs)
+// Flush flushed the data and updates checkpoint.
+func (s *CheckpointManager) Flush() {
+ // use FlushModeForceFlushNoImport to finish the flush process timely.
+ _, _, _, err := s.flushCtrl.Flush(FlushModeForceFlushNoImport)
if err != nil {
- logutil.DDLIngestLogger().Warn("flush local engine failed", zap.Error(err))
+ s.logger.Warn("flush local engine failed", zap.Error(err))
}
s.mu.Lock()
- s.progressLocalSyncMinKey()
+ s.afterFlush()
s.mu.Unlock()
- wg := sync.WaitGroup{}
- wg.Add(1)
- s.updaterCh <- &wg
- wg.Wait()
+
+ err = s.updateCheckpoint()
+ if err != nil {
+ s.logger.Error("update checkpoint failed", zap.Error(err))
+ }
}
// Reset resets the checkpoint manager between two partitions.
func (s *CheckpointManager) Reset(newPhysicalID int64, start, end kv.Key) {
s.mu.Lock()
defer s.mu.Unlock()
- logutil.DDLIngestLogger().Info("reset checkpoint manager",
- zap.Int64("newPhysicalID", newPhysicalID), zap.Int64("oldPhysicalID", s.pidLocal),
- zap.Int64s("indexIDs", s.indexIDs), zap.Int64("jobID", s.jobID), zap.Int("localCnt", s.localCnt))
- if s.pidLocal != newPhysicalID {
- s.minKeySyncLocal = nil
- s.minKeySyncGlobal = nil
- s.minTaskIDSynced = 0
- s.pidLocal = newPhysicalID
- s.startLocal = start
- s.endLocal = end
+
+ s.logger.Info("reset checkpoint manager",
+ zap.Int64("newPhysicalID", newPhysicalID),
+ zap.Int64("oldPhysicalID", s.pidFlushed),
+ zap.Int("flushedKeyCnt", s.flushedKeyCnt))
+ if s.pidFlushed != newPhysicalID {
+ s.flushedKeyLowWatermark = nil
+ s.importedKeyLowWatermark = nil
+ s.minTaskIDFinished = 0
+ s.pidFlushed = newPhysicalID
+ s.startKeyFlushed = start
+ s.endKeyFlushed = end
}
}
+// GetTS returns the TS saved in checkpoint.
+func (s *CheckpointManager) GetTS() uint64 {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return s.ts
+}
+
// JobReorgMeta is the metadata for a reorg job.
type JobReorgMeta struct {
Checkpoint *ReorgCheckpoint `json:"reorg_checkpoint"`
@@ -264,6 +317,8 @@ type ReorgCheckpoint struct {
PhysicalID int64 `json:"physical_id"`
StartKey kv.Key `json:"start_key"`
EndKey kv.Key `json:"end_key"`
+ // TS of next engine ingest.
+ TS uint64 `json:"ts"`
Version int64 `json:"version"`
}
@@ -274,14 +329,14 @@ const (
JobCheckpointVersion1 = 1
)
-func (s *CheckpointManager) resumeCheckpoint() error {
+func (s *CheckpointManager) resumeOrInitCheckpoint() error {
sessCtx, err := s.sessPool.Get()
if err != nil {
return errors.Trace(err)
}
defer s.sessPool.Put(sessCtx)
ddlSess := sess.NewSession(sessCtx)
- return ddlSess.RunInTxn(func(se *sess.Session) error {
+ err = ddlSess.RunInTxn(func(se *sess.Session) error {
template := "select reorg_meta from mysql.tidb_ddl_reorg where job_id = %d and ele_type = %s;"
sql := fmt.Sprintf(template, s.jobID, util.WrapKey2String(meta.IndexElementKey))
ctx := kv.WithInternalSourceType(s.ctx, kv.InternalTxnBackfillDDLPrefix+"add_index")
@@ -289,6 +344,7 @@ func (s *CheckpointManager) resumeCheckpoint() error {
if err != nil {
return errors.Trace(err)
}
+
if len(rows) == 0 || rows[0].IsNull(0) {
return nil
}
@@ -299,47 +355,60 @@ func (s *CheckpointManager) resumeCheckpoint() error {
return errors.Trace(err)
}
if cp := reorgMeta.Checkpoint; cp != nil {
- s.minKeySyncGlobal = cp.GlobalSyncKey
- s.globalCnt = cp.GlobalKeyCount
- s.pidGlobal = cp.PhysicalID
- s.startGlobal = cp.StartKey
- s.endGlobal = cp.EndKey
- if s.instanceAddr == cp.InstanceAddr || cp.InstanceAddr == "" /* initial state */ {
+ s.importedKeyLowWatermark = cp.GlobalSyncKey
+ s.importedKeyCnt = cp.GlobalKeyCount
+ s.pidImported = cp.PhysicalID
+ s.startKeyImported = cp.StartKey
+ s.endKeyImported = cp.EndKey
+ s.ts = cp.TS
+ if util.FolderNotEmpty(s.localStoreDir) &&
+ (s.instanceAddr == cp.InstanceAddr || cp.InstanceAddr == "" /* initial state */) {
s.localDataIsValid = true
- s.minKeySyncLocal = cp.LocalSyncKey
- s.localCnt = cp.LocalKeyCount
+ s.flushedKeyLowWatermark = cp.LocalSyncKey
+ s.flushedKeyCnt = cp.LocalKeyCount
}
- logutil.DDLIngestLogger().Info("resume checkpoint",
- zap.Int64("job ID", s.jobID), zap.Int64s("index IDs", s.indexIDs),
- zap.String("local checkpoint", hex.EncodeToString(s.minKeySyncLocal)),
- zap.String("global checkpoint", hex.EncodeToString(s.minKeySyncGlobal)),
+ s.logger.Info("resume checkpoint",
+ zap.String("flushed key low watermark", hex.EncodeToString(s.flushedKeyLowWatermark)),
+ zap.String("imported key low watermark", hex.EncodeToString(s.importedKeyLowWatermark)),
zap.Int64("physical table ID", cp.PhysicalID),
zap.String("previous instance", cp.InstanceAddr),
zap.String("current instance", s.instanceAddr))
return nil
}
- logutil.DDLIngestLogger().Info("checkpoint is empty",
- zap.Int64("job ID", s.jobID), zap.Int64s("index IDs", s.indexIDs))
+ s.logger.Info("checkpoint is empty")
return nil
})
+ if err != nil {
+ return errors.Trace(err)
+ }
+
+ if s.ts > 0 {
+ return nil
+ }
+ // if TS is not set, we need to allocate a TS and save it to the storage before
+ // continue.
+ p, l, err := s.pdCli.GetTS(s.ctx)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ ts := oracle.ComposeTS(p, l)
+ s.ts = ts
+ return s.updateCheckpointImpl()
}
-func (s *CheckpointManager) updateCheckpoint() error {
+// updateCheckpointImpl is only used by updateCheckpointLoop goroutine or in
+// NewCheckpointManager. In other cases, use updateCheckpoint instead.
+func (s *CheckpointManager) updateCheckpointImpl() error {
s.mu.Lock()
- currentLocalKey := s.minKeySyncLocal
- currentGlobalKey := s.minKeySyncGlobal
- currentLocalCnt := s.localCnt
- currentGlobalCnt := s.globalCnt
- currentGlobalPID := s.pidGlobal
- currentGlobalStart := s.startGlobal
- currentGlobalEnd := s.endGlobal
- s.updating = true
+ flushedKeyLowWatermark := s.flushedKeyLowWatermark
+ importedKeyLowWatermark := s.importedKeyLowWatermark
+ flushedKeyCnt := s.flushedKeyCnt
+ importedKeyCnt := s.importedKeyCnt
+ pidImported := s.pidImported
+ startKeyImported := s.startKeyImported
+ endKeyImported := s.endKeyImported
+ ts := s.ts
s.mu.Unlock()
- defer func() {
- s.mu.Lock()
- s.updating = false
- s.mu.Unlock()
- }()
sessCtx, err := s.sessPool.Get()
if err != nil {
@@ -350,14 +419,15 @@ func (s *CheckpointManager) updateCheckpoint() error {
err = ddlSess.RunInTxn(func(se *sess.Session) error {
template := "update mysql.tidb_ddl_reorg set reorg_meta = %s where job_id = %d and ele_type = %s;"
cp := &ReorgCheckpoint{
- LocalSyncKey: currentLocalKey,
- GlobalSyncKey: currentGlobalKey,
- LocalKeyCount: currentLocalCnt,
- GlobalKeyCount: currentGlobalCnt,
+ LocalSyncKey: flushedKeyLowWatermark,
+ GlobalSyncKey: importedKeyLowWatermark,
+ LocalKeyCount: flushedKeyCnt,
+ GlobalKeyCount: importedKeyCnt,
InstanceAddr: s.instanceAddr,
- PhysicalID: currentGlobalPID,
- StartKey: currentGlobalStart,
- EndKey: currentGlobalEnd,
+ PhysicalID: pidImported,
+ StartKey: startKeyImported,
+ EndKey: endKeyImported,
+ TS: ts,
Version: JobCheckpointVersionCurrent,
}
rawReorgMeta, err := json.Marshal(JobReorgMeta{Checkpoint: cp})
@@ -375,39 +445,74 @@ func (s *CheckpointManager) updateCheckpoint() error {
s.mu.Unlock()
return nil
})
- logutil.DDLIngestLogger().Info("update checkpoint",
- zap.Int64("job ID", s.jobID), zap.Int64s("index IDs", s.indexIDs),
- zap.String("local checkpoint", hex.EncodeToString(currentLocalKey)),
- zap.String("global checkpoint", hex.EncodeToString(currentGlobalKey)),
- zap.Int64("global physical ID", currentGlobalPID),
+ s.logger.Info("update checkpoint",
+ zap.String("local checkpoint", hex.EncodeToString(flushedKeyLowWatermark)),
+ zap.String("global checkpoint", hex.EncodeToString(importedKeyLowWatermark)),
+ zap.Int64("global physical ID", pidImported),
zap.Error(err))
return err
}
func (s *CheckpointManager) updateCheckpointLoop() {
+ failpoint.Inject("checkpointLoopExit", func() {
+ // used in a manual test
+ failpoint.Return()
+ })
ticker := time.NewTicker(10 * time.Second)
defer ticker.Stop()
for {
select {
- case wg := <-s.updaterCh:
- err := s.updateCheckpoint()
+ case finishCh := <-s.updaterCh:
+ err := s.updateCheckpointImpl()
if err != nil {
- logutil.DDLIngestLogger().Error("update checkpoint failed", zap.Error(err))
+ s.logger.Error("update checkpoint failed", zap.Error(err))
}
- wg.Done()
+ close(finishCh)
case <-ticker.C:
s.mu.Lock()
- if !s.dirty || s.updating {
+ if !s.dirty {
s.mu.Unlock()
continue
}
s.mu.Unlock()
- err := s.updateCheckpoint()
+ err := s.updateCheckpointImpl()
if err != nil {
- logutil.DDLIngestLogger().Error("update checkpoint failed", zap.Error(err))
+ s.logger.Error("periodically update checkpoint failed", zap.Error(err))
}
- case <-s.updaterExitCh:
+ case <-s.ctx.Done():
return
}
}
}
+
+func (s *CheckpointManager) updateCheckpoint() error {
+ failpoint.Inject("checkpointLoopExit", func() {
+ // used in a manual test
+ failpoint.Return(errors.New("failpoint triggered so can't update checkpoint"))
+ })
+ finishCh := make(chan struct{})
+ select {
+ case s.updaterCh <- finishCh:
+ case <-s.ctx.Done():
+ return s.ctx.Err()
+ }
+ // wait updateCheckpointLoop to finish checkpoint update.
+ select {
+ case <-finishCh:
+ case <-s.ctx.Done():
+ return s.ctx.Err()
+ }
+ return nil
+}
+
+func (s *CheckpointManager) refreshTSAndUpdateCP() (uint64, error) {
+ p, l, err := s.pdCli.GetTS(s.ctx)
+ if err != nil {
+ return 0, errors.Trace(err)
+ }
+ newTS := oracle.ComposeTS(p, l)
+ s.mu.Lock()
+ s.ts = newTS
+ s.mu.Unlock()
+ return newTS, s.updateCheckpoint()
+}
diff --git a/pkg/ddl/ingest/checkpoint_test.go b/pkg/ddl/ingest/checkpoint_test.go
index 55c701b229ae2..64a9cf5543e29 100644
--- a/pkg/ddl/ingest/checkpoint_test.go
+++ b/pkg/ddl/ingest/checkpoint_test.go
@@ -17,6 +17,8 @@ package ingest_test
import (
"context"
"encoding/json"
+ "os"
+ "path/filepath"
"testing"
"github.com/ngaut/pools"
@@ -24,8 +26,30 @@ import (
"github.com/pingcap/tidb/pkg/ddl/internal/session"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/stretchr/testify/require"
+ "github.com/tikv/client-go/v2/oracle"
+ pd "github.com/tikv/pd/client"
)
+func createDummyFile(t *testing.T, folder string) {
+ f, err := os.Create(filepath.Join(folder, "test-file"))
+ require.NoError(t, err)
+ require.NoError(t, f.Close())
+}
+
+var (
+ mockPTS int64 = 12
+ mockLTS int64 = 34
+ expectedTS = oracle.ComposeTS(mockPTS, mockLTS)
+)
+
+type mockGetTSClient struct {
+ pd.Client
+}
+
+func (m mockGetTSClient) GetTS(context.Context) (int64, int64, error) {
+ return mockPTS, mockLTS, nil
+}
+
func TestCheckpointManager(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
@@ -38,45 +62,47 @@ func TestCheckpointManager(t *testing.T) {
ctx := context.Background()
sessPool := session.NewSessionPool(rs, store)
flushCtrl := &dummyFlushCtrl{imported: false}
- mgr, err := ingest.NewCheckpointManager(ctx, flushCtrl, sessPool, 1, []int64{1})
+ tmpFolder := t.TempDir()
+ createDummyFile(t, tmpFolder)
+ mgr, err := ingest.NewCheckpointManager(ctx, flushCtrl, sessPool, 1, []int64{1}, tmpFolder, mockGetTSClient{})
require.NoError(t, err)
defer mgr.Close()
mgr.Register(1, []byte{'1', '9'})
mgr.Register(2, []byte{'2', '9'})
- mgr.UpdateTotal(1, 100, false)
- require.False(t, mgr.IsComplete([]byte{'1', '9'}))
- require.NoError(t, mgr.UpdateCurrent(1, 100))
- require.False(t, mgr.IsComplete([]byte{'1', '9'}))
- mgr.UpdateTotal(1, 100, true)
- require.NoError(t, mgr.UpdateCurrent(1, 100))
+ mgr.UpdateTotalKeys(1, 100, false)
+ require.False(t, mgr.IsKeyProcessed([]byte{'1', '9'}))
+ require.NoError(t, mgr.UpdateWrittenKeys(1, 100))
+ require.False(t, mgr.IsKeyProcessed([]byte{'1', '9'}))
+ mgr.UpdateTotalKeys(1, 100, true)
+ require.NoError(t, mgr.UpdateWrittenKeys(1, 100))
// The data is not imported to the storage yet.
- require.False(t, mgr.IsComplete([]byte{'1', '9'}))
+ require.False(t, mgr.IsKeyProcessed([]byte{'1', '9'}))
flushCtrl.imported = true // Mock the data is imported to the storage.
- require.NoError(t, mgr.UpdateCurrent(2, 0))
- require.True(t, mgr.IsComplete([]byte{'1', '9'}))
+ require.NoError(t, mgr.UpdateWrittenKeys(2, 0))
+ require.True(t, mgr.IsKeyProcessed([]byte{'1', '9'}))
// Only when the last batch is completed, the job can be completed.
- mgr.UpdateTotal(2, 50, false)
- mgr.UpdateTotal(2, 50, true)
- require.NoError(t, mgr.UpdateCurrent(2, 50))
- require.True(t, mgr.IsComplete([]byte{'1', '9'}))
- require.False(t, mgr.IsComplete([]byte{'2', '9'}))
- require.NoError(t, mgr.UpdateCurrent(2, 50))
- require.True(t, mgr.IsComplete([]byte{'1', '9'}))
- require.True(t, mgr.IsComplete([]byte{'2', '9'}))
+ mgr.UpdateTotalKeys(2, 50, false)
+ mgr.UpdateTotalKeys(2, 50, true)
+ require.NoError(t, mgr.UpdateWrittenKeys(2, 50))
+ require.True(t, mgr.IsKeyProcessed([]byte{'1', '9'}))
+ require.False(t, mgr.IsKeyProcessed([]byte{'2', '9'}))
+ require.NoError(t, mgr.UpdateWrittenKeys(2, 50))
+ require.True(t, mgr.IsKeyProcessed([]byte{'1', '9'}))
+ require.True(t, mgr.IsKeyProcessed([]byte{'2', '9'}))
// Only when the subsequent job is completed, the previous job can be completed.
mgr.Register(3, []byte{'3', '9'})
mgr.Register(4, []byte{'4', '9'})
mgr.Register(5, []byte{'5', '9'})
- mgr.UpdateTotal(3, 100, true)
- mgr.UpdateTotal(4, 100, true)
- mgr.UpdateTotal(5, 100, true)
- require.NoError(t, mgr.UpdateCurrent(5, 100))
- require.NoError(t, mgr.UpdateCurrent(4, 100))
- require.False(t, mgr.IsComplete([]byte{'3', '9'}))
- require.False(t, mgr.IsComplete([]byte{'4', '9'}))
+ mgr.UpdateTotalKeys(3, 100, true)
+ mgr.UpdateTotalKeys(4, 100, true)
+ mgr.UpdateTotalKeys(5, 100, true)
+ require.NoError(t, mgr.UpdateWrittenKeys(5, 100))
+ require.NoError(t, mgr.UpdateWrittenKeys(4, 100))
+ require.False(t, mgr.IsKeyProcessed([]byte{'3', '9'}))
+ require.False(t, mgr.IsKeyProcessed([]byte{'4', '9'}))
}
func TestCheckpointManagerUpdateReorg(t *testing.T) {
@@ -91,14 +117,16 @@ func TestCheckpointManagerUpdateReorg(t *testing.T) {
ctx := context.Background()
sessPool := session.NewSessionPool(rs, store)
flushCtrl := &dummyFlushCtrl{imported: true}
- mgr, err := ingest.NewCheckpointManager(ctx, flushCtrl, sessPool, 1, []int64{1})
+ tmpFolder := t.TempDir()
+ createDummyFile(t, tmpFolder)
+ mgr, err := ingest.NewCheckpointManager(ctx, flushCtrl, sessPool, 1, []int64{1}, tmpFolder, mockGetTSClient{})
require.NoError(t, err)
defer mgr.Close()
mgr.Register(1, []byte{'1', '9'})
- mgr.UpdateTotal(1, 100, true)
- require.NoError(t, mgr.UpdateCurrent(1, 100))
- mgr.Sync() // Wait the global checkpoint to be updated to the reorg table.
+ mgr.UpdateTotalKeys(1, 100, true)
+ require.NoError(t, mgr.UpdateWrittenKeys(1, 100))
+ mgr.Flush() // Wait the global checkpoint to be updated to the reorg table.
r, err := tk.Exec("select reorg_meta from mysql.tidb_ddl_reorg where job_id = 1 and ele_id = 1;")
require.NoError(t, err)
req := r.NewChunk(nil)
@@ -114,6 +142,7 @@ func TestCheckpointManagerUpdateReorg(t *testing.T) {
require.Equal(t, 100, reorgMeta.Checkpoint.LocalKeyCount)
require.EqualValues(t, []byte{'1', '9'}, reorgMeta.Checkpoint.GlobalSyncKey)
require.EqualValues(t, []byte{'1', '9'}, reorgMeta.Checkpoint.LocalSyncKey)
+ require.EqualValues(t, expectedTS, reorgMeta.Checkpoint.TS)
}
func TestCheckpointManagerResumeReorg(t *testing.T) {
@@ -122,12 +151,13 @@ func TestCheckpointManagerResumeReorg(t *testing.T) {
tk.MustExec("use test")
reorgMeta := &ingest.JobReorgMeta{
Checkpoint: &ingest.ReorgCheckpoint{
- LocalSyncKey: []byte{'1', '9'},
+ LocalSyncKey: []byte{'2', '9'},
LocalKeyCount: 100,
- GlobalSyncKey: []byte{'2', '9'},
+ GlobalSyncKey: []byte{'1', '9'},
GlobalKeyCount: 200,
- InstanceAddr: ingest.InitInstanceAddr(),
+ InstanceAddr: ingest.InstanceAddr(),
Version: 1,
+ TS: 123456,
},
}
reorgMetaRaw, err := json.Marshal(reorgMeta)
@@ -140,20 +170,34 @@ func TestCheckpointManagerResumeReorg(t *testing.T) {
ctx := context.Background()
sessPool := session.NewSessionPool(rs, store)
flushCtrl := &dummyFlushCtrl{imported: false}
- mgr, err := ingest.NewCheckpointManager(ctx, flushCtrl, sessPool, 1, []int64{1})
+ tmpFolder := t.TempDir()
+ // checkpoint manager should not use local checkpoint if the folder is empty
+ mgr, err := ingest.NewCheckpointManager(ctx, flushCtrl, sessPool, 1, []int64{1}, tmpFolder, nil)
require.NoError(t, err)
defer mgr.Close()
- require.True(t, mgr.IsComplete([]byte{'1', '9'}))
- require.True(t, mgr.IsComplete([]byte{'2', '9'}))
+ require.True(t, mgr.IsKeyProcessed([]byte{'1', '9'}))
+ require.False(t, mgr.IsKeyProcessed([]byte{'2', '9'}))
localCnt, globalNextKey := mgr.Status()
+ require.Equal(t, 0, localCnt)
+ require.EqualValues(t, []byte{'1', '9'}, globalNextKey)
+ require.EqualValues(t, 123456, mgr.GetTS())
+
+ createDummyFile(t, tmpFolder)
+ mgr2, err := ingest.NewCheckpointManager(ctx, flushCtrl, sessPool, 1, []int64{1}, tmpFolder, nil)
+ require.NoError(t, err)
+ defer mgr2.Close()
+ require.True(t, mgr2.IsKeyProcessed([]byte{'1', '9'}))
+ require.True(t, mgr2.IsKeyProcessed([]byte{'2', '9'}))
+ localCnt, globalNextKey = mgr2.Status()
require.Equal(t, 100, localCnt)
- require.EqualValues(t, []byte{'2', '9'}, globalNextKey)
+ require.EqualValues(t, []byte{'1', '9'}, globalNextKey)
+ require.EqualValues(t, 123456, mgr.GetTS())
}
type dummyFlushCtrl struct {
imported bool
}
-func (d *dummyFlushCtrl) Flush(_ int64, _ ingest.FlushMode) (bool, bool, error) {
- return true, d.imported, nil
+func (d *dummyFlushCtrl) Flush(_ ingest.FlushMode) (bool, bool, int64, error) {
+ return true, d.imported, 0, nil
}
diff --git a/pkg/ddl/ingest/config.go b/pkg/ddl/ingest/config.go
index fc0ca11ed83fe..c6b59a531c5cd 100644
--- a/pkg/ddl/ingest/config.go
+++ b/pkg/ddl/ingest/config.go
@@ -109,7 +109,7 @@ var (
compactConcurrency = 4
)
-func generateLocalEngineConfig(id int64, dbName, tbName string) *backend.EngineConfig {
+func generateLocalEngineConfig(ts uint64) *backend.EngineConfig {
return &backend.EngineConfig{
Local: backend.LocalEngineConfig{
Compact: true,
@@ -117,12 +117,9 @@ func generateLocalEngineConfig(id int64, dbName, tbName string) *backend.EngineC
CompactConcurrency: compactConcurrency,
BlockSize: 16 * 1024, // using default for DDL
},
- TableInfo: &checkpoints.TidbTableInfo{
- ID: id,
- DB: dbName,
- Name: tbName,
- },
+ TableInfo: &checkpoints.TidbTableInfo{},
KeepSortDir: true,
+ TS: ts,
}
}
diff --git a/pkg/ddl/ingest/engine.go b/pkg/ddl/ingest/engine.go
index 1a66d9ef0f4b1..1d3181f9b9359 100644
--- a/pkg/ddl/ingest/engine.go
+++ b/pkg/ddl/ingest/engine.go
@@ -18,7 +18,6 @@ import (
"context"
"strconv"
"sync"
- "sync/atomic"
"github.com/google/uuid"
tidbkv "github.com/pingcap/tidb/pkg/kv"
@@ -45,7 +44,6 @@ type Writer interface {
// To enable uniqueness check, the handle should be non-empty.
WriteRow(ctx context.Context, idxKey, idxVal []byte, handle tidbkv.Handle) error
LockForWrite() (unlock func())
- Close(ctx context.Context) error
}
// engineInfo is the engine for one index reorg task, each task will create several new writers under the
@@ -55,28 +53,36 @@ type engineInfo struct {
jobID int64
indexID int64
openedEngine *backend.OpenedEngine
+ // closedEngine is set only when all data is finished written and all writers are
+ // closed.
closedEngine *backend.ClosedEngine
uuid uuid.UUID
cfg *backend.EngineConfig
- writerCount int
+ litCfg *config.Config
writerCache generic.SyncMap[int, backend.EngineWriter]
memRoot MemRoot
flushLock *sync.RWMutex
- flushing atomic.Bool
}
// newEngineInfo create a new engineInfo struct.
-func newEngineInfo(ctx context.Context, jobID, indexID int64, cfg *backend.EngineConfig,
- en *backend.OpenedEngine, uuid uuid.UUID, wCnt int, memRoot MemRoot) *engineInfo {
+func newEngineInfo(
+ ctx context.Context,
+ jobID, indexID int64,
+ cfg *backend.EngineConfig,
+ litCfg *config.Config,
+ en *backend.OpenedEngine,
+ uuid uuid.UUID,
+ memRoot MemRoot,
+) *engineInfo {
return &engineInfo{
ctx: ctx,
jobID: jobID,
indexID: indexID,
cfg: cfg,
+ litCfg: litCfg,
openedEngine: en,
uuid: uuid,
- writerCount: wCnt,
- writerCache: generic.NewSyncMap[int, backend.EngineWriter](wCnt),
+ writerCache: generic.NewSyncMap[int, backend.EngineWriter](4),
memRoot: memRoot,
flushLock: &sync.RWMutex{},
}
@@ -181,9 +187,9 @@ type writerContext struct {
// CreateWriter creates a new writerContext.
func (ei *engineInfo) CreateWriter(id int) (Writer, error) {
ei.memRoot.RefreshConsumption()
- ok := ei.memRoot.CheckConsume(StructSizeWriterCtx)
+ ok := ei.memRoot.CheckConsume(structSizeWriterCtx)
if !ok {
- return nil, genEngineAllocMemFailedErr(ei.ctx, ei.memRoot, ei.jobID, ei.indexID)
+ return nil, genWriterAllocMemFailedErr(ei.ctx, ei.memRoot, ei.jobID, ei.indexID)
}
wCtx, err := ei.newWriterContext(id)
@@ -194,10 +200,10 @@ func (ei *engineInfo) CreateWriter(id int) (Writer, error) {
return nil, err
}
- ei.memRoot.Consume(StructSizeWriterCtx)
+ ei.memRoot.Consume(structSizeWriterCtx)
logutil.Logger(ei.ctx).Info(LitInfoCreateWrite, zap.Int64("job ID", ei.jobID),
zap.Int64("index ID", ei.indexID), zap.Int("worker ID", id),
- zap.Int64("allocate memory", StructSizeWriterCtx),
+ zap.Int64("allocate memory", structSizeWriterCtx),
zap.Int64("current memory usage", ei.memRoot.CurrentUsage()),
zap.Int64("max memory quota", ei.memRoot.MaxMemoryQuota()))
return wCtx, err
@@ -210,6 +216,10 @@ func (ei *engineInfo) CreateWriter(id int) (Writer, error) {
func (ei *engineInfo) newWriterContext(workerID int) (*writerContext, error) {
lWrite, exist := ei.writerCache.Load(workerID)
if !exist {
+ ok := ei.memRoot.CheckConsume(int64(ei.litCfg.TikvImporter.LocalWriterMemCacheSize))
+ if !ok {
+ return nil, genWriterAllocMemFailedErr(ei.ctx, ei.memRoot, ei.jobID, ei.indexID)
+ }
var err error
lWrite, err = ei.openedEngine.LocalWriter(ei.ctx, &backend.LocalWriterConfig{})
if err != nil {
@@ -217,6 +227,7 @@ func (ei *engineInfo) newWriterContext(workerID int) (*writerContext, error) {
}
// Cache the local writer.
ei.writerCache.Store(workerID, lWrite)
+ ei.memRoot.Consume(int64(ei.litCfg.TikvImporter.LocalWriterMemCacheSize))
}
wc := &writerContext{
ctx: ei.ctx,
@@ -236,8 +247,10 @@ func (ei *engineInfo) closeWriters() error {
firstErr = err
}
}
+ ei.memRoot.Release(int64(ei.litCfg.TikvImporter.LocalWriterMemCacheSize))
}
ei.writerCache.Delete(wid)
+ ei.memRoot.Release(structSizeWriterCtx)
}
return firstErr
}
@@ -261,8 +274,3 @@ func (wCtx *writerContext) LockForWrite() (unlock func()) {
wCtx.fLock.RUnlock()
}
}
-
-// Close implements ingest.Writer interface.
-func (*writerContext) Close(_ context.Context) error {
- return nil
-}
diff --git a/pkg/ddl/ingest/engine_mgr.go b/pkg/ddl/ingest/engine_mgr.go
index de2483c383504..68c6167e3cde9 100644
--- a/pkg/ddl/ingest/engine_mgr.go
+++ b/pkg/ddl/ingest/engine_mgr.go
@@ -15,114 +15,111 @@
package ingest
import (
- "fmt"
-
"github.com/pingcap/errors"
"github.com/pingcap/tidb/pkg/lightning/backend"
- "github.com/pingcap/tidb/pkg/util/dbterror"
"github.com/pingcap/tidb/pkg/util/logutil"
"go.uber.org/zap"
)
-// maxWriterCount is the max number of writers that can be created for a single engine.
-const maxWriterCount = 16
+// Register implements BackendCtx.
+func (bc *litBackendCtx) Register(indexIDs []int64, tableName string) ([]Engine, error) {
+ ret := make([]Engine, 0, len(indexIDs))
+
+ for _, indexID := range indexIDs {
+ en, ok := bc.engines[indexID]
+ if !ok {
+ continue
+ }
+ ret = append(ret, en)
+ }
+ if l := len(ret); l > 0 {
+ if l != len(indexIDs) {
+ return nil, errors.Errorf(
+ "engines index ID number mismatch: job ID %d, required number of index IDs: %d, actual number of engines: %d",
+ bc.jobID, len(indexIDs), l,
+ )
+ }
+ return ret, nil
+ }
-// Register create a new engineInfo and register it to the backend context.
-func (bc *litBackendCtx) Register(jobID, indexID int64, schemaName, tableName string) (Engine, error) {
- // Calculate lightning concurrency degree and set memory usage
- // and pre-allocate memory usage for worker.
- bc.MemRoot.RefreshConsumption()
- ok := bc.MemRoot.CheckConsume(int64(bc.cfg.TikvImporter.LocalWriterMemCacheSize))
+ bc.memRoot.RefreshConsumption()
+ numIdx := int64(len(indexIDs))
+ engineCacheSize := int64(bc.cfg.TikvImporter.EngineMemCacheSize)
+ ok := bc.memRoot.CheckConsume(numIdx * (structSizeEngineInfo + engineCacheSize))
if !ok {
- return nil, genEngineAllocMemFailedErr(bc.ctx, bc.MemRoot, bc.jobID, indexID)
+ return nil, genEngineAllocMemFailedErr(bc.ctx, bc.memRoot, bc.jobID, indexIDs)
}
- var info string
- en, exist := bc.Load(indexID)
- if !exist || en.openedEngine == nil {
- if exist && en.closedEngine != nil {
- // Import failed before, try to import again.
- err := en.ImportAndClean()
- if err != nil {
- return nil, errors.Trace(err)
- }
- }
- engineCacheSize := int64(bc.cfg.TikvImporter.EngineMemCacheSize)
- ok := bc.MemRoot.CheckConsume(StructSizeEngineInfo + engineCacheSize)
- if !ok {
- return nil, genEngineAllocMemFailedErr(bc.ctx, bc.MemRoot, bc.jobID, indexID)
- }
+ mgr := backend.MakeEngineManager(bc.backend)
+ ts := uint64(0)
+ if c := bc.checkpointMgr; c != nil {
+ ts = c.GetTS()
+ }
+ cfg := generateLocalEngineConfig(ts)
+
+ openedEngines := make(map[int64]*engineInfo, numIdx)
- mgr := backend.MakeEngineManager(bc.backend)
- cfg := generateLocalEngineConfig(jobID, schemaName, tableName)
- openedEn, err := mgr.OpenEngine(bc.ctx, cfg, tableName, int32(indexID))
+ for _, indexID := range indexIDs {
+ openedEngine, err := mgr.OpenEngine(bc.ctx, cfg, tableName, int32(indexID))
if err != nil {
- logutil.Logger(bc.ctx).Warn(LitErrCreateEngineFail, zap.Int64("job ID", jobID),
- zap.Int64("index ID", indexID), zap.Error(err))
- return nil, errors.Trace(err)
- }
- id := openedEn.GetEngineUUID()
- en = newEngineInfo(bc.ctx, jobID, indexID, cfg, openedEn, id, 1, bc.MemRoot)
- bc.Store(indexID, en)
- bc.MemRoot.Consume(StructSizeEngineInfo)
- bc.MemRoot.ConsumeWithTag(encodeEngineTag(jobID, indexID), engineCacheSize)
- info = LitInfoOpenEngine
- } else {
- if en.writerCount+1 > maxWriterCount {
- logutil.Logger(bc.ctx).Warn(LitErrExceedConcurrency, zap.Int64("job ID", jobID),
+ logutil.Logger(bc.ctx).Warn(LitErrCreateEngineFail,
+ zap.Int64("job ID", bc.jobID),
zap.Int64("index ID", indexID),
- zap.Int("concurrency", bc.cfg.TikvImporter.RangeConcurrency))
- return nil, dbterror.ErrIngestFailed.FastGenByArgs("concurrency quota exceeded")
+ zap.Error(err))
+
+ for _, e := range openedEngines {
+ e.Clean()
+ }
+ return nil, errors.Trace(err)
}
- en.writerCount++
- info = LitInfoAddWriter
+
+ openedEngines[indexID] = newEngineInfo(
+ bc.ctx,
+ bc.jobID,
+ indexID,
+ cfg,
+ bc.cfg,
+ openedEngine,
+ openedEngine.GetEngineUUID(),
+ bc.memRoot,
+ )
}
- bc.MemRoot.ConsumeWithTag(encodeEngineTag(jobID, indexID), int64(bc.cfg.TikvImporter.LocalWriterMemCacheSize))
- logutil.Logger(bc.ctx).Info(info, zap.Int64("job ID", jobID),
- zap.Int64("index ID", indexID),
- zap.Int64("current memory usage", bc.MemRoot.CurrentUsage()),
- zap.Int64("memory limitation", bc.MemRoot.MaxMemoryQuota()),
- zap.Int("current writer count", en.writerCount))
- return en, nil
-}
-// Unregister delete the engineInfo from the engineManager.
-func (bc *litBackendCtx) Unregister(jobID, indexID int64) {
- ei, exist := bc.Load(indexID)
- if !exist {
- return
+ for _, indexID := range indexIDs {
+ ei := openedEngines[indexID]
+ ret = append(ret, ei)
+ bc.engines[indexID] = ei
}
+ bc.memRoot.Consume(numIdx * (structSizeEngineInfo + engineCacheSize))
- ei.Clean()
- bc.Delete(indexID)
- bc.MemRoot.ReleaseWithTag(encodeEngineTag(jobID, indexID))
- bc.MemRoot.Release(StructSizeWriterCtx * int64(ei.writerCount))
- bc.MemRoot.Release(StructSizeEngineInfo)
+ logutil.Logger(bc.ctx).Info(LitInfoOpenEngine, zap.Int64("job ID", bc.jobID),
+ zap.Int64s("index IDs", indexIDs),
+ zap.Int64("current memory usage", bc.memRoot.CurrentUsage()),
+ zap.Int64("memory limitation", bc.memRoot.MaxMemoryQuota()))
+ return ret, nil
}
-// ResetWorkers reset the writer count of the engineInfo because
-// the goroutines of backfill workers have been terminated.
-func (bc *litBackendCtx) ResetWorkers(jobID int64) {
- for _, indexID := range bc.Keys() {
- ei, exist := bc.Load(indexID)
- if !exist {
- continue
- }
- bc.MemRoot.Release(StructSizeWriterCtx * int64(ei.writerCount))
- bc.MemRoot.ReleaseWithTag(encodeEngineTag(jobID, indexID))
- engineCacheSize := int64(bc.cfg.TikvImporter.EngineMemCacheSize)
- bc.MemRoot.ConsumeWithTag(encodeEngineTag(jobID, indexID), engineCacheSize)
- ei.writerCount = 0
+// UnregisterEngines implements BackendCtx.
+func (bc *litBackendCtx) UnregisterEngines() {
+ numIdx := int64(len(bc.engines))
+ for _, ei := range bc.engines {
+ ei.Clean()
}
-}
+ bc.engines = make(map[int64]*engineInfo, 10)
-// unregisterAll delete all engineInfo from the engineManager.
-func (bc *litBackendCtx) unregisterAll(jobID int64) {
- for _, idxID := range bc.Keys() {
- bc.Unregister(jobID, idxID)
- }
+ engineCacheSize := int64(bc.cfg.TikvImporter.EngineMemCacheSize)
+ bc.memRoot.Release(numIdx * (structSizeEngineInfo + engineCacheSize))
}
-func encodeEngineTag(jobID, indexID int64) string {
- return fmt.Sprintf("%d-%d", jobID, indexID)
+// ImportStarted implements BackendCtx.
+func (bc *litBackendCtx) ImportStarted() bool {
+ if len(bc.engines) == 0 {
+ return false
+ }
+ for _, ei := range bc.engines {
+ if ei.openedEngine == nil {
+ return true
+ }
+ }
+ return false
}
diff --git a/pkg/ddl/ingest/mem_root.go b/pkg/ddl/ingest/mem_root.go
index 0d37b2ef6096c..e254620a69704 100644
--- a/pkg/ddl/ingest/mem_root.go
+++ b/pkg/ddl/ingest/mem_root.go
@@ -20,6 +20,7 @@ import (
)
// MemRoot is used to track the memory usage for the lightning backfill process.
+// TODO(lance6716): change API to prevent TOCTOU.
type MemRoot interface {
Consume(size int64)
Release(size int64)
@@ -35,18 +36,18 @@ type MemRoot interface {
}
var (
- // StructSizeBackendCtx is the size of litBackendCtx.
- StructSizeBackendCtx int64
- // StructSizeEngineInfo is the size of engineInfo.
- StructSizeEngineInfo int64
- // StructSizeWriterCtx is the size of writerContext.
- StructSizeWriterCtx int64
+ // structSizeBackendCtx is the size of litBackendCtx.
+ structSizeBackendCtx int64
+ // structSizeEngineInfo is the size of engineInfo.
+ structSizeEngineInfo int64
+ // structSizeWriterCtx is the size of writerContext.
+ structSizeWriterCtx int64
)
func init() {
- StructSizeBackendCtx = int64(unsafe.Sizeof(litBackendCtx{}))
- StructSizeEngineInfo = int64(unsafe.Sizeof(engineInfo{}))
- StructSizeWriterCtx = int64(unsafe.Sizeof(writerContext{}))
+ structSizeBackendCtx = int64(unsafe.Sizeof(litBackendCtx{}))
+ structSizeEngineInfo = int64(unsafe.Sizeof(engineInfo{}))
+ structSizeWriterCtx = int64(unsafe.Sizeof(writerContext{}))
}
// memRootImpl is an implementation of MemRoot.
diff --git a/pkg/ddl/ingest/message.go b/pkg/ddl/ingest/message.go
index 1217244c6f2ac..374575d3a4b4e 100644
--- a/pkg/ddl/ingest/message.go
+++ b/pkg/ddl/ingest/message.go
@@ -67,8 +67,18 @@ func genBackendAllocMemFailedErr(ctx context.Context, memRoot MemRoot, jobID int
return dbterror.ErrIngestFailed.FastGenByArgs("memory used up")
}
-func genEngineAllocMemFailedErr(ctx context.Context, memRoot MemRoot, jobID, idxID int64) error {
- logutil.Logger(ctx).Warn(LitErrAllocMemFail, zap.Int64("job ID", jobID),
+func genEngineAllocMemFailedErr(ctx context.Context, memRoot MemRoot, jobID int64, idxIDs []int64) error {
+ logutil.Logger(ctx).Warn(LitErrAllocMemFail,
+ zap.Int64("job ID", jobID),
+ zap.Int64s("index IDs", idxIDs),
+ zap.Int64("current memory usage", memRoot.CurrentUsage()),
+ zap.Int64("max memory quota", memRoot.MaxMemoryQuota()))
+ return dbterror.ErrIngestFailed.FastGenByArgs("memory used up")
+}
+
+func genWriterAllocMemFailedErr(ctx context.Context, memRoot MemRoot, jobID int64, idxID int64) error {
+ logutil.Logger(ctx).Warn(LitErrAllocMemFail,
+ zap.Int64("job ID", jobID),
zap.Int64("index ID", idxID),
zap.Int64("current memory usage", memRoot.CurrentUsage()),
zap.Int64("max memory quota", memRoot.MaxMemoryQuota()))
diff --git a/pkg/ddl/ingest/mock.go b/pkg/ddl/ingest/mock.go
index 8829a6ca7c2fa..80d1cb652f4c6 100644
--- a/pkg/ddl/ingest/mock.go
+++ b/pkg/ddl/ingest/mock.go
@@ -17,6 +17,9 @@ package ingest
import (
"context"
"encoding/hex"
+ "os"
+ "path/filepath"
+ "strconv"
"sync"
"github.com/pingcap/tidb/pkg/ddl/logutil"
@@ -58,6 +61,7 @@ func (m *MockBackendCtxMgr) Register(ctx context.Context, jobID int64, unique bo
mockCtx := &MockBackendCtx{
mu: sync.Mutex{},
sessCtx: sessCtx,
+ jobID: jobID,
}
m.runningJobs[jobID] = mockCtx
return mockCtx, nil
@@ -96,18 +100,28 @@ func (m *MockBackendCtxMgr) ResetSessCtx() {
type MockBackendCtx struct {
sessCtx sessionctx.Context
mu sync.Mutex
+ jobID int64
checkpointMgr *CheckpointManager
}
// Register implements BackendCtx.Register interface.
-func (m *MockBackendCtx) Register(jobID, indexID int64, _, _ string) (Engine, error) {
- logutil.DDLIngestLogger().Info("mock backend ctx register", zap.Int64("jobID", jobID), zap.Int64("indexID", indexID))
- return &MockEngineInfo{sessCtx: m.sessCtx, mu: &m.mu}, nil
+func (m *MockBackendCtx) Register(indexIDs []int64, _ string) ([]Engine, error) {
+ logutil.DDLIngestLogger().Info("mock backend ctx register", zap.Int64("jobID", m.jobID), zap.Int64s("indexIDs", indexIDs))
+ ret := make([]Engine, 0, len(indexIDs))
+ for range indexIDs {
+ ret = append(ret, &MockEngineInfo{sessCtx: m.sessCtx, mu: &m.mu})
+ }
+ return ret, nil
}
-// Unregister implements BackendCtx.Unregister interface.
-func (*MockBackendCtx) Unregister(jobID, indexID int64) {
- logutil.DDLIngestLogger().Info("mock backend ctx unregister", zap.Int64("jobID", jobID), zap.Int64("indexID", indexID))
+// UnregisterEngines implements BackendCtx.UnregisterEngines interface.
+func (*MockBackendCtx) UnregisterEngines() {
+ logutil.DDLIngestLogger().Info("mock backend ctx unregister")
+}
+
+// ImportStarted implements BackendCtx interface.
+func (*MockBackendCtx) ImportStarted() bool {
+ return false
}
// CollectRemoteDuplicateRows implements BackendCtx.CollectRemoteDuplicateRows interface.
@@ -122,13 +136,9 @@ func (*MockBackendCtx) FinishImport(indexID int64, _ bool, _ table.Table) error
return nil
}
-// ResetWorkers implements BackendCtx.ResetWorkers interface.
-func (*MockBackendCtx) ResetWorkers(_ int64) {
-}
-
// Flush implements BackendCtx.Flush interface.
-func (*MockBackendCtx) Flush(_ int64, _ FlushMode) (flushed bool, imported bool, err error) {
- return false, false, nil
+func (*MockBackendCtx) Flush(_ FlushMode) (flushed bool, imported bool, errIdxID int64, err error) {
+ return false, false, 0, nil
}
// Done implements BackendCtx.Done interface.
@@ -151,8 +161,10 @@ func (m *MockBackendCtx) GetCheckpointManager() *CheckpointManager {
}
// GetLocalBackend returns the local backend.
-func (*MockBackendCtx) GetLocalBackend() *local.Backend {
- return nil
+func (m *MockBackendCtx) GetLocalBackend() *local.Backend {
+ b := &local.Backend{}
+ b.LocalStoreDir = filepath.Join(os.TempDir(), "mock_backend", strconv.FormatInt(m.jobID, 10))
+ return b
}
// MockWriteHook the hook for write in mock engine.
@@ -236,10 +248,5 @@ func (*MockWriter) LockForWrite() func() {
return func() {}
}
-// Close implements Writer.Close interface.
-func (*MockWriter) Close(_ context.Context) error {
- return nil
-}
-
// MockExecAfterWriteRow is only used for test.
var MockExecAfterWriteRow func()
diff --git a/pkg/ddl/ingest/tests/BUILD.bazel b/pkg/ddl/ingest/tests/BUILD.bazel
index 8ee0e08783e45..642e29a57b7a5 100644
--- a/pkg/ddl/ingest/tests/BUILD.bazel
+++ b/pkg/ddl/ingest/tests/BUILD.bazel
@@ -10,8 +10,8 @@ go_test(
"//pkg/config",
"//pkg/ddl/ingest",
"//pkg/ddl/ingest/testutil",
- "//pkg/ddl/util/callback",
"//pkg/parser/model",
"//pkg/testkit",
+ "//pkg/testkit/testfailpoint",
],
)
diff --git a/pkg/ddl/ingest/tests/partition_table_test.go b/pkg/ddl/ingest/tests/partition_table_test.go
index 8cc593ebf0c79..1c07d724af7ac 100644
--- a/pkg/ddl/ingest/tests/partition_table_test.go
+++ b/pkg/ddl/ingest/tests/partition_table_test.go
@@ -20,12 +20,14 @@ import (
"github.com/pingcap/tidb/pkg/config"
"github.com/pingcap/tidb/pkg/ddl/ingest"
ingesttestutil "github.com/pingcap/tidb/pkg/ddl/ingest/testutil"
- "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
)
func TestAddIndexIngestRecoverPartition(t *testing.T) {
+ // TODO we are unregistering LitBackCtxMgr when owner changes, but another owner
+ // might access it, so this case is unstable by nature.
port := config.GetGlobalConfig().Port
tc := testkit.NewDistExecutionContext(t, 3)
defer tc.Close()
@@ -37,39 +39,22 @@ func TestAddIndexIngestRecoverPartition(t *testing.T) {
tk.MustExec("insert into t values (2, 3), (3, 3), (5, 5);")
partCnt := 0
- changeOwner0To1 := func(job *model.Job, _ int64) {
- partCnt++
- if partCnt == 3 {
- tc.SetOwner(1)
- // TODO(tangenta): mock multiple backends in a better way.
- //nolint: forcetypeassert
- // TODO(tangenta): When owner changes, wait last ddl owner's DDL scheduling loop exits.
- ingest.LitBackCtxMgr.(*ingest.MockBackendCtxMgr).ResetSessCtx()
- bc, _ := ingest.LitBackCtxMgr.Load(job.ID)
- bc.GetCheckpointManager().Close()
- bc.AttachCheckpointManager(nil)
- config.GetGlobalConfig().Port = port + 1
- }
- }
- changeOwner1To2 := func(job *model.Job, _ int64) {
- partCnt++
- if partCnt == 6 {
- tc.SetOwner(2)
- //nolint: forcetypeassert
- ingest.LitBackCtxMgr.(*ingest.MockBackendCtxMgr).ResetSessCtx()
- bc, _ := ingest.LitBackCtxMgr.Load(job.ID)
- bc.GetCheckpointManager().Close()
- bc.AttachCheckpointManager(nil)
- config.GetGlobalConfig().Port = port + 2
- }
- }
- tc.SetOwner(0)
- hook0 := &callback.TestDDLCallback{}
- hook0.OnUpdateReorgInfoExported = changeOwner0To1
- hook1 := &callback.TestDDLCallback{}
- hook1.OnUpdateReorgInfoExported = changeOwner1To2
- tc.GetDomain(0).DDL().SetHook(hook0)
- tc.GetDomain(1).DDL().SetHook(hook1)
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/beforeUpdateReorgInfo-addTableIndex",
+ func(job *model.Job) {
+ partCnt++
+ if partCnt == 3 || partCnt == 6 {
+ tc.TriggerOwnerChange()
+ // TODO(tangenta): mock multiple backends in a better way.
+ //nolint: forcetypeassert
+ // TODO(tangenta): When owner changes, wait last ddl owner's DDL scheduling loop exits.
+ ingest.LitBackCtxMgr.(*ingest.MockBackendCtxMgr).ResetSessCtx()
+ bc, _ := ingest.LitBackCtxMgr.Load(job.ID)
+ bc.GetCheckpointManager().Close()
+ bc.AttachCheckpointManager(nil)
+ config.GetGlobalConfig().Port = port + 1
+ }
+ },
+ )
tk.MustExec("alter table t add index idx(b);")
tk.MustExec("admin check table t;")
}
diff --git a/pkg/ddl/internal/session/BUILD.bazel b/pkg/ddl/internal/session/BUILD.bazel
index 62c006a7b1e4c..0221a13ca544d 100644
--- a/pkg/ddl/internal/session/BUILD.bazel
+++ b/pkg/ddl/internal/session/BUILD.bazel
@@ -18,7 +18,7 @@ go_library(
"//pkg/sessionctx",
"//pkg/sessiontxn",
"//pkg/util/chunk",
- "//pkg/util/mock",
+ "//pkg/util/intest",
"//pkg/util/sqlexec",
"@com_github_ngaut_pools//:pools",
"@com_github_pingcap_errors//:errors",
diff --git a/pkg/ddl/internal/session/session.go b/pkg/ddl/internal/session/session.go
index 01b8d83afbc11..0242ca3f52c74 100644
--- a/pkg/ddl/internal/session/session.go
+++ b/pkg/ddl/internal/session/session.go
@@ -39,6 +39,8 @@ func NewSession(s sessionctx.Context) *Session {
return &Session{s}
}
+// TODO(lance6716): provide a NewSessionWithCtx
+
// Begin starts a transaction.
func (s *Session) Begin() error {
err := sessiontxn.NewTxn(context.Background(), s.Context)
diff --git a/pkg/ddl/internal/session/session_pool.go b/pkg/ddl/internal/session/session_pool.go
index 66b8cef0372b8..6a2feb91edc55 100644
--- a/pkg/ddl/internal/session/session_pool.go
+++ b/pkg/ddl/internal/session/session_pool.go
@@ -25,7 +25,7 @@ import (
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/sessionctx"
- "github.com/pingcap/tidb/pkg/util/mock"
+ "github.com/pingcap/tidb/pkg/util/intest"
)
// Pool is used to new Session.
@@ -40,18 +40,14 @@ type Pool struct {
// NewSessionPool creates a new Session pool.
func NewSessionPool(resPool *pools.ResourcePool, store kv.Storage) *Pool {
+ intest.AssertNotNil(resPool)
+ intest.AssertNotNil(store)
return &Pool{resPool: resPool, store: store}
}
// Get gets sessionCtx from context resource pool.
// Please remember to call Put after you finished using sessionCtx.
func (sg *Pool) Get() (sessionctx.Context, error) {
- if sg.resPool == nil {
- ctx := mock.NewContext()
- ctx.Store = sg.store
- return ctx, nil
- }
-
sg.mu.Lock()
if sg.mu.closed {
sg.mu.Unlock()
@@ -78,10 +74,6 @@ func (sg *Pool) Get() (sessionctx.Context, error) {
// Put returns sessionCtx to context resource pool.
func (sg *Pool) Put(ctx sessionctx.Context) {
- if sg.resPool == nil {
- return
- }
-
// no need to protect sg.resPool, even the sg.resPool is closed, the ctx still need to
// Put into resPool, because when resPool is closing, it will wait all the ctx returns, then resPool finish closing.
sg.resPool.Put(ctx.(pools.Resource))
@@ -93,7 +85,7 @@ func (sg *Pool) Close() {
sg.mu.Lock()
defer sg.mu.Unlock()
// prevent closing resPool twice.
- if sg.mu.closed || sg.resPool == nil {
+ if sg.mu.closed {
return
}
logutil.DDLLogger().Info("closing session pool")
diff --git a/pkg/ddl/job_table.go b/pkg/ddl/job_table.go
index 64b74ed9b0aef..083c71a81f692 100644
--- a/pkg/ddl/job_table.go
+++ b/pkg/ddl/job_table.go
@@ -20,11 +20,13 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
+ "runtime"
"slices"
"strconv"
"strings"
"time"
+ "github.com/ngaut/pools"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/kvproto/pkg/kvrpcpb"
@@ -41,7 +43,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/table"
- tidb_util "github.com/pingcap/tidb/pkg/util"
+ tidbutil "github.com/pingcap/tidb/pkg/util"
"github.com/pingcap/tidb/pkg/util/dbterror"
"github.com/pingcap/tidb/pkg/util/intest"
tidblogutil "github.com/pingcap/tidb/pkg/util/logutil"
@@ -50,7 +52,9 @@ import (
)
var (
- addingDDLJobConcurrent = "/tidb/ddl/add_ddl_job_general"
+ // addingDDLJobNotifyKey is the key in etcd to notify DDL scheduler that there
+ // is a new DDL job.
+ addingDDLJobNotifyKey = "/tidb/ddl/add_ddl_job_general"
dispatchLoopWaitingDuration = 1 * time.Second
localWorkerWaitingDuration = 10 * time.Millisecond
)
@@ -82,7 +86,99 @@ const (
jobTypeLocal
)
-func (d *ddl) getJob(se *sess.Session, tp jobType, filter func(*model.Job) (bool, error)) (*model.Job, error) {
+type ownerListener struct {
+ ddl *ddl
+ scheduler *jobScheduler
+}
+
+var _ owner.Listener = (*ownerListener)(nil)
+
+func (l *ownerListener) OnBecomeOwner() {
+ ctx, cancelFunc := context.WithCancel(l.ddl.ddlCtx.ctx)
+ l.scheduler = &jobScheduler{
+ schCtx: ctx,
+ cancel: cancelFunc,
+ runningJobs: newRunningJobs(),
+
+ ddlCtx: l.ddl.ddlCtx,
+ ddlJobNotifyCh: l.ddl.ddlJobNotifyCh,
+ sessPool: l.ddl.sessPool,
+ delRangeMgr: l.ddl.delRangeMgr,
+ }
+ l.scheduler.start()
+}
+
+func (l *ownerListener) OnRetireOwner() {
+ if l.scheduler == nil {
+ return
+ }
+ l.scheduler.close()
+}
+
+// jobScheduler is used to schedule the DDL jobs, it's only run on the DDL owner.
+type jobScheduler struct {
+ // *ddlCtx already have context named as "ctx", so we use "schCtx" here to avoid confusion.
+ schCtx context.Context
+ cancel context.CancelFunc
+ wg tidbutil.WaitGroupWrapper
+ runningJobs *runningJobs
+
+ // those fields are created on start
+ reorgWorkerPool *workerPool
+ generalDDLWorkerPool *workerPool
+
+ // those fields are shared with 'ddl' instance
+ // TODO ddlCtx is too large for here, we should remove dependency on it.
+ *ddlCtx
+ ddlJobNotifyCh chan struct{}
+ sessPool *sess.Pool
+ delRangeMgr delRangeManager
+}
+
+func (s *jobScheduler) start() {
+ var err error
+ s.ddlCtx.ddlSeqNumMu.Lock()
+ defer s.ddlCtx.ddlSeqNumMu.Unlock()
+ s.ddlCtx.ddlSeqNumMu.seqNum, err = s.GetNextDDLSeqNum()
+ if err != nil {
+ logutil.DDLLogger().Error("error when getting the ddl history count", zap.Error(err))
+ }
+
+ workerFactory := func(tp workerType) func() (pools.Resource, error) {
+ return func() (pools.Resource, error) {
+ wk := newWorker(s.schCtx, tp, s.sessPool, s.delRangeMgr, s.ddlCtx)
+ sessForJob, err := s.sessPool.Get()
+ if err != nil {
+ return nil, err
+ }
+ sessForJob.GetSessionVars().SetDiskFullOpt(kvrpcpb.DiskFullOpt_AllowedOnAlmostFull)
+ wk.sess = sess.NewSession(sessForJob)
+ metrics.DDLCounter.WithLabelValues(fmt.Sprintf("%s_%s", metrics.CreateDDL, wk.String())).Inc()
+ return wk, nil
+ }
+ }
+ // reorg worker count at least 1 at most 10.
+ reorgCnt := min(max(runtime.GOMAXPROCS(0)/4, 1), reorgWorkerCnt)
+ s.reorgWorkerPool = newDDLWorkerPool(pools.NewResourcePool(workerFactory(addIdxWorker), reorgCnt, reorgCnt, 0), jobTypeReorg)
+ s.generalDDLWorkerPool = newDDLWorkerPool(pools.NewResourcePool(workerFactory(generalWorker), generalWorkerCnt, generalWorkerCnt, 0), jobTypeGeneral)
+ s.wg.RunWithLog(s.startDispatchLoop)
+ s.wg.RunWithLog(func() {
+ s.schemaSyncer.SyncJobSchemaVerLoop(s.schCtx)
+ })
+}
+
+func (s *jobScheduler) close() {
+ s.cancel()
+ s.wg.Wait()
+ if s.reorgWorkerPool != nil {
+ s.reorgWorkerPool.close()
+ }
+ if s.generalDDLWorkerPool != nil {
+ s.generalDDLWorkerPool.close()
+ }
+}
+
+func (s *jobScheduler) getJob(se *sess.Session, tp jobType, filter func(*model.Job) (bool, error)) (*model.Job, error) {
not := "not"
label := "get_job_general"
if tp == jobTypeReorg {
@@ -93,7 +189,7 @@ func (d *ddl) getJob(se *sess.Session, tp jobType, filter func(*model.Job) (bool
(select min(job_id) from mysql.tidb_ddl_job group by schema_ids, table_ids, processing)
and %s reorg %s order by processing desc, job_id`
var excludedJobIDs string
- if ids := d.runningJobs.allIDs(); len(ids) > 0 {
+ if ids := s.runningJobs.allIDs(); len(ids) > 0 {
excludedJobIDs = fmt.Sprintf("and job_id not in (%s)", ids)
}
sql := fmt.Sprintf(getJobSQL, not, excludedJobIDs)
@@ -111,7 +207,7 @@ func (d *ddl) getJob(se *sess.Session, tp jobType, filter func(*model.Job) (bool
return nil, errors.Trace(err)
}
- isRunnable, err := d.processJobDuringUpgrade(se, &job)
+ isRunnable, err := s.processJobDuringUpgrade(se, &job)
if err != nil {
return nil, errors.Trace(err)
}
@@ -129,7 +225,7 @@ func (d *ddl) getJob(se *sess.Session, tp jobType, filter func(*model.Job) (bool
return nil, errors.Trace(err)
}
if b {
- if err = d.markJobProcessing(se, &job); err != nil {
+ if err = s.markJobProcessing(se, &job); err != nil {
logutil.DDLLogger().Warn(
"[ddl] handle ddl job failed: mark job is processing meet error",
zap.Error(err),
@@ -144,15 +240,15 @@ func (d *ddl) getJob(se *sess.Session, tp jobType, filter func(*model.Job) (bool
func hasSysDB(job *model.Job) bool {
for _, info := range job.GetInvolvingSchemaInfo() {
- if tidb_util.IsSysDB(info.Database) {
+ if tidbutil.IsSysDB(info.Database) {
return true
}
}
return false
}
-func (d *ddl) processJobDuringUpgrade(sess *sess.Session, job *model.Job) (isRunnable bool, err error) {
- if d.stateSyncer.IsUpgradingState() {
+func (s *jobScheduler) processJobDuringUpgrade(sess *sess.Session, job *model.Job) (isRunnable bool, err error) {
+ if s.stateSyncer.IsUpgradingState() {
if job.IsPaused() {
return false, nil
}
@@ -204,29 +300,29 @@ func (d *ddl) processJobDuringUpgrade(sess *sess.Session, job *model.Job) (isRun
return true, nil
}
-func (d *ddl) getGeneralJob(sess *sess.Session) (*model.Job, error) {
- return d.getJob(sess, jobTypeGeneral, func(job *model.Job) (bool, error) {
- if !d.runningJobs.checkRunnable(job) {
+func (s *jobScheduler) getGeneralJob(sess *sess.Session) (*model.Job, error) {
+ return s.getJob(sess, jobTypeGeneral, func(job *model.Job) (bool, error) {
+ if !s.runningJobs.checkRunnable(job) {
return false, nil
}
if job.Type == model.ActionDropSchema {
// Check if there is any reorg job on this schema.
sql := fmt.Sprintf("select job_id from mysql.tidb_ddl_job where CONCAT(',', schema_ids, ',') REGEXP CONCAT(',', %s, ',') != 0 and processing limit 1", strconv.Quote(strconv.FormatInt(job.SchemaID, 10)))
- rows, err := sess.Execute(d.ctx, sql, "check conflict jobs")
+ rows, err := sess.Execute(s.schCtx, sql, "check conflict jobs")
return len(rows) == 0, err
}
// Check if there is any running job works on the same table.
sql := fmt.Sprintf("select job_id from mysql.tidb_ddl_job t1, (select table_ids from mysql.tidb_ddl_job where job_id = %d) t2 where "+
- "(processing and CONCAT(',', t2.table_ids, ',') REGEXP CONCAT(',', REPLACE(t1.table_ids, ',', '|'), ',') != 0)"+
+ "(processing and CONCAT(',', t2.table_ids, ',') REGEXP CONCAT(',(', REPLACE(t1.table_ids, ',', '|'), '),') != 0)"+
"or (type = %d and processing)", job.ID, model.ActionFlashbackCluster)
- rows, err := sess.Execute(d.ctx, sql, "check conflict jobs")
+ rows, err := sess.Execute(s.schCtx, sql, "check conflict jobs")
return len(rows) == 0, err
})
}
-func (d *ddl) getReorgJob(sess *sess.Session) (*model.Job, error) {
- return d.getJob(sess, jobTypeReorg, func(job *model.Job) (bool, error) {
- if !d.runningJobs.checkRunnable(job) {
+func (s *jobScheduler) getReorgJob(sess *sess.Session) (*model.Job, error) {
+ return s.getJob(sess, jobTypeReorg, func(job *model.Job) (bool, error) {
+ if !s.runningJobs.checkRunnable(job) {
return false, nil
}
// Check if there is any block ddl running, like drop schema and flashback cluster.
@@ -235,7 +331,7 @@ func (d *ddl) getReorgJob(sess *sess.Session) (*model.Job, error) {
"or (CONCAT(',', table_ids, ',') REGEXP CONCAT(',', %s, ',') != 0 and processing) "+
"or (type = %d and processing) limit 1",
strconv.Quote(strconv.FormatInt(job.SchemaID, 10)), model.ActionDropSchema, strconv.Quote(strconv.FormatInt(job.TableID, 10)), model.ActionFlashbackCluster)
- rows, err := sess.Execute(d.ctx, sql, "check conflict jobs")
+ rows, err := sess.Execute(s.schCtx, sql, "check conflict jobs")
return len(rows) == 0, err
})
}
@@ -255,36 +351,31 @@ func (d *ddl) startLocalWorkerLoop() {
}
}
-func (d *ddl) startDispatchLoop() {
- sessCtx, err := d.sessPool.Get()
+func (s *jobScheduler) startDispatchLoop() {
+ sessCtx, err := s.sessPool.Get()
if err != nil {
logutil.DDLLogger().Fatal("dispatch loop get session failed, it should not happen, please try restart TiDB", zap.Error(err))
}
- defer d.sessPool.Put(sessCtx)
+ defer s.sessPool.Put(sessCtx)
se := sess.NewSession(sessCtx)
var notifyDDLJobByEtcdCh clientv3.WatchChan
- if d.etcdCli != nil {
- notifyDDLJobByEtcdCh = d.etcdCli.Watch(d.ctx, addingDDLJobConcurrent)
+ if s.etcdCli != nil {
+ notifyDDLJobByEtcdCh = s.etcdCli.Watch(s.schCtx, addingDDLJobNotifyKey)
}
- if err := d.checkAndUpdateClusterState(true); err != nil {
+ if err := s.checkAndUpdateClusterState(true); err != nil {
logutil.DDLLogger().Fatal("dispatch loop get cluster state failed, it should not happen, please try restart TiDB", zap.Error(err))
}
ticker := time.NewTicker(dispatchLoopWaitingDuration)
defer ticker.Stop()
- isOnce := false
+ // TODO move waitSchemaSyncedController out of ddlCtx.
+ s.clearOnceMap()
for {
- if d.ctx.Err() != nil {
+ if s.schCtx.Err() != nil {
return
}
- if !d.isOwner() {
- isOnce = true
- d.onceMap = make(map[int64]struct{}, jobOnceCapacity)
- time.Sleep(dispatchLoopWaitingDuration)
- continue
- }
failpoint.Inject("ownerResignAfterDispatchLoopCheck", func() {
if ingest.ResignOwnerForTest.Load() {
- err2 := d.ownerManager.ResignOwner(context.Background())
+ err2 := s.ownerManager.ResignOwner(context.Background())
if err2 != nil {
logutil.DDLLogger().Info("resign meet error", zap.Error(err2))
}
@@ -292,32 +383,34 @@ func (d *ddl) startDispatchLoop() {
}
})
select {
- case <-d.ddlJobCh:
+ case <-s.ddlJobNotifyCh:
case <-ticker.C:
case _, ok := <-notifyDDLJobByEtcdCh:
if !ok {
- logutil.DDLLogger().Warn("start worker watch channel closed", zap.String("watch key", addingDDLJobConcurrent))
- notifyDDLJobByEtcdCh = d.etcdCli.Watch(d.ctx, addingDDLJobConcurrent)
+ logutil.DDLLogger().Warn("start worker watch channel closed", zap.String("watch key", addingDDLJobNotifyKey))
+ notifyDDLJobByEtcdCh = s.etcdCli.Watch(s.schCtx, addingDDLJobNotifyKey)
time.Sleep(time.Second)
continue
}
- case <-d.ctx.Done():
+ case <-s.schCtx.Done():
return
}
- if err := d.checkAndUpdateClusterState(isOnce); err != nil {
+ if err := s.checkAndUpdateClusterState(false); err != nil {
continue
}
- isOnce = false
- d.loadDDLJobAndRun(se, d.generalDDLWorkerPool, d.getGeneralJob)
- d.loadDDLJobAndRun(se, d.reorgWorkerPool, d.getReorgJob)
+ s.loadDDLJobAndRun(se, s.generalDDLWorkerPool, s.getGeneralJob)
+ s.loadDDLJobAndRun(se, s.reorgWorkerPool, s.getReorgJob)
}
}
-func (d *ddl) checkAndUpdateClusterState(needUpdate bool) error {
+// TODO make it run in a separate routine.
+func (s *jobScheduler) checkAndUpdateClusterState(needUpdate bool) error {
select {
- case _, ok := <-d.stateSyncer.WatchChan():
+ case _, ok := <-s.stateSyncer.WatchChan():
if !ok {
- d.stateSyncer.Rewatch(d.ctx)
+ // TODO stateSyncer should only be started when we are the owner, and use
+ // the context of scheduler, will refactor it later.
+ s.stateSyncer.Rewatch(s.ddlCtx.ctx)
}
default:
if !needUpdate {
@@ -325,23 +418,20 @@ func (d *ddl) checkAndUpdateClusterState(needUpdate bool) error {
}
}
- oldState := d.stateSyncer.IsUpgradingState()
- stateInfo, err := d.stateSyncer.GetGlobalState(d.ctx)
+ oldState := s.stateSyncer.IsUpgradingState()
+ stateInfo, err := s.stateSyncer.GetGlobalState(s.schCtx)
if err != nil {
logutil.DDLLogger().Warn("get global state failed", zap.Error(err))
return errors.Trace(err)
}
logutil.DDLLogger().Info("get global state and global state change",
- zap.Bool("oldState", oldState), zap.Bool("currState", d.stateSyncer.IsUpgradingState()))
- if !d.isOwner() {
- return nil
- }
+ zap.Bool("oldState", oldState), zap.Bool("currState", s.stateSyncer.IsUpgradingState()))
ownerOp := owner.OpNone
if stateInfo.State == syncer.StateUpgrading {
ownerOp = owner.OpSyncUpgradingState
}
- err = d.ownerManager.SetOwnerOpValue(d.ctx, ownerOp)
+ err = s.ownerManager.SetOwnerOpValue(s.schCtx, ownerOp)
if err != nil {
logutil.DDLLogger().Warn("the owner sets global state to owner operator value failed", zap.Error(err))
return errors.Trace(err)
@@ -350,16 +440,16 @@ func (d *ddl) checkAndUpdateClusterState(needUpdate bool) error {
return nil
}
-func (d *ddl) loadDDLJobAndRun(se *sess.Session, pool *workerPool, getJob func(*sess.Session) (*model.Job, error)) {
+func (s *jobScheduler) loadDDLJobAndRun(se *sess.Session, pool *workerPool, getJob func(*sess.Session) (*model.Job, error)) {
wk, err := pool.get()
if err != nil || wk == nil {
logutil.DDLLogger().Debug(fmt.Sprintf("[ddl] no %v worker available now", pool.tp()), zap.Error(err))
return
}
- d.mu.RLock()
- d.mu.hook.OnGetJobBefore(pool.tp().String())
- d.mu.RUnlock()
+ s.mu.RLock()
+ s.mu.hook.OnGetJobBefore(pool.tp().String())
+ s.mu.RUnlock()
startTime := time.Now()
job, err := getJob(se)
@@ -370,11 +460,11 @@ func (d *ddl) loadDDLJobAndRun(se *sess.Session, pool *workerPool, getJob func(*
pool.put(wk)
return
}
- d.mu.RLock()
- d.mu.hook.OnGetJobAfter(pool.tp().String(), job)
- d.mu.RUnlock()
+ s.mu.RLock()
+ s.mu.hook.OnGetJobAfter(pool.tp().String(), job)
+ s.mu.RUnlock()
- d.delivery2Worker(wk, pool, job)
+ s.delivery2Worker(wk, pool, job)
}
// delivery2LocalWorker runs the DDL job of v2 in local.
@@ -415,20 +505,29 @@ func (d *ddl) delivery2LocalWorker(pool *workerPool, task *limitJobTask) {
}
// delivery2Worker owns the worker, need to put it back to the pool in this function.
-func (d *ddl) delivery2Worker(wk *worker, pool *workerPool, job *model.Job) {
+func (s *jobScheduler) delivery2Worker(wk *worker, pool *workerPool, job *model.Job) {
injectFailPointForGetJob(job)
- d.runningJobs.add(job)
- d.wg.Run(func() {
+ s.runningJobs.add(job)
+ s.wg.Run(func() {
metrics.DDLRunningJobCount.WithLabelValues(pool.tp().String()).Inc()
defer func() {
- d.runningJobs.remove(job)
- asyncNotify(d.ddlJobCh)
+ failpoint.InjectCall("afterDelivery2Worker", job)
+ s.runningJobs.remove(job)
+ asyncNotify(s.ddlJobNotifyCh)
metrics.DDLRunningJobCount.WithLabelValues(pool.tp().String()).Dec()
+ if wk.ctx.Err() != nil && ingest.LitBackCtxMgr != nil {
+ // if ctx cancelled, i.e. owner changed, we need to Unregister the backend
+ // as litBackendCtx is holding this very 'ctx', and it cannot reuse now.
+ // TODO make LitBackCtxMgr a local value of the job scheduler, it makes
+ // it much harder to test multiple owners in 1 unit test.
+ ingest.LitBackCtxMgr.Unregister(job.ID)
+ }
}()
+ ownerID := s.ownerManager.ID()
// check if this ddl job is synced to all servers.
- if !job.NotStarted() && (!d.isSynced(job) || !d.maybeAlreadyRunOnce(job.ID)) {
+ if !job.NotStarted() && (!s.isSynced(job) || !s.maybeAlreadyRunOnce(job.ID)) {
if variable.EnableMDL.Load() {
- exist, version, err := checkMDLInfo(job.ID, d.sessPool)
+ exist, version, err := checkMDLInfo(job.ID, s.sessPool)
if err != nil {
wk.jobLogger(job).Warn("check MDL info failed", zap.Error(err))
// Release the worker resource.
@@ -437,28 +536,28 @@ func (d *ddl) delivery2Worker(wk *worker, pool *workerPool, job *model.Job) {
} else if exist {
// Release the worker resource.
pool.put(wk)
- err = waitSchemaSyncedForMDL(d.ddlCtx, job, version)
+ err = waitSchemaSyncedForMDL(wk.ctx, s.ddlCtx, job, version)
if err != nil {
return
}
- d.setAlreadyRunOnce(job.ID)
- cleanMDLInfo(d.sessPool, job.ID, d.etcdCli, job.State == model.JobStateSynced)
+ s.setAlreadyRunOnce(job.ID)
+ cleanMDLInfo(s.sessPool, job, s.etcdCli, ownerID, job.State == model.JobStateSynced)
// Don't have a worker now.
return
}
} else {
- err := waitSchemaSynced(d.ddlCtx, job, 2*d.lease)
+ err := waitSchemaSynced(wk.ctx, s.ddlCtx, job, 2*s.lease)
if err != nil {
time.Sleep(time.Second)
// Release the worker resource.
pool.put(wk)
return
}
- d.setAlreadyRunOnce(job.ID)
+ s.setAlreadyRunOnce(job.ID)
}
}
- schemaVer, err := wk.HandleDDLJobTable(d.ddlCtx, job)
+ schemaVer, err := wk.HandleDDLJobTable(s.ddlCtx, job)
logCtx := wk.logCtx
pool.put(wk)
if err != nil {
@@ -476,28 +575,28 @@ func (d *ddl) delivery2Worker(wk *worker, pool *workerPool, job *model.Job) {
// Here means the job enters another state (delete only, write only, public, etc...) or is cancelled.
// If the job is done or still running or rolling back, we will wait 2 * lease time or util MDL synced to guarantee other servers to update
// the newest schema.
- err := waitSchemaChanged(d.ddlCtx, d.lease*2, schemaVer, job)
+ err := waitSchemaChanged(wk.ctx, s.ddlCtx, s.lease*2, schemaVer, job)
if err != nil {
return
}
- cleanMDLInfo(d.sessPool, job.ID, d.etcdCli, job.State == model.JobStateSynced)
- d.synced(job)
+ cleanMDLInfo(s.sessPool, job, s.etcdCli, ownerID, job.State == model.JobStateSynced)
+ s.synced(job)
if RunInGoTest {
- // d.mu.hook is initialed from domain / test callback, which will force the owner host update schema diff synchronously.
- d.mu.RLock()
- d.mu.hook.OnSchemaStateChanged(schemaVer)
- d.mu.RUnlock()
+ // s.mu.hook is initialed from domain / test callback, which will force the owner host update schema diff synchronously.
+ s.mu.RLock()
+ s.mu.hook.OnSchemaStateChanged(schemaVer)
+ s.mu.RUnlock()
}
- d.mu.RLock()
- d.mu.hook.OnJobUpdated(job)
- d.mu.RUnlock()
+ s.mu.RLock()
+ s.mu.hook.OnJobUpdated(job)
+ s.mu.RUnlock()
}
})
}
-func (*ddl) markJobProcessing(se *sess.Session, job *model.Job) error {
+func (*jobScheduler) markJobProcessing(se *sess.Session, job *model.Job) error {
se.GetSessionVars().SetDiskFullOpt(kvrpcpb.DiskFullOpt_AllowedOnAlmostFull)
_, err := se.Execute(context.Background(), fmt.Sprintf(
"update mysql.tidb_ddl_job set processing = 1 where job_id = %d", job.ID),
diff --git a/pkg/ddl/mock.go b/pkg/ddl/mock.go
index edfe2abaa7eba..7be8f499fa01e 100644
--- a/pkg/ddl/mock.go
+++ b/pkg/ddl/mock.go
@@ -146,6 +146,10 @@ func (s *MockSchemaSyncer) OwnerCheckAllVersions(ctx context.Context, jobID int6
}
}
+// SyncJobSchemaVerLoop implements SchemaSyncer.SyncJobSchemaVerLoop interface.
+func (*MockSchemaSyncer) SyncJobSchemaVerLoop(context.Context) {
+}
+
// Close implements SchemaSyncer.Close interface.
func (*MockSchemaSyncer) Close() {}
diff --git a/pkg/ddl/partition.go b/pkg/ddl/partition.go
index 15ea2271e73e6..4e6562b6be99c 100644
--- a/pkg/ddl/partition.go
+++ b/pkg/ddl/partition.go
@@ -695,7 +695,7 @@ func getPartitionIntervalFromTable(ctx expression.BuildContext, tbInfo *model.Ta
return nil
}
} else {
- if !isPartExprUnsigned(tbInfo) {
+ if !isPartExprUnsigned(ctx.GetEvalCtx(), tbInfo) {
minVal = "-9223372036854775808"
}
}
@@ -985,7 +985,7 @@ func generatePartitionDefinitionsFromInterval(ctx expression.BuildContext, partO
if partCol != nil {
min = getLowerBoundInt(partCol)
} else {
- if !isPartExprUnsigned(tbInfo) {
+ if !isPartExprUnsigned(ctx.GetEvalCtx(), tbInfo) {
min = math.MinInt64
}
}
@@ -1476,7 +1476,7 @@ func buildRangePartitionDefinitions(ctx expression.BuildContext, defs []*ast.Par
func checkPartitionValuesIsInt(ctx expression.BuildContext, defName any, exprs []ast.ExprNode, tbInfo *model.TableInfo) error {
tp := types.NewFieldType(mysql.TypeLonglong)
- if isPartExprUnsigned(tbInfo) {
+ if isPartExprUnsigned(ctx.GetEvalCtx(), tbInfo) {
tp.AddFlag(mysql.UnsignedFlag)
}
for _, exp := range exprs {
@@ -1642,7 +1642,7 @@ func checkPartitionFuncType(ctx sessionctx.Context, expr ast.ExprNode, schema st
if err != nil {
return errors.Trace(err)
}
- if e.GetType().EvalType() == types.ETInt {
+ if e.GetType(ctx.GetExprCtx().GetEvalCtx()).EvalType() == types.ETInt {
return nil
}
@@ -1665,7 +1665,7 @@ func checkRangePartitionValue(ctx sessionctx.Context, tblInfo *model.TableInfo)
if strings.EqualFold(defs[len(defs)-1].LessThan[0], partitionMaxValue) {
defs = defs[:len(defs)-1]
}
- isUnsigned := isPartExprUnsigned(tblInfo)
+ isUnsigned := isPartExprUnsigned(ctx.GetExprCtx().GetEvalCtx(), tblInfo)
var prevRangeValue any
for i := 0; i < len(defs); i++ {
if strings.EqualFold(defs[i].LessThan[0], partitionMaxValue) {
@@ -1728,7 +1728,7 @@ func formatListPartitionValue(ctx expression.BuildContext, tblInfo *model.TableI
cols := make([]*model.ColumnInfo, 0, len(pi.Columns))
if len(pi.Columns) == 0 {
tp := types.NewFieldType(mysql.TypeLonglong)
- if isPartExprUnsigned(tblInfo) {
+ if isPartExprUnsigned(ctx.GetEvalCtx(), tblInfo) {
tp.AddFlag(mysql.UnsignedFlag)
}
colTps = []*types.FieldType{tp}
@@ -1964,14 +1964,14 @@ func getTableInfoWithOriginalPartitions(t *model.TableInfo, oldIDs []int64, newI
return nt
}
-func dropLabelRules(d *ddlCtx, schemaName, tableName string, partNames []string) error {
+func dropLabelRules(ctx context.Context, schemaName, tableName string, partNames []string) error {
deleteRules := make([]string, 0, len(partNames))
for _, partName := range partNames {
deleteRules = append(deleteRules, fmt.Sprintf(label.PartitionIDFormat, label.IDPrefix, schemaName, tableName, partName))
}
// delete batch rules
patch := label.NewRulePatch([]*label.Rule{}, deleteRules)
- return infosync.UpdateLabelRules(d.ctx, patch)
+ return infosync.UpdateLabelRules(ctx, patch)
}
// onDropTablePartition deletes old partition meta.
@@ -1998,7 +1998,7 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (
return ver, errors.Wrapf(err, "failed to notify PD the placement rules")
}
// TODO: Will this drop LabelRules for existing partitions, if the new partitions have the same name?
- err = dropLabelRules(d, job.SchemaName, tblInfo.Name.L, pNames)
+ err = dropLabelRules(w.ctx, job.SchemaName, tblInfo.Name.L, pNames)
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Wrapf(err, "failed to notify PD the label rules")
@@ -2046,7 +2046,7 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (
return ver, errors.Trace(err)
}
physicalTableIDs = updateDroppingPartitionInfo(tblInfo, partNames)
- err = dropLabelRules(d, job.SchemaName, tblInfo.Name.L, partNames)
+ err = dropLabelRules(w.ctx, job.SchemaName, tblInfo.Name.L, partNames)
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Wrapf(err, "failed to notify PD the label rules")
@@ -3238,7 +3238,11 @@ type reorgPartitionWorker struct {
reorgedTbl table.PartitionedTable
}
-func newReorgPartitionWorker(sessCtx sessionctx.Context, i int, t table.PhysicalTable, decodeColMap map[int64]decoder.Column, reorgInfo *reorgInfo, jc *JobContext) (*reorgPartitionWorker, error) {
+func newReorgPartitionWorker(i int, t table.PhysicalTable, decodeColMap map[int64]decoder.Column, reorgInfo *reorgInfo, jc *JobContext) (*reorgPartitionWorker, error) {
+ bCtx, err := newBackfillCtx(i, reorgInfo, reorgInfo.SchemaName, t, jc, "reorg_partition_rate", false)
+ if err != nil {
+ return nil, err
+ }
reorgedTbl, err := tables.GetReorganizedPartitionedTable(t)
if err != nil {
return nil, errors.Trace(err)
@@ -3262,7 +3266,7 @@ func newReorgPartitionWorker(sessCtx sessionctx.Context, i int, t table.Physical
maxOffset = mathutil.Max[int](maxOffset, offset)
}
return &reorgPartitionWorker{
- backfillCtx: newBackfillCtx(reorgInfo.d, i, sessCtx, reorgInfo.SchemaName, t, jc, "reorg_partition_rate", false),
+ backfillCtx: bCtx,
rowDecoder: decoder.NewRowDecoder(t, t.WritableCols(), decodeColMap),
rowMap: make(map[int64]types.Datum, len(decodeColMap)),
writeColOffsetMap: writeColOffsetMap,
@@ -3274,7 +3278,7 @@ func newReorgPartitionWorker(sessCtx sessionctx.Context, i int, t table.Physical
func (w *reorgPartitionWorker) BackfillData(handleRange reorgBackfillTask) (taskCtx backfillTaskContext, errInTxn error) {
oprStartTime := time.Now()
ctx := kv.WithInternalSourceAndTaskType(context.Background(), w.jobContext.ddlJobSourceType(), kvutil.ExplicitTypeDDL)
- errInTxn = kv.RunInNewTxn(ctx, w.sessCtx.GetStore(), true, func(_ context.Context, txn kv.Transaction) error {
+ errInTxn = kv.RunInNewTxn(ctx, w.ddlCtx.store, true, func(_ context.Context, txn kv.Transaction) error {
taskCtx.addedCount = 0
taskCtx.scanCount = 0
updateTxnEntrySizeLimitIfNeeded(txn)
@@ -3334,12 +3338,12 @@ func (w *reorgPartitionWorker) fetchRowColVals(txn kv.Transaction, taskRange reo
// taskDone means that the added handle is out of taskRange.endHandle.
taskDone := false
- sysTZ := w.sessCtx.GetSessionVars().StmtCtx.TimeZone()
+ sysTZ := w.loc
tmpRow := make([]types.Datum, w.maxOffset+1)
var lastAccessedHandle kv.Key
oprStartTime := startTime
- err := iterateSnapshotKeys(w.jobContext, w.sessCtx.GetStore(), taskRange.priority, w.table.RecordPrefix(), txn.StartTS(), taskRange.startKey, taskRange.endKey,
+ err := iterateSnapshotKeys(w.jobContext, w.ddlCtx.store, taskRange.priority, w.table.RecordPrefix(), txn.StartTS(), taskRange.startKey, taskRange.endKey,
func(handle kv.Handle, recordKey kv.Key, rawRow []byte) (bool, error) {
oprEndTime := time.Now()
logSlowOperations(oprEndTime.Sub(oprStartTime), "iterateSnapshotKeys in reorgPartitionWorker fetchRowColVals", 0)
@@ -3353,7 +3357,7 @@ func (w *reorgPartitionWorker) fetchRowColVals(txn kv.Transaction, taskRange reo
// TODO: Extend for normal tables
// TODO: Extend for REMOVE PARTITIONING
- _, err := w.rowDecoder.DecodeTheExistedColumnMap(w.sessCtx, handle, rawRow, sysTZ, w.rowMap)
+ _, err := w.rowDecoder.DecodeTheExistedColumnMap(w.exprCtx, handle, rawRow, sysTZ, w.rowMap)
if err != nil {
return false, errors.Trace(err)
}
@@ -3366,7 +3370,7 @@ func (w *reorgPartitionWorker) fetchRowColVals(txn kv.Transaction, taskRange reo
}
tmpRow[offset] = d
}
- p, err := w.reorgedTbl.GetPartitionByRow(w.sessCtx.GetExprCtx(), tmpRow)
+ p, err := w.reorgedTbl.GetPartitionByRow(w.exprCtx.GetEvalCtx(), tmpRow)
if err != nil {
return false, errors.Trace(err)
}
@@ -4047,7 +4051,7 @@ func (cns columnNameSlice) At(i int) string {
return cns[i].Name.L
}
-func isPartExprUnsigned(tbInfo *model.TableInfo) bool {
+func isPartExprUnsigned(ectx expression.EvalContext, tbInfo *model.TableInfo) bool {
// We should not rely on any configuration, system or session variables, so use a mock ctx!
// Same as in tables.newPartitionExpr
ctx := mock.NewContext()
@@ -4056,7 +4060,7 @@ func isPartExprUnsigned(tbInfo *model.TableInfo) bool {
logutil.DDLLogger().Error("isPartExpr failed parsing expression!", zap.Error(err))
return false
}
- if mysql.HasUnsignedFlag(expr.GetType().GetFlag()) {
+ if mysql.HasUnsignedFlag(expr.GetType(ectx).GetFlag()) {
return true
}
return false
diff --git a/pkg/ddl/placement/bundle.go b/pkg/ddl/placement/bundle.go
index 891365249b37e..427bbdf2a7d98 100644
--- a/pkg/ddl/placement/bundle.go
+++ b/pkg/ddl/placement/bundle.go
@@ -465,23 +465,29 @@ func (c *constraintsGroup) MergeTransformableRoles() {
c.rules = newRules
}
+// GetRangeStartAndEndKeyHex get startKeyHex and endKeyHex of range by rangeBundleID.
+func GetRangeStartAndEndKeyHex(rangeBundleID string) (startKey string, endKey string) {
+ startKey, endKey = "", ""
+ if rangeBundleID == TiDBBundleRangePrefixForMeta {
+ startKey = hex.EncodeToString(metaPrefix)
+ endKey = hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(0)))
+ }
+ return startKey, endKey
+}
+
// RebuildForRange rebuilds the bundle for system range.
func (b *Bundle) RebuildForRange(rangeName string, policyName string) *Bundle {
rule := b.Rules
- startKey := ""
- endKey := ""
switch rangeName {
case KeyRangeGlobal:
b.ID = TiDBBundleRangePrefixForGlobal
b.Index = RuleIndexKeyRangeForGlobal
case KeyRangeMeta:
- // change range
- startKey = hex.EncodeToString(metaPrefix)
- endKey = hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(0)))
b.ID = TiDBBundleRangePrefixForMeta
b.Index = RuleIndexKeyRangeForMeta
}
+ startKey, endKey := GetRangeStartAndEndKeyHex(b.ID)
b.Override = true
newRules := make([]*pd.Rule, 0, len(rule))
for i, r := range b.Rules {
diff --git a/pkg/ddl/placement_policy.go b/pkg/ddl/placement_policy.go
index 4ab6495754f12..799da52e5efb3 100644
--- a/pkg/ddl/placement_policy.go
+++ b/pkg/ddl/placement_policy.go
@@ -119,7 +119,7 @@ func getPlacementPolicyByName(d *ddlCtx, t *meta.Meta, policyName model.CIStr) (
}
is := d.infoCache.GetLatest()
- if is.SchemaMetaVersion() == currVer {
+ if is != nil && is.SchemaMetaVersion() == currVer {
// Use cached policy.
policy, ok := is.PolicyByName(policyName)
if ok {
@@ -276,41 +276,67 @@ func updateExistPlacementPolicy(t *meta.Meta, policy *model.PolicyInfo) error {
return errors.Trace(err)
}
- dbIDs, partIDs, tblInfos, err := getPlacementPolicyDependedObjectsIDs(t, policy)
+ _, partIDs, tblInfos, err := getPlacementPolicyDependedObjectsIDs(t, policy)
if err != nil {
return errors.Trace(err)
}
- if len(dbIDs)+len(tblInfos)+len(partIDs) != 0 {
- // build bundle from new placement policy.
- bundle, err := placement.NewBundleFromOptions(policy.PlacementSettings)
- if err != nil {
- return errors.Trace(err)
- }
- // Do the http request only when the rules is existed.
- bundles := make([]*placement.Bundle, 0, len(tblInfos)+len(partIDs))
- // Reset bundle for tables (including the default rule for partition).
- for _, tbl := range tblInfos {
- cp := bundle.Clone()
- ids := []int64{tbl.ID}
- if tbl.Partition != nil {
- for _, pDef := range tbl.Partition.Definitions {
- ids = append(ids, pDef.ID)
- }
+ // build bundle from new placement policy.
+ bundle, err := placement.NewBundleFromOptions(policy.PlacementSettings)
+ if err != nil {
+ return errors.Trace(err)
+ }
+ // Do the http request only when the rules is existed.
+ bundles := make([]*placement.Bundle, 0, len(tblInfos)+len(partIDs)+2)
+ // Reset bundle for tables (including the default rule for partition).
+ for _, tbl := range tblInfos {
+ cp := bundle.Clone()
+ ids := []int64{tbl.ID}
+ if tbl.Partition != nil {
+ for _, pDef := range tbl.Partition.Definitions {
+ ids = append(ids, pDef.ID)
}
- bundles = append(bundles, cp.Reset(placement.RuleIndexTable, ids))
}
- // Reset bundle for partitions.
- for _, id := range partIDs {
+ bundles = append(bundles, cp.Reset(placement.RuleIndexTable, ids))
+ }
+ // Reset bundle for partitions.
+ for _, id := range partIDs {
+ cp := bundle.Clone()
+ bundles = append(bundles, cp.Reset(placement.RuleIndexPartition, []int64{id}))
+ }
+
+ resetRangeFn := func(ctx context.Context, rangeName string) error {
+ rangeBundleID := placement.TiDBBundleRangePrefixForGlobal
+ if rangeName == placement.KeyRangeMeta {
+ rangeBundleID = placement.TiDBBundleRangePrefixForMeta
+ }
+ policyName, err := GetRangePlacementPolicyName(ctx, rangeBundleID)
+ if err != nil {
+ return err
+ }
+ if policyName == policy.Name.L {
cp := bundle.Clone()
- bundles = append(bundles, cp.Reset(placement.RuleIndexPartition, []int64{id}))
+ bundles = append(bundles, cp.RebuildForRange(rangeName, policyName))
}
+ return nil
+ }
+ // Reset range "global".
+ err = resetRangeFn(context.TODO(), placement.KeyRangeGlobal)
+ if err != nil {
+ return err
+ }
+ // Reset range "meta".
+ err = resetRangeFn(context.TODO(), placement.KeyRangeMeta)
+ if err != nil {
+ return err
+ }
+
+ if len(bundles) > 0 {
err = infosync.PutRuleBundlesWithDefaultRetry(context.TODO(), bundles)
if err != nil {
return errors.Wrapf(err, "failed to notify PD the placement rules")
}
}
-
return nil
}
@@ -320,7 +346,7 @@ func checkPlacementPolicyNotInUse(d *ddlCtx, t *meta.Meta, policy *model.PolicyI
return err
}
is := d.infoCache.GetLatest()
- if is.SchemaMetaVersion() == currVer {
+ if is != nil && is.SchemaMetaVersion() == currVer {
err = CheckPlacementPolicyNotInUseFromInfoSchema(is, policy)
} else {
err = CheckPlacementPolicyNotInUseFromMeta(t, policy)
@@ -350,18 +376,13 @@ func CheckPlacementPolicyNotInUseFromInfoSchema(is infoschema.InfoSchema, policy
// checkPlacementPolicyNotInUseFromRange checks whether the placement policy is used by the special range.
func checkPlacementPolicyNotInUseFromRange(policy *model.PolicyInfo) error {
- checkFn := func(rangeName string) error {
- bundle, err := infosync.GetRuleBundle(context.TODO(), rangeName)
+ checkFn := func(rangeBundleID string) error {
+ policyName, err := GetRangePlacementPolicyName(context.TODO(), rangeBundleID)
if err != nil {
return err
}
- if bundle == nil {
- return nil
- }
- for _, rule := range bundle.Rules {
- if strings.Contains(rule.ID, policy.Name.L) {
- return dbterror.ErrPlacementPolicyInUse.GenWithStackByArgs(policy.Name)
- }
+ if policyName == policy.Name.L {
+ return dbterror.ErrPlacementPolicyInUse.GenWithStackByArgs(policy.Name)
}
return nil
}
@@ -447,3 +468,21 @@ func checkPlacementPolicyNotUsedByTable(tblInfo *model.TableInfo, policy *model.
return nil
}
+
+// GetRangePlacementPolicyName get the placement policy name used by range.
+// rangeBundleID is limited to TiDBBundleRangePrefixForGlobal and TiDBBundleRangePrefixForMeta.
+func GetRangePlacementPolicyName(ctx context.Context, rangeBundleID string) (string, error) {
+ bundle, err := infosync.GetRuleBundle(ctx, rangeBundleID)
+ if err != nil {
+ return "", err
+ }
+ if bundle == nil || len(bundle.Rules) == 0 {
+ return "", nil
+ }
+ rule := bundle.Rules[0]
+ pos := strings.LastIndex(rule.ID, "_rule_")
+ if pos > 0 {
+ return rule.ID[:pos], nil
+ }
+ return "", nil
+}
diff --git a/pkg/ddl/placement_policy_test.go b/pkg/ddl/placement_policy_test.go
index c3a95cce9bc1f..f6c4f6e9078d2 100644
--- a/pkg/ddl/placement_policy_test.go
+++ b/pkg/ddl/placement_policy_test.go
@@ -851,10 +851,37 @@ func TestAlterRangePlacementPolicy(t *testing.T) {
bundle, err := infosync.GetRuleBundle(context.TODO(), placement.TiDBBundleRangePrefixForGlobal)
require.NoError(t, err)
require.Equal(t, 1, len(bundle.Rules))
+ require.Equal(t, 0, len(bundle.Rules[0].LocationLabels))
tk.MustExec("alter range meta placement policy fiveReplicas")
+ tk.MustQuery(`show placement;`).Sort().Check(testkit.Rows(
+ "POLICY fiveReplicas FOLLOWERS=4 NULL",
+ "RANGE TiDB_GLOBAL FOLLOWERS=4 PENDING",
+ "RANGE TiDB_META FOLLOWERS=4 PENDING"))
bundle, err = infosync.GetRuleBundle(context.TODO(), placement.TiDBBundleRangePrefixForMeta)
require.NoError(t, err)
require.Equal(t, 1, len(bundle.Rules))
+ require.Equal(t, 0, len(bundle.Rules[0].LocationLabels))
+
+ // Test Issue #51712
+ tk.MustExec("alter placement policy fiveReplicas followers=4 SURVIVAL_PREFERENCES=\"[region]\"")
+ tk.MustQuery(`show placement;`).Sort().Check(testkit.Rows(
+ "POLICY fiveReplicas FOLLOWERS=4 SURVIVAL_PREFERENCES=\"[region]\" NULL",
+ "RANGE TiDB_GLOBAL FOLLOWERS=4 SURVIVAL_PREFERENCES=\"[region]\" PENDING",
+ "RANGE TiDB_META FOLLOWERS=4 SURVIVAL_PREFERENCES=\"[region]\" PENDING"))
+ bundle, err = infosync.GetRuleBundle(context.TODO(), placement.TiDBBundleRangePrefixForGlobal)
+ require.NoError(t, err)
+ require.Equal(t, 1, len(bundle.Rules))
+ require.Equal(t, 1, len(bundle.Rules[0].LocationLabels))
+ require.Equal(t, "region", bundle.Rules[0].LocationLabels[0])
+ bundle, err = infosync.GetRuleBundle(context.TODO(), placement.TiDBBundleRangePrefixForMeta)
+ require.NoError(t, err)
+ require.Equal(t, 1, len(bundle.Rules))
+ require.Equal(t, 1, len(bundle.Rules[0].LocationLabels))
+ require.Equal(t, "region", bundle.Rules[0].LocationLabels[0])
+ // Test Issue #52257
+ tk.MustExec("create placement policy fiveRepl followers=4 SURVIVAL_PREFERENCES=\"[region]\"")
+ tk.MustExec("drop placement policy fiveRepl")
+
err = tk.ExecToErr("drop placement policy fiveReplicas")
require.EqualError(t, err, "[ddl:8241]Placement policy 'fiveReplicas' is still in use")
tk.MustExec("alter range global placement policy default")
diff --git a/pkg/ddl/reorg.go b/pkg/ddl/reorg.go
index 51e7b3893e3b0..6fb878e2ccf2a 100644
--- a/pkg/ddl/reorg.go
+++ b/pkg/ddl/reorg.go
@@ -29,6 +29,10 @@ import (
sess "github.com/pingcap/tidb/pkg/ddl/internal/session"
"github.com/pingcap/tidb/pkg/ddl/logutil"
"github.com/pingcap/tidb/pkg/distsql"
+ distsqlctx "github.com/pingcap/tidb/pkg/distsql/context"
+ "github.com/pingcap/tidb/pkg/errctx"
+ exprctx "github.com/pingcap/tidb/pkg/expression/context"
+ "github.com/pingcap/tidb/pkg/expression/contextstatic"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/meta"
"github.com/pingcap/tidb/pkg/metrics"
@@ -47,6 +51,7 @@ import (
"github.com/pingcap/tidb/pkg/util/dbterror"
"github.com/pingcap/tidb/pkg/util/mock"
"github.com/pingcap/tidb/pkg/util/ranger"
+ "github.com/pingcap/tidb/pkg/util/timeutil"
"github.com/pingcap/tipb/go-tipb"
atomicutil "go.uber.org/atomic"
"go.uber.org/zap"
@@ -73,6 +78,46 @@ type reorgCtx struct {
references atomicutil.Int32
}
+func newReorgExprCtx() exprctx.ExprContext {
+ evalCtx := contextstatic.NewStaticEvalContext(
+ contextstatic.WithSQLMode(mysql.ModeNone),
+ contextstatic.WithTypeFlags(types.DefaultStmtFlags),
+ contextstatic.WithErrLevelMap(stmtctx.DefaultStmtErrLevels),
+ )
+
+ return contextstatic.NewStaticExprContext(
+ contextstatic.WithEvalCtx(evalCtx),
+ contextstatic.WithUseCache(false),
+ )
+}
+
+func reorgTypeFlagsWithSQLMode(mode mysql.SQLMode) types.Flags {
+ return types.StrictFlags.
+ WithTruncateAsWarning(!mode.HasStrictMode()).
+ WithIgnoreInvalidDateErr(mode.HasAllowInvalidDatesMode()).
+ WithIgnoreZeroInDate(!mode.HasStrictMode() || mode.HasAllowInvalidDatesMode()).
+ WithCastTimeToYearThroughConcat(true)
+}
+
+func reorgErrLevelsWithSQLMode(mode mysql.SQLMode) errctx.LevelMap {
+ return errctx.LevelMap{
+ errctx.ErrGroupTruncate: errctx.ResolveErrLevel(false, !mode.HasStrictMode()),
+ errctx.ErrGroupBadNull: errctx.ResolveErrLevel(false, !mode.HasStrictMode()),
+ errctx.ErrGroupDividedByZero: errctx.ResolveErrLevel(
+ !mode.HasErrorForDivisionByZeroMode(),
+ !mode.HasStrictMode(),
+ ),
+ }
+}
+
+func reorgTimeZoneWithTzLoc(tzLoc *model.TimeZoneLocation) (*time.Location, error) {
+ if tzLoc == nil {
+ // It is set to SystemLocation to be compatible with nil LocationInfo.
+ return timeutil.SystemLocation(), nil
+ }
+ return tzLoc.GetLocation()
+}
+
func newReorgSessCtx(store kv.Storage) sessionctx.Context {
c := mock.NewContext()
c.Store = store
@@ -244,11 +289,6 @@ func (w *worker) runReorgJob(reorgInfo *reorgInfo, tblInfo *model.TableInfo,
if err != nil {
return errors.Trace(err)
}
- case <-w.ctx.Done():
- logutil.DDLLogger().Info("run reorg job quit")
- d.removeReorgCtx(job.ID)
- // We return dbterror.ErrWaitReorgTimeout here too, so that outer loop will break.
- return dbterror.ErrWaitReorgTimeout
case <-time.After(waitTimeout):
rowCount := rc.getRowCount()
job.SetRowCount(rowCount)
@@ -289,7 +329,15 @@ func overwriteReorgInfoFromGlobalCheckpoint(w *worker, sess *sess.Session, job *
if ok {
// We create the checkpoint manager here because we need to wait for the reorg meta to be initialized.
if bc.GetCheckpointManager() == nil {
- mgr, err := ingest.NewCheckpointManager(w.ctx, bc, w.sessPool, job.ID, extractElemIDs(reorgInfo))
+ mgr, err := ingest.NewCheckpointManager(
+ w.ctx,
+ bc,
+ w.sessPool,
+ job.ID,
+ extractElemIDs(reorgInfo),
+ bc.GetLocalBackend().LocalStoreDir,
+ w.store.(kv.StorageWithPD).GetPDClient(),
+ )
if err != nil {
logutil.DDLIngestLogger().Warn("create checkpoint manager failed", zap.Error(err))
}
@@ -487,7 +535,7 @@ func constructLimitPB(count uint64) *tipb.Executor {
return &tipb.Executor{Tp: tipb.ExecType_TypeLimit, Limit: limitExec}
}
-func buildDescTableScanDAG(ctx sessionctx.Context, tbl table.PhysicalTable, handleCols []*model.ColumnInfo, limit uint64) (*tipb.DAGRequest, error) {
+func buildDescTableScanDAG(distSQLCtx *distsqlctx.DistSQLContext, tbl table.PhysicalTable, handleCols []*model.ColumnInfo, limit uint64) (*tipb.DAGRequest, error) {
dagReq := &tipb.DAGRequest{}
_, timeZoneOffset := time.Now().In(time.UTC).Zone()
dagReq.TimeZoneOffset = int64(timeZoneOffset)
@@ -499,7 +547,7 @@ func buildDescTableScanDAG(ctx sessionctx.Context, tbl table.PhysicalTable, hand
tblScanExec := constructDescTableScanPB(tbl.GetPhysicalID(), tbl.Meta(), handleCols)
dagReq.Executors = append(dagReq.Executors, tblScanExec)
dagReq.Executors = append(dagReq.Executors, constructLimitPB(limit))
- distsql.SetEncodeType(ctx.GetDistSQLCtx(), dagReq)
+ distsql.SetEncodeType(distSQLCtx, dagReq)
return dagReq, nil
}
@@ -514,8 +562,8 @@ func getColumnsTypes(columns []*model.ColumnInfo) []*types.FieldType {
// buildDescTableScan builds a desc table scan upon tblInfo.
func (dc *ddlCtx) buildDescTableScan(ctx *JobContext, startTS uint64, tbl table.PhysicalTable,
handleCols []*model.ColumnInfo, limit uint64) (distsql.SelectResult, error) {
- sctx := newReorgSessCtx(dc.store)
- dagPB, err := buildDescTableScanDAG(sctx, tbl, handleCols, limit)
+ distSQLCtx := newDefaultReorgDistSQLCtx(dc.store.GetClient())
+ dagPB, err := buildDescTableScanDAG(distSQLCtx, tbl, handleCols, limit)
if err != nil {
return nil, errors.Trace(err)
}
@@ -527,7 +575,7 @@ func (dc *ddlCtx) buildDescTableScan(ctx *JobContext, startTS uint64, tbl table.
} else {
ranges = ranger.FullIntRange(false)
}
- builder = b.SetHandleRanges(sctx.GetDistSQLCtx(), tbl.GetPhysicalID(), tbl.Meta().IsCommonHandle, ranges)
+ builder = b.SetHandleRanges(distSQLCtx, tbl.GetPhysicalID(), tbl.Meta().IsCommonHandle, ranges)
builder.SetDAGRequest(dagPB).
SetStartTS(startTS).
SetKeepOrder(true).
@@ -546,7 +594,7 @@ func (dc *ddlCtx) buildDescTableScan(ctx *JobContext, startTS uint64, tbl table.
return nil, errors.Trace(err)
}
- result, err := distsql.Select(ctx.ddlJobCtx, sctx.GetDistSQLCtx(), kvReq, getColumnsTypes(handleCols))
+ result, err := distsql.Select(ctx.ddlJobCtx, distSQLCtx, kvReq, getColumnsTypes(handleCols))
if err != nil {
return nil, errors.Trace(err)
}
@@ -593,16 +641,15 @@ func (dc *ddlCtx) GetTableMaxHandle(ctx *JobContext, startTS uint64, tbl table.P
// empty table
return nil, true, nil
}
- sessCtx := newReorgSessCtx(dc.store)
row := chk.GetRow(0)
if tblInfo.IsCommonHandle {
- maxHandle, err = buildCommonHandleFromChunkRow(sessCtx.GetSessionVars().StmtCtx, tblInfo, pkIdx, handleCols, row)
+ maxHandle, err = buildCommonHandleFromChunkRow(time.UTC, tblInfo, pkIdx, handleCols, row)
return maxHandle, false, err
}
return kv.IntHandle(row.GetInt64(0)), false, nil
}
-func buildCommonHandleFromChunkRow(sctx *stmtctx.StatementContext, tblInfo *model.TableInfo, idxInfo *model.IndexInfo,
+func buildCommonHandleFromChunkRow(loc *time.Location, tblInfo *model.TableInfo, idxInfo *model.IndexInfo,
cols []*model.ColumnInfo, row chunk.Row) (kv.Handle, error) {
fieldTypes := make([]*types.FieldType, 0, len(cols))
for _, col := range cols {
@@ -612,8 +659,7 @@ func buildCommonHandleFromChunkRow(sctx *stmtctx.StatementContext, tblInfo *mode
tablecodec.TruncateIndexValues(tblInfo, idxInfo, datumRow)
var handleBytes []byte
- handleBytes, err := codec.EncodeKey(sctx.TimeZone(), nil, datumRow...)
- err = sctx.HandleError(err)
+ handleBytes, err := codec.EncodeKey(loc, nil, datumRow...)
if err != nil {
return nil, err
}
diff --git a/pkg/ddl/resource_group.go b/pkg/ddl/resource_group.go
index caca77ec438e4..39265a46dee6a 100644
--- a/pkg/ddl/resource_group.go
+++ b/pkg/ddl/resource_group.go
@@ -38,7 +38,7 @@ const (
alreadyExists = "already exists"
)
-func onCreateResourceGroup(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
+func onCreateResourceGroup(ctx context.Context, d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) {
groupInfo := &model.ResourceGroupInfo{}
if err := job.DecodeArgs(groupInfo); err != nil {
job.State = model.JobStateCancelled
@@ -63,7 +63,7 @@ func onCreateResourceGroup(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64,
return ver, errors.Trace(err)
}
- ctx, cancel := context.WithTimeout(d.ctx, defaultInfosyncTimeout)
+ ctx, cancel := context.WithTimeout(ctx, defaultInfosyncTimeout)
defer cancel()
err = infosync.AddResourceGroup(ctx, protoGroup)
if err != nil {
diff --git a/pkg/ddl/schema.go b/pkg/ddl/schema.go
index 3be32a14a26de..412169b08e11f 100644
--- a/pkg/ddl/schema.go
+++ b/pkg/ddl/schema.go
@@ -76,7 +76,7 @@ func checkSchemaNotExists(d *ddlCtx, t *meta.Meta, schemaID int64, dbInfo *model
return err
}
is := d.infoCache.GetLatest()
- if is.SchemaMetaVersion() == currVer {
+ if is != nil && is.SchemaMetaVersion() == currVer {
return checkSchemaNotExistsFromInfoSchema(is, schemaID, dbInfo)
}
return checkSchemaNotExistsFromStore(t, schemaID, dbInfo)
diff --git a/pkg/ddl/schema_version.go b/pkg/ddl/schema_version.go
index 760c1d3b3d888..ec169386843b7 100644
--- a/pkg/ddl/schema_version.go
+++ b/pkg/ddl/schema_version.go
@@ -15,14 +15,15 @@
package ddl
import (
+ "context"
"time"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
+ "github.com/pingcap/tidb/pkg/ddl/logutil"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/meta"
"github.com/pingcap/tidb/pkg/parser/model"
- "github.com/pingcap/tidb/pkg/util/logutil"
"github.com/pingcap/tidb/pkg/util/mathutil"
"go.uber.org/zap"
)
@@ -359,7 +360,7 @@ func updateSchemaVersion(d *ddlCtx, t *meta.Meta, job *model.Job, multiInfos ...
return schemaVersion, errors.Trace(err)
}
-func checkAllVersions(d *ddlCtx, job *model.Job, latestSchemaVersion int64, timeStart time.Time) error {
+func checkAllVersions(ctx context.Context, d *ddlCtx, job *model.Job, latestSchemaVersion int64, timeStart time.Time) error {
failpoint.Inject("checkDownBeforeUpdateGlobalVersion", func(val failpoint.Value) {
if val.(bool) {
if mockDDLErrOnce > 0 && mockDDLErrOnce != latestSchemaVersion {
@@ -370,13 +371,13 @@ func checkAllVersions(d *ddlCtx, job *model.Job, latestSchemaVersion int64, time
})
// OwnerCheckAllVersions returns only when all TiDB schemas are synced(exclude the isolated TiDB).
- err := d.schemaSyncer.OwnerCheckAllVersions(d.ctx, job.ID, latestSchemaVersion)
+ err := d.schemaSyncer.OwnerCheckAllVersions(ctx, job.ID, latestSchemaVersion)
if err != nil {
- logutil.Logger(d.ctx).Info("wait latest schema version encounter error", zap.String("category", "ddl"), zap.Int64("ver", latestSchemaVersion),
+ logutil.DDLLogger().Info("wait latest schema version encounter error", zap.Int64("ver", latestSchemaVersion),
zap.Int64("jobID", job.ID), zap.Duration("take time", time.Since(timeStart)), zap.Error(err))
return err
}
- logutil.Logger(d.ctx).Info("wait latest schema version changed(get the metadata lock if tidb_enable_metadata_lock is true)", zap.String("category", "ddl"),
+ logutil.DDLLogger().Info("wait latest schema version changed(get the metadata lock if tidb_enable_metadata_lock is true)",
zap.Int64("ver", latestSchemaVersion),
zap.Duration("take time", time.Since(timeStart)),
zap.String("job", job.String()))
@@ -389,7 +390,7 @@ func checkAllVersions(d *ddlCtx, job *model.Job, latestSchemaVersion int64, time
// but in this case we don't wait enough 2 * lease time to let other servers update the schema.
// So here we get the latest schema version to make sure all servers' schema version update to the latest schema version
// in a cluster, or to wait for 2 * lease time.
-func waitSchemaSynced(d *ddlCtx, job *model.Job, waitTime time.Duration) error {
+func waitSchemaSynced(ctx context.Context, d *ddlCtx, job *model.Job, waitTime time.Duration) error {
if !job.IsRunning() && !job.IsRollingback() && !job.IsDone() && !job.IsRollbackDone() {
return nil
}
@@ -399,7 +400,7 @@ func waitSchemaSynced(d *ddlCtx, job *model.Job, waitTime time.Duration) error {
m := meta.NewSnapshotMeta(snapshot)
latestSchemaVersion, err := m.GetSchemaVersionWithNonEmptyDiff()
if err != nil {
- logutil.Logger(d.ctx).Warn("get global version failed", zap.String("category", "ddl"), zap.Int64("jobID", job.ID), zap.Error(err))
+ logutil.DDLLogger().Warn("get global version failed", zap.Int64("jobID", job.ID), zap.Error(err))
return err
}
@@ -412,5 +413,5 @@ func waitSchemaSynced(d *ddlCtx, job *model.Job, waitTime time.Duration) error {
}
})
- return waitSchemaChanged(d, waitTime, latestSchemaVersion, job)
+ return waitSchemaChanged(ctx, d, waitTime, latestSchemaVersion, job)
}
diff --git a/pkg/ddl/schematracker/dm_tracker.go b/pkg/ddl/schematracker/dm_tracker.go
index 4d80800060e83..e4079b29fb5d5 100644
--- a/pkg/ddl/schematracker/dm_tracker.go
+++ b/pkg/ddl/schematracker/dm_tracker.go
@@ -725,7 +725,7 @@ func (d SchemaTracker) handleModifyColumn(
tblInfo.AutoRandomBits = updatedAutoRandomBits
oldCol := table.FindCol(t.Cols(), originalColName.L).ColumnInfo
- originDefVal, err := ddl.GetOriginDefaultValueForModifyColumn(sctx, newColInfo, oldCol)
+ originDefVal, err := ddl.GetOriginDefaultValueForModifyColumn(sctx.GetExprCtx(), newColInfo, oldCol)
if err != nil {
return errors.Trace(err)
}
diff --git a/pkg/ddl/sequence.go b/pkg/ddl/sequence.go
index efd0b698fada0..341cdd94e1552 100644
--- a/pkg/ddl/sequence.go
+++ b/pkg/ddl/sequence.go
@@ -45,41 +45,40 @@ func onCreateSequence(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ err
return ver, errors.Trace(err)
}
+ err = createSequenceWithCheck(t, job, tbInfo)
+ if err != nil {
+ return ver, errors.Trace(err)
+ }
+
ver, err = updateSchemaVersion(d, t, job)
if err != nil {
return ver, errors.Trace(err)
}
+ job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tbInfo)
+ return ver, nil
+}
+func createSequenceWithCheck(t *meta.Meta, job *model.Job, tbInfo *model.TableInfo) error {
switch tbInfo.State {
- case model.StateNone:
+ case model.StateNone, model.StatePublic:
// none -> public
tbInfo.State = model.StatePublic
tbInfo.UpdateTS = t.StartTS
- err = createSequenceWithCheck(t, job, schemaID, tbInfo)
+ err := checkTableInfoValid(tbInfo)
if err != nil {
- return ver, errors.Trace(err)
+ job.State = model.JobStateCancelled
+ return errors.Trace(err)
+ }
+ var sequenceBase int64
+ if tbInfo.Sequence.Increment >= 0 {
+ sequenceBase = tbInfo.Sequence.Start - 1
+ } else {
+ sequenceBase = tbInfo.Sequence.Start + 1
}
- // Finish this job.
- job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tbInfo)
- return ver, nil
+ return t.CreateSequenceAndSetSeqValue(job.SchemaID, job.SchemaName, tbInfo, sequenceBase)
default:
- return ver, dbterror.ErrInvalidDDLState.GenWithStackByArgs("sequence", tbInfo.State)
- }
-}
-
-func createSequenceWithCheck(t *meta.Meta, job *model.Job, schemaID int64, tbInfo *model.TableInfo) error {
- err := checkTableInfoValid(tbInfo)
- if err != nil {
- job.State = model.JobStateCancelled
- return errors.Trace(err)
- }
- var sequenceBase int64
- if tbInfo.Sequence.Increment >= 0 {
- sequenceBase = tbInfo.Sequence.Start - 1
- } else {
- sequenceBase = tbInfo.Sequence.Start + 1
+ return dbterror.ErrInvalidDDLState.GenWithStackByArgs("sequence", tbInfo.State)
}
- return t.CreateSequenceAndSetSeqValue(schemaID, job.SchemaName, tbInfo, sequenceBase)
}
func handleSequenceOptions(seqOptions []*ast.SequenceOption, sequenceInfo *model.SequenceInfo) {
diff --git a/pkg/ddl/syncer/BUILD.bazel b/pkg/ddl/syncer/BUILD.bazel
index a18b5f63e9a4b..2c7aa54314196 100644
--- a/pkg/ddl/syncer/BUILD.bazel
+++ b/pkg/ddl/syncer/BUILD.bazel
@@ -15,6 +15,7 @@ go_library(
"//pkg/metrics",
"//pkg/sessionctx/variable",
"//pkg/util",
+ "//pkg/util/disttask",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@io_etcd_go_etcd_api_v3//mvccpb",
@@ -30,20 +31,23 @@ go_test(
timeout = "short",
srcs = [
"state_syncer_test.go",
+ "syncer_nokit_test.go",
"syncer_test.go",
],
+ embed = [":syncer"],
flaky = True,
- shard_count = 3,
+ shard_count = 6,
deps = [
- ":syncer",
"//pkg/ddl",
"//pkg/ddl/util",
+ "//pkg/domain/infosync",
"//pkg/infoschema",
"//pkg/parser/terror",
"//pkg/sessionctx/variable",
"//pkg/store/mockstore",
"//pkg/util",
"@com_github_pingcap_errors//:errors",
+ "@com_github_pingcap_failpoint//:failpoint",
"@com_github_stretchr_testify//require",
"@io_etcd_go_etcd_api_v3//mvccpb",
"@io_etcd_go_etcd_client_v3//:client",
diff --git a/pkg/ddl/syncer/syncer.go b/pkg/ddl/syncer/syncer.go
index 8401fdd9666f9..818d4747eed06 100644
--- a/pkg/ddl/syncer/syncer.go
+++ b/pkg/ddl/syncer/syncer.go
@@ -33,6 +33,8 @@ import (
"github.com/pingcap/tidb/pkg/metrics"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
tidbutil "github.com/pingcap/tidb/pkg/util"
+ disttaskutil "github.com/pingcap/tidb/pkg/util/disttask"
+ "go.etcd.io/etcd/api/v3/mvccpb"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/concurrency"
"go.uber.org/zap"
@@ -126,16 +128,98 @@ type SchemaSyncer interface {
// the latest schema version. (exclude the isolated TiDB)
// It returns until all servers' versions are equal to the latest version.
OwnerCheckAllVersions(ctx context.Context, jobID int64, latestVer int64) error
+ // SyncJobSchemaVerLoop syncs the schema versions on all TiDB nodes for DDL jobs.
+ SyncJobSchemaVerLoop(ctx context.Context)
// Close ends SchemaSyncer.
Close()
}
+// nodeVersions is used to record the schema versions of all TiDB nodes for a DDL job.
+type nodeVersions struct {
+ sync.Mutex
+ nodeVersions map[string]int64
+ // onceMatchFn is used to check if all the servers report the least version.
+ // If all the servers report the least version, i.e. return true, it will be
+ // set to nil.
+ onceMatchFn func(map[string]int64) bool
+}
+
+func newNodeVersions(initialCap int, fn func(map[string]int64) bool) *nodeVersions {
+ return &nodeVersions{
+ nodeVersions: make(map[string]int64, initialCap),
+ onceMatchFn: fn,
+ }
+}
+
+func (v *nodeVersions) add(nodeID string, ver int64) {
+ v.Lock()
+ defer v.Unlock()
+ v.nodeVersions[nodeID] = ver
+ if v.onceMatchFn != nil {
+ if ok := v.onceMatchFn(v.nodeVersions); ok {
+ v.onceMatchFn = nil
+ }
+ }
+}
+
+func (v *nodeVersions) del(nodeID string) {
+ v.Lock()
+ defer v.Unlock()
+ delete(v.nodeVersions, nodeID)
+ // we don't call onceMatchFn here, for only "add" can cause onceMatchFn return
+ // true currently.
+}
+
+func (v *nodeVersions) len() int {
+ v.Lock()
+ defer v.Unlock()
+ return len(v.nodeVersions)
+}
+
+// matchOrSet onceMatchFn must be nil before calling this method.
+func (v *nodeVersions) matchOrSet(fn func(nodeVersions map[string]int64) bool) {
+ v.Lock()
+ defer v.Unlock()
+ if ok := fn(v.nodeVersions); !ok {
+ v.onceMatchFn = fn
+ }
+}
+
+func (v *nodeVersions) clearData() {
+ v.Lock()
+ defer v.Unlock()
+ v.nodeVersions = make(map[string]int64, len(v.nodeVersions))
+}
+
+func (v *nodeVersions) clearMatchFn() {
+ v.Lock()
+ defer v.Unlock()
+ v.onceMatchFn = nil
+}
+
+func (v *nodeVersions) emptyAndNotUsed() bool {
+ v.Lock()
+ defer v.Unlock()
+ return len(v.nodeVersions) == 0 && v.onceMatchFn == nil
+}
+
+// for test
+func (v *nodeVersions) getMatchFn() func(map[string]int64) bool {
+ v.Lock()
+ defer v.Unlock()
+ return v.onceMatchFn
+}
+
type schemaVersionSyncer struct {
selfSchemaVerPath string
etcdCli *clientv3.Client
session unsafe.Pointer
globalVerWatcher watcher
ddlID string
+
+ mu sync.RWMutex
+ jobNodeVersions map[int64]*nodeVersions
+ jobNodeVerPrefix string
}
// NewSchemaSyncer creates a new SchemaSyncer.
@@ -144,6 +228,9 @@ func NewSchemaSyncer(etcdCli *clientv3.Client, id string) SchemaSyncer {
etcdCli: etcdCli,
selfSchemaVerPath: fmt.Sprintf("%s/%s", util.DDLAllSchemaVersions, id),
ddlID: id,
+
+ jobNodeVersions: make(map[int64]*nodeVersions),
+ jobNodeVerPrefix: util.DDLAllSchemaVersionsByJob + "/",
}
}
@@ -275,7 +362,9 @@ func (s *schemaVersionSyncer) removeSelfVersionPath() error {
// OwnerCheckAllVersions implements SchemaSyncer.OwnerCheckAllVersions interface.
func (s *schemaVersionSyncer) OwnerCheckAllVersions(ctx context.Context, jobID int64, latestVer int64) error {
startTime := time.Now()
- time.Sleep(CheckVersFirstWaitTime)
+ if !variable.EnableMDL.Load() {
+ time.Sleep(CheckVersFirstWaitTime)
+ }
notMatchVerCnt := 0
intervalCnt := int(time.Second / checkVersInterval)
@@ -296,10 +385,7 @@ func (s *schemaVersionSyncer) OwnerCheckAllVersions(ctx context.Context, jobID i
return errors.Trace(err)
}
- // Prepare path and updatedMap.
- path := util.DDLAllSchemaVersions
if variable.EnableMDL.Load() {
- path = fmt.Sprintf("%s/%d/", util.DDLAllSchemaVersionsByJob, jobID)
serverInfos, err := infosync.GetAllServerInfo(ctx)
if err != nil {
return err
@@ -307,9 +393,10 @@ func (s *schemaVersionSyncer) OwnerCheckAllVersions(ctx context.Context, jobID i
updatedMap = make(map[string]string)
instance2id := make(map[string]string)
- // Set updatedMap according to the serverInfos, and remove some invalid serverInfos.
for _, info := range serverInfos {
- instance := fmt.Sprintf("%s:%d", info.IP, info.Port)
+ instance := disttaskutil.GenerateExecID(info)
+ // if some node shutdown abnormally and start, we might see some
+ // instance with different id, we should use the latest one.
if id, ok := instance2id[instance]; ok {
if info.StartTimestamp > serverInfos[id].StartTimestamp {
// Replace it.
@@ -324,39 +411,52 @@ func (s *schemaVersionSyncer) OwnerCheckAllVersions(ctx context.Context, jobID i
}
}
- // Get all the schema versions from ETCD.
- resp, err := s.etcdCli.Get(ctx, path, clientv3.WithPrefix())
- if err != nil {
- logutil.DDLLogger().Info("syncer check all versions failed, continue checking.", zap.Error(err))
- continue
- }
-
// Check all schema versions.
- succ := true
if variable.EnableMDL.Load() {
- for _, kv := range resp.Kvs {
- key := string(kv.Key)
- tidbIDInResp := key[strings.LastIndex(key, "/")+1:]
- // We need to check if the tidb ID is in the updatedMap, in case that deleting etcd is failed, and tidb server is down.
- nodeAlive := updatedMap[tidbIDInResp] != ""
- succ = isUpdatedLatestVersion(string(kv.Key), string(kv.Value), latestVer, notMatchVerCnt, intervalCnt, nodeAlive)
- if !succ {
- break
+ notifyCh := make(chan struct{})
+ var unmatchedNodeID atomic.Pointer[string]
+ matchFn := func(nodeVersions map[string]int64) bool {
+ if len(nodeVersions) < len(updatedMap) {
+ return false
}
- delete(updatedMap, tidbIDInResp)
- }
- if len(updatedMap) > 0 {
- succ = false
- if notMatchVerCnt%intervalCnt == 0 {
- for _, info := range updatedMap {
- logutil.DDLLogger().Info("syncer check all versions, someone is not synced",
- zap.String("info", info),
- zap.Int64("ddl job id", jobID),
- zap.Int64("ver", latestVer))
+ for tidbID := range updatedMap {
+ if nodeVer, ok := nodeVersions[tidbID]; !ok || nodeVer < latestVer {
+ id := tidbID
+ unmatchedNodeID.Store(&id)
+ return false
}
}
+ close(notifyCh)
+ return true
+ }
+ item := s.jobSchemaVerMatchOrSet(jobID, matchFn)
+ select {
+ case <-notifyCh:
+ return nil
+ case <-ctx.Done():
+ item.clearMatchFn()
+ return errors.Trace(ctx.Err())
+ case <-time.After(time.Second):
+ item.clearMatchFn()
+ if id := unmatchedNodeID.Load(); id != nil {
+ logutil.DDLLogger().Info("syncer check all versions, someone is not synced",
+ zap.String("info", *id),
+ zap.Int64("ddl job id", jobID),
+ zap.Int64("ver", latestVer))
+ } else {
+ logutil.DDLLogger().Info("syncer check all versions, all nodes are not synced",
+ zap.Int64("ddl job id", jobID),
+ zap.Int64("ver", latestVer))
+ }
}
} else {
+ // Get all the schema versions from ETCD.
+ resp, err := s.etcdCli.Get(ctx, util.DDLAllSchemaVersions, clientv3.WithPrefix())
+ if err != nil {
+ logutil.DDLLogger().Info("syncer check all versions failed, continue checking.", zap.Error(err))
+ continue
+ }
+ succ := true
for _, kv := range resp.Kvs {
if _, ok := updatedMap[string(kv.Key)]; ok {
continue
@@ -368,14 +468,140 @@ func (s *schemaVersionSyncer) OwnerCheckAllVersions(ctx context.Context, jobID i
}
updatedMap[string(kv.Key)] = ""
}
+
+ if succ {
+ return nil
+ }
+ time.Sleep(checkVersInterval)
+ notMatchVerCnt++
+ }
+ }
+}
+
+// SyncJobSchemaVerLoop implements SchemaSyncer.SyncJobSchemaVerLoop interface.
+func (s *schemaVersionSyncer) SyncJobSchemaVerLoop(ctx context.Context) {
+ for {
+ s.syncJobSchemaVer(ctx)
+ logutil.DDLLogger().Info("schema version sync loop interrupted, retrying...")
+ select {
+ case <-ctx.Done():
+ return
+ case <-time.After(time.Second):
+ }
+ }
+}
+
+func (s *schemaVersionSyncer) syncJobSchemaVer(ctx context.Context) {
+ resp, err := s.etcdCli.Get(ctx, s.jobNodeVerPrefix, clientv3.WithPrefix())
+ if err != nil {
+ logutil.DDLLogger().Info("get all job versions failed", zap.Error(err))
+ return
+ }
+ s.mu.Lock()
+ for jobID, item := range s.jobNodeVersions {
+ item.clearData()
+ // we might miss some DELETE events during retry, some items might be emptyAndNotUsed, remove them.
+ if item.emptyAndNotUsed() {
+ delete(s.jobNodeVersions, jobID)
}
+ }
+ s.mu.Unlock()
+ for _, oneKV := range resp.Kvs {
+ s.handleJobSchemaVerKV(oneKV, mvccpb.PUT)
+ }
- if succ {
- return nil
+ startRev := resp.Header.Revision + 1
+ watchCtx, watchCtxCancel := context.WithCancel(ctx)
+ defer watchCtxCancel()
+ watchCtx = clientv3.WithRequireLeader(watchCtx)
+ watchCh := s.etcdCli.Watch(watchCtx, s.jobNodeVerPrefix, clientv3.WithPrefix(), clientv3.WithRev(startRev))
+ for {
+ var (
+ wresp clientv3.WatchResponse
+ ok bool
+ )
+ select {
+ case <-watchCtx.Done():
+ return
+ case wresp, ok = <-watchCh:
+ if !ok {
+ // ctx must be cancelled, else we should have received a response
+ // with err and caught by below err check.
+ return
+ }
+ }
+ failpoint.Inject("mockCompaction", func() {
+ wresp.CompactRevision = 123
+ })
+ if err := wresp.Err(); err != nil {
+ logutil.DDLLogger().Warn("watch job version failed", zap.Error(err))
+ return
+ }
+ for _, ev := range wresp.Events {
+ s.handleJobSchemaVerKV(ev.Kv, ev.Type)
+ }
+ }
+}
+
+func (s *schemaVersionSyncer) handleJobSchemaVerKV(kv *mvccpb.KeyValue, tp mvccpb.Event_EventType) {
+ jobID, tidbID, schemaVer, valid := decodeJobVersionEvent(kv, tp, s.jobNodeVerPrefix)
+ if !valid {
+ logutil.DDLLogger().Error("invalid job version kv", zap.Stringer("kv", kv), zap.Stringer("type", tp))
+ return
+ }
+ if tp == mvccpb.PUT {
+ s.mu.Lock()
+ item, exists := s.jobNodeVersions[jobID]
+ if !exists {
+ item = newNodeVersions(1, nil)
+ s.jobNodeVersions[jobID] = item
+ }
+ s.mu.Unlock()
+ item.add(tidbID, schemaVer)
+ } else { // DELETE
+ s.mu.Lock()
+ if item, exists := s.jobNodeVersions[jobID]; exists {
+ item.del(tidbID)
+ if item.len() == 0 {
+ delete(s.jobNodeVersions, jobID)
+ }
+ }
+ s.mu.Unlock()
+ }
+}
+
+func (s *schemaVersionSyncer) jobSchemaVerMatchOrSet(jobID int64, matchFn func(map[string]int64) bool) *nodeVersions {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ item, exists := s.jobNodeVersions[jobID]
+ if exists {
+ item.matchOrSet(matchFn)
+ } else {
+ item = newNodeVersions(1, matchFn)
+ s.jobNodeVersions[jobID] = item
+ }
+ return item
+}
+
+func decodeJobVersionEvent(kv *mvccpb.KeyValue, tp mvccpb.Event_EventType, prefix string) (jobID int64, tidbID string, schemaVer int64, valid bool) {
+ left := strings.TrimPrefix(string(kv.Key), prefix)
+ parts := strings.Split(left, "/")
+ if len(parts) != 2 {
+ return 0, "", 0, false
+ }
+ jobID, err := strconv.ParseInt(parts[0], 10, 64)
+ if err != nil {
+ return 0, "", 0, false
+ }
+ // there is no Value in DELETE event, so we need to check it.
+ if tp == mvccpb.PUT {
+ schemaVer, err = strconv.ParseInt(string(kv.Value), 10, 64)
+ if err != nil {
+ return 0, "", 0, false
}
- time.Sleep(checkVersInterval)
- notMatchVerCnt++
}
+ return jobID, parts[1], schemaVer, true
}
func isUpdatedLatestVersion(key, val string, latestVer int64, notMatchVerCnt, intervalCnt int, nodeAlive bool) bool {
diff --git a/pkg/ddl/syncer/syncer_nokit_test.go b/pkg/ddl/syncer/syncer_nokit_test.go
new file mode 100644
index 0000000000000..15d0cbf0eac44
--- /dev/null
+++ b/pkg/ddl/syncer/syncer_nokit_test.go
@@ -0,0 +1,191 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package syncer
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "sync"
+ "testing"
+
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/tidb/pkg/ddl/util"
+ "github.com/pingcap/tidb/pkg/domain/infosync"
+ "github.com/pingcap/tidb/pkg/sessionctx/variable"
+ "github.com/stretchr/testify/require"
+ "go.etcd.io/etcd/api/v3/mvccpb"
+ "go.etcd.io/etcd/tests/v3/integration"
+)
+
+func TestNodeVersions(t *testing.T) {
+ nv := newNodeVersions(1, nil)
+ require.True(t, nv.emptyAndNotUsed())
+ nv.add("a", 10)
+ nv.add("b", 20)
+ require.False(t, nv.emptyAndNotUsed())
+ require.EqualValues(t, 2, nv.len())
+ var waterMark int64 = 10
+ fn := func(nodeVersions map[string]int64) bool {
+ for _, v := range nodeVersions {
+ if v < waterMark {
+ return false
+ }
+ }
+ return true
+ }
+ nv.matchOrSet(fn)
+ require.Nil(t, nv.onceMatchFn)
+ waterMark = 20
+ nv.matchOrSet(fn)
+ require.NotNil(t, nv.onceMatchFn)
+ // matched and cleared
+ nv.add("a", 20)
+ require.Nil(t, nv.onceMatchFn)
+ nv.del("a")
+ require.EqualValues(t, 1, nv.len())
+ nv.del("b")
+ require.True(t, nv.emptyAndNotUsed())
+ nv.matchOrSet(func(map[string]int64) bool { return false })
+ require.False(t, nv.emptyAndNotUsed())
+}
+
+func TestDecodeJobVersionEvent(t *testing.T) {
+ prefix := util.DDLAllSchemaVersionsByJob + "/"
+ _, _, _, valid := decodeJobVersionEvent(&mvccpb.KeyValue{Key: []byte(prefix + "1")}, mvccpb.PUT, prefix)
+ require.False(t, valid)
+ _, _, _, valid = decodeJobVersionEvent(&mvccpb.KeyValue{Key: []byte(prefix + "a/aa")}, mvccpb.PUT, prefix)
+ require.False(t, valid)
+ _, _, _, valid = decodeJobVersionEvent(&mvccpb.KeyValue{
+ Key: []byte(prefix + "1/aa"), Value: []byte("aa")}, mvccpb.PUT, prefix)
+ require.False(t, valid)
+ jobID, tidbID, schemaVer, valid := decodeJobVersionEvent(&mvccpb.KeyValue{
+ Key: []byte(prefix + "1/aa"), Value: []byte("123")}, mvccpb.PUT, prefix)
+ require.True(t, valid)
+ require.EqualValues(t, 1, jobID)
+ require.EqualValues(t, "aa", tidbID)
+ require.EqualValues(t, 123, schemaVer)
+ // value is not used on delete
+ jobID, tidbID, schemaVer, valid = decodeJobVersionEvent(&mvccpb.KeyValue{
+ Key: []byte(prefix + "1/aa"), Value: []byte("aaaa")}, mvccpb.DELETE, prefix)
+ require.True(t, valid)
+ require.EqualValues(t, 1, jobID)
+ require.EqualValues(t, "aa", tidbID)
+ require.EqualValues(t, 0, schemaVer)
+}
+
+func TestSyncJobSchemaVerLoop(t *testing.T) {
+ integration.BeforeTestExternal(t)
+ mockCluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
+ defer mockCluster.Terminate(t)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ etcdCli := mockCluster.RandClient()
+ _, err := etcdCli.Put(ctx, util.DDLAllSchemaVersionsByJob+"/1/aa", "123")
+ require.NoError(t, err)
+ s := NewSchemaSyncer(etcdCli, "1111").(*schemaVersionSyncer)
+ var wg sync.WaitGroup
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ s.SyncJobSchemaVerLoop(ctx)
+ }()
+
+ // job 1 is matched
+ notifyCh := make(chan struct{})
+ item := s.jobSchemaVerMatchOrSet(1, func(m map[string]int64) bool {
+ for _, v := range m {
+ if v < 123 {
+ return false
+ }
+ }
+ close(notifyCh)
+ return true
+ })
+ <-notifyCh
+ require.Nil(t, item.getMatchFn())
+ _, err = etcdCli.Delete(ctx, util.DDLAllSchemaVersionsByJob+"/1/aa")
+ require.NoError(t, err)
+
+ // job 2 requires aa and bb
+ notifyCh = make(chan struct{}, 1)
+ item = s.jobSchemaVerMatchOrSet(2, func(m map[string]int64) bool {
+ for _, nodeID := range []string{"aa", "bb"} {
+ if v, ok := m[nodeID]; !ok || v < 123 {
+ return false
+ }
+ }
+ close(notifyCh)
+ return true
+ })
+ require.NotNil(t, item.getMatchFn())
+ require.Len(t, notifyCh, 0)
+ _, err = etcdCli.Put(ctx, util.DDLAllSchemaVersionsByJob+"/2/aa", "123")
+ require.NoError(t, err)
+ _, err = etcdCli.Put(ctx, util.DDLAllSchemaVersionsByJob+"/2/bb", "124")
+ require.NoError(t, err)
+ <-notifyCh
+ require.Nil(t, item.getMatchFn())
+ jobNodeVersionCnt := func() int {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return len(s.jobNodeVersions)
+ }
+ require.EqualValues(t, 1, jobNodeVersionCnt())
+ _, err = etcdCli.Delete(ctx, util.DDLAllSchemaVersionsByJob+"/2/aa")
+ require.NoError(t, err)
+ _, err = etcdCli.Delete(ctx, util.DDLAllSchemaVersionsByJob+"/2/bb")
+ require.NoError(t, err)
+
+ // job 3 is matched after restart from a compaction error
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/syncer/mockCompaction", `1*return(true)`))
+ t.Cleanup(func() {
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/syncer/mockCompaction"))
+ })
+ notifyCh = make(chan struct{}, 1)
+ item = s.jobSchemaVerMatchOrSet(3, func(m map[string]int64) bool {
+ for _, nodeID := range []string{"aa"} {
+ if v, ok := m[nodeID]; !ok || v < 123 {
+ return false
+ }
+ }
+ close(notifyCh)
+ return true
+ })
+ require.NotNil(t, item.getMatchFn())
+ require.Len(t, notifyCh, 0)
+ _, err = etcdCli.Put(ctx, util.DDLAllSchemaVersionsByJob+"/3/aa", "123")
+ require.NoError(t, err)
+ <-notifyCh
+ require.Nil(t, item.getMatchFn())
+ _, err = etcdCli.Delete(ctx, util.DDLAllSchemaVersionsByJob+"/3/aa")
+ require.NoError(t, err)
+
+ // job 4 is matched using OwnerCheckAllVersions
+ variable.EnableMDL.Store(true)
+ serverInfos := map[string]*infosync.ServerInfo{"aa": {ID: "aa", IP: "test", Port: 4000}}
+ bytes, err := json.Marshal(serverInfos)
+ require.NoError(t, err)
+ inTerms := fmt.Sprintf("return(`%s`)", string(bytes))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/domain/infosync/mockGetAllServerInfo", inTerms))
+ _, err = etcdCli.Put(ctx, util.DDLAllSchemaVersionsByJob+"/4/aa", "333")
+ require.NoError(t, err)
+ require.NoError(t, s.OwnerCheckAllVersions(ctx, 4, 333))
+ _, err = etcdCli.Delete(ctx, util.DDLAllSchemaVersionsByJob+"/4/aa")
+ require.NoError(t, err)
+
+ cancel()
+ wg.Wait()
+}
diff --git a/pkg/ddl/table.go b/pkg/ddl/table.go
index 63523481509dd..892a3130039cf 100644
--- a/pkg/ddl/table.go
+++ b/pkg/ddl/table.go
@@ -217,6 +217,11 @@ func createTableWithForeignKeys(d *ddlCtx, t *meta.Meta, job *model.Job, tbInfo
return ver, errors.Trace(err)
}
job.FinishTableJob(model.JobStateDone, model.StatePublic, ver, tbInfo)
+ createTableEvent := statsutil.NewCreateTableEvent(
+ job.SchemaID,
+ tbInfo,
+ )
+ asyncNotifyEvent(d, createTableEvent)
return ver, nil
default:
return ver, errors.Trace(dbterror.ErrInvalidDDLJob.GenWithStackByArgs("table", tbInfo.State))
@@ -239,18 +244,26 @@ func onCreateTables(d *ddlCtx, t *meta.Meta, job *model.Job) (int64, error) {
// We don't construct jobs for every table, but only tableInfo
// The following loop creates a stub job for every table
//
- // &*job clones a stub job from the ActionCreateTables job
- stubJob := &*job
+ // it clones a stub job from the ActionCreateTables job
+ stubJob := job.Clone()
stubJob.Args = make([]any, 1)
for i := range args {
stubJob.TableID = args[i].ID
stubJob.Args[0] = args[i]
- tbInfo, err := createTable(d, t, stubJob, fkCheck)
- if err != nil {
- job.State = model.JobStateCancelled
- return ver, errors.Trace(err)
+ if args[i].Sequence != nil {
+ err := createSequenceWithCheck(t, stubJob, args[i])
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+ } else {
+ tbInfo, err := createTable(d, t, stubJob, fkCheck)
+ if err != nil {
+ job.State = model.JobStateCancelled
+ return ver, errors.Trace(err)
+ }
+ args[i] = tbInfo
}
- args[i] = tbInfo
}
ver, err = updateSchemaVersion(d, t, job)
@@ -295,14 +308,19 @@ func onCreateView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
schemaID := job.SchemaID
tbInfo := &model.TableInfo{}
var orReplace bool
- var oldTbInfoID int64
- if err := job.DecodeArgs(tbInfo, &orReplace, &oldTbInfoID); err != nil {
+ var _placeholder int64 // oldTblInfoID
+ if err := job.DecodeArgs(tbInfo, &orReplace, &_placeholder); err != nil {
// Invalid arguments, cancel this job.
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
tbInfo.State = model.StateNone
- err := checkTableNotExists(d, t, schemaID, tbInfo.Name.L)
+
+ oldTableID, err := findTableIDByName(d, t, schemaID, tbInfo.Name.L)
+ if infoschema.ErrTableNotExists.Equal(err) {
+ err = nil
+ }
+ failpoint.InjectCall("onDDLCreateView", job)
if err != nil {
if infoschema.ErrDatabaseNotExists.Equal(err) {
job.State = model.JobStateCancelled
@@ -324,13 +342,13 @@ func onCreateView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error)
// none -> public
tbInfo.State = model.StatePublic
tbInfo.UpdateTS = t.StartTS
- if oldTbInfoID > 0 && orReplace {
- err = t.DropTableOrView(schemaID, job.SchemaName, oldTbInfoID, tbInfo.Name.L)
+ if oldTableID > 0 && orReplace {
+ err = t.DropTableOrView(schemaID, job.SchemaName, oldTableID, tbInfo.Name.L)
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
}
- err = t.GetAutoIDAccessors(schemaID, oldTbInfoID).Del()
+ err = t.GetAutoIDAccessors(schemaID, oldTableID).Del()
if err != nil {
job.State = model.JobStateCancelled
return ver, errors.Trace(err)
@@ -1500,7 +1518,7 @@ func checkTableNotExists(d *ddlCtx, t *meta.Meta, schemaID int64, tableName stri
return err
}
is := d.infoCache.GetLatest()
- if is.SchemaMetaVersion() == currVer {
+ if is != nil && is.SchemaMetaVersion() == currVer {
return checkTableNotExistsFromInfoSchema(is, schemaID, tableName)
}
@@ -1514,7 +1532,7 @@ func checkTableNotExistsByName(d *ddlCtx, t *meta.Meta, schemaID int64, schemaNa
return err
}
is := d.infoCache.GetLatest()
- if is.SchemaMetaVersion() == currVer {
+ if is != nil && is.SchemaMetaVersion() == currVer {
return checkTableNotExistsFromInfoSchema(is, schemaID, tableName)
}
return t.CheckTableNameNotExists(t.TableNameKey(schemaName, tableName))
@@ -1588,6 +1606,48 @@ func checkTableNotExistsFromStore(t *meta.Meta, schemaID int64, tableName string
return nil
}
+func findTableIDByName(d *ddlCtx, t *meta.Meta, schemaID int64, tableName string) (int64, error) {
+ // Try to use memory schema info to check first.
+ currVer, err := t.GetSchemaVersion()
+ if err != nil {
+ return 0, err
+ }
+ is := d.infoCache.GetLatest()
+ if is != nil && is.SchemaMetaVersion() == currVer {
+ return findTableIDFromInfoSchema(is, schemaID, tableName)
+ }
+
+ return findTableIDFromStore(t, schemaID, tableName)
+}
+
+func findTableIDFromInfoSchema(is infoschema.InfoSchema, schemaID int64, tableName string) (int64, error) {
+ schema, ok := is.SchemaByID(schemaID)
+ if !ok {
+ return 0, infoschema.ErrDatabaseNotExists.GenWithStackByArgs("")
+ }
+ tbl, err := is.TableByName(schema.Name, model.NewCIStr(tableName))
+ if err != nil {
+ return 0, err
+ }
+ return tbl.Meta().ID, nil
+}
+
+func findTableIDFromStore(t *meta.Meta, schemaID int64, tableName string) (int64, error) {
+ tbls, err := t.ListSimpleTables(schemaID)
+ if err != nil {
+ if meta.ErrDBNotExists.Equal(err) {
+ return 0, infoschema.ErrDatabaseNotExists.GenWithStackByArgs("")
+ }
+ return 0, errors.Trace(err)
+ }
+ for _, tbl := range tbls {
+ if tbl.Name.L == tableName {
+ return tbl.ID, nil
+ }
+ }
+ return 0, infoschema.ErrTableNotExists.FastGenByArgs(tableName)
+}
+
// updateVersionAndTableInfoWithCheck checks table info validate and updates the schema version and the table information
func updateVersionAndTableInfoWithCheck(d *ddlCtx, t *meta.Meta, job *model.Job, tblInfo *model.TableInfo, shouldUpdateVer bool, multiInfos ...schemaIDAndTableInfo) (
ver int64, err error) {
diff --git a/pkg/ddl/table_test.go b/pkg/ddl/table_test.go
index 61fc39a943de4..13d37572e5cef 100644
--- a/pkg/ddl/table_test.go
+++ b/pkg/ddl/table_test.go
@@ -309,7 +309,8 @@ func TestCreateView(t *testing.T) {
}
ctx.SetValue(sessionctx.QueryString, "skip")
err = d.DoDDLJob(ctx, job)
- require.Error(t, err)
+ // The non-existing table id in job args will not be considered anymore.
+ require.NoError(t, err)
}
func checkTableCacheTest(t *testing.T, store kv.Storage, dbInfo *model.DBInfo, tblInfo *model.TableInfo) {
diff --git a/pkg/ddl/tests/partition/db_partition_test.go b/pkg/ddl/tests/partition/db_partition_test.go
index 38de84b003b18..4683a74b0bdd9 100644
--- a/pkg/ddl/tests/partition/db_partition_test.go
+++ b/pkg/ddl/tests/partition/db_partition_test.go
@@ -1259,6 +1259,7 @@ func TestCreateTableWithKeyPartition(t *testing.T) {
tk.MustExec(`drop table if exists tm2`)
tk.MustGetErrMsg(`create table tm2 (a char(5), unique key(a(5))) partition by key() partitions 5`,
"Table partition metadata not correct, neither partition expression or list of partition columns")
+ tk.MustExec(`create table tm2 (a char(5) not null, unique key(a(5))) partition by key() partitions 5`)
}
func TestDropPartitionWithGlobalIndex(t *testing.T) {
@@ -1512,20 +1513,20 @@ func TestGlobalIndexUpdateInTruncatePartition(t *testing.T) {
originalHook := dom.DDL().GetHook()
defer dom.DDL().SetHook(originalHook)
- var err error
hook := &callback.TestDDLCallback{Do: dom}
hook.OnJobRunBeforeExported = func(job *model.Job) {
assert.Equal(t, model.ActionTruncateTablePartition, job.Type)
if job.SchemaState == model.StateDeleteOnly {
tk1 := testkit.NewTestKit(t, store)
tk1.MustExec("use test")
- err = tk1.ExecToErr("update test_global set a = 2 where a = 11")
+ err := tk1.ExecToErr("update test_global set a = 2 where a = 11")
assert.NotNil(t, err)
}
}
dom.DDL().SetHook(hook)
tk.MustExec("alter table test_global truncate partition p1")
+ tk.MustQuery("select * from test_global use index(idx_b) order by a").Check(testkit.Rows("11 11 11", "12 12 12"))
}
func TestGlobalIndexUpdateInTruncatePartition4Hash(t *testing.T) {
@@ -1564,7 +1565,7 @@ func TestGlobalIndexUpdateInTruncatePartition4Hash(t *testing.T) {
tk.MustExec("alter table test_global truncate partition p1")
}
-func TestGlobalIndexReaderInTruncatePartition(t *testing.T) {
+func TestGlobalIndexReaderAndIndexLookUpInTruncatePartition(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
@@ -1591,6 +1592,9 @@ func TestGlobalIndexReaderInTruncatePartition(t *testing.T) {
tk1.MustExec("use test")
tk1.MustQuery("select b from test_global use index(idx_b)").Sort().Check(testkit.Rows("11", "12"))
+ tk1.MustQuery("select * from test_global use index(idx_b)").Sort().Check(testkit.Rows("11 11 11", "12 12 12"))
+ tk1.MustQuery("select * from test_global use index(idx_b) order by a").Check(testkit.Rows("11 11 11", "12 12 12"))
+ tk1.MustQuery("select * from test_global use index(idx_b) order by b").Check(testkit.Rows("11 11 11", "12 12 12"))
}
}
dom.DDL().SetHook(hook)
diff --git a/pkg/ddl/ttl.go b/pkg/ddl/ttl.go
index 7b6b1c4a7ec04..a21b4e801f56e 100644
--- a/pkg/ddl/ttl.go
+++ b/pkg/ddl/ttl.go
@@ -15,23 +15,25 @@
package ddl
import (
- "fmt"
"strings"
+ "time"
"github.com/pingcap/errors"
- "github.com/pingcap/tidb/pkg/expression"
"github.com/pingcap/tidb/pkg/meta"
- "github.com/pingcap/tidb/pkg/parser"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/format"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessiontxn"
+ "github.com/pingcap/tidb/pkg/ttl/cache"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/dbterror"
)
+// DefaultTTLJobInterval is the default value for ttl job interval.
+const DefaultTTLJobInterval = "1h"
+
func onTTLInfoRemove(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) {
tblInfo, err := GetTableInfoAndCancelFaultJob(t, job, job.SchemaID)
if err != nil {
@@ -97,7 +99,7 @@ func onTTLInfoChange(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err er
}
func checkTTLInfoValid(ctx sessionctx.Context, schema model.CIStr, tblInfo *model.TableInfo) error {
- if err := checkTTLIntervalExpr(ctx.GetExprCtx(), tblInfo.TTLInfo); err != nil {
+ if err := checkTTLIntervalExpr(tblInfo.TTLInfo); err != nil {
return err
}
@@ -108,20 +110,9 @@ func checkTTLInfoValid(ctx sessionctx.Context, schema model.CIStr, tblInfo *mode
return checkTTLInfoColumnType(tblInfo)
}
-func checkTTLIntervalExpr(ctx expression.BuildContext, ttlInfo *model.TTLInfo) error {
- // FIXME: use a better way to validate the interval expression in ttl
- var nowAddIntervalExpr ast.ExprNode
-
- unit := ast.TimeUnitType(ttlInfo.IntervalTimeUnit)
- expr := fmt.Sprintf("select NOW() + INTERVAL %s %s", ttlInfo.IntervalExprStr, unit.String())
- stmts, _, err := parser.New().ParseSQL(expr)
- if err != nil {
- // FIXME: the error information can be wrong, as it could indicate an unknown position to user.
- return errors.Trace(err)
- }
- nowAddIntervalExpr = stmts[0].(*ast.SelectStmt).Fields.Fields[0].Expr
- _, err = expression.EvalSimpleAst(ctx, nowAddIntervalExpr)
- return err
+func checkTTLIntervalExpr(ttlInfo *model.TTLInfo) error {
+ _, err := cache.EvalExpireTime(time.Now(), ttlInfo.IntervalExprStr, ast.TimeUnitType(ttlInfo.IntervalTimeUnit))
+ return errors.Trace(err)
}
func checkTTLInfoColumnType(tblInfo *model.TableInfo) error {
@@ -213,7 +204,7 @@ func getTTLInfoInOptions(options []*ast.TableOption) (ttlInfo *model.TTLInfo, tt
IntervalExprStr: intervalExpr,
IntervalTimeUnit: int(op.TimeUnitValue.Unit),
Enable: true,
- JobInterval: "1h",
+ JobInterval: DefaultTTLJobInterval,
}
case ast.TableOptionTTLEnable:
ttlEnable = &op.BoolValue
diff --git a/pkg/ddl/ttl_test.go b/pkg/ddl/ttl_test.go
index 1fa6ca6acc269..7e56f0199009a 100644
--- a/pkg/ddl/ttl_test.go
+++ b/pkg/ddl/ttl_test.go
@@ -55,7 +55,7 @@ func Test_getTTLInfoInOptions(t *testing.T) {
IntervalExprStr: "5",
IntervalTimeUnit: int(ast.TimeUnitYear),
Enable: true,
- JobInterval: "1h",
+ JobInterval: DefaultTTLJobInterval,
},
nil,
nil,
@@ -79,7 +79,7 @@ func Test_getTTLInfoInOptions(t *testing.T) {
IntervalExprStr: "5",
IntervalTimeUnit: int(ast.TimeUnitYear),
Enable: false,
- JobInterval: "1h",
+ JobInterval: DefaultTTLJobInterval,
},
&falseValue,
nil,
@@ -107,7 +107,7 @@ func Test_getTTLInfoInOptions(t *testing.T) {
IntervalExprStr: "5",
IntervalTimeUnit: int(ast.TimeUnitYear),
Enable: true,
- JobInterval: "1h",
+ JobInterval: DefaultTTLJobInterval,
},
&trueValue,
nil,
diff --git a/pkg/ddl/util/BUILD.bazel b/pkg/ddl/util/BUILD.bazel
index 71bef12a86dcd..abff61676b963 100644
--- a/pkg/ddl/util/BUILD.bazel
+++ b/pkg/ddl/util/BUILD.bazel
@@ -31,11 +31,15 @@ go_library(
go_test(
name = "util_test",
timeout = "short",
- srcs = ["main_test.go"],
+ srcs = [
+ "main_test.go",
+ "util_test.go",
+ ],
embed = [":util"],
flaky = True,
deps = [
"//pkg/testkit/testsetup",
+ "@com_github_stretchr_testify//require",
"@org_uber_go_goleak//:goleak",
],
)
diff --git a/pkg/ddl/util/util.go b/pkg/ddl/util/util.go
index a8c6bef2c86fc..6dc7b576cc1bc 100644
--- a/pkg/ddl/util/util.go
+++ b/pkg/ddl/util/util.go
@@ -19,6 +19,7 @@ import (
"context"
"encoding/hex"
"fmt"
+ "os"
"strings"
"time"
@@ -55,6 +56,7 @@ const (
// DDLAllSchemaVersions is the path on etcd that is used to store all servers current schema versions.
DDLAllSchemaVersions = "/tidb/ddl/all_schema_versions"
// DDLAllSchemaVersionsByJob is the path on etcd that is used to store all servers current schema versions.
+ // /tidb/ddl/all_schema_by_job_versions// --->
DDLAllSchemaVersionsByJob = "/tidb/ddl/all_schema_by_job_versions"
// DDLGlobalSchemaVersion is the path on etcd that is used to store the latest schema versions.
DDLGlobalSchemaVersion = "/tidb/ddl/global_schema_version"
@@ -403,3 +405,9 @@ func IsRaftKv2(ctx context.Context, sctx sessionctx.Context) (bool, error) {
raftVersion := rows[0].GetString(0)
return raftVersion == raftKv2, nil
}
+
+// FolderNotEmpty returns true only when the folder is not empty.
+func FolderNotEmpty(path string) bool {
+ entries, _ := os.ReadDir(path)
+ return len(entries) > 0
+}
diff --git a/br/pkg/restore/utils/key.go b/pkg/ddl/util/util_test.go
similarity index 54%
rename from br/pkg/restore/utils/key.go
rename to pkg/ddl/util/util_test.go
index 8a4ad05b0d9d8..679bed1852379 100644
--- a/br/pkg/restore/utils/key.go
+++ b/pkg/ddl/util/util_test.go
@@ -12,23 +12,23 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package utils
+package util
-import "github.com/pingcap/tidb/pkg/util/codec"
+import (
+ "os"
+ "path/filepath"
+ "testing"
-func TruncateTS(key []byte) []byte {
- if len(key) == 0 {
- return nil
- }
- if len(key) < 8 {
- return key
- }
- return key[:len(key)-8]
-}
+ "github.com/stretchr/testify/require"
+)
+
+func TestFolderNotEmpty(t *testing.T) {
+ tmp := t.TempDir()
+ require.False(t, FolderNotEmpty(tmp))
+ require.False(t, FolderNotEmpty(filepath.Join(tmp, "not-exist")))
-func EncodeKeyPrefix(key []byte) []byte {
- encodedPrefix := make([]byte, 0)
- ungroupedLen := len(key) % 8
- encodedPrefix = append(encodedPrefix, codec.EncodeBytes([]byte{}, key[:len(key)-ungroupedLen])...)
- return append(encodedPrefix[:len(encodedPrefix)-9], key[len(key)-ungroupedLen:]...)
+ f, err := os.Create(filepath.Join(tmp, "test-file"))
+ require.NoError(t, err)
+ require.NoError(t, f.Close())
+ require.True(t, FolderNotEmpty(tmp))
}
diff --git a/pkg/distsql/context/BUILD.bazel b/pkg/distsql/context/BUILD.bazel
index fe2811f78585f..6ab89d5a7422a 100644
--- a/pkg/distsql/context/BUILD.bazel
+++ b/pkg/distsql/context/BUILD.bazel
@@ -10,6 +10,7 @@ go_library(
"//pkg/errctx",
"//pkg/kv",
"//pkg/parser/mysql",
+ "//pkg/util/context",
"//pkg/util/execdetails",
"//pkg/util/memory",
"//pkg/util/nocopy",
diff --git a/pkg/distsql/context/context.go b/pkg/distsql/context/context.go
index 6625cdeac7376..797ec00666126 100644
--- a/pkg/distsql/context/context.go
+++ b/pkg/distsql/context/context.go
@@ -21,6 +21,7 @@ import (
"github.com/pingcap/tidb/pkg/errctx"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/mysql"
+ contextutil "github.com/pingcap/tidb/pkg/util/context"
"github.com/pingcap/tidb/pkg/util/execdetails"
"github.com/pingcap/tidb/pkg/util/memory"
"github.com/pingcap/tidb/pkg/util/nocopy"
@@ -38,7 +39,8 @@ type DistSQLContext struct {
// the next execution. They'll need to be handled specially.
_ nocopy.NoCopy
- AppendWarning func(error)
+ WarnHandler contextutil.WarnAppender
+
InRestrictedSQL bool
Client kv.Client
@@ -88,3 +90,8 @@ type DistSQLContext struct {
ExecDetails *execdetails.SyncExecDetails
}
+
+// AppendWarning appends the warning to the warning handler.
+func (dctx *DistSQLContext) AppendWarning(warn error) {
+ dctx.WarnHandler.AppendWarning(warn)
+}
diff --git a/pkg/distsql/context_test.go b/pkg/distsql/context_test.go
index bca80c67a1e7c..d81568e0373ab 100644
--- a/pkg/distsql/context_test.go
+++ b/pkg/distsql/context_test.go
@@ -24,7 +24,7 @@ import (
// NewDistSQLContextForTest creates a new dist sql context for test
func NewDistSQLContextForTest() *distsqlctx.DistSQLContext {
return &distsqlctx.DistSQLContext{
- AppendWarning: func(error) {},
+ WarnHandler: contextutil.NewFuncWarnAppenderForTest(func(err error) {}),
TiFlashMaxThreads: variable.DefTiFlashMaxThreads,
TiFlashMaxBytesBeforeExternalJoin: variable.DefTiFlashMaxBytesBeforeExternalJoin,
TiFlashMaxBytesBeforeExternalGroupBy: variable.DefTiFlashMaxBytesBeforeExternalGroupBy,
diff --git a/pkg/distsql/select_result.go b/pkg/distsql/select_result.go
index ae4a0b44f2c74..5d485d4cf1483 100644
--- a/pkg/distsql/select_result.go
+++ b/pkg/distsql/select_result.go
@@ -100,14 +100,14 @@ func (h *chunkRowHeap) Pop() any {
// NewSortedSelectResults is only for partition table
// If schema == nil, sort by first few columns.
-func NewSortedSelectResults(selectResult []SelectResult, schema *expression.Schema, byitems []*util.ByItems, memTracker *memory.Tracker) SelectResult {
+func NewSortedSelectResults(ectx expression.EvalContext, selectResult []SelectResult, schema *expression.Schema, byitems []*util.ByItems, memTracker *memory.Tracker) SelectResult {
s := &sortedSelectResults{
schema: schema,
selectResult: selectResult,
byItems: byitems,
memTracker: memTracker,
}
- s.initCompareFuncs()
+ s.initCompareFuncs(ectx)
s.buildKeyColumns()
s.heap = &chunkRowHeap{s}
s.cachedChunks = make([]*chunk.Chunk, len(selectResult))
@@ -141,10 +141,10 @@ func (ssr *sortedSelectResults) updateCachedChunk(ctx context.Context, idx uint3
return nil
}
-func (ssr *sortedSelectResults) initCompareFuncs() {
+func (ssr *sortedSelectResults) initCompareFuncs(ectx expression.EvalContext) {
ssr.compareFuncs = make([]chunk.CompareFunc, len(ssr.byItems))
for i, item := range ssr.byItems {
- keyType := item.Expr.GetType()
+ keyType := item.Expr.GetType(ectx)
ssr.compareFuncs[i] = chunk.GetCompareFunc(keyType)
}
}
diff --git a/pkg/disttask/framework/handle/BUILD.bazel b/pkg/disttask/framework/handle/BUILD.bazel
index dd0697e8d2f72..7cfb6a03dc5d3 100644
--- a/pkg/disttask/framework/handle/BUILD.bazel
+++ b/pkg/disttask/framework/handle/BUILD.bazel
@@ -26,6 +26,7 @@ go_test(
"//pkg/disttask/framework/proto",
"//pkg/disttask/framework/storage",
"//pkg/testkit",
+ "//pkg/testkit/testfailpoint",
"//pkg/util/backoff",
"@com_github_ngaut_pools//:pools",
"@com_github_pingcap_errors//:errors",
diff --git a/pkg/disttask/framework/handle/handle_test.go b/pkg/disttask/framework/handle/handle_test.go
index 048223debc87a..dc73bba383bb5 100644
--- a/pkg/disttask/framework/handle/handle_test.go
+++ b/pkg/disttask/framework/handle/handle_test.go
@@ -28,13 +28,14 @@ import (
"github.com/pingcap/tidb/pkg/disttask/framework/proto"
"github.com/pingcap/tidb/pkg/disttask/framework/storage"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/util/backoff"
"github.com/stretchr/testify/require"
"github.com/tikv/client-go/v2/util"
)
func TestHandle(t *testing.T) {
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", "return(8)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", "return(8)")
ctx := util.WithInternalSourceType(context.Background(), "handle_test")
diff --git a/pkg/disttask/framework/integrationtests/BUILD.bazel b/pkg/disttask/framework/integrationtests/BUILD.bazel
index b633e56cbde8f..b9f8bc2d4a502 100644
--- a/pkg/disttask/framework/integrationtests/BUILD.bazel
+++ b/pkg/disttask/framework/integrationtests/BUILD.bazel
@@ -29,6 +29,7 @@ go_test(
"//pkg/session",
"//pkg/store/driver",
"//pkg/testkit",
+ "//pkg/testkit/testfailpoint",
"//pkg/testkit/testsetup",
"//pkg/util",
"@com_github_pingcap_failpoint//:failpoint",
diff --git a/pkg/disttask/framework/integrationtests/bench_test.go b/pkg/disttask/framework/integrationtests/bench_test.go
index 75150f0a91037..fd0b0d325b077 100644
--- a/pkg/disttask/framework/integrationtests/bench_test.go
+++ b/pkg/disttask/framework/integrationtests/bench_test.go
@@ -32,6 +32,7 @@ import (
"github.com/pingcap/tidb/pkg/session"
"github.com/pingcap/tidb/pkg/store/driver"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/stretchr/testify/require"
"go.opencensus.io/stats/view"
"go.uber.org/mock/gomock"
@@ -106,7 +107,7 @@ func BenchmarkSchedulerOverhead(b *testing.B) {
}
func prepareForBenchTest(b *testing.B) {
- testkit.EnableFailPoint(b, "github.com/pingcap/tidb/pkg/domain/MockDisableDistTask", "return(true)")
+ testfailpoint.Enable(b, "github.com/pingcap/tidb/pkg/domain/MockDisableDistTask", "return(true)")
var d driver.TiKVDriver
var err error
diff --git a/pkg/disttask/framework/integrationtests/framework_ha_test.go b/pkg/disttask/framework/integrationtests/framework_ha_test.go
index 872656d4f0f22..624c234d37998 100644
--- a/pkg/disttask/framework/integrationtests/framework_ha_test.go
+++ b/pkg/disttask/framework/integrationtests/framework_ha_test.go
@@ -23,7 +23,7 @@ import (
"github.com/pingcap/tidb/pkg/disttask/framework/proto"
"github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor"
"github.com/pingcap/tidb/pkg/disttask/framework/testutil"
- "github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/util"
"github.com/stretchr/testify/require"
)
@@ -36,7 +36,7 @@ func submitTaskAndCheckSuccessForHA(ctx context.Context, t *testing.T, taskKey s
}
func TestHANodeRandomShutdown(t *testing.T) {
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/mockTiDBShutdown", "return()")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/mockTiDBShutdown", "return()")
c := testutil.NewDXFContextWithRandomNodes(t, 4, 15)
testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockHATestSchedulerExt(c.MockCtrl), c.TestContext, nil)
@@ -55,7 +55,7 @@ func TestHANodeRandomShutdown(t *testing.T) {
}
func TestHARandomShutdownInDifferentStep(t *testing.T) {
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/mockTiDBShutdown", "return()")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/mockTiDBShutdown", "return()")
c := testutil.NewDXFContextWithRandomNodes(t, 6, 15)
testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockHATestSchedulerExt(c.MockCtrl), c.TestContext, nil)
diff --git a/pkg/disttask/framework/integrationtests/framework_pause_and_resume_test.go b/pkg/disttask/framework/integrationtests/framework_pause_and_resume_test.go
index c23c98d405fd2..c07a731332704 100644
--- a/pkg/disttask/framework/integrationtests/framework_pause_and_resume_test.go
+++ b/pkg/disttask/framework/integrationtests/framework_pause_and_resume_test.go
@@ -23,7 +23,7 @@ import (
"github.com/pingcap/tidb/pkg/disttask/framework/proto"
"github.com/pingcap/tidb/pkg/disttask/framework/storage"
"github.com/pingcap/tidb/pkg/disttask/framework/testutil"
- "github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/stretchr/testify/require"
)
@@ -50,7 +50,7 @@ func TestFrameworkPauseAndResume(t *testing.T) {
testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil)
// 1. schedule and pause one running task.
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/pauseTaskAfterRefreshTask", "2*return(true)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/pauseTaskAfterRefreshTask", "2*return(true)")
task1 := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", "", 1)
require.Equal(t, proto.TaskStatePaused, task1.State)
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/disttask/framework/scheduler/pauseTaskAfterRefreshTask"))
@@ -67,7 +67,7 @@ func TestFrameworkPauseAndResume(t *testing.T) {
require.Empty(t, errs)
// 2. pause pending task.
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/pausePendingTask", "2*return(true)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/pausePendingTask", "2*return(true)")
task2 := testutil.SubmitAndWaitTask(c.Ctx, t, "key2", "", 1)
require.Equal(t, proto.TaskStatePaused, task2.State)
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/disttask/framework/scheduler/pausePendingTask"))
diff --git a/pkg/disttask/framework/integrationtests/framework_rollback_test.go b/pkg/disttask/framework/integrationtests/framework_rollback_test.go
index 59847b40871d8..56baaaf353fa9 100644
--- a/pkg/disttask/framework/integrationtests/framework_rollback_test.go
+++ b/pkg/disttask/framework/integrationtests/framework_rollback_test.go
@@ -19,14 +19,14 @@ import (
"github.com/pingcap/tidb/pkg/disttask/framework/proto"
"github.com/pingcap/tidb/pkg/disttask/framework/testutil"
- "github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/stretchr/testify/require"
)
func TestFrameworkRollback(t *testing.T) {
c := testutil.NewTestDXFContext(t, 2, 16, true)
testutil.RegisterRollbackTaskMeta(t, c.MockCtrl, testutil.GetMockRollbackSchedulerExt(c.MockCtrl), c.TestContext)
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/cancelTaskAfterRefreshTask", "2*return(true)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/cancelTaskAfterRefreshTask", "2*return(true)")
task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", "", 1)
require.Equal(t, proto.TaskStateReverted, task.State)
diff --git a/pkg/disttask/framework/integrationtests/framework_scope_test.go b/pkg/disttask/framework/integrationtests/framework_scope_test.go
index c3a63f5589867..30f40c31cbf68 100644
--- a/pkg/disttask/framework/integrationtests/framework_scope_test.go
+++ b/pkg/disttask/framework/integrationtests/framework_scope_test.go
@@ -28,6 +28,7 @@ import (
"github.com/pingcap/tidb/pkg/disttask/framework/storage"
"github.com/pingcap/tidb/pkg/disttask/framework/testutil"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"golang.org/x/exp/rand"
@@ -80,7 +81,7 @@ func TestScopeBasic(t *testing.T) {
tk.MustQuery("select @@global.tidb_service_scope").Check(testkit.Rows("background"))
tk.MustQuery("select @@tidb_service_scope").Check(testkit.Rows("background"))
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh", "1*return()")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh", "1*return()")
<-scheduler.TestRefreshedChan
taskID = submitTaskAndCheckSuccessForScope(c.Ctx, t, "😊", nodeCnt, "background", c.TestContext)
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh"))
@@ -93,7 +94,7 @@ func TestScopeBasic(t *testing.T) {
// 3. 2 "background" role.
tk.MustExec("update mysql.dist_framework_meta set role = \"background\" where host = \":4001\"")
time.Sleep(5 * time.Second)
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh", "1*return()")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh", "1*return()")
<-scheduler.TestRefreshedChan
taskID = submitTaskAndCheckSuccessForScope(c.Ctx, t, "😆", nodeCnt, "background", c.TestContext)
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh"))
@@ -145,7 +146,7 @@ func runTargetScopeCase(t *testing.T, c *testutil.TestDXFContext, tk *testkit.Te
for i := 0; i < len(testCase.nodeScopes); i++ {
tk.MustExec(fmt.Sprintf("update mysql.dist_framework_meta set role = \"%s\" where host = \"%s\"", testCase.nodeScopes[i], c.GetNodeIDByIdx(i)))
}
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh", "3*return()")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/syncRefresh", "3*return()")
<-scheduler.TestRefreshedChan
<-scheduler.TestRefreshedChan
<-scheduler.TestRefreshedChan
diff --git a/pkg/disttask/framework/integrationtests/framework_test.go b/pkg/disttask/framework/integrationtests/framework_test.go
index a68acb1138852..ef0b97463dcdf 100644
--- a/pkg/disttask/framework/integrationtests/framework_test.go
+++ b/pkg/disttask/framework/integrationtests/framework_test.go
@@ -18,14 +18,17 @@ import (
"context"
"fmt"
"math/rand"
+ "sync"
"testing"
"time"
+ "github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/disttask/framework/proto"
"github.com/pingcap/tidb/pkg/disttask/framework/scheduler"
"github.com/pingcap/tidb/pkg/disttask/framework/storage"
"github.com/pingcap/tidb/pkg/disttask/framework/testutil"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/util"
"github.com/stretchr/testify/require"
)
@@ -126,7 +129,7 @@ func TestFrameworkCancelTask(t *testing.T) {
testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil)
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/MockExecutorRunCancel", "1*return(1)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/MockExecutorRunCancel", "1*return(1)")
task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", "", 1)
require.Equal(t, proto.TaskStateReverted, task.State)
}
@@ -135,7 +138,7 @@ func TestFrameworkSubTaskFailed(t *testing.T) {
c := testutil.NewTestDXFContext(t, 1, 16, true)
testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil)
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/MockExecutorRunErr", "1*return(true)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/MockExecutorRunErr", "1*return(true)")
task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", "", 1)
require.Equal(t, proto.TaskStateReverted, task.State)
}
@@ -143,7 +146,7 @@ func TestFrameworkSubTaskFailed(t *testing.T) {
func TestFrameworkSubTaskInitEnvFailed(t *testing.T) {
c := testutil.NewTestDXFContext(t, 1, 16, true)
testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil)
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/mockExecSubtaskInitEnvErr", "return()")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/mockExecSubtaskInitEnvErr", "return()")
task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", "", 1)
require.Equal(t, proto.TaskStateReverted, task.State)
}
@@ -151,16 +154,22 @@ func TestFrameworkSubTaskInitEnvFailed(t *testing.T) {
func TestOwnerChangeWhenSchedule(t *testing.T) {
c := testutil.NewTestDXFContext(t, 3, 16, true)
testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil)
- scheduler.MockOwnerChange = func() {
- c.AsyncChangeOwner()
- }
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/mockOwnerChange", "1*return(true)")
+ var once sync.Once
+ require.NoError(t, failpoint.EnableCall("github.com/pingcap/tidb/pkg/disttask/framework/scheduler/mockOwnerChange", func() {
+ once.Do(func() {
+ c.AsyncChangeOwner()
+ time.Sleep(time.Second)
+ })
+ }))
+ t.Cleanup(func() {
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/disttask/framework/scheduler/mockOwnerChange"))
+ })
submitTaskAndCheckSuccessForBasic(c.Ctx, t, "😊", c.TestContext)
}
func TestGC(t *testing.T) {
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/storage/subtaskHistoryKeepSeconds", "return(1)")
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/historySubtaskTableGcInterval", "return(1)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/storage/subtaskHistoryKeepSeconds", "return(1)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/historySubtaskTableGcInterval", "return(1)")
c := testutil.NewTestDXFContext(t, 3, 16, true)
testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil)
@@ -194,7 +203,7 @@ func TestFrameworkSubtaskFinishedCancel(t *testing.T) {
c := testutil.NewTestDXFContext(t, 3, 16, true)
testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil)
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/MockSubtaskFinishedCancel", "1*return(true)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/MockSubtaskFinishedCancel", "1*return(true)")
task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", "", 1)
require.Equal(t, proto.TaskStateReverted, task.State)
}
@@ -203,7 +212,7 @@ func TestFrameworkRunSubtaskCancel(t *testing.T) {
c := testutil.NewTestDXFContext(t, 3, 16, true)
testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil)
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/MockRunSubtaskCancel", "1*return(true)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor/MockRunSubtaskCancel", "1*return(true)")
task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", "", 1)
require.Equal(t, proto.TaskStateReverted, task.State)
}
@@ -216,7 +225,7 @@ func TestFrameworkCleanUpRoutine(t *testing.T) {
scheduler.DefaultCleanUpInterval = 500 * time.Millisecond
c := testutil.NewTestDXFContext(t, 3, 16, true)
testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil)
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/WaitCleanUpFinished", "return()")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/WaitCleanUpFinished", "return()")
// normal
submitTaskAndCheckSuccessForBasic(c.Ctx, t, "key1", c.TestContext)
@@ -231,7 +240,7 @@ func TestFrameworkCleanUpRoutine(t *testing.T) {
require.NotEmpty(t, subtasks)
// transfer err
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/mockTransferErr", "1*return()")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/mockTransferErr", "1*return()")
submitTaskAndCheckSuccessForBasic(c.Ctx, t, "key2", c.TestContext)
<-scheduler.WaitCleanUpFinished
mgr, err = storage.GetTaskManager()
@@ -248,7 +257,7 @@ func TestTaskCancelledBeforeUpdateTask(t *testing.T) {
c := testutil.NewTestDXFContext(t, 1, 16, true)
testutil.RegisterTaskMeta(t, c.MockCtrl, testutil.GetMockBasicSchedulerExt(c.MockCtrl), c.TestContext, nil)
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/cancelBeforeUpdateTask", "1*return(true)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/cancelBeforeUpdateTask", "1*return(true)")
task := testutil.SubmitAndWaitTask(c.Ctx, t, "key1", "", 1)
require.Equal(t, proto.TaskStateReverted, task.State)
}
diff --git a/pkg/disttask/framework/planner/BUILD.bazel b/pkg/disttask/framework/planner/BUILD.bazel
index 4876832aee915..592152e006f7f 100644
--- a/pkg/disttask/framework/planner/BUILD.bazel
+++ b/pkg/disttask/framework/planner/BUILD.bazel
@@ -12,6 +12,7 @@ go_library(
"//pkg/config",
"//pkg/disttask/framework/proto",
"//pkg/disttask/framework/storage",
+ "//pkg/kv",
"//pkg/sessionctx",
],
)
diff --git a/pkg/disttask/framework/planner/plan.go b/pkg/disttask/framework/planner/plan.go
index e3d94a7c95a10..d0d2cee557973 100644
--- a/pkg/disttask/framework/planner/plan.go
+++ b/pkg/disttask/framework/planner/plan.go
@@ -18,6 +18,7 @@ import (
"context"
"github.com/pingcap/tidb/pkg/disttask/framework/proto"
+ "github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/sessionctx"
)
@@ -38,6 +39,8 @@ type PlanCtx struct {
GlobalSort bool
NextTaskStep proto.Step
ExecuteNodesCnt int
+
+ Store kv.StorageWithPD
}
// LogicalPlan represents a logical plan in distribute framework.
diff --git a/pkg/disttask/framework/scheduler/BUILD.bazel b/pkg/disttask/framework/scheduler/BUILD.bazel
index 5abf7dd96f547..c9a34712b4860 100644
--- a/pkg/disttask/framework/scheduler/BUILD.bazel
+++ b/pkg/disttask/framework/scheduler/BUILD.bazel
@@ -69,6 +69,7 @@ go_test(
"//pkg/kv",
"//pkg/sessionctx",
"//pkg/testkit",
+ "//pkg/testkit/testfailpoint",
"//pkg/testkit/testsetup",
"//pkg/util/cpu",
"//pkg/util/disttask",
diff --git a/pkg/disttask/framework/scheduler/scheduler.go b/pkg/disttask/framework/scheduler/scheduler.go
index 3ac08f319136a..476339645762d 100644
--- a/pkg/disttask/framework/scheduler/scheduler.go
+++ b/pkg/disttask/framework/scheduler/scheduler.go
@@ -90,9 +90,6 @@ type BaseScheduler struct {
rand *rand.Rand
}
-// MockOwnerChange mock owner change in tests.
-var MockOwnerChange func()
-
// NewBaseScheduler creates a new BaseScheduler.
func NewBaseScheduler(ctx context.Context, task *proto.Task, param Param) *BaseScheduler {
logger := log.L().With(zap.Int64("task-id", task.ID), zap.Stringer("task-type", task.Type), zap.Bool("allocated-slots", param.allocatedSlots))
@@ -264,10 +261,7 @@ func (s *BaseScheduler) scheduleTask() {
s.logger.Info("schedule task meet err, reschedule it", zap.Error(err))
}
- failpoint.Inject("mockOwnerChange", func() {
- MockOwnerChange()
- time.Sleep(time.Second)
- })
+ failpoint.InjectCall("mockOwnerChange")
}
}
}
@@ -304,18 +298,11 @@ func (s *BaseScheduler) onPausing() error {
return nil
}
-// MockDMLExecutionOnPausedState is used to mock DML execution when tasks pauses.
-var MockDMLExecutionOnPausedState func(task *proto.Task)
-
// handle task in paused state.
func (s *BaseScheduler) onPaused() error {
task := s.GetTask()
s.logger.Info("on paused state", zap.Stringer("state", task.State), zap.String("step", proto.Step2Str(task.Type, task.Step)))
- failpoint.Inject("mockDMLExecutionOnPausedState", func(val failpoint.Value) {
- if val.(bool) {
- MockDMLExecutionOnPausedState(task)
- }
- })
+ failpoint.InjectCall("mockDMLExecutionOnPausedState")
return nil
}
diff --git a/pkg/disttask/framework/scheduler/scheduler_manager.go b/pkg/disttask/framework/scheduler/scheduler_manager.go
index a2f8f8789a708..186fc77702a7d 100644
--- a/pkg/disttask/framework/scheduler/scheduler_manager.go
+++ b/pkg/disttask/framework/scheduler/scheduler_manager.go
@@ -361,7 +361,7 @@ func (sm *Manager) startScheduler(basicTask *proto.TaskBase, allocateSlots bool,
sm.slotMgr.unReserve(basicTask, reservedExecID)
}
handle.NotifyTaskChange()
- sm.logger.Info("task scheduler exist", zap.Int64("task-id", task.ID))
+ sm.logger.Info("task scheduler exit", zap.Int64("task-id", task.ID))
}()
metrics.UpdateMetricsForRunTask(task)
scheduler.ScheduleTask()
diff --git a/pkg/disttask/framework/scheduler/scheduler_test.go b/pkg/disttask/framework/scheduler/scheduler_test.go
index 7820c56fea6d7..e8478b0504b65 100644
--- a/pkg/disttask/framework/scheduler/scheduler_test.go
+++ b/pkg/disttask/framework/scheduler/scheduler_test.go
@@ -34,6 +34,7 @@ import (
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
disttaskutil "github.com/pingcap/tidb/pkg/util/disttask"
"github.com/pingcap/tidb/pkg/util/logutil"
"github.com/pingcap/tidb/pkg/util/sqlexec"
@@ -150,7 +151,7 @@ func TestTaskFailInManager(t *testing.T) {
}
func checkSchedule(t *testing.T, taskCnt int, isSucc, isCancel, isSubtaskCancel, isPauseAndResume bool) {
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/domain/MockDisableDistTask", "return(true)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/domain/MockDisableDistTask", "return(true)")
// test scheduleTaskLoop
// test parallelism control
var originalConcurrency int
@@ -395,7 +396,7 @@ func TestIsCancelledErr(t *testing.T) {
func TestManagerScheduleLoop(t *testing.T) {
// Mock 16 cpu node.
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", "return(16)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", "return(16)")
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mockScheduler := mock.NewMockScheduler(ctrl)
diff --git a/pkg/disttask/framework/storage/BUILD.bazel b/pkg/disttask/framework/storage/BUILD.bazel
index 172bf921134de..f9b0edf64bc11 100644
--- a/pkg/disttask/framework/storage/BUILD.bazel
+++ b/pkg/disttask/framework/storage/BUILD.bazel
@@ -51,6 +51,7 @@ go_test(
"//pkg/sessionctx",
"//pkg/sessionctx/variable",
"//pkg/testkit",
+ "//pkg/testkit/testfailpoint",
"//pkg/testkit/testsetup",
"//pkg/util/sqlexec",
"@com_github_pingcap_errors//:errors",
diff --git a/pkg/disttask/framework/storage/table_test.go b/pkg/disttask/framework/storage/table_test.go
index 73067b79a398d..e047d53a30e54 100644
--- a/pkg/disttask/framework/storage/table_test.go
+++ b/pkg/disttask/framework/storage/table_test.go
@@ -30,6 +30,7 @@ import (
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/util/sqlexec"
"github.com/stretchr/testify/require"
"github.com/tikv/client-go/v2/util"
@@ -300,7 +301,7 @@ func TestSwitchTaskStepInBatch(t *testing.T) {
checkAfterSwitchStep(t, startTime, task1, subtasks1, proto.StepOne)
// mock another scheduler inserted some subtasks
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/storage/waitBeforeInsertSubtasks", `1*return(true)`)
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/storage/waitBeforeInsertSubtasks", `1*return(true)`)
task2, subtasks2 := prepare("key2")
go func() {
storage.TestChannel <- struct{}{}
@@ -699,14 +700,14 @@ func TestDistFrameworkMeta(t *testing.T) {
// when no node
_, err := sm.GetCPUCountOfNode(ctx)
require.ErrorContains(t, err, "no managed nodes")
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", "return(0)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", "return(0)")
require.NoError(t, sm.InitMeta(ctx, ":4000", "background"))
_, err = sm.GetCPUCountOfNode(ctx)
require.ErrorContains(t, err, "no managed node have enough resource")
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", "return(100)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", "return(100)")
require.NoError(t, sm.InitMeta(ctx, ":4000", "background"))
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", "return(8)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", "return(8)")
require.NoError(t, sm.InitMeta(ctx, ":4001", ""))
require.NoError(t, sm.InitMeta(ctx, ":4002", "background"))
nodes, err := sm.GetAllNodes(ctx)
@@ -717,7 +718,7 @@ func TestDistFrameworkMeta(t *testing.T) {
{ID: ":4002", Role: "background", CPUCount: 8},
}, nodes)
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", "return(100)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", "return(100)")
require.NoError(t, sm.InitMeta(ctx, ":4002", ""))
require.NoError(t, sm.InitMeta(ctx, ":4003", "background"))
@@ -785,10 +786,6 @@ func TestSubtaskHistoryTable(t *testing.T) {
const (
taskID = 1
taskID2 = 2
- subTask1 = 1
- subTask2 = 2
- subTask3 = 3
- subTask4 = 4 // taskID2
tidb1 = "tidb1"
tidb2 = "tidb2"
tidb3 = "tidb3"
@@ -796,11 +793,11 @@ func TestSubtaskHistoryTable(t *testing.T) {
finishedMeta = "finished"
)
- testutil.CreateSubTask(t, sm, taskID, proto.StepInit, tidb1, []byte(meta), proto.TaskTypeExample, 11)
+ subTask1 := testutil.CreateSubTask(t, sm, taskID, proto.StepInit, tidb1, []byte(meta), proto.TaskTypeExample, 11)
require.NoError(t, sm.FinishSubtask(ctx, tidb1, subTask1, []byte(finishedMeta)))
- testutil.CreateSubTask(t, sm, taskID, proto.StepInit, tidb2, []byte(meta), proto.TaskTypeExample, 11)
+ subTask2 := testutil.CreateSubTask(t, sm, taskID, proto.StepInit, tidb2, []byte(meta), proto.TaskTypeExample, 11)
require.NoError(t, sm.UpdateSubtaskStateAndError(ctx, tidb2, subTask2, proto.SubtaskStateCanceled, nil))
- testutil.CreateSubTask(t, sm, taskID, proto.StepInit, tidb3, []byte(meta), proto.TaskTypeExample, 11)
+ subTask3 := testutil.CreateSubTask(t, sm, taskID, proto.StepInit, tidb3, []byte(meta), proto.TaskTypeExample, 11)
require.NoError(t, sm.UpdateSubtaskStateAndError(ctx, tidb3, subTask3, proto.SubtaskStateFailed, nil))
subTasks, err := testutil.GetSubtasksByTaskID(ctx, sm, taskID)
@@ -827,10 +824,10 @@ func TestSubtaskHistoryTable(t *testing.T) {
require.Len(t, subTasks, 3)
// test GC history table.
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/framework/storage/subtaskHistoryKeepSeconds", "return(1)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/framework/storage/subtaskHistoryKeepSeconds", "return(1)")
time.Sleep(2 * time.Second)
- testutil.CreateSubTask(t, sm, taskID2, proto.StepInit, tidb1, []byte(meta), proto.TaskTypeExample, 11)
+ subTask4 := testutil.CreateSubTask(t, sm, taskID2, proto.StepInit, tidb1, []byte(meta), proto.TaskTypeExample, 11)
require.NoError(t, sm.UpdateSubtaskStateAndError(ctx, tidb1, subTask4, proto.SubtaskStateFailed, nil))
require.NoError(t, testutil.TransferSubTasks2History(ctx, sm, taskID2))
diff --git a/pkg/disttask/framework/storage/task_state_test.go b/pkg/disttask/framework/storage/task_state_test.go
index 196baddc6b460..460a197fda910 100644
--- a/pkg/disttask/framework/storage/task_state_test.go
+++ b/pkg/disttask/framework/storage/task_state_test.go
@@ -67,12 +67,12 @@ func TestTaskState(t *testing.T) {
id, err = gm.CreateTask(ctx, "key4", "test", 4, "", []byte("test"))
require.NoError(t, err)
// require.Equal(t, int64(4), id) TODO: unstable for infoschema v2
- task, err = gm.GetTaskByID(ctx, 4)
+ task, err = gm.GetTaskByID(ctx, id)
require.NoError(t, err)
checkTaskStateStep(t, task, proto.TaskStatePending, proto.StepInit)
err = gm.RevertTask(ctx, task.ID, proto.TaskStatePending, nil)
require.NoError(t, err)
- task, err = gm.GetTaskByID(ctx, 4)
+ task, err = gm.GetTaskByID(ctx, id)
require.NoError(t, err)
checkTaskStateStep(t, task, proto.TaskStateReverting, proto.StepInit)
diff --git a/pkg/disttask/framework/storage/task_table.go b/pkg/disttask/framework/storage/task_table.go
index aa8f82262faff..1fb3e1b0746cd 100644
--- a/pkg/disttask/framework/storage/task_table.go
+++ b/pkg/disttask/framework/storage/task_table.go
@@ -802,7 +802,7 @@ func (mgr *TaskManager) GetAllSubtasks(ctx context.Context) ([]*proto.SubtaskBas
// a stuck issue if the new version TiDB has less than 16 CPU count.
// We don't adjust the concurrency in subtask table because this field does not exist in v7.5.0.
// For details, see https://github.com/pingcap/tidb/issues/50894.
-// For the following versions, there is a check when submiting a new task. This function should be a no-op.
+// For the following versions, there is a check when submitting a new task. This function should be a no-op.
func (mgr *TaskManager) AdjustTaskOverflowConcurrency(ctx context.Context, se sessionctx.Context) error {
cpuCount, err := mgr.getCPUCountOfNode(ctx, se)
if err != nil {
diff --git a/pkg/disttask/framework/taskexecutor/BUILD.bazel b/pkg/disttask/framework/taskexecutor/BUILD.bazel
index a960e6586c76b..e65cc685e02a2 100644
--- a/pkg/disttask/framework/taskexecutor/BUILD.bazel
+++ b/pkg/disttask/framework/taskexecutor/BUILD.bazel
@@ -62,6 +62,7 @@ go_test(
"//pkg/disttask/framework/testutil",
"//pkg/kv",
"//pkg/testkit",
+ "//pkg/testkit/testfailpoint",
"//pkg/testkit/testsetup",
"//pkg/util/logutil",
"//pkg/util/memory",
diff --git a/pkg/disttask/framework/taskexecutor/task_executor.go b/pkg/disttask/framework/taskexecutor/task_executor.go
index ad9e444ee3859..2551173673f38 100644
--- a/pkg/disttask/framework/taskexecutor/task_executor.go
+++ b/pkg/disttask/framework/taskexecutor/task_executor.go
@@ -57,8 +57,6 @@ var (
// so cannot be run again.
ErrNonIdempotentSubtask = errors.New("subtask in running state and is not idempotent")
- // TestSyncChan is used to sync the test.
- TestSyncChan = make(chan struct{})
// MockTiDBDown is used to mock TiDB node down, return true if it's chosen.
MockTiDBDown func(execID string, task *proto.TaskBase) bool
)
@@ -131,14 +129,6 @@ func (e *BaseTaskExecutor) checkBalanceSubtask(ctx context.Context) {
e.logger.Error("get subtasks failed", zap.Error(err))
continue
}
- if ctx.Err() != nil {
- // workaround for https://github.com/pingcap/tidb/issues/50089
- // timeline to trigger this:
- // - this routine runs GetSubtasksByExecIDAndStepAndStates
- // - outer runSubtask finishes and cancel check-context
- // - GetSubtasksByExecIDAndStepAndStates returns with no err and no result
- return
- }
if len(subtasks) == 0 {
e.logger.Info("subtask is scheduled away, cancel running")
// cancels runStep, but leave the subtask state unchanged.
@@ -484,10 +474,7 @@ func (e *BaseTaskExecutor) onSubtaskFinished(ctx context.Context, executor execu
return
}
- failpoint.Inject("syncAfterSubtaskFinish", func() {
- TestSyncChan <- struct{}{}
- <-TestSyncChan
- })
+ failpoint.InjectCall("syncAfterSubtaskFinish")
}
// GetTaskBase implements TaskExecutor.GetTaskBase.
diff --git a/pkg/disttask/framework/taskexecutor/task_executor_testkit_test.go b/pkg/disttask/framework/taskexecutor/task_executor_testkit_test.go
index 56b786d308e6e..5c364844ac49c 100644
--- a/pkg/disttask/framework/taskexecutor/task_executor_testkit_test.go
+++ b/pkg/disttask/framework/taskexecutor/task_executor_testkit_test.go
@@ -27,6 +27,7 @@ import (
"github.com/pingcap/tidb/pkg/disttask/framework/testutil"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/util/logutil"
"github.com/stretchr/testify/require"
"github.com/tikv/client-go/v2/util"
@@ -73,7 +74,7 @@ func runOneTask(ctx context.Context, t *testing.T, mgr *storage.TaskManager, tas
func TestTaskExecutorBasic(t *testing.T) {
// must disable disttask framework to ensure the test pure.
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/domain/MockDisableDistTask", "return(true)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/domain/MockDisableDistTask", "return(true)")
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
pool := pools.NewResourcePool(func() (pools.Resource, error) {
diff --git a/pkg/disttask/framework/testutil/BUILD.bazel b/pkg/disttask/framework/testutil/BUILD.bazel
index 8e86cc0fb9ad4..a606528a86bcd 100644
--- a/pkg/disttask/framework/testutil/BUILD.bazel
+++ b/pkg/disttask/framework/testutil/BUILD.bazel
@@ -25,6 +25,7 @@ go_library(
"//pkg/sessionctx",
"//pkg/store/mockstore",
"//pkg/testkit",
+ "//pkg/testkit/testfailpoint",
"//pkg/util",
"//pkg/util/logutil",
"//pkg/util/sqlexec",
diff --git a/pkg/disttask/framework/testutil/context.go b/pkg/disttask/framework/testutil/context.go
index a8ee04aee6147..dbd46ff041916 100644
--- a/pkg/disttask/framework/testutil/context.go
+++ b/pkg/disttask/framework/testutil/context.go
@@ -30,6 +30,7 @@ import (
"github.com/pingcap/tidb/pkg/disttask/framework/taskexecutor"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
tidbutil "github.com/pingcap/tidb/pkg/util"
"github.com/stretchr/testify/require"
"github.com/tikv/client-go/v2/util"
@@ -107,9 +108,9 @@ func (c *TestDXFContext) init(nodeNum, cpuCount int, reduceCheckInterval bool) {
}
// all nodes are isometric
term := fmt.Sprintf("return(%d)", cpuCount)
- testkit.EnableFailPoint(c.T, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", term)
- testkit.EnableFailPoint(c.T, "github.com/pingcap/tidb/pkg/domain/MockDisableDistTask", "return(true)")
- testkit.EnableFailPoint(c.T, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/mockTaskExecutorNodes", "return()")
+ testfailpoint.Enable(c.T, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", term)
+ testfailpoint.Enable(c.T, "github.com/pingcap/tidb/pkg/domain/MockDisableDistTask", "return(true)")
+ testfailpoint.Enable(c.T, "github.com/pingcap/tidb/pkg/disttask/framework/scheduler/mockTaskExecutorNodes", "return()")
store := testkit.CreateMockStore(c.T)
pool := pools.NewResourcePool(func() (pools.Resource, error) {
return testkit.NewSession(c.T, store), nil
@@ -390,20 +391,6 @@ type TestContext struct {
CallTime int
}
-// InitTestContext inits test context for disttask tests.
-func InitTestContext(t *testing.T, nodeNum int) (context.Context, *gomock.Controller, *TestContext, *testkit.DistExecutionContext) {
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
- ctx := util.WithInternalSourceType(context.Background(), "scheduler")
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", "return(8)")
-
- executionContext := testkit.NewDistExecutionContext(t, nodeNum)
- testCtx := &TestContext{
- subtasksHasRun: make(map[string]map[int64]struct{}),
- }
- return ctx, ctrl, testCtx, executionContext
-}
-
// CollectSubtask collects subtask info
func (c *TestContext) CollectSubtask(subtask *proto.Subtask) {
key := getTaskStepKey(subtask.TaskID, subtask.Step)
diff --git a/pkg/disttask/framework/testutil/table_util.go b/pkg/disttask/framework/testutil/table_util.go
index 2b356aa763c7c..74969e5a9963c 100644
--- a/pkg/disttask/framework/testutil/table_util.go
+++ b/pkg/disttask/framework/testutil/table_util.go
@@ -28,6 +28,7 @@ import (
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/store/mockstore"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/pingcap/tidb/pkg/util/logutil"
"github.com/stretchr/testify/require"
"github.com/tikv/client-go/v2/util"
@@ -39,7 +40,7 @@ func InitTableTest(t *testing.T) (kv.Storage, *storage.TaskManager, context.Cont
store, pool := getResourcePool(t)
ctx := context.Background()
ctx = util.WithInternalSourceType(ctx, "table_test")
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", "return(8)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/util/cpu/mockNumCpu", "return(8)")
return store, getTaskManager(t, pool), ctx
}
@@ -52,7 +53,7 @@ func InitTableTestWithCancel(t *testing.T) (*storage.TaskManager, context.Contex
}
func getResourcePool(t *testing.T) (kv.Storage, *pools.ResourcePool) {
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/domain/MockDisableDistTask", "return(true)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/domain/MockDisableDistTask", "return(true)")
store := testkit.CreateMockStore(t, mockstore.WithStoreType(mockstore.EmbedUnistore))
tk := testkit.NewTestKit(t, store)
pool := pools.NewResourcePool(func() (pools.Resource, error) {
diff --git a/pkg/disttask/framework/testutil/task_util.go b/pkg/disttask/framework/testutil/task_util.go
index c80a6fc079b6b..e40786d37c07a 100644
--- a/pkg/disttask/framework/testutil/task_util.go
+++ b/pkg/disttask/framework/testutil/task_util.go
@@ -28,19 +28,29 @@ import (
// CreateSubTask adds a new task to subtask table.
// used for testing.
-func CreateSubTask(t *testing.T, gm *storage.TaskManager, taskID int64, step proto.Step, execID string, meta []byte, tp proto.TaskType, concurrency int) {
- InsertSubtask(t, gm, taskID, step, execID, meta, proto.SubtaskStatePending, tp, concurrency)
+func CreateSubTask(t *testing.T, gm *storage.TaskManager, taskID int64, step proto.Step, execID string, meta []byte, tp proto.TaskType, concurrency int) int64 {
+ return InsertSubtask(t, gm, taskID, step, execID, meta, proto.SubtaskStatePending, tp, concurrency)
}
// InsertSubtask adds a new subtask of any state to subtask table.
-func InsertSubtask(t *testing.T, gm *storage.TaskManager, taskID int64, step proto.Step, execID string, meta []byte, state proto.SubtaskState, tp proto.TaskType, concurrency int) {
+func InsertSubtask(t *testing.T, gm *storage.TaskManager, taskID int64, step proto.Step, execID string, meta []byte, state proto.SubtaskState, tp proto.TaskType, concurrency int) int64 {
ctx := context.Background()
ctx = util.WithInternalSourceType(ctx, "table_test")
+ var id int64
require.NoError(t, gm.WithNewSession(func(se sessionctx.Context) error {
_, err := sqlexec.ExecSQL(ctx, se.GetSQLExecutor(), `
insert into mysql.tidb_background_subtask(`+storage.InsertSubtaskColumns+`) values`+
`(%?, %?, %?, %?, %?, %?, %?, NULL, CURRENT_TIMESTAMP(), '{}', '{}')`,
step, taskID, execID, meta, state, proto.Type2Int(tp), concurrency)
- return err
+ if err != nil {
+ return err
+ }
+ rs, err := sqlexec.ExecSQL(ctx, se.GetSQLExecutor(), "select @@last_insert_id")
+ if err != nil {
+ return err
+ }
+ id = rs[0].GetInt64(0)
+ return nil
}))
+ return id
}
diff --git a/pkg/disttask/importinto/BUILD.bazel b/pkg/disttask/importinto/BUILD.bazel
index b22b92a439217..70087e97adcaf 100644
--- a/pkg/disttask/importinto/BUILD.bazel
+++ b/pkg/disttask/importinto/BUILD.bazel
@@ -64,6 +64,7 @@ go_library(
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_prometheus_client_golang//prometheus",
+ "@com_github_tikv_client_go_v2//oracle",
"@com_github_tikv_client_go_v2//util",
"@org_uber_go_atomic//:atomic",
"@org_uber_go_zap//:zap",
@@ -112,6 +113,7 @@ go_test(
"//pkg/parser/model",
"//pkg/session",
"//pkg/testkit",
+ "//pkg/testkit/testfailpoint",
"//pkg/util/mock",
"@com_github_docker_go_units//:go-units",
"@com_github_ngaut_pools//:pools",
diff --git a/pkg/disttask/importinto/planner.go b/pkg/disttask/importinto/planner.go
index 83d2294a96853..8fda8686facbb 100644
--- a/pkg/disttask/importinto/planner.go
+++ b/pkg/disttask/importinto/planner.go
@@ -38,6 +38,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/table/tables"
"github.com/pingcap/tidb/pkg/util/logutil"
+ "github.com/tikv/client-go/v2/oracle"
"go.uber.org/zap"
)
@@ -362,6 +363,13 @@ func generateWriteIngestSpecs(planCtx planner.PlanCtx, p *LogicalPlan) ([]planne
},
}, nil)
})
+
+ pTS, lTS, err := planCtx.Store.GetPDClient().GetTS(ctx)
+ if err != nil {
+ return nil, err
+ }
+ ts := oracle.ComposeTS(pTS, lTS)
+
specs := make([]planner.PipelineSpec, 0, 16)
for kvGroup, kvMeta := range kvMetas {
splitter, err1 := getRangeSplitter(ctx, controller.GlobalSortStore, kvMeta)
@@ -409,6 +417,7 @@ func generateWriteIngestSpecs(planCtx planner.PlanCtx, p *LogicalPlan) ([]planne
StatFiles: statFiles,
RangeSplitKeys: rangeSplitKeys,
RangeSplitSize: splitter.GetRangeSplitSize(),
+ TS: ts,
}
specs = append(specs, &WriteIngestSpec{m})
diff --git a/pkg/disttask/importinto/proto.go b/pkg/disttask/importinto/proto.go
index 05d09712798db..1545bcdf446a6 100644
--- a/pkg/disttask/importinto/proto.go
+++ b/pkg/disttask/importinto/proto.go
@@ -88,6 +88,7 @@ type WriteIngestStepMeta struct {
StatFiles []string `json:"stat-files"`
RangeSplitKeys [][]byte `json:"range-split-keys"`
RangeSplitSize int64 `json:"range-split-size"`
+ TS uint64 `json:"ts"`
Result Result
}
diff --git a/pkg/disttask/importinto/scheduler.go b/pkg/disttask/importinto/scheduler.go
index 27ab664787100..1dd1c0021ae7f 100644
--- a/pkg/disttask/importinto/scheduler.go
+++ b/pkg/disttask/importinto/scheduler.go
@@ -302,6 +302,7 @@ func (sch *ImportSchedulerExt) OnNextSubtasksBatch(
GlobalSort: sch.GlobalSort,
NextTaskStep: nextStep,
ExecuteNodesCnt: len(execIDs),
+ Store: sch.storeWithPD,
}
logicalPlan := &LogicalPlan{}
if err := logicalPlan.FromTaskMeta(task.Meta); err != nil {
@@ -613,10 +614,7 @@ func getLoadedRowCountOnGlobalSort(handle storage.TaskHandle, task *proto.Task)
}
func startJob(ctx context.Context, logger *zap.Logger, taskHandle storage.TaskHandle, taskMeta *TaskMeta, jobStep string) error {
- failpoint.Inject("syncBeforeJobStarted", func() {
- TestSyncChan <- struct{}{}
- <-TestSyncChan
- })
+ failpoint.InjectCall("syncBeforeJobStarted", taskMeta.JobID)
// retry for 3+6+12+24+(30-4)*30 ~= 825s ~= 14 minutes
// we consider all errors as retryable errors, except context done.
// the errors include errors happened when communicate with PD and TiKV.
@@ -630,9 +628,7 @@ func startJob(ctx context.Context, logger *zap.Logger, taskHandle storage.TaskHa
})
},
)
- failpoint.Inject("syncAfterJobStarted", func() {
- TestSyncChan <- struct{}{}
- })
+ failpoint.InjectCall("syncAfterJobStarted")
return err
}
diff --git a/pkg/disttask/importinto/scheduler_testkit_test.go b/pkg/disttask/importinto/scheduler_testkit_test.go
index 5efcfa01e0600..dc7de3662f7b3 100644
--- a/pkg/disttask/importinto/scheduler_testkit_test.go
+++ b/pkg/disttask/importinto/scheduler_testkit_test.go
@@ -31,6 +31,7 @@ import (
"github.com/pingcap/tidb/pkg/lightning/backend/external"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/stretchr/testify/require"
"github.com/tikv/client-go/v2/util"
)
@@ -169,7 +170,7 @@ func TestSchedulerExtLocalSort(t *testing.T) {
func TestSchedulerExtGlobalSort(t *testing.T) {
// Domain start scheduler manager automatically, we need to disable it as
// we test import task management in this case.
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/domain/MockDisableDistTask", "return(true)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/domain/MockDisableDistTask", "return(true)")
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
pool := pools.NewResourcePool(func() (pools.Resource, error) {
@@ -290,7 +291,7 @@ func TestSchedulerExtGlobalSort(t *testing.T) {
}
// to merge-sort stage
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/importinto/forceMergeSort", `return("data")`)
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/importinto/forceMergeSort", `return("data")`)
subtaskMetas, err = ext.OnNextSubtasksBatch(ctx, d, task, []string{":4000"}, ext.GetNextStep(&task.TaskBase))
require.NoError(t, err)
require.Len(t, subtaskMetas, 1)
@@ -326,7 +327,7 @@ func TestSchedulerExtGlobalSort(t *testing.T) {
}
// to write-and-ingest stage
- testkit.EnableFailPoint(t, "github.com/pingcap/tidb/pkg/disttask/importinto/mockWriteIngestSpecs", "return(true)")
+ testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/disttask/importinto/mockWriteIngestSpecs", "return(true)")
subtaskMetas, err = ext.OnNextSubtasksBatch(ctx, d, task, []string{":4000"}, ext.GetNextStep(&task.TaskBase))
require.NoError(t, err)
require.Len(t, subtaskMetas, 2)
diff --git a/pkg/disttask/importinto/subtask_executor.go b/pkg/disttask/importinto/subtask_executor.go
index 2592c90e8a179..2ca7fc241cc0b 100644
--- a/pkg/disttask/importinto/subtask_executor.go
+++ b/pkg/disttask/importinto/subtask_executor.go
@@ -33,9 +33,6 @@ import (
"go.uber.org/zap"
)
-// TestSyncChan is used to test.
-var TestSyncChan = make(chan struct{})
-
// MiniTaskExecutor is the interface for a minimal task executor.
// exported for testing.
type MiniTaskExecutor interface {
@@ -64,10 +61,7 @@ func (e *importMinimalTaskExecutor) Run(ctx context.Context, dataWriter, indexWr
failpoint.Inject("errorWhenSortChunk", func() {
failpoint.Return(errors.New("occur an error when sort chunk"))
})
- failpoint.Inject("syncBeforeSortChunk", func() {
- TestSyncChan <- struct{}{}
- <-TestSyncChan
- })
+ failpoint.InjectCall("syncBeforeSortChunk")
chunkCheckpoint := toChunkCheckpoint(e.mTtask.Chunk)
sharedVars := e.mTtask.SharedVars
checksum := verify.NewKVGroupChecksumWithKeyspace(sharedVars.TableImporter.GetKeySpace())
@@ -107,10 +101,7 @@ func (e *importMinimalTaskExecutor) Run(ctx context.Context, dataWriter, indexWr
// postProcess does the post-processing for the task.
func postProcess(ctx context.Context, store kv.Storage, taskMeta *TaskMeta, subtaskMeta *PostProcessStepMeta, logger *zap.Logger) (err error) {
- failpoint.Inject("syncBeforePostProcess", func() {
- TestSyncChan <- struct{}{}
- <-TestSyncChan
- })
+ failpoint.InjectCall("syncBeforePostProcess", taskMeta.JobID)
callLog := log.BeginTask(logger, "post process")
defer func() {
diff --git a/pkg/disttask/importinto/task_executor.go b/pkg/disttask/importinto/task_executor.go
index aba2028a1cbd5..78f41ff03c0f7 100644
--- a/pkg/disttask/importinto/task_executor.go
+++ b/pkg/disttask/importinto/task_executor.go
@@ -417,6 +417,7 @@ func (e *writeAndIngestStepExecutor) RunSubtask(ctx context.Context, subtask *pr
TotalKVCount: 0,
CheckHotspot: false,
},
+ TS: sm.TS,
}, engineUUID)
if err != nil {
return err
diff --git a/pkg/domain/domain.go b/pkg/domain/domain.go
index 0eb177f35cdd3..5fc35ace87bac 100644
--- a/pkg/domain/domain.go
+++ b/pkg/domain/domain.go
@@ -240,20 +240,27 @@ func (do *Domain) loadInfoSchema(startTS uint64) (infoschema.InfoSchema, bool, i
}
// fetch the commit timestamp of the schema diff
schemaTs, err := do.getTimestampForSchemaVersionWithNonEmptyDiff(m, neededSchemaVersion, startTS)
- schemaTsOrStartTs := schemaTs
if err != nil {
logutil.BgLogger().Warn("failed to get schema version", zap.Error(err), zap.Int64("version", neededSchemaVersion))
schemaTs = 0
- schemaTsOrStartTs = startTS
}
if is := do.infoCache.GetByVersion(neededSchemaVersion); is != nil {
+ isV2, raw := infoschema.IsV2(is)
+ if isV2 {
+ // Copy the infoschema V2 instance and update its ts.
+ // For example, the DDL run 30 minutes ago, GC happened 10 minutes ago. If we use
+ // that infoschema it would get error "GC life time is shorter than transaction
+ // duration" when visiting TiKV.
+ // So we keep updating the ts of the infoschema v2.
+ is = raw.CloneAndUpdateTS(startTS)
+ }
+
// try to insert here as well to correct the schemaTs if previous is wrong
// the insert method check if schemaTs is zero
do.infoCache.Insert(is, schemaTs)
enableV2 := variable.SchemaCacheSize.Load() > 0
- isV2 := infoschema.IsV2(is)
if enableV2 == isV2 {
return is, true, 0, nil, nil
}
@@ -270,14 +277,16 @@ func (do *Domain) loadInfoSchema(startTS uint64) (infoschema.InfoSchema, bool, i
// 1. Not first time bootstrap loading, which needs a full load.
// 2. It is newer than the current one, so it will be "the current one" after this function call.
// 3. There are less 100 diffs.
- // 4. No regenrated schema diff.
+ // 4. No regenerated schema diff.
startTime := time.Now()
if currentSchemaVersion != 0 && neededSchemaVersion > currentSchemaVersion && neededSchemaVersion-currentSchemaVersion < LoadSchemaDiffVersionGapThreshold {
- is, relatedChanges, diffTypes, err := do.tryLoadSchemaDiffs(m, currentSchemaVersion, neededSchemaVersion, schemaTsOrStartTs)
+ is, relatedChanges, diffTypes, err := do.tryLoadSchemaDiffs(m, currentSchemaVersion, neededSchemaVersion, startTS)
if err == nil {
infoschema_metrics.LoadSchemaDurationLoadDiff.Observe(time.Since(startTime).Seconds())
+ isV2, _ := infoschema.IsV2(is)
do.infoCache.Insert(is, schemaTs)
logutil.BgLogger().Info("diff load InfoSchema success",
+ zap.Bool("isV2", isV2),
zap.Int64("currentSchemaVersion", currentSchemaVersion),
zap.Int64("neededSchemaVersion", neededSchemaVersion),
zap.Duration("start time", time.Since(startTime)),
@@ -305,8 +314,6 @@ func (do *Domain) loadInfoSchema(startTS uint64) (infoschema.InfoSchema, bool, i
if err != nil {
return nil, false, currentSchemaVersion, nil, err
}
- // clear data
- do.infoCache.Data = infoschema.NewData()
newISBuilder, err := infoschema.NewBuilder(do, do.sysFacHack, do.infoCache.Data).InitWithDBInfos(schemas, policies, resourceGroups, neededSchemaVersion)
if err != nil {
return nil, false, currentSchemaVersion, nil, err
@@ -317,7 +324,7 @@ func (do *Domain) loadInfoSchema(startTS uint64) (infoschema.InfoSchema, bool, i
zap.Int64("neededSchemaVersion", neededSchemaVersion),
zap.Duration("start time", time.Since(startTime)))
- is := newISBuilder.Build(schemaTs)
+ is := newISBuilder.Build(startTS)
do.infoCache.Insert(is, schemaTs)
return is, false, currentSchemaVersion, nil, nil
}
@@ -429,7 +436,14 @@ func (*Domain) fetchSchemasWithTables(schemas []*model.DBInfo, m *meta.Meta, don
infoschema.ConvertCharsetCollateToLowerCaseIfNeed(tbl)
// Check whether the table is in repair mode.
if domainutil.RepairInfo.InRepairMode() && domainutil.RepairInfo.CheckAndFetchRepairedTable(di, tbl) {
- continue
+ if tbl.State != model.StatePublic {
+ // Do not load it because we are reparing the table and the table info could be `bad`
+ // before repair is done.
+ continue
+ }
+ // If the state is public, it means that the DDL job is done, but the table
+ // haven't been deleted from the repair table list.
+ // Since the repairment is done and table is visible, we should load it.
}
di.Tables = append(di.Tables, tbl)
}
@@ -441,7 +455,7 @@ func (*Domain) fetchSchemasWithTables(schemas []*model.DBInfo, m *meta.Meta, don
// Return true if the schema is loaded successfully.
// Return false if the schema can not be loaded by schema diff, then we need to do full load.
// The second returned value is the delta updated table and partition IDs.
-func (do *Domain) tryLoadSchemaDiffs(m *meta.Meta, usedVersion, newVersion int64, schemaTS uint64) (infoschema.InfoSchema, *transaction.RelatedSchemaChange, []string, error) {
+func (do *Domain) tryLoadSchemaDiffs(m *meta.Meta, usedVersion, newVersion int64, startTS uint64) (infoschema.InfoSchema, *transaction.RelatedSchemaChange, []string, error) {
var diffs []*model.SchemaDiff
for usedVersion < newVersion {
usedVersion++
@@ -452,6 +466,8 @@ func (do *Domain) tryLoadSchemaDiffs(m *meta.Meta, usedVersion, newVersion int64
if diff == nil {
// Empty diff means the txn of generating schema version is committed, but the txn of `runDDLJob` is not or fail.
// It is safe to skip the empty diff because the infoschema is new enough and consistent.
+ logutil.BgLogger().Info("diff load InfoSchema get empty schema diff", zap.Int64("version", usedVersion))
+ do.infoCache.InsertEmptySchemaVersion(usedVersion)
continue
}
diffs = append(diffs, diff)
@@ -466,6 +482,7 @@ func (do *Domain) tryLoadSchemaDiffs(m *meta.Meta, usedVersion, newVersion int64
diffTypes := make([]string, 0, len(diffs))
for _, diff := range diffs {
if diff.RegenerateSchemaMap {
+ do.infoCache.Data = infoschema.NewData()
return nil, nil, nil, errors.Errorf("Meets a schema diff with RegenerateSchemaMap flag")
}
ids, err := builder.ApplyDiff(m, diff)
@@ -482,7 +499,7 @@ func (do *Domain) tryLoadSchemaDiffs(m *meta.Meta, usedVersion, newVersion int64
}
}
- is := builder.Build(schemaTS)
+ is := builder.Build(startTS)
relatedChange := transaction.RelatedSchemaChange{}
relatedChange.PhyTblIDS = phyTblIDs
relatedChange.ActionTypes = actions
@@ -504,7 +521,7 @@ func (do *Domain) InfoSchema() infoschema.InfoSchema {
// GetSnapshotInfoSchema gets a snapshot information schema.
func (do *Domain) GetSnapshotInfoSchema(snapshotTS uint64) (infoschema.InfoSchema, error) {
- // if the snapshotTS is new enough, we can get infoschema directly through sanpshotTS.
+ // if the snapshotTS is new enough, we can get infoschema directly through snapshotTS.
if is := do.infoCache.GetBySnapshotTS(snapshotTS); is != nil {
return is, nil
}
@@ -606,7 +623,7 @@ func (do *Domain) Reload() error {
is, hitCache, oldSchemaVersion, changes, err := do.loadInfoSchema(version)
if err != nil {
if version = getFlashbackStartTSFromErrorMsg(err); version != 0 {
- // use the lastest available version to create domain
+ // use the latest available version to create domain
version--
is, hitCache, oldSchemaVersion, changes, err = do.loadInfoSchema(version)
}
@@ -942,7 +959,7 @@ func (do *Domain) loadSchemaInLoop(ctx context.Context, lease time.Duration) {
}
// The schema maybe changed, must reload schema then the schema validator can restart.
exitLoop := do.mustReload()
- // domain is cosed.
+ // domain is closed.
if exitLoop {
logutil.BgLogger().Error("domain is closed, exit loadSchemaInLoop")
return
@@ -961,7 +978,7 @@ func (do *Domain) loadSchemaInLoop(ctx context.Context, lease time.Duration) {
}
// mustRestartSyncer tries to restart the SchemaSyncer.
-// It returns until it's successful or the domain is stoped.
+// It returns until it's successful or the domain is stopped.
func (do *Domain) mustRestartSyncer(ctx context.Context) error {
syncer := do.ddl.SchemaSyncer()
@@ -2091,7 +2108,7 @@ func (do *Domain) GetHistoricalStatsWorker() *HistoricalStatsWorker {
return do.historicalStatsWorker
}
-// EnableDumpHistoricalStats used to control whether enbale dump stats for unit test
+// EnableDumpHistoricalStats used to control whether enable dump stats for unit test
var enableDumpHistoricalStats atomic.Bool
// StartHistoricalStatsWorker start historical workers running
@@ -2249,6 +2266,25 @@ func (do *Domain) UpdateTableStatsLoop(ctx, initStatsCtx sessionctx.Context) err
},
"analyzeJobsCleanupWorker",
)
+ do.wg.Run(
+ func() {
+ // The initStatsCtx is used to store the internal session for initializing stats,
+ // so we need the gc min start ts calculation to track it as an internal session.
+ // Since the session manager may not be ready at this moment, `infosync.StoreInternalSession` can fail.
+ // we need to retry until the session manager is ready or the init stats completes.
+ for !infosync.StoreInternalSession(initStatsCtx) {
+ waitRetry := time.After(time.Second)
+ select {
+ case <-do.StatsHandle().InitStatsDone:
+ return
+ case <-waitRetry:
+ }
+ }
+ <-do.StatsHandle().InitStatsDone
+ infosync.DeleteInternalSession(initStatsCtx)
+ },
+ "RemoveInitStatsFromInternalSessions",
+ )
return nil
}
@@ -2267,6 +2303,7 @@ func (do *Domain) StartLoadStatsSubWorkers(ctxList []sessionctx.Context) {
do.wg.Add(1)
go statsHandle.SubLoadWorker(ctx, do.exit, do.wg)
}
+ logutil.BgLogger().Info("start load stats sub workers", zap.Int("worker count", len(ctxList)))
}
func (do *Domain) newOwnerManager(prompt, ownerKey string) owner.Manager {
diff --git a/pkg/domain/infosync/info.go b/pkg/domain/infosync/info.go
index 51dfbcac6e16b..908d84df8aced 100644
--- a/pkg/domain/infosync/info.go
+++ b/pkg/domain/infosync/info.go
@@ -1243,16 +1243,18 @@ func ConfigureTiFlashPDForPartitions(accel bool, definitions *[]model.PartitionD
}
// StoreInternalSession is the entry function for store an internal session to SessionManager.
-func StoreInternalSession(se any) {
+// return whether the session is stored successfully.
+func StoreInternalSession(se any) bool {
is, err := getGlobalInfoSyncer()
if err != nil {
- return
+ return false
}
sm := is.GetSessionManager()
if sm == nil {
- return
+ return false
}
sm.StoreInternalSession(se)
+ return true
}
// DeleteInternalSession is the entry function for delete an internal session from SessionManager.
diff --git a/pkg/domain/plan_replayer.go b/pkg/domain/plan_replayer.go
index ff67d64663929..9e0dd80efa40c 100644
--- a/pkg/domain/plan_replayer.go
+++ b/pkg/domain/plan_replayer.go
@@ -501,7 +501,7 @@ func (h *planReplayerTaskDumpHandle) GetWorker() *planReplayerTaskDumpWorker {
return h.workers[0]
}
-// Close make finished flag ture
+// Close make finished flag true
func (h *planReplayerTaskDumpHandle) Close() {
close(h.taskCH)
}
diff --git a/pkg/domain/resourcegroup/runaway.go b/pkg/domain/resourcegroup/runaway.go
index b593776df15ec..045286236ec80 100644
--- a/pkg/domain/resourcegroup/runaway.go
+++ b/pkg/domain/resourcegroup/runaway.go
@@ -317,7 +317,7 @@ func (rm *RunawayManager) addWatchList(record *QuarantineRecord, ttl time.Durati
rm.queryLock.Lock()
defer rm.queryLock.Unlock()
if item != nil {
- // check the ID because of the eariler scan.
+ // check the ID because of the earlier scan.
if item.ID == record.ID {
return
}
diff --git a/pkg/domain/ru_stats.go b/pkg/domain/ru_stats.go
index 42dd328f922be..68a308ac3bfc9 100644
--- a/pkg/domain/ru_stats.go
+++ b/pkg/domain/ru_stats.go
@@ -34,7 +34,7 @@ import (
const (
maxRetryCount int = 10
ruStatsInterval time.Duration = 24 * time.Hour
- // only keep stats rows for last 3 monthes(92 days at most).
+ // only keep stats rows for last 3 months(92 days at most).
ruStatsGCDuration time.Duration = 92 * ruStatsInterval
gcBatchSize int64 = 1000
)
diff --git a/pkg/domain/runaway.go b/pkg/domain/runaway.go
index e5fa14e85198d..b67d941bd2d18 100644
--- a/pkg/domain/runaway.go
+++ b/pkg/domain/runaway.go
@@ -240,7 +240,7 @@ func (do *Domain) RemoveRunawayWatch(recordID int64) error {
func (do *Domain) runawayRecordFlushLoop() {
defer util.Recover(metrics.LabelDomain, "runawayRecordFlushLoop", nil, false)
- // this times is used to batch flushing rocords, with 1s duration,
+ // this times is used to batch flushing records, with 1s duration,
// we can guarantee a watch record can be seen by the user within 1s.
runawayRecordFluashTimer := time.NewTimer(runawayRecordFlushInterval)
runawayRecordGCTicker := time.NewTicker(runawayRecordGCInterval)
diff --git a/pkg/domain/schema_validator.go b/pkg/domain/schema_validator.go
index c018085d0141f..5a5b1e3a22618 100644
--- a/pkg/domain/schema_validator.go
+++ b/pkg/domain/schema_validator.go
@@ -112,7 +112,7 @@ func (s *schemaValidator) Restart() {
defer s.mux.Unlock()
s.isStarted = true
if s.do != nil {
- // When this instance reconnects PD, we should record the latest schema verion after mustReload(),
+ // When this instance reconnects PD, we should record the latest schema version after mustReload(),
// to prevent write txns using a stale schema version by aborting them before commit.
// However, the problem still exists for read-only txns.
s.restartSchemaVer = s.do.InfoSchema().SchemaMetaVersion()
diff --git a/pkg/executor/BUILD.bazel b/pkg/executor/BUILD.bazel
index 5d4e30394b25c..b3b89fd9f2a2d 100644
--- a/pkg/executor/BUILD.bazel
+++ b/pkg/executor/BUILD.bazel
@@ -168,8 +168,8 @@ go_library(
"//pkg/statistics",
"//pkg/statistics/handle",
"//pkg/statistics/handle/cache",
- "//pkg/statistics/handle/globalstats",
"//pkg/statistics/handle/storage",
+ "//pkg/statistics/handle/types",
"//pkg/statistics/handle/util",
"//pkg/store/driver/backoff",
"//pkg/store/driver/txn",
@@ -426,6 +426,7 @@ go_test(
"//pkg/testkit",
"//pkg/testkit/external",
"//pkg/testkit/testdata",
+ "//pkg/testkit/testfailpoint",
"//pkg/testkit/testmain",
"//pkg/testkit/testsetup",
"//pkg/types",
@@ -454,12 +455,15 @@ go_test(
"//pkg/util/syncutil",
"//pkg/util/tableutil",
"//pkg/util/topsql/state",
+ "@com_github_docker_go_units//:go-units",
"@com_github_gorilla_mux//:mux",
"@com_github_hashicorp_go_version//:go-version",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_fn//:fn",
+ "@com_github_pingcap_kvproto//pkg/brpb",
"@com_github_pingcap_kvproto//pkg/diagnosticspb",
+ "@com_github_pingcap_kvproto//pkg/encryptionpb",
"@com_github_pingcap_kvproto//pkg/kvrpcpb",
"@com_github_pingcap_kvproto//pkg/metapb",
"@com_github_pingcap_log//:log",
diff --git a/pkg/executor/adapter.go b/pkg/executor/adapter.go
index 71fba67f307e2..b4cf452688b4c 100644
--- a/pkg/executor/adapter.go
+++ b/pkg/executor/adapter.go
@@ -1858,8 +1858,17 @@ func (a *ExecStmt) SummaryStmt(succ bool) {
sessVars.SetPrevStmtDigest(digest.String())
// No need to encode every time, so encode lazily.
- planGenerator := func() (string, string) {
- return getEncodedPlan(stmtCtx, !sessVars.InRestrictedSQL)
+ planGenerator := func() (p string, h string, e any) {
+ defer func() {
+ e = recover()
+ if e != nil {
+ logutil.BgLogger().Warn("fail to generate plan info",
+ zap.Stack("backtrace"),
+ zap.Any("error", e))
+ }
+ }()
+ p, h = getEncodedPlan(stmtCtx, !sessVars.InRestrictedSQL)
+ return
}
var binPlanGen func() string
if variable.GenerateBinaryPlan.Load() {
diff --git a/pkg/executor/adapter_test.go b/pkg/executor/adapter_test.go
index bc5d08453aa8e..4310fefe17a14 100644
--- a/pkg/executor/adapter_test.go
+++ b/pkg/executor/adapter_test.go
@@ -15,11 +15,17 @@
package executor_test
import (
+ "context"
+ "sync"
"testing"
+ "github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/executor"
+ "github.com/pingcap/tidb/pkg/session"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
+ "github.com/pingcap/tidb/pkg/testkit"
"github.com/stretchr/testify/require"
+ "github.com/tikv/client-go/v2/util"
)
func TestFormatSQL(t *testing.T) {
@@ -32,3 +38,38 @@ func TestFormatSQL(t *testing.T) {
val = executor.FormatSQL("aaaaaaaaaaaaaaaaaaaa")
require.Equal(t, "aaaaa(len:20)", val.String())
}
+
+func TestContextCancelWhenReadFromCopIterator(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t(a int)")
+ tk.MustExec("insert into t values(1)")
+
+ syncCh := make(chan struct{})
+ require.NoError(t, failpoint.EnableCall("github.com/pingcap/tidb/pkg/store/copr/CtxCancelBeforeReceive",
+ func(ctx context.Context) {
+ if ctx.Value("TestContextCancel") == "test" {
+ syncCh <- struct{}{}
+ <-syncCh
+ }
+ },
+ ))
+ ctx := context.WithValue(context.Background(), "TestContextCancel", "test")
+ ctx, cancelFunc := context.WithCancel(ctx)
+ defer cancelFunc()
+ var wg sync.WaitGroup
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ ctx = util.WithInternalSourceType(ctx, "scheduler")
+ rs, err := tk.Session().ExecuteInternal(ctx, "select * from test.t")
+ require.NoError(t, err)
+ _, err2 := session.ResultSetToStringSlice(ctx, tk.Session(), rs)
+ require.ErrorIs(t, err2, context.Canceled)
+ }()
+ <-syncCh
+ cancelFunc()
+ syncCh <- struct{}{}
+ wg.Wait()
+}
diff --git a/pkg/executor/admin.go b/pkg/executor/admin.go
index a859de119e8e2..f38b8bb42eba7 100644
--- a/pkg/executor/admin.go
+++ b/pkg/executor/admin.go
@@ -28,7 +28,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/terror"
- plannercore "github.com/pingcap/tidb/pkg/planner/core"
+ plannercore "github.com/pingcap/tidb/pkg/planner/util"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/table/tables"
"github.com/pingcap/tidb/pkg/tablecodec"
@@ -285,7 +285,7 @@ func (e *RecoverIndexExec) buildTableScan(ctx context.Context, txn kv.Transactio
return nil, err
}
- // Actually, with limitCnt, the match datas maybe only in one region, so let the concurrency to be 1,
+ // Actually, with limitCnt, the match data maybe only in one region, so let the concurrency to be 1,
// avoid unnecessary region scan.
kvReq.Concurrency = 1
result, err := distsql.Select(ctx, e.Ctx().GetDistSQLCtx(), kvReq, e.columnsTypes())
@@ -381,6 +381,9 @@ func (e *RecoverIndexExec) fetchRecoverRows(ctx context.Context, srcResult dists
if err != nil {
return nil, err
}
+ if e.index.Meta().Global {
+ handle = kv.NewPartitionHandle(e.physicalID, handle)
+ }
idxVals, err := e.buildIndexedValues(row, e.idxValsBufs[result.scanRowCount], e.colFieldTypes, idxValLen)
if err != nil {
return nil, err
@@ -472,12 +475,11 @@ func (e *RecoverIndexExec) batchMarkDup(txn kv.Transaction, rows []recoverRows)
// 1. unique-key is duplicate and the handle is equal, skip it.
// 2. unique-key is duplicate and the handle is not equal, data is not consistent, log it and skip it.
// 3. non-unique-key is duplicate, skip it.
- isCommonHandle := e.table.Meta().IsCommonHandle
for i, key := range e.batchKeys {
val, found := values[string(key)]
if found {
if distinctFlags[i] {
- handle, err1 := tablecodec.DecodeHandleInUniqueIndexValue(val, isCommonHandle)
+ handle, err1 := tablecodec.DecodeHandleInIndexValue(val)
if err1 != nil {
return err1
}
@@ -627,10 +629,13 @@ func (e *CleanupIndexExec) batchGetRecord(txn kv.Transaction) (map[string][]byte
func (e *CleanupIndexExec) deleteDanglingIdx(txn kv.Transaction, values map[string][]byte) error {
for _, k := range e.batchKeys {
if _, found := values[string(k)]; !found {
- _, handle, err := tablecodec.DecodeRecordKey(k)
+ pid, handle, err := tablecodec.DecodeRecordKey(k)
if err != nil {
return err
}
+ if e.index.Meta().Global {
+ handle = kv.NewPartitionHandle(pid, handle)
+ }
handleIdxValsGroup, ok := e.idxValues.Get(handle)
if !ok {
return errors.Trace(errors.Errorf("batch keys are inconsistent with handles"))
@@ -688,6 +693,9 @@ func (e *CleanupIndexExec) fetchIndex(ctx context.Context, txn kv.Transaction) e
if err != nil {
return err
}
+ if e.index.Meta().Global {
+ handle = kv.NewPartitionHandle(row.GetInt64(row.Len()-1), handle)
+ }
idxVals := extractIdxVals(row, e.idxValsBufs[e.scanRowCnt], e.idxColFieldTypes, idxColLen)
e.idxValsBufs[e.scanRowCnt] = idxVals
existingIdxVals, ok := e.idxValues.Get(handle)
@@ -724,7 +732,7 @@ func (e *CleanupIndexExec) Next(ctx context.Context, req *chunk.Chunk) error {
}
var err error
- if tbl, ok := e.table.(table.PartitionedTable); ok {
+ if tbl, ok := e.table.(table.PartitionedTable); ok && !e.index.Meta().Global {
pi := e.table.Meta().GetPartitionInfo()
for _, p := range pi.Definitions {
e.table = tbl.GetPartition(p.ID)
diff --git a/pkg/executor/aggfuncs/aggfunc_test.go b/pkg/executor/aggfuncs/aggfunc_test.go
index 4f8f1f0e8a354..9e57e51e91ce8 100644
--- a/pkg/executor/aggfuncs/aggfunc_test.go
+++ b/pkg/executor/aggfuncs/aggfunc_test.go
@@ -389,7 +389,7 @@ func testMultiArgsMergePartialResult(t *testing.T, ctx *mock.Context, p multiArg
{Expr: args[0], Desc: true},
}
}
- ctor := collate.GetCollator(args[0].GetType().GetCollate())
+ ctor := collate.GetCollator(args[0].GetType(ctx).GetCollate())
partialDesc, finalDesc := desc.Split([]int{0, 1})
// build partial func for partial phase.
@@ -684,7 +684,7 @@ func testMultiArgsAggFunc(t *testing.T, ctx *mock.Context, p multiArgsAggTest) {
{Expr: args[0], Desc: true},
}
}
- ctor := collate.GetCollator(args[0].GetType().GetCollate())
+ ctor := collate.GetCollator(args[0].GetType(ctx).GetCollate())
finalFunc := aggfuncs.Build(ctx, desc, 0)
finalPr, _ := finalFunc.AllocPartialResult()
resultChk := chunk.NewChunkWithCapacity([]*types.FieldType{desc.RetTp}, 1)
diff --git a/pkg/executor/aggfuncs/builder.go b/pkg/executor/aggfuncs/builder.go
index 2f33019850d21..abbff30bae6fb 100644
--- a/pkg/executor/aggfuncs/builder.go
+++ b/pkg/executor/aggfuncs/builder.go
@@ -37,7 +37,7 @@ type AggFuncBuildContext = exprctx.ExprContext
func Build(ctx AggFuncBuildContext, aggFuncDesc *aggregation.AggFuncDesc, ordinal int) AggFunc {
switch aggFuncDesc.Name {
case ast.AggFuncCount:
- return buildCount(aggFuncDesc, ordinal)
+ return buildCount(ctx.GetEvalCtx(), aggFuncDesc, ordinal)
case ast.AggFuncSum:
return buildSum(ctx, aggFuncDesc, ordinal)
case ast.AggFuncAvg:
@@ -103,9 +103,9 @@ func BuildWindowFunctions(ctx AggFuncBuildContext, windowFuncDesc *aggregation.A
return buildLag(ctx, windowFuncDesc, ordinal)
case ast.AggFuncMax:
// The max/min aggFunc using in the window function will using the sliding window algo.
- return buildMaxMinInWindowFunction(windowFuncDesc, ordinal, true)
+ return buildMaxMinInWindowFunction(ctx, windowFuncDesc, ordinal, true)
case ast.AggFuncMin:
- return buildMaxMinInWindowFunction(windowFuncDesc, ordinal, false)
+ return buildMaxMinInWindowFunction(ctx, windowFuncDesc, ordinal, false)
default:
return Build(ctx, windowFuncDesc, ordinal)
}
@@ -144,9 +144,9 @@ func buildApproxCountDistinct(aggFuncDesc *aggregation.AggFuncDesc, ordinal int)
return nil
}
-func getEvalTypeForApproxPercentile(aggFuncDesc *aggregation.AggFuncDesc) types.EvalType {
- evalType := aggFuncDesc.Args[0].GetType().EvalType()
- argType := aggFuncDesc.Args[0].GetType().GetType()
+func getEvalTypeForApproxPercentile(ctx expression.EvalContext, aggFuncDesc *aggregation.AggFuncDesc) types.EvalType {
+ evalType := aggFuncDesc.Args[0].GetType(ctx).EvalType()
+ argType := aggFuncDesc.Args[0].GetType(ctx).GetType()
// Sometimes `mysql.EnumSetAsIntFlag` may be set to true, such as when join,
// which is unexpected for `buildApproxPercentile` and `mysql.TypeEnum` and `mysql.TypeSet` will return unexpected `ETInt` here,
@@ -174,7 +174,7 @@ func buildApproxPercentile(sctx AggFuncBuildContext, aggFuncDesc *aggregation.Ag
base := basePercentile{percent: int(percent), baseAggFunc: baseAggFunc{args: aggFuncDesc.Args, ordinal: ordinal}}
- evalType := getEvalTypeForApproxPercentile(aggFuncDesc)
+ evalType := getEvalTypeForApproxPercentile(sctx.GetEvalCtx(), aggFuncDesc)
switch aggFuncDesc.Mode {
case aggregation.CompleteMode, aggregation.Partial1Mode, aggregation.FinalMode:
switch evalType {
@@ -198,7 +198,7 @@ func buildApproxPercentile(sctx AggFuncBuildContext, aggFuncDesc *aggregation.Ag
}
// buildCount builds the AggFunc implementation for function "COUNT".
-func buildCount(aggFuncDesc *aggregation.AggFuncDesc, ordinal int) AggFunc {
+func buildCount(ctx expression.EvalContext, aggFuncDesc *aggregation.AggFuncDesc, ordinal int) AggFunc {
// If mode is DedupMode, we return nil for not implemented.
if aggFuncDesc.Mode == aggregation.DedupMode {
return nil // not implemented yet.
@@ -220,7 +220,7 @@ func buildCount(aggFuncDesc *aggregation.AggFuncDesc, ordinal int) AggFunc {
// so they're in exception for now.
// TODO: add hashCode method for all evaluate types (decimal, Time, Duration, JSON).
// https://github.com/pingcap/tidb/issues/15857
- switch aggFuncDesc.Args[0].GetType().EvalType() {
+ switch aggFuncDesc.Args[0].GetType(ctx).EvalType() {
case types.ETInt:
return &countOriginalWithDistinct4Int{baseCount{base}}
case types.ETReal:
@@ -238,7 +238,7 @@ func buildCount(aggFuncDesc *aggregation.AggFuncDesc, ordinal int) AggFunc {
switch aggFuncDesc.Mode {
case aggregation.CompleteMode, aggregation.Partial1Mode:
- switch aggFuncDesc.Args[0].GetType().EvalType() {
+ switch aggFuncDesc.Args[0].GetType(ctx).EvalType() {
case types.ETInt:
return &countOriginal4Int{baseCount{base}}
case types.ETReal:
@@ -437,7 +437,7 @@ func buildMaxMin(aggFuncDesc *aggregation.AggFuncDesc, ordinal int, isMax bool)
}
// buildMaxMin builds the AggFunc implementation for function "MAX" and "MIN" using by window function.
-func buildMaxMinInWindowFunction(aggFuncDesc *aggregation.AggFuncDesc, ordinal int, isMax bool) AggFunc {
+func buildMaxMinInWindowFunction(ctx AggFuncBuildContext, aggFuncDesc *aggregation.AggFuncDesc, ordinal int, isMax bool) AggFunc {
base := buildMaxMin(aggFuncDesc, ordinal, isMax)
// build max/min aggFunc for window function using sliding window
switch baseAggFunc := base.(type) {
@@ -452,7 +452,7 @@ func buildMaxMinInWindowFunction(aggFuncDesc *aggregation.AggFuncDesc, ordinal i
case *maxMin4Decimal:
return &maxMin4DecimalSliding{*baseAggFunc, windowInfo{}}
case *maxMin4String:
- return &maxMin4StringSliding{*baseAggFunc, windowInfo{}}
+ return &maxMin4StringSliding{*baseAggFunc, windowInfo{}, baseAggFunc.args[0].GetType(ctx.GetEvalCtx()).GetCollate()}
case *maxMin4Time:
return &maxMin4TimeSliding{*baseAggFunc, windowInfo{}}
case *maxMin4Duration:
@@ -488,12 +488,25 @@ func buildGroupConcat(ctx AggFuncBuildContext, aggFuncDesc *aggregation.AggFuncD
}
if aggFuncDesc.HasDistinct {
if len(aggFuncDesc.OrderByItems) > 0 {
- return &groupConcatDistinctOrder{base}
+ desc := make([]bool, len(base.byItems))
+ ctors := make([]collate.Collator, 0, len(base.byItems))
+ for i, byItem := range base.byItems {
+ desc[i] = byItem.Desc
+ ctors = append(ctors, collate.GetCollator(byItem.Expr.GetType(ctx.GetEvalCtx()).GetCollate()))
+ }
+ return &groupConcatDistinctOrder{base, ctors, desc}
}
return &groupConcatDistinct{base}
}
if len(aggFuncDesc.OrderByItems) > 0 {
- return &groupConcatOrder{base}
+ desc := make([]bool, len(base.byItems))
+ ctors := make([]collate.Collator, 0, len(base.byItems))
+ for i, byItem := range base.byItems {
+ desc[i] = byItem.Desc
+ ctors = append(ctors, collate.GetCollator(byItem.Expr.GetType(ctx.GetEvalCtx()).GetCollate()))
+ }
+
+ return &groupConcatOrder{base, ctors, desc}
}
return &groupConcat{base}
}
diff --git a/pkg/executor/aggfuncs/func_count_distinct.go b/pkg/executor/aggfuncs/func_count_distinct.go
index 8913b0cb69a98..13c6a2f53e808 100644
--- a/pkg/executor/aggfuncs/func_count_distinct.go
+++ b/pkg/executor/aggfuncs/func_count_distinct.go
@@ -267,7 +267,7 @@ func (e *countOriginalWithDistinct4String) AppendFinalResult2Chunk(_ AggFuncUpda
func (e *countOriginalWithDistinct4String) UpdatePartialResult(sctx AggFuncUpdateContext, rowsInGroup []chunk.Row, pr PartialResult) (memDelta int64, err error) {
p := (*partialResult4CountDistinctString)(pr)
- collator := collate.GetCollator(e.args[0].GetType().GetCollate())
+ collator := collate.GetCollator(e.args[0].GetType(sctx).GetCollate())
for _, row := range rowsInGroup {
input, isNull, err := e.args[0].EvalString(sctx, row)
@@ -322,7 +322,7 @@ func (e *countOriginalWithDistinct) UpdatePartialResult(sctx AggFuncUpdateContex
encodedBytes := make([]byte, 0)
collators := make([]collate.Collator, 0, len(e.args))
for _, arg := range e.args {
- collators = append(collators, collate.GetCollator(arg.GetType().GetCollate()))
+ collators = append(collators, collate.GetCollator(arg.GetType(sctx).GetCollate()))
}
// decimal struct is the biggest type we will use.
buf := make([]byte, types.MyDecimalStructSize)
@@ -358,7 +358,7 @@ func evalAndEncode(
sctx expression.EvalContext, arg expression.Expression, collator collate.Collator,
row chunk.Row, buf, encodedBytes []byte,
) (_ []byte, isNull bool, err error) {
- switch tp := arg.GetType().EvalType(); tp {
+ switch tp := arg.GetType(sctx).EvalType(); tp {
case types.ETInt:
var val int64
val, isNull, err = arg.EvalInt(sctx, row)
@@ -778,7 +778,7 @@ func (e *approxCountDistinctOriginal) UpdatePartialResult(sctx AggFuncUpdateCont
buf := make([]byte, types.MyDecimalStructSize)
collators := make([]collate.Collator, 0, len(e.args))
for _, arg := range e.args {
- collators = append(collators, collate.GetCollator(arg.GetType().GetCollate()))
+ collators = append(collators, collate.GetCollator(arg.GetType(sctx).GetCollate()))
}
for _, row := range rowsInGroup {
diff --git a/pkg/executor/aggfuncs/func_group_concat.go b/pkg/executor/aggfuncs/func_group_concat.go
index b041078651d23..8444fdfc98f6e 100644
--- a/pkg/executor/aggfuncs/func_group_concat.go
+++ b/pkg/executor/aggfuncs/func_group_concat.go
@@ -242,7 +242,7 @@ func (e *groupConcatDistinct) UpdatePartialResult(sctx AggFuncUpdateContext, row
collators := make([]collate.Collator, 0, len(e.args))
for _, arg := range e.args {
- collators = append(collators, collate.GetCollator(arg.GetType().GetCollate()))
+ collators = append(collators, collate.GetCollator(arg.GetType(sctx).GetCollate()))
}
for _, row := range rowsInGroup {
@@ -429,6 +429,8 @@ type partialResult4GroupConcatOrder struct {
type groupConcatOrder struct {
baseGroupConcat4String
+ ctors []collate.Collator
+ desc []bool
}
func (e *groupConcatOrder) AppendFinalResult2Chunk(_ AggFuncUpdateContext, pr PartialResult, chk *chunk.Chunk) error {
@@ -442,20 +444,14 @@ func (e *groupConcatOrder) AppendFinalResult2Chunk(_ AggFuncUpdateContext, pr Pa
}
func (e *groupConcatOrder) AllocPartialResult() (pr PartialResult, memDelta int64) {
- desc := make([]bool, len(e.byItems))
- ctors := make([]collate.Collator, 0, len(e.byItems))
- for i, byItem := range e.byItems {
- desc[i] = byItem.Desc
- ctors = append(ctors, collate.GetCollator(byItem.Expr.GetType().GetCollate()))
- }
p := &partialResult4GroupConcatOrder{
topN: &topNRows{
- desc: desc,
+ desc: e.desc,
currSize: 0,
limitSize: e.maxLen,
sepSize: uint64(len(e.sep)),
isSepTruncated: false,
- collators: ctors,
+ collators: e.ctors,
},
}
return PartialResult(p), DefPartialResult4GroupConcatOrderSize + DefTopNRowsSize
@@ -534,6 +530,8 @@ type partialResult4GroupConcatOrderDistinct struct {
type groupConcatDistinctOrder struct {
baseGroupConcat4String
+ ctors []collate.Collator
+ desc []bool
}
func (e *groupConcatDistinctOrder) AppendFinalResult2Chunk(_ AggFuncUpdateContext, pr PartialResult, chk *chunk.Chunk) error {
@@ -547,21 +545,15 @@ func (e *groupConcatDistinctOrder) AppendFinalResult2Chunk(_ AggFuncUpdateContex
}
func (e *groupConcatDistinctOrder) AllocPartialResult() (pr PartialResult, memDelta int64) {
- desc := make([]bool, len(e.byItems))
- ctors := make([]collate.Collator, 0, len(e.byItems))
- for i, byItem := range e.byItems {
- desc[i] = byItem.Desc
- ctors = append(ctors, collate.GetCollator(byItem.Expr.GetType().GetCollate()))
- }
valSet, setSize := set.NewStringSetWithMemoryUsage()
p := &partialResult4GroupConcatOrderDistinct{
topN: &topNRows{
- desc: desc,
+ desc: e.desc,
currSize: 0,
limitSize: e.maxLen,
sepSize: uint64(len(e.sep)),
isSepTruncated: false,
- collators: ctors,
+ collators: e.ctors,
},
valSet: valSet,
}
@@ -583,7 +575,7 @@ func (e *groupConcatDistinctOrder) UpdatePartialResult(sctx AggFuncUpdateContext
collators := make([]collate.Collator, 0, len(e.args))
for _, arg := range e.args {
- collators = append(collators, collate.GetCollator(arg.GetType().GetCollate()))
+ collators = append(collators, collate.GetCollator(arg.GetType(sctx).GetCollate()))
}
for _, row := range rowsInGroup {
diff --git a/pkg/executor/aggfuncs/func_json_arrayagg.go b/pkg/executor/aggfuncs/func_json_arrayagg.go
index ecc795e8706b1..c473ca64b1904 100644
--- a/pkg/executor/aggfuncs/func_json_arrayagg.go
+++ b/pkg/executor/aggfuncs/func_json_arrayagg.go
@@ -69,7 +69,7 @@ func (e *jsonArrayagg) UpdatePartialResult(ctx AggFuncUpdateContext, rowsInGroup
return 0, errors.Trace(err)
}
- realItem, err := getRealJSONValue(item, e.args[0].GetType())
+ realItem, err := getRealJSONValue(item, e.args[0].GetType(ctx))
if err != nil {
return 0, errors.Trace(err)
}
diff --git a/pkg/executor/aggfuncs/func_json_objectagg.go b/pkg/executor/aggfuncs/func_json_objectagg.go
index 262da4cad5770..9e2f23a0c7f45 100644
--- a/pkg/executor/aggfuncs/func_json_objectagg.go
+++ b/pkg/executor/aggfuncs/func_json_objectagg.go
@@ -80,8 +80,8 @@ func (e *jsonObjectAgg) UpdatePartialResult(sctx AggFuncUpdateContext, rowsInGro
return 0, types.ErrJSONDocumentNULLKey
}
- if e.args[0].GetType().GetCharset() == charset.CharsetBin {
- return 0, types.ErrInvalidJSONCharset.GenWithStackByArgs(e.args[0].GetType().GetCharset())
+ if e.args[0].GetType(sctx).GetCharset() == charset.CharsetBin {
+ return 0, types.ErrInvalidJSONCharset.GenWithStackByArgs(e.args[0].GetType(sctx).GetCharset())
}
key = strings.Clone(key)
@@ -90,7 +90,7 @@ func (e *jsonObjectAgg) UpdatePartialResult(sctx AggFuncUpdateContext, rowsInGro
return 0, errors.Trace(err)
}
- realVal, err := getRealJSONValue(value, e.args[1].GetType())
+ realVal, err := getRealJSONValue(value, e.args[1].GetType(sctx))
if err != nil {
return 0, errors.Trace(err)
}
diff --git a/pkg/executor/aggfuncs/func_max_min.go b/pkg/executor/aggfuncs/func_max_min.go
index 7ff3962e96133..9903a880b2523 100644
--- a/pkg/executor/aggfuncs/func_max_min.go
+++ b/pkg/executor/aggfuncs/func_max_min.go
@@ -1081,7 +1081,7 @@ func (e *maxMin4String) AppendFinalResult2Chunk(_ AggFuncUpdateContext, pr Parti
func (e *maxMin4String) UpdatePartialResult(sctx AggFuncUpdateContext, rowsInGroup []chunk.Row, pr PartialResult) (memDelta int64, err error) {
p := (*partialResult4MaxMinString)(pr)
- tp := e.args[0].GetType()
+ tp := e.args[0].GetType(sctx)
for _, row := range rowsInGroup {
input, isNull, err := e.args[0].EvalString(sctx, row)
if err != nil {
@@ -1111,7 +1111,7 @@ func (e *maxMin4String) UpdatePartialResult(sctx AggFuncUpdateContext, rowsInGro
return memDelta, nil
}
-func (e *maxMin4String) MergePartialResult(_ AggFuncUpdateContext, src, dst PartialResult) (memDelta int64, err error) {
+func (e *maxMin4String) MergePartialResult(ctx AggFuncUpdateContext, src, dst PartialResult) (memDelta int64, err error) {
p1, p2 := (*partialResult4MaxMinString)(src), (*partialResult4MaxMinString)(dst)
if p1.isNull {
return 0, nil
@@ -1120,7 +1120,7 @@ func (e *maxMin4String) MergePartialResult(_ AggFuncUpdateContext, src, dst Part
*p2 = *p1
return 0, nil
}
- tp := e.args[0].GetType()
+ tp := e.args[0].GetType(ctx)
cmp := types.CompareString(p1.val, p2.val, tp.GetCollate())
if e.isMax && cmp > 0 || !e.isMax && cmp < 0 {
p2.val, p2.isNull = p1.val, false
@@ -1151,13 +1151,13 @@ func (e *maxMin4String) deserializeForSpill(helper *deserializeHelper) (PartialR
type maxMin4StringSliding struct {
maxMin4String
windowInfo
+ collate string
}
func (e *maxMin4StringSliding) AllocPartialResult() (pr PartialResult, memDelta int64) {
p, memDelta := e.maxMin4String.AllocPartialResult()
- tp := e.args[0].GetType()
(*partialResult4MaxMinString)(p).deque = NewDeque(e.isMax, func(i, j any) int {
- return types.CompareString(i.(string), j.(string), tp.GetCollate())
+ return types.CompareString(i.(string), j.(string), e.collate)
})
return p, memDelta + DefMaxMinDequeSize
}
diff --git a/pkg/executor/aggregate/agg_hash_executor.go b/pkg/executor/aggregate/agg_hash_executor.go
index fe994fb5015d4..c0b0122460cac 100644
--- a/pkg/executor/aggregate/agg_hash_executor.go
+++ b/pkg/executor/aggregate/agg_hash_executor.go
@@ -291,6 +291,7 @@ func (e *HashAggExec) initPartialWorkers(partialConcurrency int, finalConcurrenc
partialResultsMap[i] = make(aggfuncs.AggPartialResultMapper)
}
+ partialResultsBuffer, groupKeyBuf := getBuffer()
e.partialWorkers[i] = HashAggPartialWorker{
baseHashAggWorker: newBaseHashAggWorker(e.finishCh, e.PartialAggFuncs, e.MaxChunkSize(), e.memTracker),
idForTest: i,
@@ -299,12 +300,12 @@ func (e *HashAggExec) initPartialWorkers(partialConcurrency int, finalConcurrenc
outputChs: e.partialOutputChs,
giveBackCh: e.inputCh,
BInMaps: make([]int, finalConcurrency),
- partialResultsBuffer: make([][]aggfuncs.PartialResult, 0, 2048),
+ partialResultsBuffer: *partialResultsBuffer,
globalOutputCh: e.finalOutputCh,
partialResultsMap: partialResultsMap,
groupByItems: e.GroupByItems,
chk: exec.TryNewCacheChunk(e.Children(0)),
- groupKey: make([][]byte, 0, 8),
+ groupKeyBuf: *groupKeyBuf,
serializeHelpers: aggfuncs.NewSerializeHelper(),
isSpillPrepared: false,
spillHelper: e.spillHelper,
@@ -342,9 +343,7 @@ func (e *HashAggExec) initFinalWorkers(finalConcurrency int) {
inputCh: e.partialOutputChs[i],
outputCh: e.finalOutputCh,
finalResultHolderCh: make(chan *chunk.Chunk, 1),
- rowBuffer: make([]types.Datum, 0, e.Schema().Len()),
mutableRow: chunk.MutRowFromTypes(exec.RetTypes(e)),
- groupKeys: make([][]byte, 0, 8),
spillHelper: e.spillHelper,
restoredAggResultMapperMem: 0,
}
diff --git a/pkg/executor/aggregate/agg_hash_final_worker.go b/pkg/executor/aggregate/agg_hash_final_worker.go
index 684ab110d137b..d2cc2cb047f10 100644
--- a/pkg/executor/aggregate/agg_hash_final_worker.go
+++ b/pkg/executor/aggregate/agg_hash_final_worker.go
@@ -23,7 +23,6 @@ import (
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/executor/aggfuncs"
"github.com/pingcap/tidb/pkg/sessionctx"
- "github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/chunk"
"github.com/pingcap/tidb/pkg/util/hack"
"github.com/pingcap/tidb/pkg/util/logutil"
@@ -42,14 +41,12 @@ type AfFinalResult struct {
type HashAggFinalWorker struct {
baseHashAggWorker
- rowBuffer []types.Datum
mutableRow chunk.MutRow
partialResultMap aggfuncs.AggPartialResultMapper
BInMap int
inputCh chan *aggfuncs.AggPartialResultMapper
outputCh chan *AfFinalResult
finalResultHolderCh chan *chunk.Chunk
- groupKeys [][]byte
spillHelper *parallelHashAggSpillHelper
diff --git a/pkg/executor/aggregate/agg_hash_partial_worker.go b/pkg/executor/aggregate/agg_hash_partial_worker.go
index bacd5491f19e5..17498e1a9cbba 100644
--- a/pkg/executor/aggregate/agg_hash_partial_worker.go
+++ b/pkg/executor/aggregate/agg_hash_partial_worker.go
@@ -56,7 +56,7 @@ type HashAggPartialWorker struct {
partialResultsMapMem atomic.Int64
groupByItems []expression.Expression
- groupKey [][]byte
+ groupKeyBuf [][]byte
// chk stores the input data from child,
// and is reused by childExec and partial worker.
chk *chunk.Chunk
@@ -201,6 +201,8 @@ func (w *HashAggPartialWorker) run(ctx sessionctx.Context, waitGroup *sync.WaitG
// We must ensure that there is no panic before `waitGroup.Done()` or there will be hang
waitGroup.Done()
+
+ tryRecycleBuffer(&w.partialResultsBuffer, &w.groupKeyBuf)
}()
intestBeforePartialWorkerRun()
@@ -254,15 +256,15 @@ func (w *HashAggPartialWorker) getPartialResultsOfEachRow(groupKey [][]byte, fin
}
func (w *HashAggPartialWorker) updatePartialResult(ctx sessionctx.Context, chk *chunk.Chunk, finalConcurrency int) (err error) {
- memSize := getGroupKeyMemUsage(w.groupKey)
- w.groupKey, err = GetGroupKey(w.ctx, chk, w.groupKey, w.groupByItems)
+ memSize := getGroupKeyMemUsage(w.groupKeyBuf)
+ w.groupKeyBuf, err = GetGroupKey(w.ctx, chk, w.groupKeyBuf, w.groupByItems)
failpoint.Inject("ConsumeRandomPanic", nil)
- w.memTracker.Consume(getGroupKeyMemUsage(w.groupKey) - memSize)
+ w.memTracker.Consume(getGroupKeyMemUsage(w.groupKeyBuf) - memSize)
if err != nil {
return err
}
- partialResultOfEachRow := w.getPartialResultsOfEachRow(w.groupKey, finalConcurrency)
+ partialResultOfEachRow := w.getPartialResultsOfEachRow(w.groupKeyBuf, finalConcurrency)
numRows := chk.NumRows()
rows := make([]chunk.Row, 1)
diff --git a/pkg/executor/aggregate/agg_util.go b/pkg/executor/aggregate/agg_util.go
index 69daafb1f84d5..81b06147ee1d0 100644
--- a/pkg/executor/aggregate/agg_util.go
+++ b/pkg/executor/aggregate/agg_util.go
@@ -20,6 +20,7 @@ import (
"fmt"
"math/rand"
"slices"
+ "sync"
"sync/atomic"
"time"
@@ -40,6 +41,43 @@ import (
"go.uber.org/zap"
)
+const defaultPartialResultsBufferCap = 2048
+const defaultGroupKeyCap = 8
+
+var partialResultsBufferPool = sync.Pool{
+ New: func() any {
+ s := make([][]aggfuncs.PartialResult, 0, defaultPartialResultsBufferCap)
+ return &s
+ },
+}
+
+var groupKeyPool = sync.Pool{
+ New: func() any {
+ s := make([][]byte, 0, defaultGroupKeyCap)
+ return &s
+ },
+}
+
+func getBuffer() (*[][]aggfuncs.PartialResult, *[][]byte) {
+ partialResultsBuffer := partialResultsBufferPool.Get().(*[][]aggfuncs.PartialResult)
+ *partialResultsBuffer = (*partialResultsBuffer)[:0]
+ groupKey := groupKeyPool.Get().(*[][]byte)
+ *groupKey = (*groupKey)[:0]
+ return partialResultsBuffer, groupKey
+}
+
+// tryRecycleBuffer recycles small buffers only. This approach reduces the CPU pressure
+// from memory allocation during high concurrency aggregation computations (like DDL's scheduled tasks),
+// and also prevents the pool from holding too much memory and causing memory pressure.
+func tryRecycleBuffer(buf *[][]aggfuncs.PartialResult, groupKey *[][]byte) {
+ if cap(*buf) <= defaultPartialResultsBufferCap {
+ partialResultsBufferPool.Put(buf)
+ }
+ if cap(*groupKey) <= defaultGroupKeyCap {
+ groupKeyPool.Put(groupKey)
+ }
+}
+
func closeBaseExecutor(b *exec.BaseExecutor) {
if r := recover(); r != nil {
// Release the resource, but throw the panic again and let the top level handle it.
@@ -78,7 +116,7 @@ func GetGroupKey(ctx sessionctx.Context, input *chunk.Chunk, groupKey [][]byte,
errCtx := ctx.GetSessionVars().StmtCtx.ErrCtx()
exprCtx := ctx.GetExprCtx()
for _, item := range groupByItems {
- tp := item.GetType()
+ tp := item.GetType(ctx.GetExprCtx().GetEvalCtx())
buf, err := expression.GetColumn(tp.EvalType(), numRows)
if err != nil {
@@ -91,7 +129,7 @@ func GetGroupKey(ctx sessionctx.Context, input *chunk.Chunk, groupKey [][]byte,
// Ref to issue #26885.
// This check is used to handle invalid enum name same with user defined enum name.
// Use enum value as groupKey instead of enum name.
- if item.GetType().GetType() == mysql.TypeEnum {
+ if item.GetType(ctx.GetExprCtx().GetEvalCtx()).GetType() == mysql.TypeEnum {
newTp := *tp
newTp.AddFlag(mysql.EnumSetAsIntFlag)
tp = &newTp
@@ -102,7 +140,7 @@ func GetGroupKey(ctx sessionctx.Context, input *chunk.Chunk, groupKey [][]byte,
return nil, err
}
// This check is used to avoid error during the execution of `EncodeDecimal`.
- if item.GetType().GetType() == mysql.TypeNewDecimal {
+ if item.GetType(ctx.GetExprCtx().GetEvalCtx()).GetType() == mysql.TypeNewDecimal {
newTp := *tp
newTp.SetFlen(0)
tp = &newTp
diff --git a/pkg/executor/analyze.go b/pkg/executor/analyze.go
index 2f87ce7116ca5..5d4454fe210e5 100644
--- a/pkg/executor/analyze.go
+++ b/pkg/executor/analyze.go
@@ -39,6 +39,7 @@ import (
"github.com/pingcap/tidb/pkg/sessiontxn"
"github.com/pingcap/tidb/pkg/statistics"
"github.com/pingcap/tidb/pkg/statistics/handle"
+ statstypes "github.com/pingcap/tidb/pkg/statistics/handle/types"
handleutil "github.com/pingcap/tidb/pkg/statistics/handle/util"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util"
@@ -117,7 +118,7 @@ func (e *AnalyzeExec) Next(ctx context.Context, _ *chunk.Chunk) error {
pruneMode := variable.PartitionPruneMode(sessionVars.PartitionPruneMode.Load())
// needGlobalStats used to indicate whether we should merge the partition-level stats to global-level stats.
needGlobalStats := pruneMode == variable.Dynamic
- globalStatsMap := make(map[globalStatsKey]globalStatsInfo)
+ globalStatsMap := make(map[globalStatsKey]statstypes.GlobalStatsInfo)
g, gctx := errgroup.WithContext(ctx)
g.Go(func() error {
return e.handleResultsError(ctx, concurrency, needGlobalStats, globalStatsMap, resultsCh, len(tasks))
@@ -160,7 +161,7 @@ TASKLOOP:
})
// If we enabled dynamic prune mode, then we need to generate global stats here for partition tables.
if needGlobalStats {
- err = e.handleGlobalStats(ctx, globalStatsMap)
+ err = e.handleGlobalStats(globalStatsMap)
if err != nil {
return err
}
@@ -769,11 +770,11 @@ func handleGlobalStats(needGlobalStats bool, globalStatsMap globalStatsMap, resu
}
histIDs = append(histIDs, hg.ID)
}
- globalStatsMap[globalStatsID] = globalStatsInfo{isIndex: result.IsIndex, histIDs: histIDs, statsVersion: results.StatsVer}
+ globalStatsMap[globalStatsID] = statstypes.GlobalStatsInfo{IsIndex: result.IsIndex, HistIDs: histIDs, StatsVersion: results.StatsVer}
} else {
for _, hg := range result.Hist {
globalStatsID := globalStatsKey{tableID: results.TableID.TableID, indexID: hg.ID}
- globalStatsMap[globalStatsID] = globalStatsInfo{isIndex: result.IsIndex, histIDs: []int64{hg.ID}, statsVersion: results.StatsVer}
+ globalStatsMap[globalStatsID] = statstypes.GlobalStatsInfo{IsIndex: result.IsIndex, HistIDs: []int64{hg.ID}, StatsVersion: results.StatsVer}
}
}
}
diff --git a/pkg/executor/analyze_col.go b/pkg/executor/analyze_col.go
index 3d699b5e26af5..c472002647fec 100644
--- a/pkg/executor/analyze_col.go
+++ b/pkg/executor/analyze_col.go
@@ -30,6 +30,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/planner/core"
+ plannerutil "github.com/pingcap/tidb/pkg/planner/util"
"github.com/pingcap/tidb/pkg/statistics"
"github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/types"
@@ -46,7 +47,7 @@ type AnalyzeColumnsExec struct {
tableInfo *model.TableInfo
colsInfo []*model.ColumnInfo
- handleCols core.HandleCols
+ handleCols plannerutil.HandleCols
commonHandle *model.IndexInfo
resultHandler *tableResultHandler
indexes []*model.IndexInfo
@@ -379,7 +380,7 @@ func (e *AnalyzeColumnsExecV1) analyzeColumnsPushDownV1() *statistics.AnalyzeRes
}
}
-func hasPkHist(handleCols core.HandleCols) bool {
+func hasPkHist(handleCols plannerutil.HandleCols) bool {
return handleCols != nil && handleCols.IsInt()
}
diff --git a/pkg/executor/analyze_col_v2.go b/pkg/executor/analyze_col_v2.go
index 07f14a5819d30..3f0b4ced2d0af 100644
--- a/pkg/executor/analyze_col_v2.go
+++ b/pkg/executor/analyze_col_v2.go
@@ -528,7 +528,7 @@ func (e *AnalyzeColumnsExecV2) buildSubIndexJobForSpecialIndex(indexInfos []*mod
_, offset := timeutil.Zone(e.ctx.GetSessionVars().Location())
tasks := make([]*analyzeTask, 0, len(indexInfos))
sc := e.ctx.GetSessionVars().StmtCtx
- concurrency := e.ctx.GetSessionVars().AnalyzeDistSQLScanConcurrency()
+ concurrency := adaptiveAnlayzeDistSQLConcurrency(context.Background(), e.ctx)
for _, indexInfo := range indexInfos {
base := baseAnalyzeExec{
ctx: e.ctx,
diff --git a/pkg/executor/analyze_global_stats.go b/pkg/executor/analyze_global_stats.go
index 74be6daf71d4d..29801ca35fda8 100644
--- a/pkg/executor/analyze_global_stats.go
+++ b/pkg/executor/analyze_global_stats.go
@@ -15,15 +15,12 @@
package executor
import (
- "context"
"fmt"
"github.com/pingcap/tidb/pkg/domain"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/statistics"
- "github.com/pingcap/tidb/pkg/statistics/handle/globalstats"
- "github.com/pingcap/tidb/pkg/statistics/handle/util"
- "github.com/pingcap/tidb/pkg/types"
+ statstypes "github.com/pingcap/tidb/pkg/statistics/handle/types"
"github.com/pingcap/tidb/pkg/util/logutil"
"go.uber.org/zap"
)
@@ -33,20 +30,12 @@ type globalStatsKey struct {
indexID int64
}
-type globalStatsInfo struct {
- isIndex int
- // When the `isIndex == 0`, histIDs will be the column IDs.
- // Otherwise, histIDs will only contain the index ID.
- histIDs []int64
- statsVersion int
-}
-
// globalStatsMap is a map used to store which partition tables and the corresponding indexes need global-level stats.
// The meaning of key in map is the structure that used to store the tableID and indexID.
// The meaning of value in map is some additional information needed to build global-level stats.
-type globalStatsMap map[globalStatsKey]globalStatsInfo
+type globalStatsMap map[globalStatsKey]statstypes.GlobalStatsInfo
-func (e *AnalyzeExec) handleGlobalStats(ctx context.Context, globalStatsMap globalStatsMap) error {
+func (e *AnalyzeExec) handleGlobalStats(globalStatsMap globalStatsMap) error {
globalStatsTableIDs := make(map[int64]struct{}, len(globalStatsMap))
for globalStatsID := range globalStatsMap {
globalStatsTableIDs[globalStatsID.tableID] = struct{}{}
@@ -75,47 +64,15 @@ func (e *AnalyzeExec) handleGlobalStats(ctx context.Context, globalStatsMap glob
globalOpts = v2Options.FilledOpts
}
}
- globalStatsI, err := statsHandle.MergePartitionStats2GlobalStatsByTableID(
+ err := statsHandle.MergePartitionStats2GlobalStatsByTableID(
e.Ctx(),
globalOpts, e.Ctx().GetInfoSchema().(infoschema.InfoSchema),
+ &info,
globalStatsID.tableID,
- info.isIndex == 1,
- info.histIDs,
)
if err != nil {
logutil.BgLogger().Warn("merge global stats failed",
zap.String("info", job.JobInfo), zap.Error(err), zap.Int64("tableID", tableID))
- if types.ErrPartitionStatsMissing.Equal(err) || types.ErrPartitionColumnStatsMissing.Equal(err) {
- // When we find some partition-level stats are missing, we need to report warning.
- e.Ctx().GetSessionVars().StmtCtx.AppendWarning(err)
- }
- return err
- }
- globalStats := globalStatsI.(*globalstats.GlobalStats)
- // Dump global-level stats to kv.
- for i := 0; i < globalStats.Num; i++ {
- hg, cms, topN := globalStats.Hg[i], globalStats.Cms[i], globalStats.TopN[i]
- if hg == nil {
- // All partitions have no stats so global stats are not created.
- continue
- }
- // fms for global stats doesn't need to dump to kv.
- err = statsHandle.SaveStatsToStorage(globalStatsID.tableID,
- globalStats.Count,
- globalStats.ModifyCount,
- info.isIndex,
- hg,
- cms,
- topN,
- info.statsVersion,
- 1,
- true,
- util.StatsMetaHistorySourceAnalyze,
- )
- if err != nil {
- logutil.Logger(ctx).Error("save global-level stats to storage failed", zap.String("info", job.JobInfo),
- zap.Int64("histID", hg.ID), zap.Error(err), zap.Int64("tableID", tableID))
- }
}
return err
}()
diff --git a/pkg/executor/analyze_utils.go b/pkg/executor/analyze_utils.go
index 7c8617b2a5dda..13fd3d508c07d 100644
--- a/pkg/executor/analyze_utils.go
+++ b/pkg/executor/analyze_utils.go
@@ -24,11 +24,50 @@ import (
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/statistics"
+ "github.com/pingcap/tidb/pkg/store/helper"
"github.com/pingcap/tidb/pkg/util"
+ "github.com/pingcap/tidb/pkg/util/logutil"
"github.com/tiancaiamao/gp"
"go.uber.org/atomic"
+ "go.uber.org/zap"
)
+func adaptiveAnlayzeDistSQLConcurrency(ctx context.Context, sctx sessionctx.Context) int {
+ concurrency := sctx.GetSessionVars().AnalyzeDistSQLScanConcurrency()
+ if concurrency > 0 {
+ return concurrency
+ }
+ tikvStore, ok := sctx.GetStore().(helper.Storage)
+ if !ok {
+ logutil.BgLogger().Warn("Information about TiKV store status can be gotten only when the storage is TiKV")
+ return variable.DefAnalyzeDistSQLScanConcurrency
+ }
+ tikvHelper := &helper.Helper{
+ Store: tikvStore,
+ RegionCache: tikvStore.GetRegionCache(),
+ }
+ pdCli, err := tikvHelper.TryGetPDHTTPClient()
+ if err != nil {
+ logutil.BgLogger().Warn("fail to TryGetPDHTTPClient", zap.Error(err))
+ return variable.DefAnalyzeDistSQLScanConcurrency
+ }
+ storesStat, err := pdCli.GetStores(ctx)
+ if err != nil {
+ logutil.BgLogger().Warn("fail to get stores info", zap.Error(err))
+ return variable.DefAnalyzeDistSQLScanConcurrency
+ }
+ if storesStat.Count <= 5 {
+ return variable.DefAnalyzeDistSQLScanConcurrency
+ } else if storesStat.Count <= 10 {
+ return storesStat.Count
+ } else if storesStat.Count <= 20 {
+ return storesStat.Count * 2
+ } else if storesStat.Count <= 50 {
+ return storesStat.Count * 3
+ }
+ return storesStat.Count * 4
+}
+
func getIntFromSessionVars(ctx sessionctx.Context, name string) (int, error) {
sessionVars := ctx.GetSessionVars()
concurrency, err := sessionVars.GetSessionOrGlobalSystemVar(context.Background(), name)
diff --git a/pkg/executor/batch_checker.go b/pkg/executor/batch_checker.go
index 1c5b67972e7dd..d7f83edeb459f 100644
--- a/pkg/executor/batch_checker.go
+++ b/pkg/executor/batch_checker.go
@@ -37,9 +37,8 @@ import (
)
type keyValueWithDupInfo struct {
- newKey kv.Key
- dupErr error
- commonHandle bool
+ newKey kv.Key
+ dupErr error
}
type toBeCheckedRow struct {
@@ -98,7 +97,7 @@ func getKeysNeedCheckOneRow(ctx sessionctx.Context, t table.Table, row []types.D
pkIdxInfo *model.IndexInfo, result []toBeCheckedRow) ([]toBeCheckedRow, error) {
var err error
if p, ok := t.(table.PartitionedTable); ok {
- t, err = p.GetPartitionByRow(ctx.GetExprCtx(), row)
+ t, err = p.GetPartitionByRow(ctx.GetExprCtx().GetEvalCtx(), row)
if err != nil {
if terr, ok := errors.Cause(err).(*terror.Error); ok && (terr.Code() == errno.ErrNoPartitionForGivenValue || terr.Code() == errno.ErrRowDoesNotMatchGivenPartitionSet) {
ec := ctx.GetSessionVars().StmtCtx.ErrCtx()
@@ -213,9 +212,8 @@ func getKeysNeedCheckOneRow(ctx sessionctx.Context, t table.Table, row []types.D
return nil, err1
}
uniqueKeys = append(uniqueKeys, &keyValueWithDupInfo{
- newKey: key,
- dupErr: kv.ErrKeyExists.FastGenByArgs(colValStr, fmt.Sprintf("%s.%s", v.TableMeta().Name.String(), v.Meta().Name.String())),
- commonHandle: t.Meta().IsCommonHandle,
+ newKey: key,
+ dupErr: kv.ErrKeyExists.FastGenByArgs(colValStr, fmt.Sprintf("%s.%s", v.TableMeta().Name.String(), v.Meta().Name.String())),
})
}
}
diff --git a/pkg/executor/batch_point_get.go b/pkg/executor/batch_point_get.go
index 1a4a110061767..4b0262b44014d 100644
--- a/pkg/executor/batch_point_get.go
+++ b/pkg/executor/batch_point_get.go
@@ -193,16 +193,24 @@ func (e *BatchPointGetExec) Next(ctx context.Context, req *chunk.Chunk) error {
if e.index >= len(e.values) {
return nil
}
+
+ schema := e.Schema()
+ sctx := e.BaseExecutor.Ctx()
+ start := e.index
for !req.IsFull() && e.index < len(e.values) {
handle, val := e.handles[e.index], e.values[e.index]
- err := DecodeRowValToChunk(e.BaseExecutor.Ctx(), e.Schema(), e.tblInfo, handle, val, req, e.rowDecoder)
+ err := DecodeRowValToChunk(sctx, schema, e.tblInfo, handle, val, req, e.rowDecoder)
if err != nil {
return err
}
e.index++
}
- err := table.FillVirtualColumnValue(e.virtualColumnRetFieldTypes, e.virtualColumnIndex, e.Schema().Columns, e.columns, e.Ctx().GetExprCtx(), req)
+ err := fillRowChecksum(sctx, start, e.index, schema, e.tblInfo, e.values, e.handles, req, nil)
+ if err != nil {
+ return err
+ }
+ err = table.FillVirtualColumnValue(e.virtualColumnRetFieldTypes, e.virtualColumnIndex, schema.Columns, e.columns, sctx.GetExprCtx(), req)
if err != nil {
return err
}
@@ -282,15 +290,14 @@ func (e *BatchPointGetExec) initialize(ctx context.Context) error {
if len(handleVal) == 0 {
continue
}
- handle, err1 := tablecodec.DecodeHandleInUniqueIndexValue(handleVal, e.tblInfo.IsCommonHandle)
+ handle, err1 := tablecodec.DecodeHandleInIndexValue(handleVal)
if err1 != nil {
return err1
}
if e.tblInfo.Partition != nil {
var pid int64
if e.idxInfo.Global {
- segs := tablecodec.SplitIndexValue(handleVal)
- _, pid, err = codec.DecodeInt(segs.PartitionID)
+ _, pid, err = codec.DecodeInt(tablecodec.SplitIndexValue(handleVal).PartitionID)
if err != nil {
return err
}
diff --git a/pkg/executor/brie.go b/pkg/executor/brie.go
index 1e5316881819e..c3dad72e0328a 100644
--- a/pkg/executor/brie.go
+++ b/pkg/executor/brie.go
@@ -312,6 +312,9 @@ func (b *executorBuilder) buildBRIE(s *ast.BRIEStmt, schema *expression.Schema)
default:
}
+ failpoint.Inject("modifyStore", func(v failpoint.Value) {
+ tidbCfg.Store = v.(string)
+ })
if tidbCfg.Store != "tikv" {
b.err = errors.Errorf("%s requires tikv store, not %s", s.Kind, tidbCfg.Store)
return nil
@@ -330,6 +333,28 @@ func (b *executorBuilder) buildBRIE(s *ast.BRIEStmt, schema *expression.Schema)
cfg.Checksum = opt.UintValue != 0
case ast.BRIEOptionSendCreds:
cfg.SendCreds = opt.UintValue != 0
+ case ast.BRIEOptionChecksumConcurrency:
+ cfg.ChecksumConcurrency = uint(opt.UintValue)
+ case ast.BRIEOptionEncryptionKeyFile:
+ cfg.CipherInfo.CipherKey, err = task.GetCipherKeyContent("", opt.StrValue)
+ if err != nil {
+ b.err = err
+ return nil
+ }
+ case ast.BRIEOptionEncryptionMethod:
+ switch opt.StrValue {
+ case "aes128-ctr":
+ cfg.CipherInfo.CipherType = encryptionpb.EncryptionMethod_AES128_CTR
+ case "aes192-ctr":
+ cfg.CipherInfo.CipherType = encryptionpb.EncryptionMethod_AES192_CTR
+ case "aes256-ctr":
+ cfg.CipherInfo.CipherType = encryptionpb.EncryptionMethod_AES256_CTR
+ case "plaintext":
+ cfg.CipherInfo.CipherType = encryptionpb.EncryptionMethod_PLAINTEXT
+ default:
+ b.err = errors.Errorf("unsupported encryption method: %s", opt.StrValue)
+ return nil
+ }
}
}
@@ -383,6 +408,22 @@ func (b *executorBuilder) buildBRIE(s *ast.BRIEStmt, schema *expression.Schema)
return nil
}
e.backupCfg.BackupTS = tso
+ case ast.BRIEOptionCompression:
+ switch opt.StrValue {
+ case "zstd":
+ e.backupCfg.CompressionConfig.CompressionType = backuppb.CompressionType_ZSTD
+ case "snappy":
+ e.backupCfg.CompressionConfig.CompressionType = backuppb.CompressionType_SNAPPY
+ case "lz4":
+ e.backupCfg.CompressionConfig.CompressionType = backuppb.CompressionType_LZ4
+ default:
+ b.err = errors.Errorf("unsupported compression type: %s", opt.StrValue)
+ return nil
+ }
+ case ast.BRIEOptionCompressionLevel:
+ e.backupCfg.CompressionConfig.CompressionLevel = int32(opt.UintValue)
+ case ast.BRIEOptionIgnoreStats:
+ e.backupCfg.IgnoreStats = opt.UintValue != 0
}
}
@@ -391,8 +432,15 @@ func (b *executorBuilder) buildBRIE(s *ast.BRIEStmt, schema *expression.Schema)
rcfg.Config = cfg
e.restoreCfg = &rcfg
for _, opt := range s.Options {
- if opt.Tp == ast.BRIEOptionOnline {
+ switch opt.Tp {
+ case ast.BRIEOptionOnline:
e.restoreCfg.Online = opt.UintValue != 0
+ case ast.BRIEOptionWaitTiflashReady:
+ e.restoreCfg.WaitTiflashReady = opt.UintValue != 0
+ case ast.BRIEOptionWithSysTable:
+ e.restoreCfg.WithSysTable = opt.UintValue != 0
+ case ast.BRIEOptionLoadStats:
+ e.restoreCfg.LoadStats = opt.UintValue != 0
}
}
diff --git a/pkg/executor/brie_test.go b/pkg/executor/brie_test.go
index 266144bcf2c10..151bc5e09efb5 100644
--- a/pkg/executor/brie_test.go
+++ b/pkg/executor/brie_test.go
@@ -17,11 +17,15 @@ package executor
import (
"context"
"fmt"
+ "os"
"strconv"
"strings"
"testing"
"time"
+ "github.com/pingcap/failpoint"
+ backuppb "github.com/pingcap/kvproto/pkg/brpb"
+ "github.com/pingcap/kvproto/pkg/encryptionpb"
"github.com/pingcap/tidb/pkg/executor/internal/exec"
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/parser"
@@ -140,3 +144,82 @@ func TestFetchShowBRIE(t *testing.T) {
globalBRIEQueue.clearTask(e.Ctx().GetSessionVars().StmtCtx)
require.Equal(t, info2Res, fetchShowBRIEResult(t, e, brieColTypes))
}
+
+func TestBRIEBuilderOPtions(t *testing.T) {
+ sctx := mock.NewContext()
+ sctx.GetSessionVars().User = &auth.UserIdentity{Username: "test"}
+ is := infoschema.MockInfoSchema([]*model.TableInfo{core.MockSignedTable(), core.MockUnsignedTable()})
+ ResetGlobalBRIEQueueForTest()
+ builder := NewMockExecutorBuilderForTest(sctx, is)
+ ctx := context.Background()
+ p := parser.New()
+ p.SetParserConfig(parser.ParserConfig{EnableWindowFunction: true, EnableStrictDoubleTypeCheck: true})
+ failpoint.Enable("github.com/pingcap/tidb/pkg/executor/modifyStore", `return("tikv")`)
+ defer failpoint.Disable("github.com/pingcap/tidb/pkg/executor/modifyStore")
+ err := os.WriteFile("/tmp/keyfile", []byte(strings.Repeat("A", 128)), 0644)
+
+ require.NoError(t, err)
+ stmt, err := p.ParseOneStmt("BACKUP TABLE `a` TO 'noop://' CHECKSUM_CONCURRENCY = 4 IGNORE_STATS = 1 COMPRESSION_LEVEL = 4 COMPRESSION_TYPE = 'lz4' ENCRYPTION_METHOD = 'aes256-ctr' ENCRYPTION_KEYFILE = '/tmp/keyfile'", "", "")
+ require.NoError(t, err)
+ plan, err := core.BuildLogicalPlanForTest(ctx, sctx, stmt, infoschema.MockInfoSchema([]*model.TableInfo{core.MockSignedTable(), core.MockUnsignedTable(), core.MockView()}))
+ require.NoError(t, err)
+ s, ok := stmt.(*ast.BRIEStmt)
+ require.True(t, ok)
+ require.True(t, s.Kind == ast.BRIEKindBackup)
+ for _, opt := range s.Options {
+ switch opt.Tp {
+ case ast.BRIEOptionChecksumConcurrency:
+ require.Equal(t, uint64(4), opt.UintValue)
+ case ast.BRIEOptionCompressionLevel:
+ require.Equal(t, uint64(4), opt.UintValue)
+ case ast.BRIEOptionIgnoreStats:
+ require.Equal(t, uint64(1), opt.UintValue)
+ case ast.BRIEOptionCompression:
+ require.Equal(t, "lz4", opt.StrValue)
+ case ast.BRIEOptionEncryptionMethod:
+ require.Equal(t, "aes256-ctr", opt.StrValue)
+ case ast.BRIEOptionEncryptionKeyFile:
+ require.Equal(t, "/tmp/keyfile", opt.StrValue)
+ }
+ }
+ schema := plan.Schema()
+ exec := builder.buildBRIE(s, schema)
+ require.NoError(t, builder.err)
+ e, ok := exec.(*BRIEExec)
+ require.True(t, ok)
+ require.Equal(t, uint(4), e.backupCfg.ChecksumConcurrency)
+ require.Equal(t, int32(4), e.backupCfg.CompressionLevel)
+ require.Equal(t, true, e.backupCfg.IgnoreStats)
+ require.Equal(t, backuppb.CompressionType_LZ4, e.backupCfg.CompressionConfig.CompressionType)
+ require.Equal(t, encryptionpb.EncryptionMethod_AES256_CTR, e.backupCfg.CipherInfo.CipherType)
+ require.Greater(t, len(e.backupCfg.CipherInfo.CipherKey), 0)
+
+ stmt, err = p.ParseOneStmt("RESTORE TABLE `a` FROM 'noop://' CHECKSUM_CONCURRENCY = 4 WAIT_TIFLASH_READY = 1 WITH_SYS_TABLE = 1 LOAD_STATS = 1", "", "")
+ require.NoError(t, err)
+ plan, err = core.BuildLogicalPlanForTest(ctx, sctx, stmt, infoschema.MockInfoSchema([]*model.TableInfo{core.MockSignedTable(), core.MockUnsignedTable(), core.MockView()}))
+ require.NoError(t, err)
+ s, ok = stmt.(*ast.BRIEStmt)
+ require.True(t, ok)
+ require.True(t, s.Kind == ast.BRIEKindRestore)
+ for _, opt := range s.Options {
+ switch opt.Tp {
+ case ast.BRIEOptionChecksumConcurrency:
+ require.Equal(t, uint64(4), opt.UintValue)
+ case ast.BRIEOptionWaitTiflashReady:
+ require.Equal(t, uint64(1), opt.UintValue)
+ case ast.BRIEOptionWithSysTable:
+ require.Equal(t, uint64(1), opt.UintValue)
+ case ast.BRIEOptionLoadStats:
+ require.Equal(t, uint64(1), opt.UintValue)
+ }
+ }
+ schema = plan.Schema()
+ exec = builder.buildBRIE(s, schema)
+ require.NoError(t, builder.err)
+ e, ok = exec.(*BRIEExec)
+ require.True(t, ok)
+ require.Equal(t, uint(4), e.restoreCfg.ChecksumConcurrency)
+ require.True(t, e.restoreCfg.WaitTiflashReady)
+ require.True(t, e.restoreCfg.WithSysTable)
+ require.True(t, e.restoreCfg.LoadStats)
+}
diff --git a/pkg/executor/builder.go b/pkg/executor/builder.go
index 49428a19c2ba1..8e3946657bdbc 100644
--- a/pkg/executor/builder.go
+++ b/pkg/executor/builder.go
@@ -451,6 +451,9 @@ func buildIndexLookUpChecker(b *executorBuilder, p *plannercore.PhysicalIndexLoo
if !e.isCommonHandle() {
fullColLen++
}
+ if e.index.Global {
+ fullColLen++
+ }
e.dagPB.OutputOffsets = make([]uint32, fullColLen)
for i := 0; i < fullColLen; i++ {
e.dagPB.OutputOffsets[i] = uint32(i)
@@ -470,6 +473,9 @@ func buildIndexLookUpChecker(b *executorBuilder, p *plannercore.PhysicalIndexLoo
if !e.isCommonHandle() {
tps = append(tps, types.NewFieldType(mysql.TypeLonglong))
}
+ if e.index.Global {
+ tps = append(tps, types.NewFieldType(mysql.TypeLonglong))
+ }
e.checkIndexValue = &checkIndexValue{idxColTps: tps}
@@ -615,14 +621,14 @@ func (b *executorBuilder) buildRecoverIndex(v *plannercore.RecoverIndex) exec.Ex
}
func buildHandleColsForExec(sctx *stmtctx.StatementContext, tblInfo *model.TableInfo,
- allColInfo []*model.ColumnInfo) plannercore.HandleCols {
+ allColInfo []*model.ColumnInfo) plannerutil.HandleCols {
if !tblInfo.IsCommonHandle {
extraColPos := len(allColInfo) - 1
intCol := &expression.Column{
Index: extraColPos,
RetType: types.NewFieldType(mysql.TypeLonglong),
}
- return plannercore.NewIntHandleCols(intCol)
+ return plannerutil.NewIntHandleCols(intCol)
}
tblCols := make([]*expression.Column, len(tblInfo.Columns))
for i := 0; i < len(tblInfo.Columns); i++ {
@@ -640,7 +646,7 @@ func buildHandleColsForExec(sctx *stmtctx.StatementContext, tblInfo *model.Table
}
}
}
- return plannercore.NewCommonHandleCols(sctx, tblInfo, pkIdx, tblCols)
+ return plannerutil.NewCommonHandleCols(sctx, tblInfo, pkIdx, tblCols)
}
func (b *executorBuilder) buildCleanupIndex(v *plannercore.CleanupIndex) exec.Executor {
@@ -676,6 +682,9 @@ func (b *executorBuilder) buildCleanupIndex(v *plannercore.CleanupIndex) exec.Ex
}
sessCtx := e.Ctx().GetSessionVars().StmtCtx
e.handleCols = buildHandleColsForExec(sessCtx, tblInfo, e.columns)
+ if e.index.Meta().Global {
+ e.columns = append(e.columns, model.NewExtraPartitionIDColInfo())
+ }
return e
}
@@ -2168,8 +2177,9 @@ func (b *executorBuilder) buildTopN(v *plannercore.PhysicalTopN) exec.Executor {
}
executor_metrics.ExecutorCounterTopNExec.Inc()
return &sortexec.TopNExec{
- SortExec: sortExec,
- Limit: &plannercore.PhysicalLimit{Count: v.Count, Offset: v.Offset},
+ SortExec: sortExec,
+ Limit: &plannercore.PhysicalLimit{Count: v.Count, Offset: v.Offset},
+ Concurrency: b.ctx.GetSessionVars().Concurrency.ExecutorConcurrency,
}
}
@@ -2294,7 +2304,7 @@ func (b *executorBuilder) buildUnionAll(v *plannercore.PhysicalUnionAll) exec.Ex
return e
}
-func buildHandleColsForSplit(sc *stmtctx.StatementContext, tbInfo *model.TableInfo) plannercore.HandleCols {
+func buildHandleColsForSplit(sc *stmtctx.StatementContext, tbInfo *model.TableInfo) plannerutil.HandleCols {
if tbInfo.IsCommonHandle {
primaryIdx := tables.FindPrimaryIndex(tbInfo)
tableCols := make([]*expression.Column, len(tbInfo.Columns))
@@ -2307,12 +2317,12 @@ func buildHandleColsForSplit(sc *stmtctx.StatementContext, tbInfo *model.TableIn
for i, pkCol := range primaryIdx.Columns {
tableCols[pkCol.Offset].Index = i
}
- return plannercore.NewCommonHandleCols(sc, tbInfo, primaryIdx, tableCols)
+ return plannerutil.NewCommonHandleCols(sc, tbInfo, primaryIdx, tableCols)
}
intCol := &expression.Column{
RetType: types.NewFieldType(mysql.TypeLonglong),
}
- return plannercore.NewIntHandleCols(intCol)
+ return plannerutil.NewIntHandleCols(intCol)
}
func (b *executorBuilder) buildSplitRegion(v *plannercore.SplitRegion) exec.Executor {
@@ -2485,7 +2495,7 @@ func (b *executorBuilder) buildAnalyzeIndexPushdown(task plannercore.AnalyzeInde
failpoint.Inject("injectAnalyzeSnapshot", func(val failpoint.Value) {
startTS = uint64(val.(int))
})
- concurrency := b.ctx.GetSessionVars().AnalyzeDistSQLScanConcurrency()
+ concurrency := adaptiveAnlayzeDistSQLConcurrency(context.Background(), b.ctx)
base := baseAnalyzeExec{
ctx: b.ctx,
tableID: task.TableID,
@@ -2602,7 +2612,7 @@ func (b *executorBuilder) buildAnalyzeSamplingPushdown(
PartitionName: task.PartitionName,
SampleRateReason: sampleRateReason,
}
- concurrency := b.ctx.GetSessionVars().AnalyzeDistSQLScanConcurrency()
+ concurrency := adaptiveAnlayzeDistSQLConcurrency(context.Background(), b.ctx)
base := baseAnalyzeExec{
ctx: b.ctx,
tableID: task.TableID,
@@ -2736,7 +2746,7 @@ func (b *executorBuilder) buildAnalyzeColumnsPushdown(
failpoint.Inject("injectAnalyzeSnapshot", func(val failpoint.Value) {
startTS = uint64(val.(int))
})
- concurrency := b.ctx.GetSessionVars().AnalyzeDistSQLScanConcurrency()
+ concurrency := adaptiveAnlayzeDistSQLConcurrency(context.Background(), b.ctx)
base := baseAnalyzeExec{
ctx: b.ctx,
tableID: task.TableID,
@@ -2895,6 +2905,12 @@ func (*executorBuilder) corColInDistPlan(plans []base.PhysicalPlan) bool {
return true
}
}
+ case *plannercore.PhysicalTopN:
+ for _, byItem := range x.ByItems {
+ if len(expression.ExtractCorColumns(byItem.Expr)) > 0 {
+ return true
+ }
+ }
case *plannercore.PhysicalTableScan:
for _, cond := range x.LateMaterializationFilterCondition {
if len(expression.ExtractCorColumns(cond)) > 0 {
@@ -3516,7 +3532,7 @@ func (builder *dataReaderBuilder) prunePartitionForInnerExecutor(tbl table.Table
for i, data := range content.Keys {
locateKey[keyColOffsets[i]] = data
}
- p, err := partitionTbl.GetPartitionByRow(exprCtx, locateKey)
+ p, err := partitionTbl.GetPartitionByRow(exprCtx.GetEvalCtx(), locateKey)
if table.ErrNoPartitionForGivenValue.Equal(err) {
continue
}
@@ -3669,7 +3685,7 @@ func buildTableReq(b *executorBuilder, schemaLen int, plans []base.PhysicalPlan)
// buildIndexReq is designed to create a DAG for index request.
// If len(ByItems) != 0 means index request should return related columns
-// to sort result rows in TiDB side for parition tables.
+// to sort result rows in TiDB side for partition tables.
func buildIndexReq(ctx sessionctx.Context, columns []*model.IndexColumn, handleLen int, plans []base.PhysicalPlan) (dagReq *tipb.DAGRequest, err error) {
indexReq, err := builder.ConstructDAGReq(ctx, plans, kv.TiKV)
if err != nil {
@@ -4155,7 +4171,7 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte
for i, data := range content.Keys {
locateKey[keyColOffsets[i]] = data
}
- p, err := pt.GetPartitionByRow(exprCtx, locateKey)
+ p, err := pt.GetPartitionByRow(exprCtx.GetEvalCtx(), locateKey)
if table.ErrNoPartitionForGivenValue.Equal(err) {
continue
}
@@ -4203,7 +4219,7 @@ func (builder *dataReaderBuilder) buildTableReaderForIndexJoin(ctx context.Conte
for i, data := range content.Keys {
locateKey[keyColOffsets[i]] = data
}
- p, err := pt.GetPartitionByRow(exprCtx, locateKey)
+ p, err := pt.GetPartitionByRow(exprCtx.GetEvalCtx(), locateKey)
if table.ErrNoPartitionForGivenValue.Equal(err) {
continue
}
@@ -4561,7 +4577,7 @@ func buildRangesForIndexJoin(ctx sessionctx.Context, lookUpContents []*join.Inde
}
}
if cwc == nil {
- // A deep copy is need here because the old []*range.Range is overwriten
+ // A deep copy is need here because the old []*range.Range is overwritten
for _, ran := range ranges {
retRanges = append(retRanges, ran.Clone())
}
diff --git a/pkg/executor/compiler.go b/pkg/executor/compiler.go
index b9269bc7fb0a3..8771753e78626 100644
--- a/pkg/executor/compiler.go
+++ b/pkg/executor/compiler.go
@@ -88,10 +88,7 @@ func (c *Compiler) Compile(ctx context.Context, stmtNode ast.StmtNode) (_ *ExecS
sessVars := c.Ctx.GetSessionVars()
stmtCtx := sessVars.StmtCtx
// handle the execute statement
- var (
- pointGetPlanShortPathOK bool
- preparedObj *plannercore.PlanCacheStmt
- )
+ var preparedObj *plannercore.PlanCacheStmt
if execStmt, ok := stmtNode.(*ast.ExecuteStmt); ok {
if preparedObj, err = plannercore.GetPreparedStmt(execStmt, sessVars); err != nil {
@@ -128,21 +125,11 @@ func (c *Compiler) Compile(ctx context.Context, stmtNode ast.StmtNode) (_ *ExecS
Ctx: c.Ctx,
OutputNames: names,
}
-
- if _, ok := stmtNode.(*ast.ExecuteStmt); ok {
- if pointGetPlanShortPathOK, err = plannercore.IsPointGetPlanShortPathOK(c.Ctx, is, preparedObj, finalPlan); err != nil {
- return nil, err
- }
- }
-
// Use cached plan if possible.
- if pointGetPlanShortPathOK {
- if ep, ok := stmt.Plan.(*plannercore.Execute); ok {
- if pointPlan, ok := ep.Plan.(*plannercore.PointGetPlan); ok {
- stmtCtx.SetPlan(stmt.Plan)
- stmtCtx.SetPlanDigest(preparedObj.NormalizedPlan, preparedObj.PlanDigest)
- stmt.Plan = pointPlan
- stmt.PsStmt = preparedObj
+ if preparedObj != nil && plannercore.IsSafeToReusePointGetExecutor(c.Ctx, is, preparedObj) {
+ if exec, isExec := finalPlan.(*plannercore.Execute); isExec {
+ if pointPlan, isPointPlan := exec.Plan.(*plannercore.PointGetPlan); isPointPlan {
+ stmt.PsStmt, stmt.Plan = preparedObj, pointPlan // notify to re-use the cached plan
}
}
}
diff --git a/pkg/executor/delete.go b/pkg/executor/delete.go
index cbd328e14dafe..2af45fb344d0a 100644
--- a/pkg/executor/delete.go
+++ b/pkg/executor/delete.go
@@ -22,6 +22,7 @@ import (
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/model"
plannercore "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/planner/util"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/sessiontxn"
@@ -61,7 +62,7 @@ func (e *DeleteExec) Next(ctx context.Context, req *chunk.Chunk) error {
return e.deleteSingleTableByChunk(ctx)
}
-func (e *DeleteExec) deleteOneRow(tbl table.Table, handleCols plannercore.HandleCols, isExtraHandle bool, row []types.Datum) error {
+func (e *DeleteExec) deleteOneRow(tbl table.Table, handleCols util.HandleCols, isExtraHandle bool, row []types.Datum) error {
end := len(row)
if isExtraHandle {
end--
@@ -81,7 +82,7 @@ func (e *DeleteExec) deleteSingleTableByChunk(ctx context.Context) error {
var (
tbl table.Table
isExtrahandle bool
- handleCols plannercore.HandleCols
+ handleCols util.HandleCols
rowCount int
)
for _, info := range e.tblColPosInfos {
diff --git a/pkg/executor/distsql.go b/pkg/executor/distsql.go
index 41b9857b6a3e2..1d264a12fc44a 100644
--- a/pkg/executor/distsql.go
+++ b/pkg/executor/distsql.go
@@ -375,7 +375,7 @@ func (e *IndexReaderExecutor) open(ctx context.Context, kvRanges []kv.KeyRange)
}
results = append(results, result)
}
- e.result = distsql.NewSortedSelectResults(results, e.Schema(), e.byItems, e.memTracker)
+ e.result = distsql.NewSortedSelectResults(e.Ctx().GetExprCtx().GetEvalCtx(), results, e.Schema(), e.byItems, e.memTracker)
}
return nil
}
@@ -594,17 +594,16 @@ func (e *IndexLookUpExecutor) needPartitionHandle(tp getHandleType) (bool, error
outputOffsets := e.tableRequest.OutputOffsets
col = cols[outputOffsets[len(outputOffsets)-1]]
- // For TableScan, need partitionHandle in `indexOrder` when e.keepOrder == true
- needPartitionHandle = (e.index.Global || e.partitionTableMode) && e.keepOrder
+ // For TableScan, need partitionHandle in `indexOrder` when e.keepOrder == true or execute `admin check [table|index]` with global index
+ needPartitionHandle = ((e.index.Global || e.partitionTableMode) && e.keepOrder) || (e.index.Global && e.checkIndexValue != nil)
// no ExtraPidColID here, because TableScan shouldn't contain them.
hasExtraCol = col.ID == model.ExtraPhysTblID
}
- // TODO: fix global index related bugs later
// There will be two needPartitionHandle != hasExtraCol situations.
// Only `needPartitionHandle` == true and `hasExtraCol` == false are not allowed.
// `ExtraPhysTblID` will be used in `SelectLock` when `needPartitionHandle` == false and `hasExtraCol` == true.
- if needPartitionHandle && !hasExtraCol && !e.index.Global {
+ if needPartitionHandle && !hasExtraCol {
return needPartitionHandle, errors.Errorf("Internal error, needPartitionHandle != ret, tp(%d)", tp)
}
return needPartitionHandle, nil
@@ -621,7 +620,7 @@ func (e *IndexLookUpExecutor) getRetTpsForIndexReader() []*types.FieldType {
var tps []*types.FieldType
if len(e.byItems) != 0 {
for _, item := range e.byItems {
- tps = append(tps, item.Expr.GetType())
+ tps = append(tps, item.Expr.GetType(e.Ctx().GetExprCtx().GetEvalCtx()))
}
}
if e.isCommonHandle() {
@@ -651,7 +650,7 @@ func (e *IndexLookUpExecutor) startIndexWorker(ctx context.Context, workCh chan<
kvRanges = e.partitionKVRanges
}
// When len(kvrange) = 1, no sorting is required,
- // so remove byItems and non-necessary output colums
+ // so remove byItems and non-necessary output columns
if len(kvRanges) == 1 {
e.dagPB.OutputOffsets = e.dagPB.OutputOffsets[len(e.byItems):]
e.byItems = nil
@@ -720,7 +719,7 @@ func (e *IndexLookUpExecutor) startIndexWorker(ctx context.Context, workCh chan<
worker.batchSize = min(initBatchSize, worker.maxBatchSize)
if len(results) > 1 && len(e.byItems) != 0 {
// e.Schema() not the output schema for indexReader, and we put byItems related column at first in `buildIndexReq`, so use nil here.
- ssr := distsql.NewSortedSelectResults(results, nil, e.byItems, e.memTracker)
+ ssr := distsql.NewSortedSelectResults(e.Ctx().GetExprCtx().GetEvalCtx(), results, nil, e.byItems, e.memTracker)
results = []distsql.SelectResult{ssr}
}
ctx1, cancel := context.WithCancel(ctx)
@@ -739,7 +738,7 @@ func (e *IndexLookUpExecutor) startIndexWorker(ctx context.Context, workCh chan<
return nil
}
-// startTableWorker launchs some background goroutines which pick tasks from workCh and execute the task.
+// startTableWorker launches some background goroutines which pick tasks from workCh and execute the task.
func (e *IndexLookUpExecutor) startTableWorker(ctx context.Context, workCh <-chan *lookupTableTask) {
lookupConcurrencyLimit := e.Ctx().GetSessionVars().IndexLookupConcurrency()
e.tblWorkerWg.Add(lookupConcurrencyLimit)
diff --git a/pkg/executor/executor.go b/pkg/executor/executor.go
index 54bd3080fcfd7..a382e133a86ac 100644
--- a/pkg/executor/executor.go
+++ b/pkg/executor/executor.go
@@ -55,6 +55,7 @@ import (
planctx "github.com/pingcap/tidb/pkg/planner/context"
plannercore "github.com/pingcap/tidb/pkg/planner/core"
"github.com/pingcap/tidb/pkg/planner/core/base"
+ plannerutil "github.com/pingcap/tidb/pkg/planner/util"
"github.com/pingcap/tidb/pkg/planner/util/fixcontrol"
"github.com/pingcap/tidb/pkg/privilege"
"github.com/pingcap/tidb/pkg/resourcemanager/pool/workerpool"
@@ -558,7 +559,7 @@ func ts2Time(timestamp uint64, loc *time.Location) types.Time {
duration := time.Duration(math.Pow10(9-types.DefaultFsp)) * time.Nanosecond
t := model.TSConvert2Time(timestamp)
t.Truncate(duration)
- return types.NewTime(types.FromGoTime(t.In(loc)), mysql.TypeDatetime, types.DefaultFsp)
+ return types.NewTime(types.FromGoTime(t.In(loc)), mysql.TypeDatetime, types.MaxFsp)
}
// ShowDDLJobQueriesExec represents a show DDL job queries executor.
@@ -1099,7 +1100,7 @@ type SelectLockExec struct {
keys []kv.Key
// The children may be a join of multiple tables, so we need a map.
- tblID2Handle map[int64][]plannercore.HandleCols
+ tblID2Handle map[int64][]plannerutil.HandleCols
// When SelectLock work on a partition table, we need the partition ID
// (Physical Table ID) instead of the 'logical' table ID to calculate
@@ -2035,7 +2036,14 @@ func ResetContextOfStmt(ctx sessionctx.Context, s ast.StmtNode) (err error) {
sc.RuntimeStatsColl = execdetails.NewRuntimeStatsColl(reuseObj)
// also enable index usage collector
- sc.IndexUsageCollector = ctx.NewStmtIndexUsageCollector()
+ if sc.IndexUsageCollector == nil {
+ sc.IndexUsageCollector = ctx.NewStmtIndexUsageCollector()
+ } else {
+ sc.IndexUsageCollector.Reset()
+ }
+ } else {
+ // turn off the index usage collector
+ sc.IndexUsageCollector = nil
}
sc.SetForcePlanCache(fixcontrol.GetBoolWithDefault(vars.OptimizerFixControl, fixcontrol.Fix49736, false))
diff --git a/pkg/executor/executor_required_rows_test.go b/pkg/executor/executor_required_rows_test.go
index ad222b6ddf920..b2782935b5cdf 100644
--- a/pkg/executor/executor_required_rows_test.go
+++ b/pkg/executor/executor_required_rows_test.go
@@ -393,8 +393,9 @@ func buildTopNExec(ctx sessionctx.Context, offset, count int, byItems []*util.By
ExecSchema: src.Schema(),
}
return &sortexec.TopNExec{
- SortExec: sortExec,
- Limit: &plannercore.PhysicalLimit{Count: uint64(count), Offset: uint64(offset)},
+ SortExec: sortExec,
+ Limit: &plannercore.PhysicalLimit{Count: uint64(count), Offset: uint64(offset)},
+ Concurrency: 5,
}
}
@@ -733,7 +734,7 @@ func buildMergeJoinExec(ctx sessionctx.Context, joinType plannercore.JoinType, i
j.CompareFuncs = make([]expression.CompareFunc, 0, len(j.LeftJoinKeys))
for i := range j.LeftJoinKeys {
- j.CompareFuncs = append(j.CompareFuncs, expression.GetCmpFunction(nil, j.LeftJoinKeys[i], j.RightJoinKeys[i]))
+ j.CompareFuncs = append(j.CompareFuncs, expression.GetCmpFunction(ctx.GetExprCtx(), j.LeftJoinKeys[i], j.RightJoinKeys[i]))
}
b := newExecutorBuilder(ctx, nil)
diff --git a/pkg/executor/executor_txn_test.go b/pkg/executor/executor_txn_test.go
index bf63b8d072f8c..76f5773ceca7e 100644
--- a/pkg/executor/executor_txn_test.go
+++ b/pkg/executor/executor_txn_test.go
@@ -18,12 +18,15 @@ import (
"fmt"
"strconv"
"strings"
+ "sync"
"testing"
"time"
+ "github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/executor"
"github.com/pingcap/tidb/pkg/sessionctx/binloginfo"
"github.com/pingcap/tidb/pkg/testkit"
+ "github.com/pingcap/tidb/pkg/testkit/testfailpoint"
"github.com/stretchr/testify/require"
)
@@ -694,3 +697,40 @@ func TestSavepointWithBinlog(t *testing.T) {
tk.MustExec("commit")
tk.MustQuery("select * from t").Check(testkit.Rows("1 1"))
}
+
+func TestColumnNotMatchError(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.Session().GetSessionVars().BinlogClient = binloginfo.MockPumpsClient(&testkit.MockPumpClient{})
+ tk.MustExec("set @@global.tidb_enable_metadata_lock=0")
+ tk.MustExec("use test")
+ tk2 := testkit.NewTestKit(t, store)
+ tk2.MustExec("use test")
+ tk.MustExec("create table t(id int primary key, a int)")
+ tk.MustExec("insert into t values(1, 2)")
+
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onAddColumnStateWriteReorg", func() {
+ tk.MustExec("begin;")
+ })
+ var wg sync.WaitGroup
+ wg.Add(1)
+ go func() {
+ tk2.MustExec("alter table t add column wait_notify int")
+ wg.Done()
+ }()
+ wg.Wait()
+ tk.MustExec("delete from t where id=1")
+ tk.MustGetErrCode("commit", errno.ErrInfoSchemaChanged)
+
+ testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onDropColumnStateWriteOnly", func() {
+ tk.MustExec("begin;")
+ })
+ wg.Add(1)
+ go func() {
+ tk2.MustExec("alter table t drop column wait_notify")
+ wg.Done()
+ }()
+ wg.Wait()
+ tk.MustExec("delete from t where id=1")
+ tk.MustGetErrCode("commit", errno.ErrInfoSchemaChanged)
+}
diff --git a/pkg/executor/import_into.go b/pkg/executor/import_into.go
index 42f4e33c85698..7d9a4f92efd95 100644
--- a/pkg/executor/import_into.go
+++ b/pkg/executor/import_into.go
@@ -45,11 +45,6 @@ import (
"golang.org/x/sync/errgroup"
)
-var (
- // TestCancelFunc for test.
- TestCancelFunc context.CancelFunc
-)
-
const unknownImportedRowCount = -1
// ImportIntoExec represents a IMPORT INTO executor.
@@ -124,12 +119,7 @@ func (e *ImportIntoExec) Next(ctx context.Context, req *chunk.Chunk) (err error)
return err
}
- failpoint.Inject("cancellableCtx", func() {
- // KILL is not implemented in testkit, so we use a fail-point to simulate it.
- newCtx, cancel := context.WithCancel(ctx)
- ctx = newCtx
- TestCancelFunc = cancel
- })
+ failpoint.InjectCall("cancellableCtx", &ctx)
jobID, task, err := e.submitTask(ctx)
if err != nil {
diff --git a/pkg/executor/importer/import.go b/pkg/executor/importer/import.go
index 2b526953cff8a..dd8a54a36af2a 100644
--- a/pkg/executor/importer/import.go
+++ b/pkg/executor/importer/import.go
@@ -590,7 +590,7 @@ func (p *Plan) initOptions(ctx context.Context, seCtx sessionctx.Context, option
}
optAsString := func(opt *plannercore.LoadDataOpt) (string, error) {
- if opt.Value.GetType().GetType() != mysql.TypeVarString {
+ if opt.Value.GetType(seCtx.GetExprCtx().GetEvalCtx()).GetType() != mysql.TypeVarString {
return "", exeerrors.ErrInvalidOptionVal.FastGenByArgs(opt.Name)
}
val, isNull, err2 := opt.Value.EvalString(seCtx.GetExprCtx().GetEvalCtx(), chunk.Row{})
@@ -601,7 +601,7 @@ func (p *Plan) initOptions(ctx context.Context, seCtx sessionctx.Context, option
}
optAsInt64 := func(opt *plannercore.LoadDataOpt) (int64, error) {
// current parser takes integer and bool as mysql.TypeLonglong
- if opt.Value.GetType().GetType() != mysql.TypeLonglong || mysql.HasIsBooleanFlag(opt.Value.GetType().GetFlag()) {
+ if opt.Value.GetType(seCtx.GetExprCtx().GetEvalCtx()).GetType() != mysql.TypeLonglong || mysql.HasIsBooleanFlag(opt.Value.GetType(seCtx.GetExprCtx().GetEvalCtx()).GetFlag()) {
return 0, exeerrors.ErrInvalidOptionVal.FastGenByArgs(opt.Name)
}
val, isNull, err2 := opt.Value.EvalInt(seCtx.GetExprCtx().GetEvalCtx(), chunk.Row{})
@@ -758,6 +758,10 @@ func (p *Plan) initOptions(ctx context.Context, seCtx sessionctx.Context, option
return exeerrors.ErrInvalidOptionVal.FastGenByArgs("skip_rows, should be <= 1 when split-file is enabled")
}
+ if p.SplitFile && len(p.LinesTerminatedBy) == 0 {
+ return exeerrors.ErrInvalidOptionVal.FastGenByArgs("lines_terminated_by, should not be empty when use split_file")
+ }
+
p.adjustOptions(targetNodeCPUCnt)
return nil
}
diff --git a/pkg/executor/importer/precheck.go b/pkg/executor/importer/precheck.go
index db0739be0055d..5f4363ba5b900 100644
--- a/pkg/executor/importer/precheck.go
+++ b/pkg/executor/importer/precheck.go
@@ -130,7 +130,7 @@ func (*LoadDataController) checkCDCPiTRTasks(ctx context.Context) error {
return exeerrors.ErrLoadDataPreCheckFailed.FastGenByArgs(fmt.Sprintf("found PiTR log streaming task(s): %v,", names))
}
- nameSet, err := cdcutil.GetCDCChangefeedNameSet(ctx, cli.GetClient())
+ nameSet, err := cdcutil.GetRunningChangefeeds(ctx, cli.GetClient())
if err != nil {
return errors.Trace(err)
}
diff --git a/pkg/executor/index_merge_reader.go b/pkg/executor/index_merge_reader.go
index 7930f53dd8738..79b696e8966cd 100644
--- a/pkg/executor/index_merge_reader.go
+++ b/pkg/executor/index_merge_reader.go
@@ -134,7 +134,7 @@ type IndexMergeReaderExecutor struct {
partialNetDataSizes []float64
dataAvgRowSize float64
- handleCols plannercore.HandleCols
+ handleCols plannerutil.HandleCols
stats *IndexMergeRuntimeStat
// Indicates whether there is correlated column in filter or table/index range.
@@ -441,7 +441,7 @@ func (e *IndexMergeReaderExecutor) startPartialIndexWorker(ctx context.Context,
worker.batchSize = min(e.MaxChunkSize(), worker.maxBatchSize)
if len(results) > 1 && len(e.byItems) != 0 {
// e.Schema() not the output schema for partialIndexReader, and we put byItems related column at first in `buildIndexReq`, so use nil here.
- ssr := distsql.NewSortedSelectResults(results, nil, e.byItems, e.memTracker)
+ ssr := distsql.NewSortedSelectResults(e.Ctx().GetExprCtx().GetEvalCtx(), results, nil, e.byItems, e.memTracker)
results = []distsql.SelectResult{ssr}
}
ctx1, cancel := context.WithCancel(ctx)
@@ -641,7 +641,7 @@ func (w *partialTableWorker) needPartitionHandle() (bool, error) {
}
func (w *partialTableWorker) fetchHandles(ctx context.Context, exitCh <-chan struct{}, fetchCh chan<- *indexMergeTableTask,
- finished <-chan struct{}, handleCols plannercore.HandleCols, parTblIdx int, partialPlanIndex int) (count int64, err error) {
+ finished <-chan struct{}, handleCols plannerutil.HandleCols, parTblIdx int, partialPlanIndex int) (count int64, err error) {
chk := w.tableReader.NewChunkWithCapacity(w.getRetTpsForTableScan(), w.maxChunkSize, w.maxBatchSize)
for {
start := time.Now()
@@ -674,7 +674,7 @@ func (w *partialTableWorker) getRetTpsForTableScan() []*types.FieldType {
return exec.RetTypes(w.tableReader)
}
-func (w *partialTableWorker) extractTaskHandles(ctx context.Context, chk *chunk.Chunk, handleCols plannercore.HandleCols) (
+func (w *partialTableWorker) extractTaskHandles(ctx context.Context, chk *chunk.Chunk, handleCols plannerutil.HandleCols) (
handles []kv.Handle, retChk *chunk.Chunk, err error) {
handles = make([]kv.Handle, 0, w.batchSize)
if len(w.byItems) != 0 {
@@ -1012,7 +1012,7 @@ func (h *handleHeap) Pop() any {
func (w *indexMergeProcessWorker) NewHandleHeap(taskMap map[int][]*indexMergeTableTask, memTracker *memory.Tracker) *handleHeap {
compareFuncs := make([]chunk.CompareFunc, 0, len(w.indexMerge.byItems))
for _, item := range w.indexMerge.byItems {
- keyType := item.Expr.GetType()
+ keyType := item.Expr.GetType(w.indexMerge.Ctx().GetExprCtx().GetEvalCtx())
compareFuncs = append(compareFuncs, chunk.GetCompareFunc(keyType))
}
@@ -1724,7 +1724,7 @@ func (w *partialIndexWorker) fetchHandles(
exitCh <-chan struct{},
fetchCh chan<- *indexMergeTableTask,
finished <-chan struct{},
- handleCols plannercore.HandleCols,
+ handleCols plannerutil.HandleCols,
partialPlanIndex int) (count int64, err error) {
tps := w.getRetTpsForIndexScan(handleCols)
chk := chunk.NewChunkWithCapacity(tps, w.maxChunkSize)
@@ -1757,11 +1757,11 @@ func (w *partialIndexWorker) fetchHandles(
return count, nil
}
-func (w *partialIndexWorker) getRetTpsForIndexScan(handleCols plannercore.HandleCols) []*types.FieldType {
+func (w *partialIndexWorker) getRetTpsForIndexScan(handleCols plannerutil.HandleCols) []*types.FieldType {
var tps []*types.FieldType
if len(w.byItems) != 0 {
for _, item := range w.byItems {
- tps = append(tps, item.Expr.GetType())
+ tps = append(tps, item.Expr.GetType(w.sc.GetExprCtx().GetEvalCtx()))
}
}
tps = append(tps, handleCols.GetFieldsTypes()...)
@@ -1771,7 +1771,7 @@ func (w *partialIndexWorker) getRetTpsForIndexScan(handleCols plannercore.Handle
return tps
}
-func (w *partialIndexWorker) extractTaskHandles(ctx context.Context, chk *chunk.Chunk, idxResult distsql.SelectResult, handleCols plannercore.HandleCols) (
+func (w *partialIndexWorker) extractTaskHandles(ctx context.Context, chk *chunk.Chunk, idxResult distsql.SelectResult, handleCols plannerutil.HandleCols) (
handles []kv.Handle, retChk *chunk.Chunk, err error) {
handles = make([]kv.Handle, 0, w.batchSize)
if len(w.byItems) != 0 {
diff --git a/pkg/executor/infoschema_reader.go b/pkg/executor/infoschema_reader.go
index b3ddabc37fa98..ada196fcd2699 100644
--- a/pkg/executor/infoschema_reader.go
+++ b/pkg/executor/infoschema_reader.go
@@ -535,7 +535,7 @@ func (e *memtableRetriever) setDataFromReferConst(sctx sessionctx.Context, schem
func (e *memtableRetriever) updateStatsCacheIfNeed() bool {
for _, col := range e.columns {
- // only the following columns need stats cahce.
+ // only the following columns need stats cache.
if col.Name.O == "AVG_ROW_LENGTH" || col.Name.O == "DATA_LENGTH" || col.Name.O == "INDEX_LENGTH" || col.Name.O == "TABLE_ROWS" {
return true
}
@@ -922,7 +922,7 @@ ForColumnsTag:
idx := expression.FindFieldNameIdxByColName(e.viewOutputNamesMap[tbl.ID], col.Name.L)
if idx >= 0 {
col1 := e.viewSchemaMap[tbl.ID].Columns[idx]
- ft = col1.GetType()
+ ft = col1.GetType(sctx.GetExprCtx().GetEvalCtx())
}
}
e.viewMu.RUnlock()
@@ -1249,6 +1249,7 @@ func (e *memtableRetriever) setDataFromIndexes(ctx sessionctx.Context, schemas [
0, // INDEX_ID
"YES", // IS_VISIBLE
"YES", // CLUSTERED
+ 0, // IS_GLOBAL
)
rows = append(rows, record)
}
@@ -1294,6 +1295,7 @@ func (e *memtableRetriever) setDataFromIndexes(ctx sessionctx.Context, schemas [
idxInfo.ID, // INDEX_ID
visible, // IS_VISIBLE
isClustered, // CLUSTERED
+ idxInfo.Global, // IS_GLOBAL
)
rows = append(rows, record)
}
diff --git a/pkg/executor/insert.go b/pkg/executor/insert.go
index 24c7cf6066587..86895458e9ecd 100644
--- a/pkg/executor/insert.go
+++ b/pkg/executor/insert.go
@@ -161,12 +161,12 @@ func prefetchConflictedOldRows(ctx context.Context, txn kv.Transaction, rows []t
for _, uk := range r.uniqueKeys {
if val, found := values[string(uk.newKey)]; found {
if tablecodec.IsTempIndexKey(uk.newKey) {
- // If it is a temp index, the value cannot be decoded by DecodeHandleInUniqueIndexValue.
+ // If it is a temp index, the value cannot be decoded by DecodeHandleInIndexValue.
// Since this function is an optimization, we can skip prefetching the rows referenced by
// temp indexes.
continue
}
- handle, err := tablecodec.DecodeHandleInUniqueIndexValue(val, uk.commonHandle)
+ handle, err := tablecodec.DecodeHandleInIndexValue(val)
if err != nil {
return err
}
@@ -253,7 +253,7 @@ func (e *InsertExec) batchUpdateDupRows(ctx context.Context, newRows [][]types.D
}
for _, uk := range r.uniqueKeys {
- _, handle, err := tables.FetchDuplicatedHandle(ctx, uk.newKey, true, txn, e.Table.Meta().ID, uk.commonHandle)
+ _, handle, err := tables.FetchDuplicatedHandle(ctx, uk.newKey, true, txn, e.Table.Meta().ID)
if err != nil {
return err
}
diff --git a/pkg/executor/insert_common.go b/pkg/executor/insert_common.go
index 4915286e920fa..da1d9bd5f6703 100644
--- a/pkg/executor/insert_common.go
+++ b/pkg/executor/insert_common.go
@@ -1178,7 +1178,7 @@ func (e *InsertValues) handleDuplicateKey(ctx context.Context, txn kv.Transactio
}
return true, nil
}
- _, handle, err := tables.FetchDuplicatedHandle(ctx, uk.newKey, true, txn, e.Table.Meta().ID, uk.commonHandle)
+ _, handle, err := tables.FetchDuplicatedHandle(ctx, uk.newKey, true, txn, e.Table.Meta().ID)
if err != nil {
return false, err
}
@@ -1373,7 +1373,11 @@ func (e *InsertValues) removeRow(
return true, nil
}
- err = r.t.RemoveRecord(e.Ctx().GetTableCtx(), handle, oldRow)
+ if ph, ok := handle.(kv.PartitionHandle); ok {
+ err = e.Table.(table.PartitionedTable).GetPartition(ph.PartitionID).RemoveRecord(e.Ctx().GetTableCtx(), ph.Handle, oldRow)
+ } else {
+ err = r.t.RemoveRecord(e.Ctx().GetTableCtx(), handle, oldRow)
+ }
if err != nil {
return false, err
}
diff --git a/pkg/executor/inspection_result.go b/pkg/executor/inspection_result.go
index bab436ad44b5e..8596f562ca720 100644
--- a/pkg/executor/inspection_result.go
+++ b/pkg/executor/inspection_result.go
@@ -28,6 +28,7 @@ import (
"github.com/pingcap/tidb/pkg/infoschema"
"github.com/pingcap/tidb/pkg/kv"
plannercore "github.com/pingcap/tidb/pkg/planner/core"
+ plannerutil "github.com/pingcap/tidb/pkg/planner/util"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/types"
@@ -58,7 +59,7 @@ type (
inspectionFilter struct {
set set.StringSet
- timeRange plannercore.QueryTimeRange
+ timeRange plannerutil.QueryTimeRange
}
inspectionRule interface {
@@ -108,7 +109,7 @@ type inspectionResultRetriever struct {
dummyCloser
retrieved bool
extractor *plannercore.InspectionResultTableExtractor
- timeRange plannercore.QueryTimeRange
+ timeRange plannerutil.QueryTimeRange
instanceToStatusAddress map[string]string
statusToInstanceAddress map[string]string
}
@@ -490,7 +491,7 @@ func (nodeLoadInspection) inspect(ctx context.Context, sctx sessionctx.Context,
type inspectVirtualMemUsage struct{}
-func (inspectVirtualMemUsage) genSQL(timeRange plannercore.QueryTimeRange) string {
+func (inspectVirtualMemUsage) genSQL(timeRange plannerutil.QueryTimeRange) string {
sql := fmt.Sprintf("select instance, max(value) as max_usage from metrics_schema.node_memory_usage %s group by instance having max_usage >= 70", timeRange.Condition())
return sql
}
@@ -513,7 +514,7 @@ func (inspectVirtualMemUsage) getItem() string {
type inspectSwapMemoryUsed struct{}
-func (inspectSwapMemoryUsed) genSQL(timeRange plannercore.QueryTimeRange) string {
+func (inspectSwapMemoryUsed) genSQL(timeRange plannerutil.QueryTimeRange) string {
sql := fmt.Sprintf("select instance, max(value) as max_used from metrics_schema.node_memory_swap_used %s group by instance having max_used > 0", timeRange.Condition())
return sql
}
@@ -535,7 +536,7 @@ func (inspectSwapMemoryUsed) getItem() string {
type inspectDiskUsage struct{}
-func (inspectDiskUsage) genSQL(timeRange plannercore.QueryTimeRange) string {
+func (inspectDiskUsage) genSQL(timeRange plannerutil.QueryTimeRange) string {
sql := fmt.Sprintf("select instance, device, max(value) as max_usage from metrics_schema.node_disk_usage %v and device like '/%%' group by instance, device having max_usage >= 70", timeRange.Condition())
return sql
}
@@ -561,7 +562,7 @@ type inspectCPULoad struct {
tbl string
}
-func (i inspectCPULoad) genSQL(timeRange plannercore.QueryTimeRange) string {
+func (i inspectCPULoad) genSQL(timeRange plannerutil.QueryTimeRange) string {
sql := fmt.Sprintf(`select t1.instance, t1.max_load , 0.7*t2.cpu_count from
(select instance,max(value) as max_load from metrics_schema.%[1]s %[2]s group by instance) as t1 join
(select instance,max(value) as cpu_count from metrics_schema.node_virtual_cpus %[2]s group by instance) as t2
@@ -1039,7 +1040,7 @@ func (thresholdCheckInspection) inspectThreshold2(ctx context.Context, sctx sess
}
type ruleChecker interface {
- genSQL(timeRange plannercore.QueryTimeRange) string
+ genSQL(timeRange plannerutil.QueryTimeRange) string
genResult(sql string, row chunk.Row) inspectionResult
getItem() string
}
@@ -1050,10 +1051,10 @@ type compareStoreStatus struct {
threshold float64
}
-func (c compareStoreStatus) genSQL(timeRange plannercore.QueryTimeRange) string {
+func (c compareStoreStatus) genSQL(timeRange plannerutil.QueryTimeRange) string {
condition := fmt.Sprintf(`where t1.time>='%[1]s' and t1.time<='%[2]s' and
- t2.time>='%[1]s' and t2.time<='%[2]s'`, timeRange.From.Format(plannercore.MetricTableTimeFormat),
- timeRange.To.Format(plannercore.MetricTableTimeFormat))
+ t2.time>='%[1]s' and t2.time<='%[2]s'`, timeRange.From.Format(plannerutil.MetricTableTimeFormat),
+ timeRange.To.Format(plannerutil.MetricTableTimeFormat))
return fmt.Sprintf(`
SELECT t1.address,
max(t1.value),
@@ -1097,7 +1098,7 @@ func (c compareStoreStatus) getItem() string {
type checkRegionHealth struct{}
-func (checkRegionHealth) genSQL(timeRange plannercore.QueryTimeRange) string {
+func (checkRegionHealth) genSQL(timeRange plannerutil.QueryTimeRange) string {
condition := timeRange.Condition()
return fmt.Sprintf(`select instance, sum(value) as sum_value from metrics_schema.pd_region_health %s and
type in ('extra-peer-region-count','learner-peer-region-count','pending-peer-region-count') having sum_value>100`, condition)
@@ -1125,7 +1126,7 @@ func (checkRegionHealth) getItem() string {
type checkStoreRegionTooMuch struct{}
-func (checkStoreRegionTooMuch) genSQL(timeRange plannercore.QueryTimeRange) string {
+func (checkStoreRegionTooMuch) genSQL(timeRange plannerutil.QueryTimeRange) string {
condition := timeRange.Condition()
return fmt.Sprintf(`select address, max(value) from metrics_schema.pd_scheduler_store_status %s and type='region_count' and value > 20000 group by address`, condition)
}
diff --git a/pkg/executor/inspection_summary.go b/pkg/executor/inspection_summary.go
index 0a45b621e708f..9b02880df5852 100644
--- a/pkg/executor/inspection_summary.go
+++ b/pkg/executor/inspection_summary.go
@@ -24,6 +24,7 @@ import (
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/model"
plannercore "github.com/pingcap/tidb/pkg/planner/core"
+ plannerutil "github.com/pingcap/tidb/pkg/planner/util"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util"
@@ -34,7 +35,7 @@ type inspectionSummaryRetriever struct {
retrieved bool
table *model.TableInfo
extractor *plannercore.InspectionSummaryTableExtractor
- timeRange plannercore.QueryTimeRange
+ timeRange plannerutil.QueryTimeRange
}
// inspectionSummaryRules is used to maintain
diff --git a/pkg/executor/internal/mpp/local_mpp_coordinator.go b/pkg/executor/internal/mpp/local_mpp_coordinator.go
index 77fd00374e1ce..15ef85eaa2da0 100644
--- a/pkg/executor/internal/mpp/local_mpp_coordinator.go
+++ b/pkg/executor/internal/mpp/local_mpp_coordinator.go
@@ -499,9 +499,6 @@ func (c *localMppCoordinator) cancelMppTasks() {
func (c *localMppCoordinator) receiveResults(req *kv.MPPDispatchRequest, taskMeta *mpp.TaskMeta, bo *backoff.Backoffer) {
stream, err := c.sessionCtx.GetMPPClient().EstablishMPPConns(kv.EstablishMPPConnsParam{Ctx: bo.GetCtx(), Req: req, TaskMeta: taskMeta})
if err != nil {
- if stream != nil {
- stream.Close()
- }
// if NeedTriggerFallback is true, we return timeout to trigger tikv's fallback
if c.needTriggerFallback {
c.sendError(derr.ErrTiFlashServerTimeout)
diff --git a/pkg/executor/internal/vecgroupchecker/vec_group_checker.go b/pkg/executor/internal/vecgroupchecker/vec_group_checker.go
index 85a63c0538c7e..056ee8a478032 100644
--- a/pkg/executor/internal/vecgroupchecker/vec_group_checker.go
+++ b/pkg/executor/internal/vecgroupchecker/vec_group_checker.go
@@ -160,7 +160,7 @@ func (e *VecGroupChecker) SplitIntoGroups(chk *chunk.Chunk) (isFirstGroupSameAsP
func (e *VecGroupChecker) getFirstAndLastRowDatum(
item expression.Expression, chk *chunk.Chunk, numRows int) (err error) {
var firstRowDatum, lastRowDatum types.Datum
- tp := item.GetType()
+ tp := item.GetType(e.ctx)
eType := tp.EvalType()
switch eType {
case types.ETInt:
@@ -328,7 +328,7 @@ func (e *VecGroupChecker) getFirstAndLastRowDatum(
// And resolve the rows into groups according to the evaluation results
func (e *VecGroupChecker) evalGroupItemsAndResolveGroups(
item expression.Expression, vecEnabled bool, chk *chunk.Chunk, numRows int) (err error) {
- tp := item.GetType()
+ tp := item.GetType(e.ctx)
eType := tp.EvalType()
if e.allocateBuffer == nil {
e.allocateBuffer = expression.GetColumn
diff --git a/pkg/executor/join/join.go b/pkg/executor/join/join.go
index 2818929c93df2..4da5f2b5346ff 100644
--- a/pkg/executor/join/join.go
+++ b/pkg/executor/join/join.go
@@ -272,9 +272,6 @@ func (fetcher *ProbeSideTupleFetcher) fetchProbeSideChunks(ctx context.Context,
probeSideResult.Reset()
}
})
- if probeSideResult.NumRows() == 0 && !fetcher.UseOuterToBuild {
- fetcher.finished.Store(true)
- }
emptyBuild, buildErr := fetcher.wait4BuildSide()
if buildErr != nil {
fetcher.joinResultCh <- &hashjoinWorkerResult{
@@ -332,12 +329,22 @@ func (w *BuildWorker) fetchBuildSideRows(ctx context.Context, chkCh chan<- *chun
}
})
sessVars := w.HashJoinCtx.SessCtx.GetSessionVars()
+ failpoint.Inject("issue51998", func(val failpoint.Value) {
+ if val.(bool) {
+ time.Sleep(2 * time.Second)
+ }
+ })
for {
if w.HashJoinCtx.finished.Load() {
return
}
chk := w.HashJoinCtx.ChunkAllocPool.Alloc(w.BuildSideExec.RetFieldTypes(), sessVars.MaxChunkSize, sessVars.MaxChunkSize)
err = exec.Next(ctx, w.BuildSideExec, chk)
+ failpoint.Inject("issue51998", func(val failpoint.Value) {
+ if val.(bool) {
+ err = errors.Errorf("issue51998 build return error")
+ }
+ })
if err != nil {
errCh <- errors.Trace(err)
return
diff --git a/pkg/executor/load_data.go b/pkg/executor/load_data.go
index 45b96d092052a..e42bc778f77cb 100644
--- a/pkg/executor/load_data.go
+++ b/pkg/executor/load_data.go
@@ -211,7 +211,9 @@ func (e *LoadDataWorker) LoadLocal(ctx context.Context, r io.ReadCloser) error {
readers := []importer.LoadDataReaderInfo{{
Opener: func(_ context.Context) (io.ReadSeekCloser, error) {
addedSeekReader := NewSimpleSeekerOnReadCloser(r)
- return storage.InterceptDecompressReader(addedSeekReader, compressTp2, storage.DecompressConfig{})
+ return storage.InterceptDecompressReader(addedSeekReader, compressTp2, storage.DecompressConfig{
+ ZStdDecodeConcurrency: 1,
+ })
}}}
return e.load(ctx, readers)
}
@@ -356,7 +358,7 @@ type commitTask struct {
rows [][]types.Datum
}
-// processStream always trys to build a parser from channel and process it. When
+// processStream always tries to build a parser from channel and process it. When
// it returns nil, it means all data is read.
func (w *encodeWorker) processStream(
ctx context.Context,
diff --git a/pkg/executor/mem_reader.go b/pkg/executor/mem_reader.go
index 1f32e6b3654e8..9b287f74ee694 100644
--- a/pkg/executor/mem_reader.go
+++ b/pkg/executor/mem_reader.go
@@ -982,8 +982,10 @@ func (iter *memRowsIterForIndex) Next() ([]types.Datum, error) {
// filter key/value by partitition id
if iter.index.Global {
- seg := tablecodec.SplitIndexValue(value)
- _, pid, _ := codec.DecodeInt(seg.PartitionID)
+ _, pid, err := codec.DecodeInt(tablecodec.SplitIndexValue(value).PartitionID)
+ if err != nil {
+ return nil, err
+ }
if _, exists := iter.partitionIDMap[pid]; !exists {
continue
}
diff --git a/pkg/executor/metrics_reader.go b/pkg/executor/metrics_reader.go
index 5a214b3a9b61b..31a0073148584 100644
--- a/pkg/executor/metrics_reader.go
+++ b/pkg/executor/metrics_reader.go
@@ -30,6 +30,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
plannercore "github.com/pingcap/tidb/pkg/planner/core"
+ plannerutil "github.com/pingcap/tidb/pkg/planner/util"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util"
@@ -188,7 +189,7 @@ type MetricsSummaryRetriever struct {
dummyCloser
table *model.TableInfo
extractor *plannercore.MetricSummaryTableExtractor
- timeRange plannercore.QueryTimeRange
+ timeRange plannerutil.QueryTimeRange
retrieved bool
}
@@ -265,7 +266,7 @@ type MetricsSummaryByLabelRetriever struct {
dummyCloser
table *model.TableInfo
extractor *plannercore.MetricSummaryTableExtractor
- timeRange plannercore.QueryTimeRange
+ timeRange plannerutil.QueryTimeRange
retrieved bool
}
diff --git a/pkg/executor/point_get.go b/pkg/executor/point_get.go
index ec7f7be4245a0..c26b430a09ea8 100644
--- a/pkg/executor/point_get.go
+++ b/pkg/executor/point_get.go
@@ -17,6 +17,8 @@ package executor
import (
"context"
"fmt"
+ "sort"
+ "strconv"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
@@ -327,7 +329,7 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error {
}
var iv kv.Handle
- iv, err = tablecodec.DecodeHandleInUniqueIndexValue(e.handleVal, e.tblInfo.IsCommonHandle)
+ iv, err = tablecodec.DecodeHandleInIndexValue(e.handleVal)
if err != nil {
return err
}
@@ -347,8 +349,7 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error {
failpoint.InjectContext(ctx, "pointGetRepeatableReadTest-step2", nil)
})
if e.idxInfo.Global {
- segs := tablecodec.SplitIndexValue(e.handleVal)
- _, pid, err := codec.DecodeInt(segs.PartitionID)
+ _, pid, err := codec.DecodeInt(tablecodec.SplitIndexValue(e.handleVal).PartitionID)
if err != nil {
return err
}
@@ -387,19 +388,122 @@ func (e *PointGetExecutor) Next(ctx context.Context, req *chunk.Chunk) error {
}
return nil
}
- err = DecodeRowValToChunk(e.BaseExecutor.Ctx(), e.Schema(), e.tblInfo, e.handle, val, req, e.rowDecoder)
+
+ sctx := e.BaseExecutor.Ctx()
+ schema := e.Schema()
+ err = DecodeRowValToChunk(sctx, schema, e.tblInfo, e.handle, val, req, e.rowDecoder)
+ if err != nil {
+ return err
+ }
+
+ err = fillRowChecksum(sctx, 0, 1, schema, e.tblInfo, [][]byte{val}, []kv.Handle{e.handle}, req, nil)
if err != nil {
return err
}
err = table.FillVirtualColumnValue(e.virtualColumnRetFieldTypes, e.virtualColumnIndex,
- e.Schema().Columns, e.columns, e.Ctx().GetExprCtx(), req)
+ schema.Columns, e.columns, sctx.GetExprCtx(), req)
if err != nil {
return err
}
return nil
}
+func shouldFillRowChecksum(schema *expression.Schema) (int, bool) {
+ for idx, col := range schema.Columns {
+ if col.ID == model.ExtraRowChecksumID {
+ return idx, true
+ }
+ }
+ return 0, false
+}
+
+func fillRowChecksum(
+ sctx sessionctx.Context,
+ start, end int,
+ schema *expression.Schema, tblInfo *model.TableInfo,
+ values [][]byte, handles []kv.Handle,
+ req *chunk.Chunk, buf []byte,
+) error {
+ checksumColumnIndex, ok := shouldFillRowChecksum(schema)
+ if !ok {
+ return nil
+ }
+
+ var handleColIDs []int64
+ if tblInfo.PKIsHandle {
+ colInfo := tblInfo.GetPkColInfo()
+ handleColIDs = []int64{colInfo.ID}
+ } else if tblInfo.IsCommonHandle {
+ pkIdx := tables.FindPrimaryIndex(tblInfo)
+ for _, col := range pkIdx.Columns {
+ colInfo := tblInfo.Columns[col.Offset]
+ handleColIDs = append(handleColIDs, colInfo.ID)
+ }
+ }
+
+ columnFt := make(map[int64]*types.FieldType)
+ for idx := range tblInfo.Columns {
+ col := tblInfo.Columns[idx]
+ columnFt[col.ID] = &col.FieldType
+ }
+ tz := sctx.GetSessionVars().TimeZone
+ ft := []*types.FieldType{schema.Columns[checksumColumnIndex].GetType(sctx.GetExprCtx().GetEvalCtx())}
+ checksumCols := chunk.NewChunkWithCapacity(ft, req.Capacity())
+ for i := start; i < end; i++ {
+ handle, val := handles[i], values[i]
+ if !rowcodec.IsNewFormat(val) {
+ checksumCols.AppendNull(0)
+ continue
+ }
+ datums, err := tablecodec.DecodeRowWithMapNew(val, columnFt, tz, nil)
+ if err != nil {
+ return err
+ }
+ datums, err = tablecodec.DecodeHandleToDatumMap(handle, handleColIDs, columnFt, tz, datums)
+ if err != nil {
+ return err
+ }
+ for _, col := range tblInfo.Columns {
+ // cannot found from the datums, which means the data is not stored, this
+ // may happen after `add column` executed, filling with the default value.
+ _, ok := datums[col.ID]
+ if !ok {
+ colInfo := getColInfoByID(tblInfo, col.ID)
+ d, err := table.GetColOriginDefaultValue(sctx.GetExprCtx(), colInfo)
+ if err != nil {
+ return err
+ }
+ datums[col.ID] = d
+ }
+ }
+
+ colData := make([]rowcodec.ColData, len(tblInfo.Columns))
+ for idx, col := range tblInfo.Columns {
+ d := datums[col.ID]
+ data := rowcodec.ColData{
+ ColumnInfo: col,
+ Datum: &d,
+ }
+ colData[idx] = data
+ }
+ row := rowcodec.RowData{
+ Cols: colData,
+ Data: buf,
+ }
+ if !sort.IsSorted(row) {
+ sort.Sort(row)
+ }
+ checksum, err := row.Checksum(tz)
+ if err != nil {
+ return err
+ }
+ checksumCols.AppendString(0, strconv.FormatUint(uint64(checksum), 10))
+ }
+ req.SetCol(checksumColumnIndex, checksumCols.Column(0))
+ return nil
+}
+
func (e *PointGetExecutor) getAndLock(ctx context.Context, key kv.Key) (val []byte, err error) {
if e.Ctx().GetSessionVars().IsPessimisticReadConsistency() {
// Only Lock the existing keys in RC isolation.
diff --git a/pkg/executor/point_get_test.go b/pkg/executor/point_get_test.go
index 97be1d6f549ea..d4058f3e16f78 100644
--- a/pkg/executor/point_get_test.go
+++ b/pkg/executor/point_get_test.go
@@ -88,7 +88,7 @@ func TestReturnValues(t *testing.T) {
txnCtx := tk.Session().GetSessionVars().TxnCtx
val, ok := txnCtx.GetKeyInPessimisticLockCache(pk)
require.True(t, ok)
- handle, err := tablecodec.DecodeHandleInUniqueIndexValue(val, false)
+ handle, err := tablecodec.DecodeHandleInIndexValue(val)
require.NoError(t, err)
rowKey := tablecodec.EncodeRowKeyWithHandle(tid, handle)
_, ok = txnCtx.GetKeyInPessimisticLockCache(rowKey)
@@ -375,40 +375,3 @@ func TestWithTiDBSnapshot(t *testing.T) {
tk.MustQuery("select * from xx").Check(testkit.Rows("1", "7"))
}
-
-func TestGlobalIndexPointGet(t *testing.T) {
- store := testkit.CreateMockStore(t)
- tk := testkit.NewTestKit(t, store)
- tk.MustExec("use test")
- tk.MustExec("set tidb_enable_global_index=true")
- defer func() {
- tk.MustExec("set tidb_enable_global_index=default")
- }()
-
- tk.MustExec(`CREATE TABLE t ( a int, b int, c int default 0)
- PARTITION BY RANGE (a) (
- PARTITION p0 VALUES LESS THAN (10),
- PARTITION p1 VALUES LESS THAN (20),
- PARTITION p2 VALUES LESS THAN (30),
- PARTITION p3 VALUES LESS THAN (40))`)
- tk.MustExec("INSERT INTO t(a, b) values(1, 1), (2, 2), (3, 3), (15, 15), (25, 25), (35, 35)")
- tk.MustExec("ALTER TABLE t ADD UNIQUE INDEX idx(b)")
- tk.MustExec("analyze table t")
-
- tk.MustQuery("select * from t use index(idx) where b in (15, 25, 35)").Sort().Check(testkit.Rows("15 15 0", "25 25 0", "35 35 0"))
- tk.MustQuery("explain select * from t use index(idx) where b in (15, 25, 35)").Check(
- testkit.Rows("Batch_Point_Get_1 3.00 root table:t, index:idx(b) keep order:false, desc:false"))
-
- tk.MustQuery("select * from t use index(idx) where b in (select b from t use index(idx) where b>10)").Sort().Check(testkit.Rows("15 15 0", "25 25 0", "35 35 0"))
-
- tk.MustQuery("select * from t use index(idx) where b = 15").Check(testkit.Rows("15 15 0"))
- tk.MustQuery("explain select * from t use index(idx) where b = 15").Check(
- testkit.Rows("Point_Get_1 1.00 root table:t, index:idx(b) "))
-
- tk.MustQuery("select * from t use index(idx) where b in (select b from t use index(idx) where b > 10)").Sort().Check(
- testkit.Rows("15 15 0", "25 25 0", "35 35 0"))
-
- tk.MustQuery("explain format='brief' select * from t partition(p1) use index(idx) where b = 3").Check(testkit.Rows("Point_Get 1.00 root table:t, index:idx(b) "))
- tk.MustQuery("select * from t partition(p1) use index(idx) where b = 3").Check(testkit.Rows())
- tk.MustQuery("select * from t partition(p1) use index(idx) where b in (15, 25, 35)").Check(testkit.Rows("15 15 0"))
-}
diff --git a/pkg/executor/reload_expr_pushdown_blacklist.go b/pkg/executor/reload_expr_pushdown_blacklist.go
index 8cb6f62a07ac5..c751ec3f9623a 100644
--- a/pkg/executor/reload_expr_pushdown_blacklist.go
+++ b/pkg/executor/reload_expr_pushdown_blacklist.go
@@ -347,6 +347,7 @@ var funcName2Alias = map[string]string{
"json_merge_preserve": ast.JSONMergePreserve,
"json_pretty": ast.JSONPretty,
"json_quote": ast.JSONQuote,
+ "json_schema_valid": ast.JSONSchemaValid,
"json_search": ast.JSONSearch,
"json_storage_size": ast.JSONStorageSize,
"json_depth": ast.JSONDepth,
diff --git a/pkg/executor/replace.go b/pkg/executor/replace.go
index 687ddaf00a333..bc9a5d7447f82 100644
--- a/pkg/executor/replace.go
+++ b/pkg/executor/replace.go
@@ -120,7 +120,7 @@ func (e *ReplaceExec) replaceRow(ctx context.Context, r toBeCheckedRow) error {
// 3. error: the error.
func (e *ReplaceExec) removeIndexRow(ctx context.Context, txn kv.Transaction, r toBeCheckedRow) (rowUnchanged, foundDupKey bool, err error) {
for _, uk := range r.uniqueKeys {
- _, handle, err := tables.FetchDuplicatedHandle(ctx, uk.newKey, true, txn, e.Table.Meta().ID, uk.commonHandle)
+ _, handle, err := tables.FetchDuplicatedHandle(ctx, uk.newKey, true, txn, e.Table.Meta().ID)
if err != nil {
return false, false, err
}
diff --git a/pkg/executor/sample.go b/pkg/executor/sample.go
index 2c9d07ec0e60b..813dd2d58d527 100644
--- a/pkg/executor/sample.go
+++ b/pkg/executor/sample.go
@@ -169,7 +169,7 @@ func (s *tableRegionSampler) writeChunkFromRanges(ranges []kv.KeyRange, req *chu
}
rowDecoder := decoder.NewRowDecoder(s.table, cols, decColMap)
err = s.scanFirstKVForEachRange(ranges, func(handle kv.Handle, value []byte) error {
- _, err := rowDecoder.DecodeAndEvalRowWithMap(s.ctx, handle, value, decLoc, s.rowMap)
+ _, err := rowDecoder.DecodeAndEvalRowWithMap(s.ctx.GetExprCtx(), handle, value, decLoc, s.rowMap)
if err != nil {
return err
}
diff --git a/pkg/executor/select_into.go b/pkg/executor/select_into.go
index e4d0ab450d85b..5161291d1eada 100644
--- a/pkg/executor/select_into.go
+++ b/pkg/executor/select_into.go
@@ -149,7 +149,7 @@ func (s *SelectIntoExec) dumpToOutfile() error {
s.lineBuf = append(s.lineBuf, nullTerm...)
continue
}
- et := col.GetType().EvalType()
+ et := col.GetType(s.Ctx().GetExprCtx().GetEvalCtx()).EvalType()
if (encloseFlag && !encloseOpt) ||
(encloseFlag && encloseOpt && s.considerEncloseOpt(et)) {
s.lineBuf = append(s.lineBuf, encloseByte)
@@ -158,11 +158,11 @@ func (s *SelectIntoExec) dumpToOutfile() error {
s.enclosed = false
}
s.fieldBuf = s.fieldBuf[:0]
- switch col.GetType().GetType() {
+ switch col.GetType(s.Ctx().GetExprCtx().GetEvalCtx()).GetType() {
case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeYear:
s.fieldBuf = strconv.AppendInt(s.fieldBuf, row.GetInt64(j), 10)
case mysql.TypeLonglong:
- if mysql.HasUnsignedFlag(col.GetType().GetFlag()) {
+ if mysql.HasUnsignedFlag(col.GetType(s.Ctx().GetExprCtx().GetEvalCtx()).GetFlag()) {
s.fieldBuf = strconv.AppendUint(s.fieldBuf, row.GetUint64(j), 10)
} else {
s.fieldBuf = strconv.AppendInt(s.fieldBuf, row.GetInt64(j), 10)
@@ -182,7 +182,7 @@ func (s *SelectIntoExec) dumpToOutfile() error {
case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp:
s.fieldBuf = append(s.fieldBuf, row.GetTime(j).String()...)
case mysql.TypeDuration:
- s.fieldBuf = append(s.fieldBuf, row.GetDuration(j, col.GetType().GetDecimal()).String()...)
+ s.fieldBuf = append(s.fieldBuf, row.GetDuration(j, col.GetType(s.Ctx().GetExprCtx().GetEvalCtx()).GetDecimal()).String()...)
case mysql.TypeEnum:
s.fieldBuf = append(s.fieldBuf, row.GetEnum(j).String()...)
case mysql.TypeSet:
@@ -191,7 +191,7 @@ func (s *SelectIntoExec) dumpToOutfile() error {
s.fieldBuf = append(s.fieldBuf, row.GetJSON(j).String()...)
}
- switch col.GetType().EvalType() {
+ switch col.GetType(s.Ctx().GetExprCtx().GetEvalCtx()).EvalType() {
case types.ETString, types.ETJson:
s.lineBuf = append(s.lineBuf, s.escapeField(s.fieldBuf)...)
default:
diff --git a/pkg/executor/select_into_test.go b/pkg/executor/select_into_test.go
index 0737683fe1b44..0587c13d9711f 100644
--- a/pkg/executor/select_into_test.go
+++ b/pkg/executor/select_into_test.go
@@ -55,6 +55,18 @@ func TestSelectIntoFileExists(t *testing.T) {
require.True(t, strings.Contains(err.Error(), outfile))
}
+func TestSelectIntoOutfilePointGet(t *testing.T) {
+ outfile := randomSelectFilePath("TestSelectIntoOutfilePointGet")
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+
+ tk.MustExec(`create table t (id int not null, primary key (id) /*T![clustered_index] CLUSTERED */ );`)
+ tk.MustExec(`insert into t values(1);`)
+ tk.MustExec(fmt.Sprintf("select * from t where id = 1 into outfile %q", outfile))
+ cmpAndRm("1\n", outfile, t)
+}
+
func TestSelectIntoOutfileTypes(t *testing.T) {
outfile := randomSelectFilePath("TestSelectIntoOutfileTypes")
store := testkit.CreateMockStore(t)
diff --git a/pkg/executor/set.go b/pkg/executor/set.go
index 2c5ad1e95aff5..89a65bb77a2e4 100644
--- a/pkg/executor/set.go
+++ b/pkg/executor/set.go
@@ -97,7 +97,7 @@ func (e *SetExecutor) Next(ctx context.Context, req *chunk.Chunk) error {
sessionVars.UnsetUserVar(name)
} else {
sessionVars.SetUserVarVal(name, value)
- sessionVars.SetUserVarType(name, v.Expr.GetType())
+ sessionVars.SetUserVarType(name, v.Expr.GetType(sctx.GetExprCtx().GetEvalCtx()))
}
continue
}
diff --git a/pkg/executor/set_config.go b/pkg/executor/set_config.go
index ac02be1f85152..0b9541acbc093 100644
--- a/pkg/executor/set_config.go
+++ b/pkg/executor/set_config.go
@@ -187,7 +187,7 @@ func ConvertConfigItem2JSON(ctx sessionctx.Context, key string, val expression.E
}
isNull := false
str := ""
- switch val.GetType().EvalType() {
+ switch val.GetType(ctx.GetExprCtx().GetEvalCtx()).EvalType() {
case types.ETString:
var s string
s, isNull, err = val.EvalString(ctx.GetExprCtx().GetEvalCtx(), chunk.Row{})
@@ -198,7 +198,7 @@ func ConvertConfigItem2JSON(ctx sessionctx.Context, key string, val expression.E
var i int64
i, isNull, err = val.EvalInt(ctx.GetExprCtx().GetEvalCtx(), chunk.Row{})
if err == nil && !isNull {
- if mysql.HasIsBooleanFlag(val.GetType().GetFlag()) {
+ if mysql.HasIsBooleanFlag(val.GetType(ctx.GetExprCtx().GetEvalCtx()).GetFlag()) {
str = "true"
if i == 0 {
str = "false"
diff --git a/pkg/executor/set_test.go b/pkg/executor/set_test.go
index 11380474ad02a..4dc2256cc5af2 100644
--- a/pkg/executor/set_test.go
+++ b/pkg/executor/set_test.go
@@ -665,7 +665,7 @@ func TestSetVar(t *testing.T) {
tk.MustQuery("select @@pd_enable_follower_handle_region").Check(testkit.Rows("0"))
require.Error(t, tk.ExecToErr("set pd_enable_follower_handle_region = 1"))
- tk.MustQuery("select @@tidb_enable_historical_stats").Check(testkit.Rows("1"))
+ tk.MustQuery("select @@tidb_enable_historical_stats").Check(testkit.Rows("0"))
tk.MustExec("set global tidb_enable_historical_stats = 1")
tk.MustQuery("select @@tidb_enable_historical_stats").Check(testkit.Rows("1"))
tk.MustExec("set global tidb_enable_historical_stats = 0")
@@ -870,14 +870,14 @@ func TestSetVar(t *testing.T) {
tk.MustExec("set @@session.tidb_cdc_write_source = 0")
require.Equal(t, uint64(0), tk.Session().GetSessionVars().CDCWriteSource)
- tk.MustQuery("select @@session.tidb_analyze_skip_column_types").Check(testkit.Rows("json,blob,mediumblob,longblob,text,mediumtext,longtext"))
+ tk.MustQuery("select @@session.tidb_analyze_skip_column_types").Check(testkit.Rows("json,blob,mediumblob,longblob,mediumtext,longtext"))
tk.MustExec("set @@session.tidb_analyze_skip_column_types = 'json, text, blob'")
tk.MustQuery("select @@session.tidb_analyze_skip_column_types").Check(testkit.Rows("json,text,blob"))
tk.MustExec("set @@session.tidb_analyze_skip_column_types = ''")
tk.MustQuery("select @@session.tidb_analyze_skip_column_types").Check(testkit.Rows(""))
tk.MustGetErrMsg("set @@session.tidb_analyze_skip_column_types = 'int,json'", "[variable:1231]Variable 'tidb_analyze_skip_column_types' can't be set to the value of 'int,json'")
- tk.MustQuery("select @@global.tidb_analyze_skip_column_types").Check(testkit.Rows("json,blob,mediumblob,longblob,text,mediumtext,longtext"))
+ tk.MustQuery("select @@global.tidb_analyze_skip_column_types").Check(testkit.Rows("json,blob,mediumblob,longblob,mediumtext,longtext"))
tk.MustExec("set @@global.tidb_analyze_skip_column_types = 'json, text, blob'")
tk.MustQuery("select @@global.tidb_analyze_skip_column_types").Check(testkit.Rows("json,text,blob"))
tk.MustExec("set @@global.tidb_analyze_skip_column_types = ''")
diff --git a/pkg/executor/show.go b/pkg/executor/show.go
index 8bd86a2dacc75..58ccc651eddbd 100644
--- a/pkg/executor/show.go
+++ b/pkg/executor/show.go
@@ -2355,7 +2355,7 @@ func tryFillViewColumnType(ctx context.Context, sctx sessionctx.Context, is info
for _, col := range tbl.Columns {
idx := expression.FindFieldNameIdxByColName(viewOutputNames, col.Name.L)
if idx >= 0 {
- col.FieldType = *viewSchema.Columns[idx].GetType()
+ col.FieldType = *viewSchema.Columns[idx].GetType(sctx.GetExprCtx().GetEvalCtx())
}
if col.GetType() == mysql.TypeVarString {
col.SetType(mysql.TypeVarchar)
diff --git a/pkg/executor/show_placement.go b/pkg/executor/show_placement.go
index 194a17d45ae5b..ba69acb5f84e7 100644
--- a/pkg/executor/show_placement.go
+++ b/pkg/executor/show_placement.go
@@ -21,9 +21,9 @@ import (
gjson "encoding/json"
"fmt"
"slices"
- "strings"
"github.com/pingcap/errors"
+ "github.com/pingcap/tidb/pkg/ddl"
"github.com/pingcap/tidb/pkg/ddl/placement"
"github.com/pingcap/tidb/pkg/domain/infosync"
"github.com/pingcap/tidb/pkg/infoschema"
@@ -268,39 +268,26 @@ func (e *ShowExec) fetchAllPlacementPolicies() error {
}
func (e *ShowExec) fetchRangesPlacementPlocy(ctx context.Context) error {
- fetchFn := func(ctx context.Context, rangeName string) error {
- bundle, err := infosync.GetRuleBundle(ctx, rangeName)
+ fetchFn := func(ctx context.Context, rangeBundleID string) error {
+ policyName, err := ddl.GetRangePlacementPolicyName(ctx, rangeBundleID)
if err != nil {
return err
}
- if bundle == nil || len(bundle.Rules) == 0 {
- return nil
- }
- policyName := ""
- startKey := []byte("")
- endKey := []byte("")
- rule := bundle.Rules[0]
- pos := strings.Index(rule.ID, "_rule")
- if pos > 0 {
- policyName = rule.ID[:pos]
- }
- startKey, err = hex.DecodeString(rule.StartKeyHex)
- if err != nil {
- return err
- }
- endKey, err = hex.DecodeString(rule.EndKeyHex)
- if err != nil {
- return err
- }
- state, err := infosync.GetReplicationState(ctx, startKey, endKey)
- if err != nil {
- return err
- }
- policy, ok := e.is.PolicyByName(model.NewCIStr(policyName))
- if !ok {
- return errors.Errorf("Policy with name '%s' not found", policyName)
+ if policyName != "" {
+ startKeyHex, endKeyHex := placement.GetRangeStartAndEndKeyHex(rangeBundleID)
+ startKey, _ := hex.DecodeString(startKeyHex)
+ endKey, _ := hex.DecodeString(endKeyHex)
+ state, err := infosync.GetReplicationState(ctx, startKey, endKey)
+ if err != nil {
+ return err
+ }
+ policy, ok := e.is.PolicyByName(model.NewCIStr(policyName))
+ if !ok {
+ return errors.Errorf("Policy with name '%s' not found", policyName)
+ }
+ e.appendRow([]any{"RANGE " + rangeBundleID, policy.PlacementSettings.String(), state.String()})
}
- e.appendRow([]any{"RANGE " + rangeName, policy.PlacementSettings.String(), state.String()})
+
return nil
}
// try fetch ranges placement policy
diff --git a/pkg/executor/show_stats_test.go b/pkg/executor/show_stats_test.go
index 951e77662383e..97b2946ba5991 100644
--- a/pkg/executor/show_stats_test.go
+++ b/pkg/executor/show_stats_test.go
@@ -19,6 +19,7 @@ import (
"testing"
"time"
+ "github.com/docker/go-units"
"github.com/pingcap/tidb/pkg/domain/infosync"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
@@ -292,7 +293,7 @@ func TestShowStatusSnapshot(t *testing.T) {
UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment)
tk.MustExec(updateSafePoint)
- for _, cacheSize := range []int{1024, 0} {
+ for _, cacheSize := range []int{units.GiB, 0} {
tk.MustExec("set @@global.tidb_schema_cache_size = ?", cacheSize)
tk.MustExec("create table t (a int);")
snapshotTime := time.Now()
diff --git a/pkg/executor/slow_query.go b/pkg/executor/slow_query.go
index fb1e3373342e1..2934825240996 100644
--- a/pkg/executor/slow_query.go
+++ b/pkg/executor/slow_query.go
@@ -928,7 +928,7 @@ func (e *slowQueryRetriever) getAllFiles(ctx context.Context, sctx sessionctx.Co
}
// If we want to get the end time from a compressed file,
- // we need uncompress the whole file which is very slow and consume a lot of memeory.
+ // we need uncompress the whole file which is very slow and consume a lot of memory.
if !compressed {
// Get the file end time.
fileEndTime, err := e.getFileEndTime(ctx, file)
diff --git a/pkg/executor/sortexec/BUILD.bazel b/pkg/executor/sortexec/BUILD.bazel
index c545297a74fd7..b50318a909708 100644
--- a/pkg/executor/sortexec/BUILD.bazel
+++ b/pkg/executor/sortexec/BUILD.bazel
@@ -11,6 +11,9 @@ go_library(
"sort_spill.go",
"sort_util.go",
"topn.go",
+ "topn_chunk_heap.go",
+ "topn_spill.go",
+ "topn_worker.go",
],
importpath = "github.com/pingcap/tidb/pkg/executor/sortexec",
visibility = ["//visibility:public"],
@@ -39,7 +42,7 @@ go_test(
timeout = "short",
srcs = ["sort_test.go"],
flaky = True,
- shard_count = 10,
+ shard_count = 13,
deps = [
"//pkg/config",
"//pkg/sessionctx/variable",
@@ -60,6 +63,7 @@ go_test(
"sort_spill_test.go",
"sort_test.go",
"sortexec_pkg_test.go",
+ "topn_spill_test.go",
],
embed = [":sortexec"],
flaky = True,
@@ -69,7 +73,9 @@ go_test(
"//pkg/executor/internal/exec",
"//pkg/executor/internal/testutil",
"//pkg/expression",
+ "//pkg/expression/contextstatic",
"//pkg/parser/mysql",
+ "//pkg/planner/core",
"//pkg/planner/util",
"//pkg/sessionctx/variable",
"//pkg/testkit",
diff --git a/pkg/executor/sortexec/sort.go b/pkg/executor/sortexec/sort.go
index 923949fc3ee9f..122e7ccb86a0e 100644
--- a/pkg/executor/sortexec/sort.go
+++ b/pkg/executor/sortexec/sort.go
@@ -272,7 +272,7 @@ func (e *SortExec) InitInParallelModeForTest() {
*/
func (e *SortExec) Next(ctx context.Context, req *chunk.Chunk) error {
if e.fetched.CompareAndSwap(false, true) {
- e.initCompareFuncs()
+ e.initCompareFuncs(e.Ctx().GetExprCtx().GetEvalCtx())
e.buildKeyColumns()
err := e.fetchChunks(ctx)
if err != nil {
@@ -753,10 +753,10 @@ func (e *SortExec) fetchChunksFromChild(ctx context.Context) {
}
}
-func (e *SortExec) initCompareFuncs() {
+func (e *SortExec) initCompareFuncs(ctx expression.EvalContext) {
e.keyCmpFuncs = make([]chunk.CompareFunc, len(e.ByItems))
for i := range e.ByItems {
- keyType := e.ByItems[i].Expr.GetType()
+ keyType := e.ByItems[i].Expr.GetType(ctx)
e.keyCmpFuncs[i] = chunk.GetCompareFunc(keyType)
}
}
diff --git a/pkg/executor/sortexec/sort_spill_test.go b/pkg/executor/sortexec/sort_spill_test.go
index cb8b00642c4f1..bff19e1a2ff9c 100644
--- a/pkg/executor/sortexec/sort_spill_test.go
+++ b/pkg/executor/sortexec/sort_spill_test.go
@@ -25,6 +25,7 @@ import (
"github.com/pingcap/tidb/pkg/executor/internal/testutil"
"github.com/pingcap/tidb/pkg/executor/sortexec"
"github.com/pingcap/tidb/pkg/expression"
+ "github.com/pingcap/tidb/pkg/expression/contextstatic"
plannerutil "github.com/pingcap/tidb/pkg/planner/util"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/chunk"
@@ -98,16 +99,25 @@ func (r *resultChecker) initRowPtrs() {
}
}
-func (r *resultChecker) check(resultChunks []*chunk.Chunk) bool {
+func (r *resultChecker) check(resultChunks []*chunk.Chunk, offset int64, count int64) bool {
+ ctx := contextstatic.NewStaticEvalContext()
+
if r.rowPtrs == nil {
r.initRowPtrs()
sort.Slice(r.rowPtrs, r.keyColumnsLess)
+ if offset < 0 {
+ offset = 0
+ }
+ if count < 0 {
+ count = (int64(len(r.rowPtrs)) - offset)
+ }
+ r.rowPtrs = r.rowPtrs[offset : offset+count]
}
cursor := 0
fieldTypes := make([]*types.FieldType, 0)
for _, col := range r.schema.Columns {
- fieldTypes = append(fieldTypes, col.GetType())
+ fieldTypes = append(fieldTypes, col.GetType(ctx))
}
// Check row number
@@ -220,7 +230,7 @@ func executeSortExecutorAndManullyTriggerSpill(t *testing.T, exe *sortexec.SortE
func checkCorrectness(schema *expression.Schema, exe *sortexec.SortExec, dataSource *testutil.MockDataSource, resultChunks []*chunk.Chunk) bool {
keyColumns, keyCmpFuncs, byItemsDesc := exe.GetSortMetaForTest()
checker := newResultChecker(schema, keyColumns, keyCmpFuncs, byItemsDesc, dataSource.GenData)
- return checker.check(resultChunks)
+ return checker.check(resultChunks, -1, -1)
}
func onePartitionAndAllDataInMemoryCase(t *testing.T, ctx *mock.Context, sortCase *testutil.SortCase) {
diff --git a/pkg/executor/sortexec/sort_util.go b/pkg/executor/sortexec/sort_util.go
index 1e63dbd3c1b27..59ef17f90da2c 100644
--- a/pkg/executor/sortexec/sort_util.go
+++ b/pkg/executor/sortexec/sort_util.go
@@ -46,10 +46,10 @@ type rowWithPartition struct {
partitionID int
}
-func processPanicAndLog(errOutputChan chan rowWithError, r any) {
+func processPanicAndLog(errOutputChan chan<- rowWithError, r any) {
err := util.GetRecoverError(r)
errOutputChan <- rowWithError{err: err}
- logutil.BgLogger().Error("parallel sort panicked", zap.Error(err), zap.Stack("stack"))
+ logutil.BgLogger().Error("executor panicked", zap.Error(err), zap.Stack("stack"))
}
// chunkWithMemoryUsage contains chunk and memory usage.
diff --git a/pkg/executor/sortexec/topn.go b/pkg/executor/sortexec/topn.go
index 6c6f074064359..e6b6104d936a3 100644
--- a/pkg/executor/sortexec/topn.go
+++ b/pkg/executor/sortexec/topn.go
@@ -17,12 +17,19 @@ package sortexec
import (
"container/heap"
"context"
+ "math/rand"
"slices"
+ "sync"
"sync/atomic"
+ "github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/executor/internal/exec"
plannercore "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/sessionctx/variable"
+ "github.com/pingcap/tidb/pkg/util"
+ "github.com/pingcap/tidb/pkg/util/channel"
"github.com/pingcap/tidb/pkg/util/chunk"
+ "github.com/pingcap/tidb/pkg/util/disk"
"github.com/pingcap/tidb/pkg/util/memory"
)
@@ -30,134 +37,237 @@ import (
// Instead of sorting all the rows fetched from the table, it keeps the Top-N elements only in a heap to reduce memory usage.
type TopNExec struct {
SortExec
- Limit *plannercore.PhysicalLimit
- totalLimit uint64
+ Limit *plannercore.PhysicalLimit
+
+ // It's useful when spill is triggered and the fetcher could know when workers finish their works.
+ fetcherAndWorkerSyncer *sync.WaitGroup
+ resultChannel chan rowWithError
+ chunkChannel chan *chunk.Chunk
+
+ finishCh chan struct{}
chkHeap *topNChunkHeap
-}
-// topNChunkHeap implements heap.Interface.
-type topNChunkHeap struct {
- *TopNExec
+ spillHelper *topNSpillHelper
+ spillAction *topNSpillAction
- // rowChunks is the chunks to store row values.
- rowChunks *chunk.List
- // rowPointer store the chunk index and row index for each row.
- rowPtrs []chunk.RowPtr
+ // Normally, heap will be stored in memory after it has been built.
+ // However, other executors may trigger topn spill after the heap is built
+ // and inMemoryThenSpillFlag will be set to true at this time.
+ inMemoryThenSpillFlag bool
- Idx int
-}
+ // Topn executor has two stage:
+ // 1. Building heap, in this stage all received rows will be inserted into heap.
+ // 2. Updating heap, in this stage only rows that is smaller than the heap top could be inserted and we will drop the heap top.
+ //
+ // This variable is only used for test.
+ isSpillTriggeredInStage1ForTest bool
+ isSpillTriggeredInStage2ForTest bool
-// Less implement heap.Interface, but since we mantains a max heap,
-// this function returns true if row i is greater than row j.
-func (h *topNChunkHeap) Less(i, j int) bool {
- rowI := h.rowChunks.GetRow(h.rowPtrs[i])
- rowJ := h.rowChunks.GetRow(h.rowPtrs[j])
- return h.greaterRow(rowI, rowJ)
+ Concurrency int
}
-func (h *topNChunkHeap) greaterRow(rowI, rowJ chunk.Row) bool {
- for i, colIdx := range h.keyColumns {
- cmpFunc := h.keyCmpFuncs[i]
- cmp := cmpFunc(rowI, colIdx, rowJ, colIdx)
- if h.ByItems[i].Desc {
- cmp = -cmp
+// Open implements the Executor Open interface.
+func (e *TopNExec) Open(ctx context.Context) error {
+ e.memTracker = memory.NewTracker(e.ID(), -1)
+ e.memTracker.AttachTo(e.Ctx().GetSessionVars().StmtCtx.MemTracker)
+
+ e.fetched = &atomic.Bool{}
+ e.fetched.Store(false)
+ e.chkHeap = &topNChunkHeap{memTracker: e.memTracker}
+ e.chkHeap.idx = 0
+
+ e.finishCh = make(chan struct{}, 1)
+ e.resultChannel = make(chan rowWithError, e.MaxChunkSize())
+ e.chunkChannel = make(chan *chunk.Chunk, e.Concurrency)
+ e.inMemoryThenSpillFlag = false
+ e.isSpillTriggeredInStage1ForTest = false
+ e.isSpillTriggeredInStage2ForTest = false
+
+ if variable.EnableTmpStorageOnOOM.Load() {
+ e.diskTracker = disk.NewTracker(e.ID(), -1)
+ diskTracker := e.Ctx().GetSessionVars().StmtCtx.DiskTracker
+ if diskTracker != nil {
+ e.diskTracker.AttachTo(diskTracker)
}
- if cmp > 0 {
- return true
- } else if cmp < 0 {
- return false
+ e.fetcherAndWorkerSyncer = &sync.WaitGroup{}
+
+ workers := make([]*topNWorker, e.Concurrency)
+ for i := range workers {
+ chkHeap := &topNChunkHeap{}
+ // Offset of heap in worker should be 0, as we need to spill all data
+ chkHeap.init(e, e.memTracker, e.Limit.Offset+e.Limit.Count, 0, e.greaterRow, e.RetFieldTypes())
+ workers[i] = newTopNWorker(i, e.chunkChannel, e.fetcherAndWorkerSyncer, e.resultChannel, e.finishCh, e, chkHeap, e.memTracker)
}
+
+ e.spillHelper = newTopNSpillerHelper(
+ e,
+ e.finishCh,
+ e.resultChannel,
+ e.memTracker,
+ e.diskTracker,
+ exec.RetTypes(e),
+ workers,
+ e.Concurrency,
+ )
+ e.spillAction = &topNSpillAction{spillHelper: e.spillHelper}
+ e.Ctx().GetSessionVars().MemTracker.FallbackOldAndSetNewAction(e.spillAction)
}
- return false
-}
-func (h *topNChunkHeap) Len() int {
- return len(h.rowPtrs)
+ return exec.Open(ctx, e.Children(0))
}
-func (*topNChunkHeap) Push(any) {
- // Should never be called.
-}
+// Close implements the Executor Close interface.
+func (e *TopNExec) Close() error {
+ // `e.finishCh == nil` means that `Open` is not called.
+ if e.finishCh == nil {
+ return exec.Close(e.Children(0))
+ }
-func (h *topNChunkHeap) Pop() any {
- h.rowPtrs = h.rowPtrs[:len(h.rowPtrs)-1]
- // We don't need the popped value, return nil to avoid memory allocation.
- return nil
-}
+ close(e.finishCh)
+ if e.fetched.CompareAndSwap(false, true) {
+ close(e.resultChannel)
+ return exec.Close(e.Children(0))
+ }
-func (h *topNChunkHeap) Swap(i, j int) {
- h.rowPtrs[i], h.rowPtrs[j] = h.rowPtrs[j], h.rowPtrs[i]
-}
+ // Wait for the finish of all tasks
+ channel.Clear(e.resultChannel)
-func (e *TopNExec) keyColumnsCompare(i, j chunk.RowPtr) int {
- rowI := e.chkHeap.rowChunks.GetRow(i)
- rowJ := e.chkHeap.rowChunks.GetRow(j)
- return e.compareRow(rowI, rowJ)
-}
+ e.chkHeap = nil
+ e.spillAction = nil
-func (e *TopNExec) initPointers() {
- e.chkHeap.rowPtrs = make([]chunk.RowPtr, 0, e.chkHeap.rowChunks.Len())
- e.memTracker.Consume(int64(8 * e.chkHeap.rowChunks.Len()))
- for chkIdx := 0; chkIdx < e.chkHeap.rowChunks.NumChunks(); chkIdx++ {
- rowChk := e.chkHeap.rowChunks.GetChunk(chkIdx)
- for rowIdx := 0; rowIdx < rowChk.NumRows(); rowIdx++ {
- e.chkHeap.rowPtrs = append(e.chkHeap.rowPtrs, chunk.RowPtr{ChkIdx: uint32(chkIdx), RowIdx: uint32(rowIdx)})
- }
+ if e.spillHelper != nil {
+ e.spillHelper.close()
+ e.spillHelper = nil
}
-}
-// Open implements the Executor Open interface.
-func (e *TopNExec) Open(ctx context.Context) error {
- e.memTracker = memory.NewTracker(e.ID(), -1)
- e.memTracker.AttachTo(e.Ctx().GetSessionVars().StmtCtx.MemTracker)
+ if e.memTracker != nil {
+ e.memTracker.ReplaceBytesUsed(0)
+ }
- e.fetched = &atomic.Bool{}
- e.fetched.Store(false)
- e.chkHeap = &topNChunkHeap{TopNExec: e}
- e.chkHeap.Idx = 0
+ return exec.Close(e.Children(0))
+}
- return exec.Open(ctx, e.Children(0))
+func (e *TopNExec) greaterRow(rowI, rowJ chunk.Row) bool {
+ for i, colIdx := range e.keyColumns {
+ cmpFunc := e.keyCmpFuncs[i]
+ cmp := cmpFunc(rowI, colIdx, rowJ, colIdx)
+ if e.ByItems[i].Desc {
+ cmp = -cmp
+ }
+ if cmp > 0 {
+ return true
+ } else if cmp < 0 {
+ return false
+ }
+ }
+ return false
}
// Next implements the Executor Next interface.
+//
+// The following picture shows the procedure of topn when spill is triggered.
+/*
+Spill Stage:
+ ┌─────────┐
+ │ Child │
+ └────▲────┘
+ │
+ Fetch
+ │
+ ┌───────┴───────┐
+ │ Chunk Fetcher │
+ └───────┬───────┘
+ │
+ │
+ ▼
+ Check Spill──────►Spill Triggered─────────►Spill
+ │ │
+ ▼ │
+ Spill Not Triggered │
+ │ │
+ ▼ │
+ Push Chunk◄─────────────────────────────────┘
+ │
+ ▼
+ ┌────────────────►Channel◄───────────────────┐
+ │ ▲ │
+ │ │ │
+ Fetch Fetch Fetch
+ │ │ │
+ ┌────┴───┐ ┌───┴────┐ ┌───┴────┐
+ │ Worker │ │ Worker │ ...... │ Worker │
+ └────┬───┘ └───┬────┘ └───┬────┘
+ │ │ │
+ │ │ │
+ │ ▼ │
+ └───────────► Multi-way Merge◄───────────────┘
+ │
+ │
+ ▼
+ Output
+
+Restore Stage:
+ ┌────────┐ ┌────────┐ ┌────────┐
+ │ Heap │ │ Heap │ ...... │ Heap │
+ └────┬───┘ └───┬────┘ └───┬────┘
+ │ │ │
+ │ │ │
+ │ ▼ │
+ └───────────► Multi-way Merge◄───────────────┘
+ │
+ │
+ ▼
+ Output
+
+*/
func (e *TopNExec) Next(ctx context.Context, req *chunk.Chunk) error {
req.Reset()
if e.fetched.CompareAndSwap(false, true) {
- e.totalLimit = e.Limit.Offset + e.Limit.Count
- e.chkHeap.Idx = int(e.Limit.Offset)
- err := e.loadChunksUntilTotalLimit(ctx)
- if err != nil {
- return err
- }
- err = e.executeTopN(ctx)
+ err := e.fetchChunks(ctx)
if err != nil {
return err
}
}
- if e.chkHeap.Idx >= len(e.chkHeap.rowPtrs) {
- return nil
- }
+
if !req.IsFull() {
- numToAppend := min(len(e.chkHeap.rowPtrs)-e.chkHeap.Idx, req.RequiredRows()-req.NumRows())
- rows := make([]chunk.Row, numToAppend)
- for index := 0; index < numToAppend; index++ {
- rows[index] = e.chkHeap.rowChunks.GetRow(e.chkHeap.rowPtrs[e.chkHeap.Idx])
- e.chkHeap.Idx++
+ numToAppend := req.RequiredRows() - req.NumRows()
+ for i := 0; i < numToAppend; i++ {
+ row, ok := <-e.resultChannel
+ if !ok || row.err != nil {
+ return row.err
+ }
+ req.AppendRow(row.row)
+ }
+ }
+ return nil
+}
+
+func (e *TopNExec) fetchChunks(ctx context.Context) error {
+ defer func() {
+ if r := recover(); r != nil {
+ processPanicAndLog(e.resultChannel, r)
+ close(e.resultChannel)
}
- req.AppendRows(rows)
+ }()
+
+ err := e.loadChunksUntilTotalLimit(ctx)
+ if err != nil {
+ close(e.resultChannel)
+ return err
}
+ go e.executeTopN(ctx)
return nil
}
func (e *TopNExec) loadChunksUntilTotalLimit(ctx context.Context) error {
- e.chkHeap.rowChunks = chunk.NewList(exec.RetTypes(e), e.InitCap(), e.MaxChunkSize())
- e.chkHeap.rowChunks.GetMemTracker().AttachTo(e.memTracker)
- e.chkHeap.rowChunks.GetMemTracker().SetLabel(memory.LabelForRowChunks)
- for uint64(e.chkHeap.rowChunks.Len()) < e.totalLimit {
+ e.initCompareFuncs(e.Ctx().GetExprCtx().GetEvalCtx())
+ e.buildKeyColumns()
+ e.chkHeap.init(e, e.memTracker, e.Limit.Offset+e.Limit.Count, int(e.Limit.Offset), e.greaterRow, e.RetFieldTypes())
+ for uint64(e.chkHeap.rowChunks.Len()) < e.chkHeap.totalLimit {
srcChk := exec.TryNewCacheChunk(e.Children(0))
// adjust required rows by total limit
- srcChk.SetRequiredRows(int(e.totalLimit-uint64(e.chkHeap.rowChunks.Len())), e.MaxChunkSize())
+ srcChk.SetRequiredRows(int(e.chkHeap.totalLimit-uint64(e.chkHeap.rowChunks.Len())), e.MaxChunkSize())
err := exec.Next(ctx, e.Children(0), srcChk)
if err != nil {
return err
@@ -166,76 +276,366 @@ func (e *TopNExec) loadChunksUntilTotalLimit(ctx context.Context) error {
break
}
e.chkHeap.rowChunks.Add(srcChk)
+ if e.spillHelper.isSpillNeeded() {
+ e.isSpillTriggeredInStage1ForTest = true
+ break
+ }
+
+ injectTopNRandomFail(1)
}
- e.initPointers()
- e.initCompareFuncs()
- e.buildKeyColumns()
+
+ e.chkHeap.initPtrs()
return nil
}
const topNCompactionFactor = 4
-func (e *TopNExec) executeTopN(ctx context.Context) error {
- heap.Init(e.chkHeap)
- for uint64(len(e.chkHeap.rowPtrs)) > e.totalLimit {
- // The number of rows we loaded may exceeds total limit, remove greatest rows by Pop.
- heap.Pop(e.chkHeap)
+func (e *TopNExec) executeTopNWhenNoSpillTriggered(ctx context.Context) error {
+ if e.spillHelper.isSpillNeeded() {
+ e.isSpillTriggeredInStage2ForTest = true
+ return nil
}
+
childRowChk := exec.TryNewCacheChunk(e.Children(0))
for {
+ if e.spillHelper.isSpillNeeded() {
+ e.isSpillTriggeredInStage2ForTest = true
+ return nil
+ }
+
err := exec.Next(ctx, e.Children(0), childRowChk)
if err != nil {
return err
}
+
if childRowChk.NumRows() == 0 {
break
}
- err = e.processChildChk(childRowChk)
- if err != nil {
- return err
- }
+
+ e.chkHeap.processChk(childRowChk)
+
if e.chkHeap.rowChunks.Len() > len(e.chkHeap.rowPtrs)*topNCompactionFactor {
- err = e.doCompaction(e.chkHeap)
+ err = e.chkHeap.doCompaction(e)
if err != nil {
return err
}
}
+ injectTopNRandomFail(10)
+ }
+
+ slices.SortFunc(e.chkHeap.rowPtrs, e.chkHeap.keyColumnsCompare)
+ return nil
+}
+
+func (e *TopNExec) spillRemainingRowsWhenNeeded() error {
+ if e.spillHelper.isSpillTriggered() {
+ return e.spillHelper.spill()
}
- slices.SortFunc(e.chkHeap.rowPtrs, e.keyColumnsCompare)
return nil
}
-func (e *TopNExec) processChildChk(childRowChk *chunk.Chunk) error {
- for i := 0; i < childRowChk.NumRows(); i++ {
- heapMaxPtr := e.chkHeap.rowPtrs[0]
- var heapMax, next chunk.Row
- heapMax = e.chkHeap.rowChunks.GetRow(heapMaxPtr)
- next = childRowChk.GetRow(i)
- if e.chkHeap.greaterRow(heapMax, next) {
- // Evict heap max, keep the next row.
- e.chkHeap.rowPtrs[0] = e.chkHeap.rowChunks.AppendRow(childRowChk.GetRow(i))
- heap.Fix(e.chkHeap, 0)
+func (e *TopNExec) checkSpillAndExecute() error {
+ if e.spillHelper.isSpillNeeded() {
+ // Wait for the stop of all workers
+ e.fetcherAndWorkerSyncer.Wait()
+ return e.spillHelper.spill()
+ }
+ return nil
+}
+
+func (e *TopNExec) fetchChunksFromChild(ctx context.Context) {
+ defer func() {
+ if r := recover(); r != nil {
+ processPanicAndLog(e.resultChannel, r)
+ }
+
+ e.fetcherAndWorkerSyncer.Wait()
+ err := e.spillRemainingRowsWhenNeeded()
+ if err != nil {
+ e.resultChannel <- rowWithError{err: err}
+ }
+
+ close(e.chunkChannel)
+ }()
+
+ for {
+ chk := exec.TryNewCacheChunk(e.Children(0))
+ err := exec.Next(ctx, e.Children(0), chk)
+ if err != nil {
+ e.resultChannel <- rowWithError{err: err}
+ return
+ }
+
+ rowCount := chk.NumRows()
+ if rowCount == 0 {
+ break
}
+
+ e.fetcherAndWorkerSyncer.Add(1)
+ select {
+ case <-e.finishCh:
+ e.fetcherAndWorkerSyncer.Done()
+ return
+ case e.chunkChannel <- chk:
+ }
+
+ injectTopNRandomFail(10)
+
+ err = e.checkSpillAndExecute()
+ if err != nil {
+ e.resultChannel <- rowWithError{err: err}
+ return
+ }
+ }
+}
+
+// Spill the heap which is in TopN executor
+func (e *TopNExec) spillTopNExecHeap() error {
+ e.spillHelper.setInSpilling()
+ defer e.spillHelper.cond.Broadcast()
+ defer e.spillHelper.setNotSpilled()
+
+ err := e.spillHelper.spillHeap(e.chkHeap)
+ if err != nil {
+ return err
}
return nil
}
-// doCompaction rebuild the chunks and row pointers to release memory.
-// If we don't do compaction, in a extreme case like the child data is already ascending sorted
-// but we want descending top N, then we will keep all data in memory.
-// But if data is distributed randomly, this function will be called log(n) times.
-func (e *TopNExec) doCompaction(chkHeap *topNChunkHeap) error {
- newRowChunks := chunk.NewList(exec.RetTypes(e), e.InitCap(), e.MaxChunkSize())
- newRowPtrs := make([]chunk.RowPtr, 0, chkHeap.rowChunks.Len())
- for _, rowPtr := range chkHeap.rowPtrs {
- newRowPtr := newRowChunks.AppendRow(chkHeap.rowChunks.GetRow(rowPtr))
- newRowPtrs = append(newRowPtrs, newRowPtr)
+func (e *TopNExec) executeTopNWhenSpillTriggered(ctx context.Context) error {
+ // idx need to be set to 0 as we need to spill all data
+ e.chkHeap.idx = 0
+ err := e.spillTopNExecHeap()
+ if err != nil {
+ return err
+ }
+
+ // Wait for the finish of chunk fetcher
+ fetcherWaiter := util.WaitGroupWrapper{}
+ // Wait for the finish of all workers
+ workersWaiter := util.WaitGroupWrapper{}
+
+ for i := range e.spillHelper.workers {
+ worker := e.spillHelper.workers[i]
+ worker.initWorker()
+ workersWaiter.Run(func() {
+ worker.run()
+ })
+ }
+
+ // Fetch chunks from child and put chunks into chunkChannel
+ fetcherWaiter.Run(func() {
+ e.fetchChunksFromChild(ctx)
+ })
+
+ fetcherWaiter.Wait()
+ workersWaiter.Wait()
+ return nil
+}
+
+func (e *TopNExec) executeTopN(ctx context.Context) {
+ defer func() {
+ if r := recover(); r != nil {
+ processPanicAndLog(e.resultChannel, r)
+ }
+
+ close(e.resultChannel)
+ }()
+
+ heap.Init(e.chkHeap)
+ for uint64(len(e.chkHeap.rowPtrs)) > e.chkHeap.totalLimit {
+ // The number of rows we loaded may exceeds total limit, remove greatest rows by Pop.
+ heap.Pop(e.chkHeap)
+ }
+
+ if err := e.executeTopNWhenNoSpillTriggered(ctx); err != nil {
+ e.resultChannel <- rowWithError{err: err}
+ return
+ }
+
+ if e.spillHelper.isSpillNeeded() {
+ if err := e.executeTopNWhenSpillTriggered(ctx); err != nil {
+ e.resultChannel <- rowWithError{err: err}
+ return
+ }
+ }
+
+ e.generateTopNResults()
+}
+
+// Return true when spill is triggered
+func (e *TopNExec) generateTopNResultsWhenNoSpillTriggered() bool {
+ rowPtrNum := len(e.chkHeap.rowPtrs)
+ for ; e.chkHeap.idx < rowPtrNum; e.chkHeap.idx++ {
+ if e.chkHeap.idx%10 == 0 && e.spillHelper.isSpillNeeded() {
+ return true
+ }
+ e.resultChannel <- rowWithError{row: e.chkHeap.rowChunks.GetRow(e.chkHeap.rowPtrs[e.chkHeap.idx])}
+ }
+ return false
+}
+
+func (e *TopNExec) generateResultWithMultiWayMerge(offset int64, limit int64) error {
+ multiWayMerge := newMultiWayMerger(&diskSource{sortedRowsInDisk: e.spillHelper.sortedRowsInDisk}, e.lessRow)
+
+ err := multiWayMerge.init()
+ if err != nil {
+ return err
+ }
+
+ outputRowNum := int64(0)
+ for {
+ if outputRowNum >= limit {
+ return nil
+ }
+
+ row, err := multiWayMerge.next()
+ if err != nil {
+ return err
+ }
+
+ if row.IsEmpty() {
+ return nil
+ }
+
+ if outputRowNum >= offset {
+ select {
+ case <-e.finishCh:
+ return nil
+ case e.resultChannel <- rowWithError{row: row}:
+ }
+ }
+ outputRowNum++
+ injectParallelSortRandomFail(1)
}
- newRowChunks.GetMemTracker().SetLabel(memory.LabelForRowChunks)
- e.memTracker.ReplaceChild(chkHeap.rowChunks.GetMemTracker(), newRowChunks.GetMemTracker())
- chkHeap.rowChunks = newRowChunks
+}
+
+// GenerateTopNResultsWhenSpillOnlyOnce generates results with this function when we trigger spill only once.
+// It's a public function as we need to test it in ut.
+func (e *TopNExec) GenerateTopNResultsWhenSpillOnlyOnce() error {
+ inDisk := e.spillHelper.sortedRowsInDisk[0]
+ chunkNum := inDisk.NumChunks()
+ skippedRowNum := uint64(0)
+ offset := e.Limit.Offset
+ for i := 0; i < chunkNum; i++ {
+ chk, err := inDisk.GetChunk(i)
+ if err != nil {
+ return err
+ }
+
+ injectTopNRandomFail(10)
+
+ rowNum := chk.NumRows()
+ j := 0
+ if !e.inMemoryThenSpillFlag {
+ // When e.inMemoryThenSpillFlag == false, we need to manually set j
+ // because rows that should be ignored before offset have also been
+ // spilled to disk.
+ if skippedRowNum < offset {
+ rowNumNeedSkip := offset - skippedRowNum
+ if rowNum <= int(rowNumNeedSkip) {
+ // All rows in this chunk should be skipped
+ skippedRowNum += uint64(rowNum)
+ continue
+ }
+ j += int(rowNumNeedSkip)
+ skippedRowNum += rowNumNeedSkip
+ }
+ }
- e.memTracker.Consume(int64(8 * (len(newRowPtrs) - len(chkHeap.rowPtrs))))
- chkHeap.rowPtrs = newRowPtrs
+ for ; j < rowNum; j++ {
+ select {
+ case <-e.finishCh:
+ return nil
+ case e.resultChannel <- rowWithError{row: chk.GetRow(j)}:
+ }
+ }
+ }
return nil
}
+
+func (e *TopNExec) generateTopNResultsWhenSpillTriggered() error {
+ inDiskNum := len(e.spillHelper.sortedRowsInDisk)
+ if inDiskNum == 0 {
+ panic("inDiskNum can't be 0 when we generate result with spill triggered")
+ }
+
+ if inDiskNum == 1 {
+ return e.GenerateTopNResultsWhenSpillOnlyOnce()
+ }
+ return e.generateResultWithMultiWayMerge(int64(e.Limit.Offset), int64(e.Limit.Offset+e.Limit.Count))
+}
+
+func (e *TopNExec) generateTopNResults() {
+ if !e.spillHelper.isSpillTriggered() {
+ if !e.generateTopNResultsWhenNoSpillTriggered() {
+ return
+ }
+
+ err := e.spillTopNExecHeap()
+ if err != nil {
+ e.resultChannel <- rowWithError{err: err}
+ }
+
+ e.inMemoryThenSpillFlag = true
+ }
+
+ err := e.generateTopNResultsWhenSpillTriggered()
+ if err != nil {
+ e.resultChannel <- rowWithError{err: err}
+ }
+}
+
+// IsSpillTriggeredForTest shows if spill is triggered, used for test.
+func (e *TopNExec) IsSpillTriggeredForTest() bool {
+ return e.spillHelper.isSpillTriggered()
+}
+
+// GetIsSpillTriggeredInStage1ForTest shows if spill is triggered in stage 1, only used for test.
+func (e *TopNExec) GetIsSpillTriggeredInStage1ForTest() bool {
+ return e.isSpillTriggeredInStage1ForTest
+}
+
+// GetIsSpillTriggeredInStage2ForTest shows if spill is triggered in stage 2, only used for test.
+func (e *TopNExec) GetIsSpillTriggeredInStage2ForTest() bool {
+ return e.isSpillTriggeredInStage2ForTest
+}
+
+// GetInMemoryThenSpillFlagForTest shows if results are in memory before they are spilled, only used for test
+func (e *TopNExec) GetInMemoryThenSpillFlagForTest() bool {
+ return e.inMemoryThenSpillFlag
+}
+
+func injectTopNRandomFail(triggerFactor int32) {
+ failpoint.Inject("TopNRandomFail", func(val failpoint.Value) {
+ if val.(bool) {
+ randNum := rand.Int31n(10000)
+ if randNum < triggerFactor {
+ panic("panic is triggered by random fail")
+ }
+ }
+ })
+}
+
+// InitTopNExecForTest initializes TopN executors, only for test.
+func InitTopNExecForTest(topnExec *TopNExec, offset uint64, sortedRowsInDisk *chunk.DataInDiskByChunks) {
+ topnExec.inMemoryThenSpillFlag = false
+ topnExec.finishCh = make(chan struct{}, 1)
+ topnExec.resultChannel = make(chan rowWithError, 10000)
+ topnExec.Limit.Offset = offset
+ topnExec.spillHelper = &topNSpillHelper{}
+ topnExec.spillHelper.sortedRowsInDisk = []*chunk.DataInDiskByChunks{sortedRowsInDisk}
+}
+
+// GetResultForTest gets result, only for test.
+func GetResultForTest(topnExec *TopNExec) []int64 {
+ close(topnExec.resultChannel)
+ result := make([]int64, 0, 100)
+ for {
+ row, ok := <-topnExec.resultChannel
+ if !ok {
+ return result
+ }
+ result = append(result, row.row.GetInt64(0))
+ }
+}
diff --git a/pkg/executor/sortexec/topn_chunk_heap.go b/pkg/executor/sortexec/topn_chunk_heap.go
new file mode 100644
index 0000000000000..df19763e4693a
--- /dev/null
+++ b/pkg/executor/sortexec/topn_chunk_heap.go
@@ -0,0 +1,155 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sortexec
+
+import (
+ "container/heap"
+
+ "github.com/pingcap/tidb/pkg/executor/internal/exec"
+ "github.com/pingcap/tidb/pkg/types"
+ "github.com/pingcap/tidb/pkg/util/chunk"
+ "github.com/pingcap/tidb/pkg/util/memory"
+)
+
+// topNChunkHeap implements heap.Interface.
+type topNChunkHeap struct {
+ compareRow func(chunk.Row, chunk.Row) int
+ greaterRow func(chunk.Row, chunk.Row) bool
+
+ // rowChunks is the chunks to store row values.
+ rowChunks *chunk.List
+ // rowPointer store the chunk index and row index for each row.
+ rowPtrs []chunk.RowPtr
+
+ isInitialized bool
+ isRowPtrsInit bool
+
+ memTracker *memory.Tracker
+
+ totalLimit uint64
+ idx int
+
+ fieldTypes []*types.FieldType
+}
+
+func (h *topNChunkHeap) init(topnExec *TopNExec, memTracker *memory.Tracker, totalLimit uint64, idx int, greaterRow func(chunk.Row, chunk.Row) bool, fieldTypes []*types.FieldType) {
+ h.memTracker = memTracker
+
+ h.rowChunks = chunk.NewList(exec.RetTypes(topnExec), topnExec.InitCap(), topnExec.MaxChunkSize())
+ h.rowChunks.GetMemTracker().AttachTo(h.memTracker)
+ h.rowChunks.GetMemTracker().SetLabel(memory.LabelForRowChunks)
+
+ h.compareRow = topnExec.compareRow
+ h.greaterRow = greaterRow
+
+ h.totalLimit = totalLimit
+ h.idx = idx
+ h.isInitialized = true
+
+ h.fieldTypes = fieldTypes
+}
+
+func (h *topNChunkHeap) initPtrs() {
+ h.memTracker.Consume(int64(chunk.RowPtrSize * h.rowChunks.Len()))
+ h.initPtrsImpl()
+}
+
+func (h *topNChunkHeap) initPtrsImpl() {
+ h.rowPtrs = make([]chunk.RowPtr, 0, h.rowChunks.Len())
+ for chkIdx := 0; chkIdx < h.rowChunks.NumChunks(); chkIdx++ {
+ rowChk := h.rowChunks.GetChunk(chkIdx)
+ for rowIdx := 0; rowIdx < rowChk.NumRows(); rowIdx++ {
+ h.rowPtrs = append(h.rowPtrs, chunk.RowPtr{ChkIdx: uint32(chkIdx), RowIdx: uint32(rowIdx)})
+ }
+ }
+ h.isRowPtrsInit = true
+}
+
+func (h *topNChunkHeap) clear() {
+ h.rowChunks.Clear()
+ h.memTracker.Consume(int64(-chunk.RowPtrSize * len(h.rowPtrs)))
+ h.rowPtrs = nil
+ h.isRowPtrsInit = false
+ h.isInitialized = false
+ h.idx = 0
+}
+
+func (h *topNChunkHeap) update(heapMaxRow chunk.Row, newRow chunk.Row) {
+ if h.greaterRow(heapMaxRow, newRow) {
+ // Evict heap max, keep the next row.
+ h.rowPtrs[0] = h.rowChunks.AppendRow(newRow)
+ heap.Fix(h, 0)
+ }
+}
+
+func (h *topNChunkHeap) processChk(chk *chunk.Chunk) {
+ for i := 0; i < chk.NumRows(); i++ {
+ heapMaxRow := h.rowChunks.GetRow(h.rowPtrs[0])
+ newRow := chk.GetRow(i)
+ h.update(heapMaxRow, newRow)
+ }
+}
+
+// doCompaction rebuild the chunks and row pointers to release memory.
+// If we don't do compaction, in a extreme case like the child data is already ascending sorted
+// but we want descending top N, then we will keep all data in memory.
+// But if data is distributed randomly, this function will be called log(n) times.
+func (h *topNChunkHeap) doCompaction(topnExec *TopNExec) error {
+ newRowChunks := chunk.NewList(exec.RetTypes(topnExec), topnExec.InitCap(), topnExec.MaxChunkSize())
+ newRowPtrs := make([]chunk.RowPtr, 0, h.rowChunks.Len())
+ for _, rowPtr := range h.rowPtrs {
+ newRowPtr := newRowChunks.AppendRow(h.rowChunks.GetRow(rowPtr))
+ newRowPtrs = append(newRowPtrs, newRowPtr)
+ }
+ newRowChunks.GetMemTracker().SetLabel(memory.LabelForRowChunks)
+ h.memTracker.ReplaceChild(h.rowChunks.GetMemTracker(), newRowChunks.GetMemTracker())
+ h.rowChunks = newRowChunks
+
+ h.memTracker.Consume(int64(chunk.RowPtrSize * (len(newRowPtrs) - len(h.rowPtrs))))
+ h.rowPtrs = newRowPtrs
+ return nil
+}
+
+func (h *topNChunkHeap) keyColumnsCompare(i, j chunk.RowPtr) int {
+ rowI := h.rowChunks.GetRow(i)
+ rowJ := h.rowChunks.GetRow(j)
+ return h.compareRow(rowI, rowJ)
+}
+
+// Less implement heap.Interface, but since we mantains a max heap,
+// this function returns true if row i is greater than row j.
+func (h *topNChunkHeap) Less(i, j int) bool {
+ rowI := h.rowChunks.GetRow(h.rowPtrs[i])
+ rowJ := h.rowChunks.GetRow(h.rowPtrs[j])
+ return h.greaterRow(rowI, rowJ)
+}
+
+func (h *topNChunkHeap) Len() int {
+ return len(h.rowPtrs)
+}
+
+func (*topNChunkHeap) Push(any) {
+ // Should never be called.
+}
+
+func (h *topNChunkHeap) Pop() any {
+ h.rowPtrs = h.rowPtrs[:len(h.rowPtrs)-1]
+ // We don't need the popped value, return nil to avoid memory allocation.
+ return nil
+}
+
+func (h *topNChunkHeap) Swap(i, j int) {
+ h.rowPtrs[i], h.rowPtrs[j] = h.rowPtrs[j], h.rowPtrs[i]
+}
diff --git a/pkg/executor/sortexec/topn_spill.go b/pkg/executor/sortexec/topn_spill.go
new file mode 100644
index 0000000000000..b3cf5ff20c0e8
--- /dev/null
+++ b/pkg/executor/sortexec/topn_spill.go
@@ -0,0 +1,265 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sortexec
+
+import (
+ "slices"
+ "sync"
+ "sync/atomic"
+
+ "github.com/pingcap/tidb/pkg/executor/internal/exec"
+ "github.com/pingcap/tidb/pkg/types"
+ "github.com/pingcap/tidb/pkg/util"
+ "github.com/pingcap/tidb/pkg/util/chunk"
+ "github.com/pingcap/tidb/pkg/util/disk"
+ "github.com/pingcap/tidb/pkg/util/logutil"
+ "github.com/pingcap/tidb/pkg/util/memory"
+ "go.uber.org/zap"
+)
+
+type topNSpillHelper struct {
+ cond *sync.Cond
+ spillStatus int
+ sortedRowsInDisk []*chunk.DataInDiskByChunks
+
+ finishCh <-chan struct{}
+ errOutputChan chan<- rowWithError
+
+ memTracker *memory.Tracker
+ diskTracker *disk.Tracker
+
+ fieldTypes []*types.FieldType
+ tmpSpillChunksChan chan *chunk.Chunk
+
+ workers []*topNWorker
+
+ bytesConsumed atomic.Int64
+ bytesLimit atomic.Int64
+}
+
+func newTopNSpillerHelper(
+ topn *TopNExec,
+ finishCh <-chan struct{},
+ errOutputChan chan<- rowWithError,
+ memTracker *memory.Tracker,
+ diskTracker *disk.Tracker,
+ fieldTypes []*types.FieldType,
+ workers []*topNWorker,
+ concurrencyNum int,
+) *topNSpillHelper {
+ lock := sync.Mutex{}
+ tmpSpillChunksChan := make(chan *chunk.Chunk, concurrencyNum)
+ for i := 0; i < len(workers); i++ {
+ tmpSpillChunksChan <- exec.TryNewCacheChunk(topn.Children(0))
+ }
+
+ return &topNSpillHelper{
+ cond: sync.NewCond(&lock),
+ spillStatus: notSpilled,
+ sortedRowsInDisk: make([]*chunk.DataInDiskByChunks, 0),
+ finishCh: finishCh,
+ errOutputChan: errOutputChan,
+ memTracker: memTracker,
+ diskTracker: diskTracker,
+ fieldTypes: fieldTypes,
+ tmpSpillChunksChan: tmpSpillChunksChan,
+ workers: workers,
+ bytesConsumed: atomic.Int64{},
+ bytesLimit: atomic.Int64{},
+ }
+}
+
+func (t *topNSpillHelper) close() {
+ for _, inDisk := range t.sortedRowsInDisk {
+ inDisk.Close()
+ }
+}
+
+func (t *topNSpillHelper) isNotSpilledNoLock() bool {
+ return t.spillStatus == notSpilled
+}
+
+func (t *topNSpillHelper) isInSpillingNoLock() bool {
+ return t.spillStatus == inSpilling
+}
+
+func (t *topNSpillHelper) isSpillNeeded() bool {
+ t.cond.L.Lock()
+ defer t.cond.L.Unlock()
+ return t.spillStatus == needSpill
+}
+
+func (t *topNSpillHelper) isSpillTriggered() bool {
+ t.cond.L.Lock()
+ defer t.cond.L.Unlock()
+ return len(t.sortedRowsInDisk) > 0
+}
+
+func (t *topNSpillHelper) setInSpilling() {
+ t.cond.L.Lock()
+ defer t.cond.L.Unlock()
+ t.spillStatus = inSpilling
+ logutil.BgLogger().Info(spillInfo, zap.Int64("consumed", t.bytesConsumed.Load()), zap.Int64("quota", t.bytesLimit.Load()))
+}
+
+func (t *topNSpillHelper) setNotSpilled() {
+ t.cond.L.Lock()
+ defer t.cond.L.Unlock()
+ t.spillStatus = notSpilled
+}
+
+func (t *topNSpillHelper) setNeedSpillNoLock() {
+ t.spillStatus = needSpill
+}
+
+func (t *topNSpillHelper) addInDisk(inDisk *chunk.DataInDiskByChunks) {
+ t.cond.L.Lock()
+ defer t.cond.L.Unlock()
+ t.sortedRowsInDisk = append(t.sortedRowsInDisk, inDisk)
+}
+
+func (*topNSpillHelper) spillTmpSpillChunk(inDisk *chunk.DataInDiskByChunks, tmpSpillChunk *chunk.Chunk) error {
+ err := inDisk.Add(tmpSpillChunk)
+ if err != nil {
+ return err
+ }
+ tmpSpillChunk.Reset()
+ return nil
+}
+
+func (t *topNSpillHelper) spill() (err error) {
+ defer func() {
+ if r := recover(); r != nil {
+ err = util.GetRecoverError(r)
+ }
+ }()
+
+ select {
+ case <-t.finishCh:
+ return nil
+ default:
+ }
+
+ t.setInSpilling()
+ defer t.cond.Broadcast()
+ defer t.setNotSpilled()
+
+ workerNum := len(t.workers)
+ errChan := make(chan error, workerNum)
+ workerWaiter := &sync.WaitGroup{}
+ workerWaiter.Add(workerNum)
+ for i := 0; i < workerNum; i++ {
+ go func(idx int) {
+ defer func() {
+ if r := recover(); r != nil {
+ processPanicAndLog(t.errOutputChan, r)
+ }
+ workerWaiter.Done()
+ }()
+
+ spillErr := t.spillHeap(t.workers[idx].chkHeap)
+ if spillErr != nil {
+ errChan <- spillErr
+ }
+ }(i)
+ }
+
+ workerWaiter.Wait()
+ close(errChan)
+
+ // Fetch only one error is enough
+ spillErr := <-errChan
+ if spillErr != nil {
+ return spillErr
+ }
+ return nil
+}
+
+func (t *topNSpillHelper) spillHeap(chkHeap *topNChunkHeap) error {
+ if chkHeap.Len() <= 0 && chkHeap.rowChunks.Len() <= 0 {
+ return nil
+ }
+
+ if !chkHeap.isRowPtrsInit {
+ // Do not consume memory here, as it will hang
+ chkHeap.initPtrsImpl()
+ }
+ slices.SortFunc(chkHeap.rowPtrs, chkHeap.keyColumnsCompare)
+
+ tmpSpillChunk := <-t.tmpSpillChunksChan
+ tmpSpillChunk.Reset()
+ defer func() {
+ t.tmpSpillChunksChan <- tmpSpillChunk
+ }()
+
+ inDisk := chunk.NewDataInDiskByChunks(t.fieldTypes)
+ inDisk.GetDiskTracker().AttachTo(t.diskTracker)
+
+ rowPtrNum := chkHeap.Len()
+ for ; chkHeap.idx < rowPtrNum; chkHeap.idx++ {
+ if tmpSpillChunk.IsFull() {
+ err := t.spillTmpSpillChunk(inDisk, tmpSpillChunk)
+ if err != nil {
+ return err
+ }
+ }
+ tmpSpillChunk.AppendRow(chkHeap.rowChunks.GetRow(chkHeap.rowPtrs[chkHeap.idx]))
+ }
+
+ // Spill remaining rows in tmpSpillChunk
+ if tmpSpillChunk.NumRows() > 0 {
+ err := t.spillTmpSpillChunk(inDisk, tmpSpillChunk)
+ if err != nil {
+ return err
+ }
+ }
+
+ t.addInDisk(inDisk)
+ injectTopNRandomFail(200)
+
+ chkHeap.clear()
+ return nil
+}
+
+type topNSpillAction struct {
+ memory.BaseOOMAction
+ spillHelper *topNSpillHelper
+}
+
+// GetPriority get the priority of the Action.
+func (*topNSpillAction) GetPriority() int64 {
+ return memory.DefSpillPriority
+}
+
+func (t *topNSpillAction) Action(tracker *memory.Tracker) {
+ t.spillHelper.cond.L.Lock()
+ defer t.spillHelper.cond.L.Unlock()
+
+ for t.spillHelper.isInSpillingNoLock() {
+ t.spillHelper.cond.Wait()
+ }
+
+ hasEnoughData := hasEnoughDataToSpill(t.spillHelper.memTracker, tracker)
+ if tracker.CheckExceed() && t.spillHelper.isNotSpilledNoLock() && hasEnoughData {
+ t.spillHelper.setNeedSpillNoLock()
+ t.spillHelper.bytesConsumed.Store(tracker.BytesConsumed())
+ t.spillHelper.bytesLimit.Store(tracker.GetBytesLimit())
+ return
+ }
+
+ if tracker.CheckExceed() && !hasEnoughData {
+ t.GetFallback()
+ }
+}
diff --git a/pkg/executor/sortexec/topn_spill_test.go b/pkg/executor/sortexec/topn_spill_test.go
new file mode 100644
index 0000000000000..3c8ddda4ac2c8
--- /dev/null
+++ b/pkg/executor/sortexec/topn_spill_test.go
@@ -0,0 +1,480 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sortexec_test
+
+import (
+ "context"
+ "math/rand"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/tidb/pkg/executor/internal/exec"
+ "github.com/pingcap/tidb/pkg/executor/internal/testutil"
+ "github.com/pingcap/tidb/pkg/executor/sortexec"
+ "github.com/pingcap/tidb/pkg/expression"
+ "github.com/pingcap/tidb/pkg/parser/mysql"
+ plannercore "github.com/pingcap/tidb/pkg/planner/core"
+ plannerutil "github.com/pingcap/tidb/pkg/planner/util"
+ "github.com/pingcap/tidb/pkg/types"
+ "github.com/pingcap/tidb/pkg/util/chunk"
+ "github.com/pingcap/tidb/pkg/util/memory"
+ "github.com/pingcap/tidb/pkg/util/mock"
+ "github.com/stretchr/testify/require"
+)
+
+var totalRowNum = 10000
+var noSpillCaseHardLimit = hardLimit2
+var spillCase1HardLimit = hardLimit1
+var spillCase2HardLimit = hardLimit1
+var inMemoryThenSpillHardLimit = hardLimit1 * 2
+
+// Test is successful if there is no hang
+func executeTopNInFailpoint(t *testing.T, exe *sortexec.TopNExec, hardLimit int64, tracker *memory.Tracker) {
+ tmpCtx := context.Background()
+ err := exe.Open(tmpCtx)
+ require.NoError(t, err)
+
+ goRoutineWaiter := sync.WaitGroup{}
+ goRoutineWaiter.Add(1)
+ defer goRoutineWaiter.Wait()
+
+ once := sync.Once{}
+
+ go func() {
+ time.Sleep(time.Duration(rand.Int31n(300)) * time.Millisecond)
+ once.Do(func() {
+ exe.Close()
+ })
+ goRoutineWaiter.Done()
+ }()
+
+ chk := exec.NewFirstChunk(exe)
+ for i := 0; i >= 0; i++ {
+ err := exe.Next(tmpCtx, chk)
+ if err != nil {
+ once.Do(func() {
+ err = exe.Close()
+ require.Equal(t, nil, err)
+ })
+ break
+ }
+ if chk.NumRows() == 0 {
+ break
+ }
+
+ if i == 10 && hardLimit > 0 {
+ // Trigger the spill
+ tracker.Consume(hardLimit)
+ tracker.Consume(-hardLimit)
+ }
+ }
+ once.Do(func() {
+ err = exe.Close()
+ require.Equal(t, nil, err)
+ })
+}
+
+func initTopNNoSpillCaseParams(
+ ctx *mock.Context,
+ dataSource *testutil.MockDataSource,
+ topNCase *testutil.SortCase,
+ totalRowNum int,
+ count *uint64,
+ offset *uint64,
+ exe **sortexec.TopNExec,
+) {
+ ctx.GetSessionVars().MemTracker = memory.NewTracker(memory.LabelForSQLText, noSpillCaseHardLimit)
+ ctx.GetSessionVars().StmtCtx.MemTracker.AttachTo(ctx.GetSessionVars().MemTracker)
+
+ *count = uint64(totalRowNum / 3)
+ *offset = uint64(totalRowNum / 10)
+
+ if exe != nil {
+ *exe = buildTopNExec(topNCase, dataSource, *offset, *count)
+ }
+}
+
+func initTopNSpillCase1Params(
+ ctx *mock.Context,
+ dataSource *testutil.MockDataSource,
+ topNCase *testutil.SortCase,
+ totalRowNum int,
+ count *uint64,
+ offset *uint64,
+ exe **sortexec.TopNExec,
+) {
+ ctx.GetSessionVars().MemTracker = memory.NewTracker(memory.LabelForSQLText, spillCase1HardLimit)
+ ctx.GetSessionVars().StmtCtx.MemTracker.AttachTo(ctx.GetSessionVars().MemTracker)
+
+ *count = uint64(totalRowNum - totalRowNum/10)
+ *offset = uint64(totalRowNum / 10)
+
+ if exe != nil {
+ *exe = buildTopNExec(topNCase, dataSource, *offset, *count)
+ }
+}
+
+func initTopNSpillCase2Params(
+ ctx *mock.Context,
+ dataSource *testutil.MockDataSource,
+ topNCase *testutil.SortCase,
+ totalRowNum int,
+ count *uint64,
+ offset *uint64,
+ exe **sortexec.TopNExec,
+) {
+ ctx.GetSessionVars().MemTracker = memory.NewTracker(memory.LabelForSQLText, spillCase2HardLimit)
+ ctx.GetSessionVars().StmtCtx.MemTracker.AttachTo(ctx.GetSessionVars().MemTracker)
+
+ *count = uint64(totalRowNum / 5)
+ *offset = *count / 5
+
+ if exe != nil {
+ *exe = buildTopNExec(topNCase, dataSource, *offset, *count)
+ }
+}
+
+func initTopNInMemoryThenSpillParams(
+ ctx *mock.Context,
+ dataSource *testutil.MockDataSource,
+ topNCase *testutil.SortCase,
+ totalRowNum int,
+ count *uint64,
+ offset *uint64,
+ exe **sortexec.TopNExec,
+) {
+ ctx.GetSessionVars().MemTracker = memory.NewTracker(memory.LabelForSQLText, inMemoryThenSpillHardLimit)
+ ctx.GetSessionVars().StmtCtx.MemTracker.AttachTo(ctx.GetSessionVars().MemTracker)
+
+ *count = uint64(totalRowNum / 5)
+ *offset = *count / 5
+
+ if exe != nil {
+ *exe = buildTopNExec(topNCase, dataSource, *offset, *count)
+ }
+}
+
+func checkTopNCorrectness(schema *expression.Schema, exe *sortexec.TopNExec, dataSource *testutil.MockDataSource, resultChunks []*chunk.Chunk, offset uint64, count uint64) bool {
+ keyColumns, keyCmpFuncs, byItemsDesc := exe.GetSortMetaForTest()
+ checker := newResultChecker(schema, keyColumns, keyCmpFuncs, byItemsDesc, dataSource.GenData)
+ return checker.check(resultChunks, int64(offset), int64(count))
+}
+
+func buildTopNExec(sortCase *testutil.SortCase, dataSource *testutil.MockDataSource, offset uint64, count uint64) *sortexec.TopNExec {
+ dataSource.PrepareChunks()
+ sortExec := sortexec.SortExec{
+ BaseExecutor: exec.NewBaseExecutor(sortCase.Ctx, dataSource.Schema(), 0, dataSource),
+ ByItems: make([]*plannerutil.ByItems, 0, len(sortCase.OrderByIdx)),
+ ExecSchema: dataSource.Schema(),
+ }
+
+ for _, idx := range sortCase.OrderByIdx {
+ sortExec.ByItems = append(sortExec.ByItems, &plannerutil.ByItems{Expr: sortCase.Columns()[idx]})
+ }
+
+ topNexec := &sortexec.TopNExec{
+ SortExec: sortExec,
+ Limit: &plannercore.PhysicalLimit{Offset: offset, Count: count},
+ Concurrency: 5,
+ }
+
+ return topNexec
+}
+
+func executeTopNExecutor(t *testing.T, exe *sortexec.TopNExec) []*chunk.Chunk {
+ tmpCtx := context.Background()
+ err := exe.Open(tmpCtx)
+ require.NoError(t, err)
+
+ resultChunks := make([]*chunk.Chunk, 0)
+ chk := exec.NewFirstChunk(exe)
+ for {
+ err = exe.Next(tmpCtx, chk)
+ require.NoError(t, err)
+ if chk.NumRows() == 0 {
+ break
+ }
+ resultChunks = append(resultChunks, chk.CopyConstruct())
+ }
+ return resultChunks
+}
+
+func executeTopNAndManuallyTriggerSpill(t *testing.T, exe *sortexec.TopNExec, hardLimit int64, tracker *memory.Tracker) []*chunk.Chunk {
+ tmpCtx := context.Background()
+ err := exe.Open(tmpCtx)
+ require.NoError(t, err)
+
+ resultChunks := make([]*chunk.Chunk, 0)
+ chk := exec.NewFirstChunk(exe)
+ for i := 0; i >= 0; i++ {
+ err = exe.Next(tmpCtx, chk)
+ require.NoError(t, err)
+
+ if i == 10 {
+ // Trigger the spill
+ tracker.Consume(hardLimit)
+ tracker.Consume(-hardLimit)
+ }
+
+ if chk.NumRows() == 0 {
+ break
+ }
+ resultChunks = append(resultChunks, chk.CopyConstruct())
+ }
+ return resultChunks
+}
+
+// No spill will be triggered in this test
+func topNNoSpillCase(t *testing.T, exe *sortexec.TopNExec, sortCase *testutil.SortCase, schema *expression.Schema, dataSource *testutil.MockDataSource, offset uint64, count uint64) {
+ if exe == nil {
+ exe = buildTopNExec(sortCase, dataSource, offset, count)
+ }
+ dataSource.PrepareChunks()
+ resultChunks := executeTopNExecutor(t, exe)
+
+ require.False(t, exe.IsSpillTriggeredForTest())
+
+ err := exe.Close()
+ require.NoError(t, err)
+
+ require.True(t, checkTopNCorrectness(schema, exe, dataSource, resultChunks, offset, count))
+}
+
+// Topn executor has two stage:
+// 1. Building heap, in this stage all received rows will be inserted into heap.
+// 2. Updating heap, in this stage only rows that is smaller than the heap top could be inserted and we will drop the heap top.
+//
+// Case1 means that we will trigger spill in stage 1
+func topNSpillCase1(t *testing.T, exe *sortexec.TopNExec, sortCase *testutil.SortCase, schema *expression.Schema, dataSource *testutil.MockDataSource, offset uint64, count uint64) {
+ if exe == nil {
+ exe = buildTopNExec(sortCase, dataSource, offset, count)
+ }
+ dataSource.PrepareChunks()
+ resultChunks := executeTopNExecutor(t, exe)
+
+ require.True(t, exe.IsSpillTriggeredForTest())
+ require.True(t, exe.GetIsSpillTriggeredInStage1ForTest())
+ require.False(t, exe.GetInMemoryThenSpillFlagForTest())
+
+ err := exe.Close()
+ require.NoError(t, err)
+
+ require.True(t, checkTopNCorrectness(schema, exe, dataSource, resultChunks, offset, count))
+}
+
+// Case2 means that we will trigger spill in stage 2
+func topNSpillCase2(t *testing.T, exe *sortexec.TopNExec, sortCase *testutil.SortCase, schema *expression.Schema, dataSource *testutil.MockDataSource, offset uint64, count uint64) {
+ if exe == nil {
+ exe = buildTopNExec(sortCase, dataSource, offset, count)
+ }
+ dataSource.PrepareChunks()
+ resultChunks := executeTopNExecutor(t, exe)
+
+ require.True(t, exe.IsSpillTriggeredForTest())
+ require.False(t, exe.GetIsSpillTriggeredInStage1ForTest())
+ require.True(t, exe.GetIsSpillTriggeredInStage2ForTest())
+ require.False(t, exe.GetInMemoryThenSpillFlagForTest())
+
+ err := exe.Close()
+ require.NoError(t, err)
+
+ require.True(t, checkTopNCorrectness(schema, exe, dataSource, resultChunks, offset, count))
+}
+
+// After all sorted rows are in memory, then the spill will be triggered after some chunks have been fetched
+func topNInMemoryThenSpillCase(t *testing.T, ctx *mock.Context, exe *sortexec.TopNExec, sortCase *testutil.SortCase, schema *expression.Schema, dataSource *testutil.MockDataSource, offset uint64, count uint64) {
+ if exe == nil {
+ exe = buildTopNExec(sortCase, dataSource, offset, count)
+ }
+ dataSource.PrepareChunks()
+ resultChunks := executeTopNAndManuallyTriggerSpill(t, exe, hardLimit1*2, ctx.GetSessionVars().StmtCtx.MemTracker)
+
+ require.True(t, exe.IsSpillTriggeredForTest())
+ require.False(t, exe.GetIsSpillTriggeredInStage1ForTest())
+ require.False(t, exe.GetIsSpillTriggeredInStage2ForTest())
+ require.True(t, exe.GetInMemoryThenSpillFlagForTest())
+
+ err := exe.Close()
+ require.NoError(t, err)
+
+ require.True(t, checkTopNCorrectness(schema, exe, dataSource, resultChunks, offset, count))
+}
+
+func topNFailPointTest(t *testing.T, exe *sortexec.TopNExec, sortCase *testutil.SortCase, dataSource *testutil.MockDataSource, offset uint64, count uint64, hardLimit int64, tracker *memory.Tracker) {
+ if exe == nil {
+ exe = buildTopNExec(sortCase, dataSource, offset, count)
+ }
+ dataSource.PrepareChunks()
+ executeTopNInFailpoint(t, exe, hardLimit, tracker)
+}
+
+const spilledChunkMaxSize = 32
+
+func createAndInitDataInDiskByChunks(spilledRowNum uint64) *chunk.DataInDiskByChunks {
+ fieldType := types.FieldType{}
+ fieldType.SetType(mysql.TypeLonglong)
+ inDisk := chunk.NewDataInDiskByChunks([]*types.FieldType{&fieldType})
+ var spilledChunk *chunk.Chunk
+ for i := uint64(0); i < spilledRowNum; i++ {
+ if i%spilledChunkMaxSize == 0 {
+ if spilledChunk != nil {
+ inDisk.Add(spilledChunk)
+ }
+ spilledChunk = chunk.NewChunkWithCapacity([]*types.FieldType{&fieldType}, spilledChunkMaxSize)
+ }
+ spilledChunk.AppendInt64(0, int64(i))
+ }
+ inDisk.Add(spilledChunk)
+ return inDisk
+}
+
+func testImpl(t *testing.T, topnExec *sortexec.TopNExec, inDisk *chunk.DataInDiskByChunks, totalRowNumInDisk uint64, offset uint64) {
+ sortexec.InitTopNExecForTest(topnExec, offset, inDisk)
+ topnExec.GenerateTopNResultsWhenSpillOnlyOnce()
+ result := sortexec.GetResultForTest(topnExec)
+ require.Equal(t, int(totalRowNumInDisk-offset), len(result))
+ for i := range result {
+ require.Equal(t, int64(i+int(offset)), result[i])
+ }
+}
+
+func oneChunkInDiskCase(t *testing.T, topnExec *sortexec.TopNExec) {
+ rowNumInDisk := uint64(spilledChunkMaxSize)
+ inDisk := createAndInitDataInDiskByChunks(rowNumInDisk)
+
+ testImpl(t, topnExec, inDisk, rowNumInDisk, 0)
+ testImpl(t, topnExec, inDisk, rowNumInDisk, uint64(spilledChunkMaxSize-15))
+ testImpl(t, topnExec, inDisk, rowNumInDisk, rowNumInDisk-1)
+ testImpl(t, topnExec, inDisk, rowNumInDisk, rowNumInDisk)
+}
+
+func severalChunksInDiskCase(t *testing.T, topnExec *sortexec.TopNExec) {
+ rowNumInDisk := uint64(spilledChunkMaxSize*3 + 10)
+ inDisk := createAndInitDataInDiskByChunks(rowNumInDisk)
+
+ testImpl(t, topnExec, inDisk, rowNumInDisk, 0)
+ testImpl(t, topnExec, inDisk, rowNumInDisk, spilledChunkMaxSize-15)
+ testImpl(t, topnExec, inDisk, rowNumInDisk, spilledChunkMaxSize*2+10)
+ testImpl(t, topnExec, inDisk, rowNumInDisk, rowNumInDisk-1)
+ testImpl(t, topnExec, inDisk, rowNumInDisk, rowNumInDisk)
+}
+
+func TestGenerateTopNResultsWhenSpillOnlyOnce(t *testing.T) {
+ topnExec := &sortexec.TopNExec{}
+ topnExec.Limit = &plannercore.PhysicalLimit{}
+
+ oneChunkInDiskCase(t, topnExec)
+ severalChunksInDiskCase(t, topnExec)
+}
+
+func TestTopNSpillDisk(t *testing.T) {
+ sortexec.SetSmallSpillChunkSizeForTest()
+ ctx := mock.NewContext()
+ topNCase := &testutil.SortCase{Rows: totalRowNum, OrderByIdx: []int{0, 1}, Ndvs: []int{0, 0}, Ctx: ctx}
+
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/executor/sortexec/SlowSomeWorkers", `return(true)`))
+
+ ctx.GetSessionVars().InitChunkSize = 32
+ ctx.GetSessionVars().MaxChunkSize = 32
+ ctx.GetSessionVars().MemTracker = memory.NewTracker(memory.LabelForSQLText, hardLimit2)
+ ctx.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(memory.LabelForSQLText, -1)
+ ctx.GetSessionVars().StmtCtx.MemTracker.AttachTo(ctx.GetSessionVars().MemTracker)
+
+ offset := uint64(totalRowNum / 10)
+ count := uint64(totalRowNum / 3)
+
+ var exe *sortexec.TopNExec
+ schema := expression.NewSchema(topNCase.Columns()...)
+ dataSource := buildDataSource(topNCase, schema)
+ initTopNNoSpillCaseParams(ctx, dataSource, topNCase, totalRowNum, &count, &offset, &exe)
+ for i := 0; i < 20; i++ {
+ topNNoSpillCase(t, nil, topNCase, schema, dataSource, 0, count)
+ topNNoSpillCase(t, exe, topNCase, schema, dataSource, offset, count)
+ }
+
+ initTopNSpillCase1Params(ctx, dataSource, topNCase, totalRowNum, &count, &offset, &exe)
+ for i := 0; i < 20; i++ {
+ topNSpillCase1(t, nil, topNCase, schema, dataSource, 0, count)
+ topNSpillCase1(t, exe, topNCase, schema, dataSource, offset, count)
+ }
+
+ initTopNSpillCase2Params(ctx, dataSource, topNCase, totalRowNum, &count, &offset, &exe)
+ for i := 0; i < 20; i++ {
+ topNSpillCase2(t, nil, topNCase, schema, dataSource, 0, count)
+ topNSpillCase2(t, exe, topNCase, schema, dataSource, offset, count)
+ }
+
+ initTopNInMemoryThenSpillParams(ctx, dataSource, topNCase, totalRowNum, &count, &offset, &exe)
+ for i := 0; i < 20; i++ {
+ topNInMemoryThenSpillCase(t, ctx, nil, topNCase, schema, dataSource, 0, count)
+ topNInMemoryThenSpillCase(t, ctx, exe, topNCase, schema, dataSource, offset, count)
+ }
+
+ failpoint.Disable("github.com/pingcap/tidb/pkg/executor/sortexec/SlowSomeWorkers")
+}
+
+func TestTopNSpillDiskFailpoint(t *testing.T) {
+ sortexec.SetSmallSpillChunkSizeForTest()
+ ctx := mock.NewContext()
+ topNCase := &testutil.SortCase{Rows: totalRowNum, OrderByIdx: []int{0, 1}, Ndvs: []int{0, 0}, Ctx: ctx}
+
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/executor/sortexec/SlowSomeWorkers", `return(true)`))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/executor/sortexec/TopNRandomFail", `return(true)`))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/executor/sortexec/ParallelSortRandomFail", `return(true)`))
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/util/chunk/ChunkInDiskError", `return(true)`))
+
+ ctx.GetSessionVars().InitChunkSize = 32
+ ctx.GetSessionVars().MaxChunkSize = 32
+ ctx.GetSessionVars().MemTracker = memory.NewTracker(memory.LabelForSQLText, hardLimit1)
+ ctx.GetSessionVars().StmtCtx.MemTracker = memory.NewTracker(memory.LabelForSQLText, -1)
+ ctx.GetSessionVars().StmtCtx.MemTracker.AttachTo(ctx.GetSessionVars().MemTracker)
+
+ offset := uint64(totalRowNum / 10)
+ count := uint64(totalRowNum / 3)
+
+ var exe *sortexec.TopNExec
+ schema := expression.NewSchema(topNCase.Columns()...)
+ dataSource := buildDataSource(topNCase, schema)
+ initTopNNoSpillCaseParams(ctx, dataSource, topNCase, totalRowNum, &count, &offset, &exe)
+ for i := 0; i < 10; i++ {
+ topNFailPointTest(t, nil, topNCase, dataSource, 0, count, 0, ctx.GetSessionVars().MemTracker)
+ topNFailPointTest(t, exe, topNCase, dataSource, offset, count, 0, ctx.GetSessionVars().MemTracker)
+ }
+
+ initTopNSpillCase1Params(ctx, dataSource, topNCase, totalRowNum, &count, &offset, &exe)
+ for i := 0; i < 10; i++ {
+ topNFailPointTest(t, nil, topNCase, dataSource, 0, count, 0, ctx.GetSessionVars().MemTracker)
+ topNFailPointTest(t, exe, topNCase, dataSource, offset, count, 0, ctx.GetSessionVars().MemTracker)
+ }
+
+ initTopNSpillCase2Params(ctx, dataSource, topNCase, totalRowNum, &count, &offset, &exe)
+ for i := 0; i < 10; i++ {
+ topNFailPointTest(t, nil, topNCase, dataSource, 0, count, 0, ctx.GetSessionVars().MemTracker)
+ topNFailPointTest(t, exe, topNCase, dataSource, offset, count, 0, ctx.GetSessionVars().MemTracker)
+ }
+
+ initTopNInMemoryThenSpillParams(ctx, dataSource, topNCase, totalRowNum, &count, &offset, &exe)
+ for i := 0; i < 10; i++ {
+ topNFailPointTest(t, nil, topNCase, dataSource, 0, count, inMemoryThenSpillHardLimit, ctx.GetSessionVars().MemTracker)
+ topNFailPointTest(t, exe, topNCase, dataSource, offset, count, inMemoryThenSpillHardLimit, ctx.GetSessionVars().MemTracker)
+ }
+
+ failpoint.Disable("github.com/pingcap/tidb/pkg/executor/sortexec/SlowSomeWorkers")
+ failpoint.Disable("github.com/pingcap/tidb/pkg/executor/sortexec/TopNRandomFail")
+ failpoint.Disable("github.com/pingcap/tidb/pkg/executor/sortexec/ParallelSortRandomFail")
+ failpoint.Disable("github.com/pingcap/tidb/pkg/util/chunk/ChunkInDiskError")
+}
diff --git a/pkg/executor/sortexec/topn_worker.go b/pkg/executor/sortexec/topn_worker.go
new file mode 100644
index 0000000000000..527dcd42977e9
--- /dev/null
+++ b/pkg/executor/sortexec/topn_worker.go
@@ -0,0 +1,130 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sortexec
+
+import (
+ "container/heap"
+ "math/rand"
+ "sync"
+ "time"
+
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/tidb/pkg/util/chunk"
+ "github.com/pingcap/tidb/pkg/util/memory"
+)
+
+// topNWorker is used only when topn spill is triggered
+type topNWorker struct {
+ workerIDForTest int
+
+ chunkChannel <-chan *chunk.Chunk
+ fetcherAndWorkerSyncer *sync.WaitGroup
+ errOutputChan chan<- rowWithError
+ finishChan <-chan struct{}
+
+ topn *TopNExec
+ chkHeap *topNChunkHeap
+ memTracker *memory.Tracker
+}
+
+func newTopNWorker(
+ idForTest int,
+ chunkChannel <-chan *chunk.Chunk,
+ fetcherAndWorkerSyncer *sync.WaitGroup,
+ errOutputChan chan<- rowWithError,
+ finishChan <-chan struct{},
+ topn *TopNExec,
+ chkHeap *topNChunkHeap,
+ memTracker *memory.Tracker) *topNWorker {
+ return &topNWorker{
+ workerIDForTest: idForTest,
+ chunkChannel: chunkChannel,
+ fetcherAndWorkerSyncer: fetcherAndWorkerSyncer,
+ errOutputChan: errOutputChan,
+ finishChan: finishChan,
+ chkHeap: chkHeap,
+ topn: topn,
+ memTracker: memTracker,
+ }
+}
+
+func (t *topNWorker) initWorker() {
+ // Offset of heap in worker should be 0, as we need to spill all data
+ t.chkHeap.init(t.topn, t.memTracker, t.topn.Limit.Offset+t.topn.Limit.Count, 0, t.topn.greaterRow, t.topn.RetFieldTypes())
+}
+
+func (t *topNWorker) fetchChunksAndProcess() {
+ for t.fetchChunksAndProcessImpl() {
+ }
+}
+
+func (t *topNWorker) fetchChunksAndProcessImpl() bool {
+ select {
+ case <-t.finishChan:
+ return false
+ case chk, ok := <-t.chunkChannel:
+ if !ok {
+ return false
+ }
+ defer func() {
+ t.fetcherAndWorkerSyncer.Done()
+ }()
+
+ t.injectFailPointForTopNWorker(3)
+
+ if uint64(t.chkHeap.rowChunks.Len()) < t.chkHeap.totalLimit {
+ if !t.chkHeap.isInitialized {
+ t.chkHeap.init(t.topn, t.memTracker, t.topn.Limit.Offset+t.topn.Limit.Count, 0, t.topn.greaterRow, t.topn.RetFieldTypes())
+ }
+ t.chkHeap.rowChunks.Add(chk)
+ } else {
+ if !t.chkHeap.isRowPtrsInit {
+ t.chkHeap.initPtrs()
+ heap.Init(t.chkHeap)
+ }
+ t.chkHeap.processChk(chk)
+ }
+ }
+ return true
+}
+
+func (t *topNWorker) run() {
+ defer func() {
+ if r := recover(); r != nil {
+ processPanicAndLog(t.errOutputChan, r)
+ }
+
+ // Consume all chunks to avoid hang of fetcher
+ for range t.chunkChannel {
+ t.fetcherAndWorkerSyncer.Done()
+ }
+ }()
+
+ t.fetchChunksAndProcess()
+}
+
+func (t *topNWorker) injectFailPointForTopNWorker(triggerFactor int32) {
+ injectTopNRandomFail(triggerFactor)
+ failpoint.Inject("SlowSomeWorkers", func(val failpoint.Value) {
+ if val.(bool) {
+ if t.workerIDForTest%2 == 0 {
+ randNum := rand.Int31n(10000)
+ if randNum < 10 {
+ time.Sleep(1 * time.Millisecond)
+ }
+ }
+ }
+ })
+}
diff --git a/pkg/executor/split.go b/pkg/executor/split.go
index 9582f0bdaec64..888224ba5d506 100644
--- a/pkg/executor/split.go
+++ b/pkg/executor/split.go
@@ -27,7 +27,7 @@ import (
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
- "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/planner/util"
"github.com/pingcap/tidb/pkg/sessionctx"
"github.com/pingcap/tidb/pkg/store/helper"
"github.com/pingcap/tidb/pkg/table/tables"
@@ -330,7 +330,7 @@ type SplitTableRegionExec struct {
lower []types.Datum
upper []types.Datum
num int
- handleCols core.HandleCols
+ handleCols util.HandleCols
valueLists [][]types.Datum
splitKeys [][]byte
diff --git a/pkg/executor/split_test.go b/pkg/executor/split_test.go
index b65009502ecd5..cc15d6b321be6 100644
--- a/pkg/executor/split_test.go
+++ b/pkg/executor/split_test.go
@@ -28,7 +28,7 @@ import (
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
- "github.com/pingcap/tidb/pkg/planner/core"
+ "github.com/pingcap/tidb/pkg/planner/util"
"github.com/pingcap/tidb/pkg/sessionctx/stmtctx"
"github.com/pingcap/tidb/pkg/table/tables"
"github.com/pingcap/tidb/pkg/tablecodec"
@@ -320,7 +320,7 @@ func TestSplitTable(t *testing.T) {
e := &SplitTableRegionExec{
BaseExecutor: exec.NewBaseExecutor(ctx, nil, 0),
tableInfo: tbInfo,
- handleCols: core.NewIntHandleCols(&expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)}),
+ handleCols: util.NewIntHandleCols(&expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)}),
lower: []types.Datum{types.NewDatum(0)},
upper: []types.Datum{types.NewDatum(100)},
num: 10,
@@ -380,7 +380,7 @@ func TestStepShouldLargeThanMinStep(t *testing.T) {
e1 := &SplitTableRegionExec{
BaseExecutor: exec.NewBaseExecutor(ctx, nil, 0),
tableInfo: tbInfo,
- handleCols: core.NewIntHandleCols(&expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)}),
+ handleCols: util.NewIntHandleCols(&expression.Column{RetType: types.NewFieldType(mysql.TypeLonglong)}),
lower: []types.Datum{types.NewDatum(0)},
upper: []types.Datum{types.NewDatum(1000)},
num: 10,
diff --git a/pkg/executor/stale_txn_test.go b/pkg/executor/stale_txn_test.go
index b8ca85ff2b6a6..8dd3613c4b5bd 100644
--- a/pkg/executor/stale_txn_test.go
+++ b/pkg/executor/stale_txn_test.go
@@ -20,6 +20,7 @@ import (
"testing"
"time"
+ "github.com/docker/go-units"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/config"
@@ -893,7 +894,7 @@ func TestSetTransactionInfoSchema(t *testing.T) {
UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment)
tk.MustExec(updateSafePoint)
- for _, cacheSize := range []int{1024, 0} {
+ for _, cacheSize := range []int{units.GiB, 0} {
tk.MustExec("set @@global.tidb_schema_cache_size = ?", cacheSize)
testSetTransactionInfoSchema(t, tk)
}
diff --git a/pkg/executor/table_reader.go b/pkg/executor/table_reader.go
index 59c1043ec4cbc..62ded870ad9f1 100644
--- a/pkg/executor/table_reader.go
+++ b/pkg/executor/table_reader.go
@@ -401,7 +401,7 @@ func (e *TableReaderExecutor) buildResp(ctx context.Context, ranges []*ranger.Ra
if len(results) == 1 {
return results[0], nil
}
- return distsql.NewSortedSelectResults(results, e.Schema(), e.byItems, e.memTracker), nil
+ return distsql.NewSortedSelectResults(e.ectx.GetEvalCtx(), results, e.Schema(), e.byItems, e.memTracker), nil
}
kvReq, err := e.buildKVReq(ctx, ranges)
diff --git a/pkg/executor/test/admintest/BUILD.bazel b/pkg/executor/test/admintest/BUILD.bazel
index 40cdf88fe1826..6488fa5a84fd5 100644
--- a/pkg/executor/test/admintest/BUILD.bazel
+++ b/pkg/executor/test/admintest/BUILD.bazel
@@ -8,9 +8,11 @@ go_test(
"main_test.go",
],
flaky = True,
- shard_count = 18,
+ shard_count = 23,
deps = [
"//pkg/config",
+ "//pkg/ddl",
+ "//pkg/ddl/util/callback",
"//pkg/domain",
"//pkg/errno",
"//pkg/executor",
@@ -22,6 +24,7 @@ go_test(
"//pkg/sessionctx/variable",
"//pkg/table",
"//pkg/table/tables",
+ "//pkg/tablecodec",
"//pkg/testkit",
"//pkg/testkit/testsetup",
"//pkg/testkit/testutil",
@@ -32,6 +35,8 @@ go_test(
"//pkg/util/mock",
"//pkg/util/redact",
"@com_github_pingcap_errors//:errors",
+ "@com_github_pingcap_failpoint//:failpoint",
+ "@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",
"@com_github_tikv_client_go_v2//tikv",
"@org_uber_go_goleak//:goleak",
diff --git a/pkg/executor/test/admintest/admin_test.go b/pkg/executor/test/admintest/admin_test.go
index 470d50443f0f8..2612ebac24de3 100644
--- a/pkg/executor/test/admintest/admin_test.go
+++ b/pkg/executor/test/admintest/admin_test.go
@@ -24,6 +24,9 @@ import (
"time"
"github.com/pingcap/errors"
+ "github.com/pingcap/failpoint"
+ "github.com/pingcap/tidb/pkg/ddl"
+ "github.com/pingcap/tidb/pkg/ddl/util/callback"
"github.com/pingcap/tidb/pkg/domain"
mysql "github.com/pingcap/tidb/pkg/errno"
"github.com/pingcap/tidb/pkg/executor"
@@ -34,6 +37,7 @@ import (
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/table/tables"
+ "github.com/pingcap/tidb/pkg/tablecodec"
"github.com/pingcap/tidb/pkg/testkit"
"github.com/pingcap/tidb/pkg/testkit/testutil"
"github.com/pingcap/tidb/pkg/types"
@@ -42,6 +46,7 @@ import (
"github.com/pingcap/tidb/pkg/util/logutil/consistency"
"github.com/pingcap/tidb/pkg/util/mock"
"github.com/pingcap/tidb/pkg/util/redact"
+ "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
@@ -1773,3 +1778,329 @@ func TestAdminCheckTableErrorLocateForClusterIndex(t *testing.T) {
tk.MustExec("admin check table admin_test")
}
}
+
+func TestAdminCleanUpGlobalIndex(t *testing.T) {
+ store, domain := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists admin_test")
+
+ tk.MustExec("set tidb_enable_global_index = true")
+ tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a)) partition by hash(c) partitions 5")
+ tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)")
+ tk.MustExec("analyze table admin_test")
+
+ // Make some corrupted index. Build the index information.
+ sctx := mock.NewContext()
+ sctx.Store = store
+ is := domain.InfoSchema()
+ dbName := model.NewCIStr("test")
+ tblName := model.NewCIStr("admin_test")
+ tbl, err := is.TableByName(dbName, tblName)
+ require.NoError(t, err)
+ tblInfo := tbl.Meta()
+ idxInfo := tblInfo.Indices[0]
+ require.True(t, idxInfo.Global)
+ idx := tbl.Indices()[0]
+ require.NotNil(t, idx)
+
+ // Reduce one row of table.
+ // Index count > table count, (2, 12, 0) is deleted.
+ txn, err := store.Begin()
+ require.NoError(t, err)
+ err = txn.Delete(tablecodec.EncodeRowKey(tblInfo.GetPartitionInfo().Definitions[0].ID, kv.IntHandle(4).Encoded()))
+ require.NoError(t, err)
+ err = txn.Commit(context.Background())
+ require.NoError(t, err)
+ err = tk.ExecToErr("admin check table admin_test")
+ require.Error(t, err)
+ require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err))
+
+ r := tk.MustQuery("admin cleanup index admin_test uidx_a")
+ r.Check(testkit.Rows("1"))
+ err = tk.ExecToErr("admin check table admin_test")
+ require.NoError(t, err)
+ require.Len(t, tk.MustQuery("select * from admin_test use index(uidx_a)").Rows(), 6)
+}
+
+func TestAdminRecoverGlobalIndex(t *testing.T) {
+ store, domain := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists admin_test")
+
+ tk.MustExec("set tidb_enable_global_index = true")
+ tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a)) partition by hash(c) partitions 5")
+ tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)")
+ tk.MustExec("analyze table admin_test")
+
+ // Make some corrupted index. Build the index information.
+ sctx := mock.NewContext()
+ sctx.Store = store
+ is := domain.InfoSchema()
+ dbName := model.NewCIStr("test")
+ tblName := model.NewCIStr("admin_test")
+ tbl, err := is.TableByName(dbName, tblName)
+ require.NoError(t, err)
+ tblInfo := tbl.Meta()
+ idxInfo := tblInfo.Indices[0]
+ require.True(t, idxInfo.Global)
+ idx := tbl.Indices()[0]
+ require.NotNil(t, idx)
+
+ indexOpr := tables.NewIndex(tblInfo.GetPartitionInfo().Definitions[2].ID, tblInfo, idxInfo)
+
+ // Reduce one row of index.
+ // Index count < table count, (-1, -10, 2) is deleted.
+ txn, err := store.Begin()
+ require.NoError(t, err)
+ err = indexOpr.Delete(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(-1)}, kv.IntHandle(2))
+ require.NoError(t, err)
+ err = txn.Commit(context.Background())
+ require.NoError(t, err)
+ err = tk.ExecToErr("admin check table admin_test")
+ require.Error(t, err)
+ require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err))
+
+ r := tk.MustQuery("admin recover index admin_test uidx_a")
+ r.Check(testkit.Rows("1 7"))
+ err = tk.ExecToErr("admin check table admin_test")
+ require.NoError(t, err)
+ require.Len(t, tk.MustQuery("select * from admin_test use index(uidx_a)").Rows(), 7)
+}
+
+func TestAdminCheckGlobalIndex(t *testing.T) {
+ store, domain := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+ var enableFastCheck = []bool{false, true}
+ for _, enabled := range enableFastCheck {
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists admin_test")
+
+ tk.MustExec("set tidb_enable_global_index = true")
+ tk.MustExec(fmt.Sprintf("set tidb_enable_fast_table_check = %v", enabled))
+
+ tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a)) partition by hash(c) partitions 5")
+ tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)")
+
+ // Make some corrupted index. Build the index information.
+ sctx := mock.NewContext()
+ sctx.Store = store
+ is := domain.InfoSchema()
+ dbName := model.NewCIStr("test")
+ tblName := model.NewCIStr("admin_test")
+ tbl, err := is.TableByName(dbName, tblName)
+ require.NoError(t, err)
+ tblInfo := tbl.Meta()
+ idxInfo := tblInfo.Indices[0]
+ require.True(t, idxInfo.Global)
+ idx := tbl.Indices()[0]
+ require.NotNil(t, idx)
+
+ // Reduce one row of table.
+ // Index count > table count, (2, 12, 0) is deleted.
+ txn, err := store.Begin()
+ require.NoError(t, err)
+ err = txn.Delete(tablecodec.EncodeRowKey(tblInfo.GetPartitionInfo().Definitions[0].ID, kv.IntHandle(4).Encoded()))
+ require.NoError(t, err)
+ err = txn.Commit(context.Background())
+ require.NoError(t, err)
+ err = tk.ExecToErr("admin check table admin_test")
+ require.Error(t, err)
+ require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err))
+ require.ErrorContains(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 4, index-values:\"handle: 4, values: [KindInt64 2")
+
+ indexOpr := tables.NewIndex(tblInfo.GetPartitionInfo().Definitions[0].ID, tblInfo, idxInfo)
+ // Remove corresponding index key/value.
+ // Admin check table will success.
+ txn, err = store.Begin()
+ require.NoError(t, err)
+ err = indexOpr.Delete(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(2)}, kv.IntHandle(4))
+ require.NoError(t, err)
+ err = txn.Commit(context.Background())
+ require.NoError(t, err)
+ tk.MustExec("admin check table admin_test")
+
+ indexOpr = tables.NewIndex(tblInfo.GetPartitionInfo().Definitions[2].ID, tblInfo, idxInfo)
+
+ // Reduce one row of index.
+ // Index count < table count, (-1, -10, 2) is deleted.
+ txn, err = store.Begin()
+ require.NoError(t, err)
+ err = indexOpr.Delete(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(-1)}, kv.IntHandle(2))
+ require.NoError(t, err)
+ err = txn.Commit(context.Background())
+ require.NoError(t, err)
+ err = tk.ExecToErr("admin check table admin_test")
+ require.Error(t, err)
+ require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err))
+ require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 2, index-values:\"\" != record-values:\"handle: 2, values: [KindInt64 -1]\"")
+
+ // Add one row of index with inconsistent value.
+ // Index count = table count, but data is different.
+ txn, err = store.Begin()
+ require.NoError(t, err)
+ _, err = indexOpr.Create(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(100)}, kv.IntHandle(2), nil)
+ require.NoError(t, err)
+ err = txn.Commit(context.Background())
+ require.NoError(t, err)
+ err = tk.ExecToErr("admin check table admin_test")
+ require.Error(t, err)
+ if !enabled {
+ require.True(t, consistency.ErrAdminCheckInconsistentWithColInfo.Equal(err))
+ require.EqualError(t, err, "[executor:8134]data inconsistency in table: admin_test, index: uidx_a, col: a, handle: \"2\", index-values:\"KindInt64 100\" != record-values:\"KindInt64 -1\", compare err:")
+ } else {
+ require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err))
+ require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 2, index-values:\"handle: 2, values: [KindInt64 100]\" != record-values:\"handle: 2, values: [KindInt64 -1]\"")
+ }
+ }
+}
+
+func TestAdminCheckGlobalIndexWithClusterIndex(t *testing.T) {
+ store, domain := testkit.CreateMockStoreAndDomain(t)
+
+ tk := testkit.NewTestKit(t, store)
+
+ getCommonHandle := func(row int) *kv.CommonHandle {
+ h, err := codec.EncodeKey(tk.Session().GetSessionVars().StmtCtx.TimeZone(), nil, types.MakeDatums(row)...)
+ require.NoError(t, err)
+ ch, err := kv.NewCommonHandle(h)
+ require.NoError(t, err)
+ return ch
+ }
+
+ var enableFastCheck = []bool{false, true}
+ for _, enabled := range enableFastCheck {
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists admin_test")
+
+ tk.MustExec("set tidb_enable_global_index = true")
+ tk.MustExec(fmt.Sprintf("set tidb_enable_fast_table_check = %v", enabled))
+
+ tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a), primary key(c)) partition by hash(c) partitions 5")
+ tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)")
+
+ // Make some corrupted index. Build the index information.
+ sctx := mock.NewContext()
+ sctx.Store = store
+ is := domain.InfoSchema()
+ dbName := model.NewCIStr("test")
+ tblName := model.NewCIStr("admin_test")
+ tbl, err := is.TableByName(dbName, tblName)
+ require.NoError(t, err)
+ tblInfo := tbl.Meta()
+ idxInfo := tblInfo.Indices[0]
+ require.True(t, idxInfo.Global)
+ df := tblInfo.GetPartitionInfo().Definitions[0]
+
+ // Reduce one row of table.
+ // Index count > table count, (2, 12, 0) is deleted.
+ txn, err := store.Begin()
+ require.NoError(t, err)
+ txn.Delete(tablecodec.EncodeRowKey(df.ID, kv.IntHandle(0).Encoded()))
+ err = txn.Commit(context.Background())
+ require.NoError(t, err)
+ err = tk.ExecToErr("admin check table admin_test")
+ require.Error(t, err)
+ require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err))
+ require.ErrorContains(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 0, index-values:\"handle: 0, values: [KindInt64 2")
+
+ indexOpr := tables.NewIndex(tblInfo.GetPartitionInfo().Definitions[0].ID, tblInfo, idxInfo)
+ // Remove corresponding index key/value.
+ // Admin check table will success.
+ txn, err = store.Begin()
+ require.NoError(t, err)
+ err = indexOpr.Delete(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(2)}, getCommonHandle(0))
+ require.NoError(t, err)
+ err = txn.Commit(context.Background())
+ require.NoError(t, err)
+ tk.MustExec("admin check table admin_test")
+
+ indexOpr = tables.NewIndex(tblInfo.GetPartitionInfo().Definitions[2].ID, tblInfo, idxInfo)
+ // Reduce one row of index.
+ // Index count < table count, (-1, -10, 2) is deleted.
+ txn, err = store.Begin()
+ require.NoError(t, err)
+ err = indexOpr.Delete(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(-1)}, getCommonHandle(2))
+ require.NoError(t, err)
+ err = txn.Commit(context.Background())
+ require.NoError(t, err)
+ err = tk.ExecToErr("admin check table admin_test")
+ require.Error(t, err)
+ require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err))
+ require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 2, index-values:\"\" != record-values:\"handle: 2, values: [KindInt64 -1]\"")
+
+ // Add one row with inconsistent value.
+ // Index count = table count, but data is different.
+ txn, err = store.Begin()
+ require.NoError(t, err)
+ _, err = indexOpr.Create(tk.Session().GetTableCtx(), txn, []types.Datum{types.NewIntDatum(100)}, getCommonHandle(2), nil)
+ require.NoError(t, err)
+ err = txn.Commit(context.Background())
+ require.NoError(t, err)
+ err = tk.ExecToErr("admin check table admin_test")
+ require.Error(t, err)
+ if !enabled {
+ require.True(t, consistency.ErrAdminCheckInconsistentWithColInfo.Equal(err))
+ require.EqualError(t, err, "[executor:8134]data inconsistency in table: admin_test, index: uidx_a, col: a, handle: \"2\", index-values:\"KindInt64 100\" != record-values:\"KindInt64 -1\", compare err:")
+ } else {
+ require.True(t, consistency.ErrAdminCheckInconsistent.Equal(err))
+ require.EqualError(t, err, "[admin:8223]data inconsistency in table: admin_test, index: uidx_a, handle: 2, index-values:\"handle: 2, values: [KindInt64 100]\" != record-values:\"handle: 2, values: [KindInt64 -1]\"")
+ }
+ }
+}
+
+func TestAdminCheckGlobalIndexDuringDDL(t *testing.T) {
+ store, dom := testkit.CreateMockStoreAndDomain(t)
+ originalHook := dom.DDL().GetHook()
+ tk := testkit.NewTestKit(t, store)
+
+ var schemaMap = make(map[model.SchemaState]struct{})
+
+ hook := &callback.TestDDLCallback{Do: dom}
+ onJobUpdatedExportedFunc := func(job *model.Job) {
+ schemaMap[job.SchemaState] = struct{}{}
+ _, err := tk.Exec("admin check table admin_test")
+ assert.NoError(t, err)
+ }
+ hook.OnJobUpdatedExported.Store(&onJobUpdatedExportedFunc)
+
+ // check table after delete some index key/value pairs.
+ ddl.MockDMLExecution = func() {
+ _, err := tk.Exec("admin check table admin_test")
+ assert.NoError(t, err)
+ }
+
+ batchSize := 32
+ tk.MustExec(fmt.Sprintf("set global tidb_ddl_reorg_batch_size = %d", batchSize))
+
+ var enableFastCheck = []bool{false, true}
+ for _, enabled := range enableFastCheck {
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists admin_test")
+
+ tk.MustExec("set tidb_enable_global_index = true")
+ tk.MustExec(fmt.Sprintf("set tidb_enable_fast_table_check = %v", enabled))
+
+ tk.MustExec("create table admin_test (a int, b int, c int, unique key uidx_a(a), primary key(c)) partition by hash(c) partitions 5")
+ tk.MustExec("insert admin_test values (-10, -20, 1), (-1, -10, 2), (1, 11, 3), (2, 12, 0), (5, 15, -1), (10, 20, -2), (20, 30, -3)")
+ for i := 1; i <= batchSize*2; i++ {
+ tk.MustExec(fmt.Sprintf("insert admin_test values (%d, %d, %d)", i*5+1, i, i*5+1))
+ }
+
+ dom.DDL().SetHook(hook)
+ require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/mockDMLExecution", "1*return(true)->return(false)"))
+ tk.MustExec("alter table admin_test truncate partition p1")
+ require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/mockDMLExecution"))
+ dom.DDL().SetHook(originalHook)
+
+ // Should have 3 different schema states, `none`, `deleteOnly`, `deleteReorg`
+ require.Len(t, schemaMap, 3)
+ for ss := range schemaMap {
+ delete(schemaMap, ss)
+ }
+ }
+}
diff --git a/pkg/executor/test/analyzetest/analyze_test.go b/pkg/executor/test/analyzetest/analyze_test.go
index 90fbcff71bc9f..d44ad171abddf 100644
--- a/pkg/executor/test/analyzetest/analyze_test.go
+++ b/pkg/executor/test/analyzetest/analyze_test.go
@@ -2166,6 +2166,7 @@ func TestShowAanalyzeStatusJobInfo(t *testing.T) {
tk.MustExec("delete from mysql.analyze_jobs")
}
checkJobInfo("analyze table columns b, c, d with 2 buckets, 2 topn, 1 samplerate")
+ tk.MustExec("set global tidb_persist_analyze_options = 1")
tk.MustExec("set global tidb_enable_column_tracking = 1")
tk.MustExec("select * from t where c > 1")
h := dom.StatsHandle()
@@ -2173,8 +2174,7 @@ func TestShowAanalyzeStatusJobInfo(t *testing.T) {
tk.MustExec("analyze table t predicate columns with 2 topn, 2 buckets")
checkJobInfo("analyze table columns b, c, d with 2 buckets, 2 topn, 1 samplerate")
tk.MustExec("analyze table t")
- checkJobInfo("analyze table all columns with 256 buckets, 500 topn, 1 samplerate")
- tk.MustExec("set global tidb_persist_analyze_options = 1")
+ checkJobInfo("analyze table columns b, c, d with 2 buckets, 2 topn, 1 samplerate")
tk.MustExec("analyze table t columns a with 1 topn, 3 buckets")
checkJobInfo("analyze table columns a, b, d with 3 buckets, 1 topn, 1 samplerate")
tk.MustExec("analyze table t")
diff --git a/pkg/executor/test/executor/executor_test.go b/pkg/executor/test/executor/executor_test.go
index 40a79b3a81013..0121ab9b13d71 100644
--- a/pkg/executor/test/executor/executor_test.go
+++ b/pkg/executor/test/executor/executor_test.go
@@ -1943,8 +1943,7 @@ func TestIsPointGet(t *testing.T) {
require.NoError(t, err)
p, _, err := planner.Optimize(context.TODO(), ctx, stmtNode, preprocessorReturn.InfoSchema)
require.NoError(t, err)
- ret, err := plannercore.IsPointGetWithPKOrUniqueKeyByAutoCommit(ctx.GetSessionVars(), p)
- require.NoError(t, err)
+ ret := plannercore.IsPointGetWithPKOrUniqueKeyByAutoCommit(ctx.GetSessionVars(), p)
require.Equal(t, result, ret)
}
}
@@ -1984,8 +1983,7 @@ func TestClusteredIndexIsPointGet(t *testing.T) {
require.NoError(t, err)
p, _, err := planner.Optimize(context.TODO(), ctx, stmtNode, preprocessorReturn.InfoSchema)
require.NoError(t, err)
- ret, err := plannercore.IsPointGetWithPKOrUniqueKeyByAutoCommit(ctx.GetSessionVars(), p)
- require.NoError(t, err)
+ ret := plannercore.IsPointGetWithPKOrUniqueKeyByAutoCommit(ctx.GetSessionVars(), p)
require.Equal(t, result, ret)
}
}
diff --git a/pkg/executor/test/issuetest/BUILD.bazel b/pkg/executor/test/issuetest/BUILD.bazel
index e2a29deedee2a..584acbb753be7 100644
--- a/pkg/executor/test/issuetest/BUILD.bazel
+++ b/pkg/executor/test/issuetest/BUILD.bazel
@@ -8,7 +8,7 @@ go_test(
"main_test.go",
],
flaky = True,
- shard_count = 17,
+ shard_count = 21,
deps = [
"//pkg/autoid_service",
"//pkg/config",
diff --git a/pkg/executor/test/issuetest/executor_issue_test.go b/pkg/executor/test/issuetest/executor_issue_test.go
index 2bebef216db06..9f5416c864740 100644
--- a/pkg/executor/test/issuetest/executor_issue_test.go
+++ b/pkg/executor/test/issuetest/executor_issue_test.go
@@ -183,6 +183,21 @@ func TestIssue30289(t *testing.T) {
require.EqualError(t, err, "issue30289 build return error")
}
+func TestIssue51998(t *testing.T) {
+ fpName := "github.com/pingcap/tidb/pkg/executor/join/issue51998"
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t(a int)")
+ require.NoError(t, failpoint.Enable(fpName, `return(true)`))
+ defer func() {
+ require.NoError(t, failpoint.Disable(fpName))
+ }()
+ err := tk.QueryToErr("select /*+ hash_join(t1) */ * from t t1 join t t2 on t1.a=t2.a")
+ require.EqualError(t, err, "issue51998 build return error")
+}
+
func TestIssue29498(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
@@ -635,3 +650,53 @@ func TestIssue51874(t *testing.T) {
tk.MustExec("insert into t2 values (10), (100)")
tk.MustQuery("select (select sum(a) over () from t2 limit 1) from t;").Check(testkit.Rows("10", "2"))
}
+
+func TestIssue51777(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.Session().GetSessionVars().AllowProjectionPushDown = true
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t0, t1")
+ tk.MustExec("create table t0 (c_k int)")
+ tk.MustExec("create table t1 (c_pv int)")
+ tk.MustExec("insert into t0 values(-2127559046),(-190905159),(-171305020),(-59638845),(98004414),(2111663670),(2137868682),(2137868682),(2142611610)")
+ tk.MustExec("insert into t1 values(-2123227448), (2131706870), (-2071508387), (2135465388), (2052805244), (-2066000113)")
+ tk.MustQuery("SELECT ( select (ref_4.c_pv <= ref_3.c_k) as c0 from t1 as ref_4 order by c0 asc limit 1) as p2 FROM t0 as ref_3 order by p2;").Check(testkit.Rows("0", "0", "0", "0", "0", "0", "1", "1", "1"))
+}
+
+func TestIssue52978(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t (a int)")
+ tk.MustExec("insert into t values (-1790816583),(2049821819), (-1366665321), (536581933), (-1613686445)")
+ tk.MustQuery("select min(truncate(cast(-26340 as double), ref_11.a)) as c3 from t as ref_11;").Check(testkit.Rows("-26340"))
+ tk.MustExec("drop table if exists t")
+}
+
+func TestIssue53221(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t (a varchar(20))")
+ tk.MustExec("insert into t values ('')")
+ tk.MustExec("insert into t values ('')")
+ err := tk.QueryToErr("select regexp_like('hello', t.a) from test.t")
+ require.ErrorContains(t, err, "Empty pattern is invalid")
+
+ err = tk.QueryToErr("select regexp_instr('hello', t.a) from test.t")
+ require.ErrorContains(t, err, "Empty pattern is invalid")
+
+ err = tk.QueryToErr("select regexp_substr('hello', t.a) from test.t")
+ require.ErrorContains(t, err, "Empty pattern is invalid")
+
+ err = tk.QueryToErr("select regexp_replace('hello', t.a, 'd') from test.t")
+ require.ErrorContains(t, err, "Empty pattern is invalid")
+
+ tk.MustExec("drop table if exists t")
+}
diff --git a/pkg/executor/test/passwordtest/BUILD.bazel b/pkg/executor/test/passwordtest/BUILD.bazel
index 2f7cfb57851ef..8b165cd8ce842 100644
--- a/pkg/executor/test/passwordtest/BUILD.bazel
+++ b/pkg/executor/test/passwordtest/BUILD.bazel
@@ -8,7 +8,7 @@ go_test(
"password_management_test.go",
],
flaky = True,
- shard_count = 8,
+ shard_count = 9,
deps = [
"//pkg/domain",
"//pkg/errno",
diff --git a/pkg/executor/test/passwordtest/password_management_test.go b/pkg/executor/test/passwordtest/password_management_test.go
index 2dc435fb7ed11..1a2e04dde5b45 100644
--- a/pkg/executor/test/passwordtest/password_management_test.go
+++ b/pkg/executor/test/passwordtest/password_management_test.go
@@ -858,6 +858,46 @@ func TestPasswordExpiredAndTacking(t *testing.T) {
require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: user, Hostname: host}, sha1Password("!@#HASHhs123"), nil, nil))
}
+// TestPasswordMySQLCompatibility is to test compatibility with the output of what MySQL outputs on SHOW CREATE USER.
+func TestPasswordMySQLCompatibility(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+
+ // MySQL 8.0.37
+ //
+ // This is using mysql_native_password as that's common with this version, however it is not the default.
+ //
+ // CREATE USER 'test80037'@'%' IDENTIFIED WITH 'mysql_native_password' BY 'secret';
+ // SHOW CREATE USER 'test80037'@'%';
+ tk.MustExec(
+ "CREATE USER `test80037`@`%` " +
+ "IDENTIFIED WITH 'mysql_native_password' AS '*14E65567ABDB5135D0CFD9A70B3032C179A49EE7' " +
+ "REQUIRE NONE " +
+ "PASSWORD EXPIRE DEFAULT " +
+ "ACCOUNT UNLOCK " +
+ "PASSWORD HISTORY DEFAULT " +
+ "PASSWORD REUSE INTERVAL DEFAULT " +
+ "PASSWORD REQUIRE CURRENT DEFAULT",
+ )
+
+ // MySQL 8.4.0
+ //
+ // NOT using mysql_native_password here as that is disabled by default in this version.
+ //
+ // CREATE USER 'test80400'@'%';
+ // SHOW CREATE USER 'test80400'@'%';
+ tk.MustExec(
+ "CREATE USER `test80400`@`%` " +
+ "IDENTIFIED WITH 'caching_sha2_password' " +
+ "REQUIRE NONE " +
+ "PASSWORD EXPIRE DEFAULT " +
+ "ACCOUNT UNLOCK " +
+ "PASSWORD HISTORY DEFAULT " +
+ "PASSWORD REUSE INTERVAL DEFAULT " +
+ "PASSWORD REQUIRE CURRENT DEFAULT",
+ )
+}
+
func loginFailedAncCheck(t *testing.T, store kv.Storage, user, host, password string, failedLoginCount int64, autoAccountLocked string) {
tk := testkit.NewTestKit(t, store)
require.Error(t, tk.Session().Auth(&auth.UserIdentity{Username: user, Hostname: host}, sha1Password(password), nil, nil))
diff --git a/pkg/executor/test/tiflashtest/BUILD.bazel b/pkg/executor/test/tiflashtest/BUILD.bazel
index b60b56f408a9e..6f1da72d25ce1 100644
--- a/pkg/executor/test/tiflashtest/BUILD.bazel
+++ b/pkg/executor/test/tiflashtest/BUILD.bazel
@@ -9,7 +9,7 @@ go_test(
],
flaky = True,
race = "on",
- shard_count = 42,
+ shard_count = 43,
deps = [
"//pkg/config",
"//pkg/domain",
diff --git a/pkg/executor/test/tiflashtest/tiflash_test.go b/pkg/executor/test/tiflashtest/tiflash_test.go
index a58847cfebbed..da0bc8df5fcf6 100644
--- a/pkg/executor/test/tiflashtest/tiflash_test.go
+++ b/pkg/executor/test/tiflashtest/tiflash_test.go
@@ -1988,3 +1988,47 @@ func TestIssue50358(t *testing.T) {
tk.MustQuery("select 8 from t join t1").Check(testkit.Rows("8", "8"))
}
}
+
+func TestMppAggShouldAlignFinalMode(t *testing.T) {
+ store := testkit.CreateMockStore(t, withMockTiFlash(1))
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t (" +
+ " d date," +
+ " v int," +
+ " primary key(d, v)" +
+ ") partition by range columns (d) (" +
+ " partition p1 values less than ('2023-07-02')," +
+ " partition p2 values less than ('2023-07-03')" +
+ ");")
+ tk.MustExec("alter table t set tiflash replica 1")
+ tb := external.GetTableByName(t, tk, "test", "t")
+ err := domain.GetDomain(tk.Session()).DDL().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true)
+ require.NoError(t, err)
+ tk.MustExec(`set tidb_partition_prune_mode='static';`)
+ err = failpoint.Enable("github.com/pingcap/tidb/pkg/expression/aggregation/show-agg-mode", "return(true)")
+ require.Nil(t, err)
+
+ tk.MustExec("set @@session.tidb_isolation_read_engines=\"tiflash\"")
+ tk.MustQuery("explain format='brief' select 1 from (" +
+ " select /*+ read_from_storage(tiflash[t]) */ sum(1)" +
+ " from t where d BETWEEN '2023-07-01' and '2023-07-03' group by d" +
+ ") total;").Check(testkit.Rows("Projection 400.00 root 1->Column#4",
+ "└─HashAgg 400.00 root group by:test.t.d, funcs:count(complete,1)->Column#8",
+ " └─PartitionUnion 400.00 root ",
+ " ├─Projection 200.00 root test.t.d",
+ " │ └─HashAgg 200.00 root group by:test.t.d, funcs:firstrow(partial2,test.t.d)->test.t.d, funcs:count(final,Column#12)->Column#9",
+ " │ └─TableReader 200.00 root MppVersion: 2, data:ExchangeSender",
+ " │ └─ExchangeSender 200.00 mpp[tiflash] ExchangeType: PassThrough",
+ " │ └─HashAgg 200.00 mpp[tiflash] group by:test.t.d, funcs:count(partial1,1)->Column#12",
+ " │ └─TableRangeScan 250.00 mpp[tiflash] table:t, partition:p1 range:[2023-07-01,2023-07-03], keep order:false, stats:pseudo",
+ " └─Projection 200.00 root test.t.d",
+ " └─HashAgg 200.00 root group by:test.t.d, funcs:firstrow(partial2,test.t.d)->test.t.d, funcs:count(final,Column#14)->Column#10",
+ " └─TableReader 200.00 root MppVersion: 2, data:ExchangeSender",
+ " └─ExchangeSender 200.00 mpp[tiflash] ExchangeType: PassThrough",
+ " └─HashAgg 200.00 mpp[tiflash] group by:test.t.d, funcs:count(partial1,1)->Column#14",
+ " └─TableRangeScan 250.00 mpp[tiflash] table:t, partition:p2 range:[2023-07-01,2023-07-03], keep order:false, stats:pseudo"))
+
+ err = failpoint.Disable("github.com/pingcap/tidb/pkg/expression/aggregation/show-agg-mode")
+ require.Nil(t, err)
+}
diff --git a/pkg/executor/union_scan.go b/pkg/executor/union_scan.go
index b2f5795aa5955..c1d6a7cc0c9ef 100644
--- a/pkg/executor/union_scan.go
+++ b/pkg/executor/union_scan.go
@@ -24,7 +24,7 @@ import (
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
- plannercore "github.com/pingcap/tidb/pkg/planner/core"
+ plannerutil "github.com/pingcap/tidb/pkg/planner/util"
"github.com/pingcap/tidb/pkg/sessionctx/stmtctx"
"github.com/pingcap/tidb/pkg/table"
"github.com/pingcap/tidb/pkg/tablecodec"
@@ -294,7 +294,7 @@ type compareExec struct {
usedIndex []int
desc bool
// handleCols is the handle's position of the below scan plan.
- handleCols plannercore.HandleCols
+ handleCols plannerutil.HandleCols
}
func (ce compareExec) compare(sctx *stmtctx.StatementContext, a, b []types.Datum) (ret int, err error) {
diff --git a/pkg/executor/window_test.go b/pkg/executor/window_test.go
index 57094f5fc8ec3..43482a35a2174 100644
--- a/pkg/executor/window_test.go
+++ b/pkg/executor/window_test.go
@@ -470,3 +470,12 @@ func TestIssue45964And46050(t *testing.T) {
testReturnColumnNullableAttribute(tk, "cume_dist()", false)
testReturnColumnNullableAttribute(tk, "percent_rank()", false)
}
+
+func TestVarSampAsAWindowFunction(t *testing.T) {
+ store := testkit.CreateMockStore(t)
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("create table t1 (c1 int)")
+ tk.MustExec("select var_samp(c1) from t1")
+ tk.MustExec("select c1, var_samp(c1) over (partition by c1) from t1")
+}
diff --git a/pkg/executor/write.go b/pkg/executor/write.go
index 9004a22f28861..cfa0d63016fee 100644
--- a/pkg/executor/write.go
+++ b/pkg/executor/write.go
@@ -246,7 +246,7 @@ func addUnchangedKeysForLockByRow(
count := 0
physicalID := t.Meta().ID
if pt, ok := t.(table.PartitionedTable); ok {
- p, err := pt.GetPartitionByRow(sctx.GetExprCtx(), row)
+ p, err := pt.GetPartitionByRow(sctx.GetExprCtx().GetEvalCtx(), row)
if err != nil {
return 0, err
}
@@ -330,7 +330,7 @@ func checkRowForExchangePartition(sctx table.MutateContext, row []types.Datum, t
return errors.Errorf("exchange partition process assert table partition failed")
}
err := p.CheckForExchangePartition(
- sctx.GetExprCtx(),
+ sctx.GetExprCtx().GetEvalCtx(),
pt.Meta().Partition,
row,
tbl.ExchangePartitionInfo.ExchangePartitionDefID,
diff --git a/pkg/expression/BUILD.bazel b/pkg/expression/BUILD.bazel
index 464415390e620..8b3b9efdd4e06 100644
--- a/pkg/expression/BUILD.bazel
+++ b/pkg/expression/BUILD.bazel
@@ -123,8 +123,8 @@ go_library(
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_tipb//go-tipb",
+ "@com_github_qri_io_jsonschema//:jsonschema",
"@com_github_tikv_client_go_v2//oracle",
- "@org_golang_x_tools//container/intsets",
"@org_uber_go_atomic//:atomic",
"@org_uber_go_zap//:zap",
],
@@ -200,6 +200,7 @@ go_test(
"//pkg/errctx",
"//pkg/errno",
"//pkg/expression/context",
+ "//pkg/expression/contextstatic",
"//pkg/kv",
"//pkg/parser",
"//pkg/parser/ast",
diff --git a/pkg/expression/aggregation/BUILD.bazel b/pkg/expression/aggregation/BUILD.bazel
index 502d43d527cbc..d4d35efd5700a 100644
--- a/pkg/expression/aggregation/BUILD.bazel
+++ b/pkg/expression/aggregation/BUILD.bazel
@@ -50,6 +50,7 @@ go_library(
"//pkg/util/mvmap",
"//pkg/util/size",
"@com_github_pingcap_errors//:errors",
+ "@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_tipb//go-tipb",
],
)
diff --git a/pkg/expression/aggregation/aggregation.go b/pkg/expression/aggregation/aggregation.go
index 71567ed560534..d02625b7865d7 100644
--- a/pkg/expression/aggregation/aggregation.go
+++ b/pkg/expression/aggregation/aggregation.go
@@ -79,11 +79,11 @@ func NewDistAggFunc(expr *tipb.Expr, fieldTps []*types.FieldType, ctx expression
case tipb.ExprType_Max:
aggF := newAggFunc(ast.AggFuncMax, args, false)
aggF.Mode = AggFunctionMode(*expr.AggFuncMode)
- return &maxMinFunction{aggFunction: aggF, isMax: true, ctor: collate.GetCollator(args[0].GetType().GetCollate())}, aggF.AggFuncDesc, nil
+ return &maxMinFunction{aggFunction: aggF, isMax: true, ctor: collate.GetCollator(args[0].GetType(ctx.GetEvalCtx()).GetCollate())}, aggF.AggFuncDesc, nil
case tipb.ExprType_Min:
aggF := newAggFunc(ast.AggFuncMin, args, false)
aggF.Mode = AggFunctionMode(*expr.AggFuncMode)
- return &maxMinFunction{aggFunction: aggF, ctor: collate.GetCollator(args[0].GetType().GetCollate())}, aggF.AggFuncDesc, nil
+ return &maxMinFunction{aggFunction: aggF, ctor: collate.GetCollator(args[0].GetType(ctx.GetEvalCtx()).GetCollate())}, aggF.AggFuncDesc, nil
case tipb.ExprType_First:
aggF := newAggFunc(ast.AggFuncFirstRow, args, false)
aggF.Mode = AggFunctionMode(*expr.AggFuncMode)
@@ -134,6 +134,23 @@ const (
DedupMode
)
+// ToString show the agg mode.
+func (a AggFunctionMode) ToString() string {
+ switch a {
+ case CompleteMode:
+ return "complete"
+ case FinalMode:
+ return "final"
+ case Partial1Mode:
+ return "partial1"
+ case Partial2Mode:
+ return "partial2"
+ case DedupMode:
+ return "deduplicate"
+ }
+ return ""
+}
+
type aggFunction struct {
*AggFuncDesc
}
@@ -215,7 +232,7 @@ func IsAllFirstRow(aggFuncs []*AggFuncDesc) bool {
}
// CheckAggPushDown checks whether an agg function can be pushed to storage.
-func CheckAggPushDown(aggFunc *AggFuncDesc, storeType kv.StoreType) bool {
+func CheckAggPushDown(ctx expression.EvalContext, aggFunc *AggFuncDesc, storeType kv.StoreType) bool {
if len(aggFunc.OrderByItems) > 0 && aggFunc.Name != ast.AggFuncGroupConcat {
return false
}
@@ -225,7 +242,7 @@ func CheckAggPushDown(aggFunc *AggFuncDesc, storeType kv.StoreType) bool {
ret := true
switch storeType {
case kv.TiFlash:
- ret = CheckAggPushFlash(aggFunc)
+ ret = CheckAggPushFlash(ctx, aggFunc)
case kv.TiKV:
// TiKV does not support group_concat now
ret = aggFunc.Name != ast.AggFuncGroupConcat
@@ -237,9 +254,9 @@ func CheckAggPushDown(aggFunc *AggFuncDesc, storeType kv.StoreType) bool {
}
// CheckAggPushFlash checks whether an agg function can be pushed to flash storage.
-func CheckAggPushFlash(aggFunc *AggFuncDesc) bool {
+func CheckAggPushFlash(ctx expression.EvalContext, aggFunc *AggFuncDesc) bool {
for _, arg := range aggFunc.Args {
- if arg.GetType().GetType() == mysql.TypeDuration {
+ if arg.GetType(ctx).GetType() == mysql.TypeDuration {
return false
}
}
@@ -248,7 +265,7 @@ func CheckAggPushFlash(aggFunc *AggFuncDesc) bool {
return true
case ast.AggFuncSum, ast.AggFuncAvg, ast.AggFuncGroupConcat:
// Now tiflash doesn't support CastJsonAsReal and CastJsonAsString.
- return aggFunc.Args[0].GetType().GetType() != mysql.TypeJSON
+ return aggFunc.Args[0].GetType(ctx).GetType() != mysql.TypeJSON
}
return false
}
diff --git a/pkg/expression/aggregation/base_func.go b/pkg/expression/aggregation/base_func.go
index 43876b8888d0c..d41250474f078 100644
--- a/pkg/expression/aggregation/base_func.go
+++ b/pkg/expression/aggregation/base_func.go
@@ -94,9 +94,9 @@ func (a *baseFuncDesc) TypeInfer(ctx expression.BuildContext) error {
case ast.AggFuncApproxPercentile:
return a.typeInfer4ApproxPercentile(ctx.GetEvalCtx())
case ast.AggFuncSum:
- a.typeInfer4Sum()
+ a.typeInfer4Sum(ctx.GetEvalCtx())
case ast.AggFuncAvg:
- a.typeInfer4Avg(ctx.GetEvalCtx().GetDivPrecisionIncrement())
+ a.typeInfer4Avg(ctx.GetEvalCtx())
case ast.AggFuncGroupConcat:
a.typeInfer4GroupConcat(ctx)
case ast.AggFuncMax, ast.AggFuncMin, ast.AggFuncFirstRow,
@@ -158,7 +158,7 @@ func (a *baseFuncDesc) typeInfer4ApproxPercentile(ctx expression.EvalContext) er
return fmt.Errorf("Percentage value %d is out of range [1, 100]", percent)
}
- switch a.Args[0].GetType().GetType() {
+ switch a.Args[0].GetType(ctx).GetType() {
case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
a.RetTp = types.NewFieldType(mysql.TypeLonglong)
case mysql.TypeDouble, mysql.TypeFloat:
@@ -166,14 +166,14 @@ func (a *baseFuncDesc) typeInfer4ApproxPercentile(ctx expression.EvalContext) er
case mysql.TypeNewDecimal:
a.RetTp = types.NewFieldType(mysql.TypeNewDecimal)
a.RetTp.SetFlen(mysql.MaxDecimalWidth)
- a.RetTp.SetDecimal(a.Args[0].GetType().GetDecimal())
+ a.RetTp.SetDecimal(a.Args[0].GetType(ctx).GetDecimal())
if a.RetTp.GetDecimal() < 0 || a.RetTp.GetDecimal() > mysql.MaxDecimalScale {
a.RetTp.SetDecimal(mysql.MaxDecimalScale)
}
case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeNewDate, mysql.TypeTimestamp:
- a.RetTp = a.Args[0].GetType().Clone()
+ a.RetTp = a.Args[0].GetType(ctx).Clone()
default:
- a.RetTp = a.Args[0].GetType().Clone()
+ a.RetTp = a.Args[0].GetType(ctx).Clone()
a.RetTp.DelFlag(mysql.NotNullFlag)
}
return nil
@@ -181,22 +181,22 @@ func (a *baseFuncDesc) typeInfer4ApproxPercentile(ctx expression.EvalContext) er
// typeInfer4Sum should return a "decimal", otherwise it returns a "double".
// Because child returns integer or decimal type.
-func (a *baseFuncDesc) typeInfer4Sum() {
- switch a.Args[0].GetType().GetType() {
+func (a *baseFuncDesc) typeInfer4Sum(ctx expression.EvalContext) {
+ switch a.Args[0].GetType(ctx).GetType() {
case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong, mysql.TypeYear:
a.RetTp = types.NewFieldType(mysql.TypeNewDecimal)
- a.RetTp.SetFlenUnderLimit(a.Args[0].GetType().GetFlen() + 21)
+ a.RetTp.SetFlenUnderLimit(a.Args[0].GetType(ctx).GetFlen() + 21)
a.RetTp.SetDecimal(0)
- if a.Args[0].GetType().GetFlen() < 0 {
+ if a.Args[0].GetType(ctx).GetFlen() < 0 {
a.RetTp.SetFlen(mysql.MaxDecimalWidth)
}
case mysql.TypeNewDecimal:
a.RetTp = types.NewFieldType(mysql.TypeNewDecimal)
- a.RetTp.UpdateFlenAndDecimalUnderLimit(a.Args[0].GetType(), 0, 22)
+ a.RetTp.UpdateFlenAndDecimalUnderLimit(a.Args[0].GetType(ctx), 0, 22)
case mysql.TypeDouble, mysql.TypeFloat:
a.RetTp = types.NewFieldType(mysql.TypeDouble)
a.RetTp.SetFlen(mysql.MaxRealWidth)
- a.RetTp.SetDecimal(a.Args[0].GetType().GetDecimal())
+ a.RetTp.SetDecimal(a.Args[0].GetType(ctx).GetDecimal())
default:
a.RetTp = types.NewFieldType(mysql.TypeDouble)
a.RetTp.SetFlen(mysql.MaxRealWidth)
@@ -220,20 +220,21 @@ func (a *baseFuncDesc) TypeInfer4FinalCount(finalCountRetType *types.FieldType)
// typeInfer4Avg should returns a "decimal", otherwise it returns a "double".
// Because child returns integer or decimal type.
-func (a *baseFuncDesc) typeInfer4Avg(divPrecIncre int) {
- switch a.Args[0].GetType().GetType() {
+func (a *baseFuncDesc) typeInfer4Avg(ctx expression.EvalContext) {
+ divPrecIncre := ctx.GetDivPrecisionIncrement()
+ switch a.Args[0].GetType(ctx).GetType() {
case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong:
a.RetTp = types.NewFieldType(mysql.TypeNewDecimal)
a.RetTp.SetDecimalUnderLimit(divPrecIncre)
- flen, _ := mysql.GetDefaultFieldLengthAndDecimal(a.Args[0].GetType().GetType())
+ flen, _ := mysql.GetDefaultFieldLengthAndDecimal(a.Args[0].GetType(ctx).GetType())
a.RetTp.SetFlenUnderLimit(flen + divPrecIncre)
case mysql.TypeYear, mysql.TypeNewDecimal:
a.RetTp = types.NewFieldType(mysql.TypeNewDecimal)
- a.RetTp.UpdateFlenAndDecimalUnderLimit(a.Args[0].GetType(), divPrecIncre, divPrecIncre)
+ a.RetTp.UpdateFlenAndDecimalUnderLimit(a.Args[0].GetType(ctx), divPrecIncre, divPrecIncre)
case mysql.TypeDouble, mysql.TypeFloat:
a.RetTp = types.NewFieldType(mysql.TypeDouble)
a.RetTp.SetFlen(mysql.MaxRealWidth)
- a.RetTp.SetDecimal(a.Args[0].GetType().GetDecimal())
+ a.RetTp.SetDecimal(a.Args[0].GetType(ctx).GetDecimal())
case mysql.TypeDate, mysql.TypeDuration, mysql.TypeDatetime, mysql.TypeTimestamp:
a.RetTp = types.NewFieldType(mysql.TypeDouble)
a.RetTp.SetFlen(mysql.MaxRealWidth)
@@ -256,7 +257,7 @@ func (a *baseFuncDesc) typeInfer4GroupConcat(ctx expression.BuildContext) {
a.RetTp.SetDecimal(0)
// TODO: a.Args[i] = expression.WrapWithCastAsString(ctx, a.Args[i])
for i := 0; i < len(a.Args)-1; i++ {
- if tp := a.Args[i].GetType(); tp.GetType() == mysql.TypeNewDecimal {
+ if tp := a.Args[i].GetType(ctx.GetEvalCtx()); tp.GetType() == mysql.TypeNewDecimal {
a.Args[i] = expression.BuildCastFunction(ctx, a.Args[i], tp)
}
}
@@ -264,7 +265,7 @@ func (a *baseFuncDesc) typeInfer4GroupConcat(ctx expression.BuildContext) {
func (a *baseFuncDesc) typeInfer4MaxMin(ctx expression.BuildContext) {
_, argIsScalaFunc := a.Args[0].(*expression.ScalarFunction)
- if argIsScalaFunc && a.Args[0].GetType().GetType() == mysql.TypeFloat {
+ if argIsScalaFunc && a.Args[0].GetType(ctx.GetEvalCtx()).GetType() == mysql.TypeFloat {
// For scalar function, the result of "float32" is set to the "float64"
// field in the "Datum". If we do not wrap a cast-as-double function on a.Args[0],
// error would happen when extracting the evaluation of a.Args[0] to a ProjectionExec.
@@ -274,10 +275,10 @@ func (a *baseFuncDesc) typeInfer4MaxMin(ctx expression.BuildContext) {
types.SetBinChsClnFlag(tp)
a.Args[0] = expression.BuildCastFunction(ctx, a.Args[0], tp)
}
- a.RetTp = a.Args[0].GetType()
+ a.RetTp = a.Args[0].GetType(ctx.GetEvalCtx())
if a.Name == ast.AggFuncMax || a.Name == ast.AggFuncMin ||
a.Name == ast.WindowFuncLead || a.Name == ast.WindowFuncLag {
- a.RetTp = a.Args[0].GetType().Clone()
+ a.RetTp = a.Args[0].GetType(ctx.GetEvalCtx()).Clone()
a.RetTp.DelFlag(mysql.NotNullFlag)
}
// issue #13027, #13961
@@ -430,7 +431,7 @@ func (a *baseFuncDesc) WrapCastForAggArgs(ctx expression.BuildContext) {
if i == 1 && (a.Name == ast.WindowFuncLead || a.Name == ast.WindowFuncLag || a.Name == ast.WindowFuncNthValue) {
continue
}
- if a.Args[i].GetType().GetType() == mysql.TypeNull {
+ if a.Args[i].GetType(ctx.GetEvalCtx()).GetType() == mysql.TypeNull {
continue
}
a.Args[i] = castFunc(ctx, a.Args[i])
diff --git a/pkg/expression/aggregation/descriptor.go b/pkg/expression/aggregation/descriptor.go
index f83dd6c6bd1ae..c56c95befc025 100644
--- a/pkg/expression/aggregation/descriptor.go
+++ b/pkg/expression/aggregation/descriptor.go
@@ -226,9 +226,9 @@ func (a *AggFuncDesc) GetAggFunc(ctx expression.AggFuncBuildContext) Aggregation
case ast.AggFuncGroupConcat:
return &concatFunction{aggFunction: aggFunc, maxLen: ctx.GetGroupConcatMaxLen()}
case ast.AggFuncMax:
- return &maxMinFunction{aggFunction: aggFunc, isMax: true, ctor: collate.GetCollator(a.Args[0].GetType().GetCollate())}
+ return &maxMinFunction{aggFunction: aggFunc, isMax: true, ctor: collate.GetCollator(a.Args[0].GetType(ctx.GetEvalCtx()).GetCollate())}
case ast.AggFuncMin:
- return &maxMinFunction{aggFunction: aggFunc, isMax: false, ctor: collate.GetCollator(a.Args[0].GetType().GetCollate())}
+ return &maxMinFunction{aggFunction: aggFunc, isMax: false, ctor: collate.GetCollator(a.Args[0].GetType(ctx.GetEvalCtx()).GetCollate())}
case ast.AggFuncFirstRow:
return &firstRowFunction{aggFunction: aggFunc}
case ast.AggFuncBitOr:
diff --git a/pkg/expression/aggregation/explain.go b/pkg/expression/aggregation/explain.go
index 29f88499e1bd1..d89fc08d88dc6 100644
--- a/pkg/expression/aggregation/explain.go
+++ b/pkg/expression/aggregation/explain.go
@@ -18,6 +18,7 @@ import (
"bytes"
"fmt"
+ "github.com/pingcap/failpoint"
"github.com/pingcap/tidb/pkg/expression"
"github.com/pingcap/tidb/pkg/parser/ast"
)
@@ -25,7 +26,18 @@ import (
// ExplainAggFunc generates explain information for a aggregation function.
func ExplainAggFunc(ctx expression.EvalContext, agg *AggFuncDesc, normalized bool) string {
var buffer bytes.Buffer
- fmt.Fprintf(&buffer, "%s(", agg.Name)
+ showMode := false
+ failpoint.Inject("show-agg-mode", func(v failpoint.Value) {
+ if v.(bool) {
+ showMode = true
+ }
+ })
+ if showMode {
+ fmt.Fprintf(&buffer, "%s(%s,", agg.Name, agg.Mode.ToString())
+ } else {
+ fmt.Fprintf(&buffer, "%s(", agg.Name)
+ }
+
if agg.HasDistinct {
buffer.WriteString("distinct ")
}
diff --git a/pkg/expression/aggregation/window_func.go b/pkg/expression/aggregation/window_func.go
index b0bc52415cd53..5efa3fa5fd77f 100644
--- a/pkg/expression/aggregation/window_func.go
+++ b/pkg/expression/aggregation/window_func.go
@@ -66,8 +66,8 @@ func NewWindowFuncDesc(ctx expression.BuildContext, name string, args []expressi
base.RetTp.SetFlag(mysql.NotNullFlag)
case ast.WindowFuncLead, ast.WindowFuncLag:
if len(args) == 3 &&
- ((args[0].GetType().GetFlag() & mysql.NotNullFlag) != 0) &&
- ((args[2].GetType().GetFlag() & mysql.NotNullFlag) != 0) {
+ ((args[0].GetType(ctx.GetEvalCtx()).GetFlag() & mysql.NotNullFlag) != 0) &&
+ ((args[2].GetType(ctx.GetEvalCtx()).GetFlag() & mysql.NotNullFlag) != 0) {
base.RetTp.SetFlag(mysql.NotNullFlag)
break
}
diff --git a/pkg/expression/bench_test.go b/pkg/expression/bench_test.go
index 18bf00acca102..4d46fbcafe665 100644
--- a/pkg/expression/bench_test.go
+++ b/pkg/expression/bench_test.go
@@ -149,7 +149,7 @@ func (h *benchHelper) init() {
h.outputTypes = make([]*types.FieldType, 0, len(h.exprs))
for i := 0; i < len(h.exprs); i++ {
- h.outputTypes = append(h.outputTypes, h.exprs[i].GetType())
+ h.outputTypes = append(h.outputTypes, h.exprs[i].GetType(h.ctx))
}
h.outputChunk = chunk.NewChunkWithCapacity(h.outputTypes, numRows)
@@ -1293,7 +1293,7 @@ func genVecExprBenchCase(ctx BuildContext, funcName string, testCase vecExprBenc
panic(err)
}
- output = chunk.New([]*types.FieldType{eType2FieldType(expr.GetType().EvalType())}, testCase.chunkSize, testCase.chunkSize)
+ output = chunk.New([]*types.FieldType{eType2FieldType(expr.GetType(ctx.GetEvalCtx()).EvalType())}, testCase.chunkSize, testCase.chunkSize)
return expr, fts, input, output
}
@@ -1313,7 +1313,7 @@ func testVectorizedEvalOneVec(t *testing.T, vecExprCases vecExprBenchCases) {
require.NoErrorf(t, evalOneColumn(ctx, expr, it, output2, 0), "func: %v, case: %+v", funcName, testCase)
c1, c2 := output.Column(0), output2.Column(0)
- switch expr.GetType().EvalType() {
+ switch expr.GetType(ctx).EvalType() {
case types.ETInt:
for i := 0; i < input.NumRows(); i++ {
require.Equal(t, c1.IsNull(i), c2.IsNull(i), commentf(i))
diff --git a/pkg/expression/builtin.go b/pkg/expression/builtin.go
index d4f1e2c71f169..b38c379677ddf 100644
--- a/pkg/expression/builtin.go
+++ b/pkg/expression/builtin.go
@@ -89,7 +89,7 @@ func (b *baseBuiltinFunc) collator() collate.Collator {
return b.ctor
}
-func adjustNullFlagForReturnType(funcName string, args []Expression, bf baseBuiltinFunc) {
+func adjustNullFlagForReturnType(ctx EvalContext, funcName string, args []Expression, bf baseBuiltinFunc) {
if functionSetForReturnTypeAlwaysNotNull.Exist(funcName) {
bf.tp.AddFlag(mysql.NotNullFlag)
} else if functionSetForReturnTypeAlwaysNullable.Exist(funcName) {
@@ -97,7 +97,7 @@ func adjustNullFlagForReturnType(funcName string, args []Expression, bf baseBuil
} else if functionSetForReturnTypeNotNullOnNotNull.Exist(funcName) {
returnNullable := false
for _, arg := range args {
- if !mysql.HasNotNullFlag(arg.GetType().GetFlag()) {
+ if !mysql.HasNotNullFlag(arg.GetType(ctx).GetFlag()) {
returnNullable = true
break
}
@@ -131,7 +131,7 @@ func newBaseBuiltinFunc(ctx BuildContext, funcName string, args []Expression, tp
bf.setCollator(collate.GetCollator(ec.Collation))
bf.SetCoercibility(ec.Coer)
bf.SetRepertoire(ec.Repe)
- adjustNullFlagForReturnType(funcName, args, bf)
+ adjustNullFlagForReturnType(ctx.GetEvalCtx(), funcName, args, bf)
return bf, nil
}
@@ -193,7 +193,7 @@ func newBaseBuiltinFuncWithTp(ctx BuildContext, funcName string, args []Expressi
args[i] = WrapWithCastAsDecimal(ctx, args[i])
case types.ETString:
args[i] = WrapWithCastAsString(ctx, args[i])
- args[i] = HandleBinaryLiteral(ctx, args[i], ec, funcName)
+ args[i] = HandleBinaryLiteral(ctx, args[i], ec, funcName, false)
case types.ETDatetime:
args[i] = WrapWithCastAsTime(ctx, args[i], types.NewFieldType(mysql.TypeDatetime))
case types.ETTimestamp:
@@ -218,7 +218,7 @@ func newBaseBuiltinFuncWithTp(ctx BuildContext, funcName string, args []Expressi
bf.SetCoercibility(ec.Coer)
bf.SetRepertoire(ec.Repe)
// note this function must be called after wrap cast function to the args
- adjustNullFlagForReturnType(funcName, args, bf)
+ adjustNullFlagForReturnType(ctx.GetEvalCtx(), funcName, args, bf)
return bf, nil
}
@@ -253,14 +253,14 @@ func newBaseBuiltinFuncWithFieldTypes(ctx BuildContext, funcName string, args []
args[i] = WrapWithCastAsReal(ctx, args[i])
case types.ETString:
args[i] = WrapWithCastAsString(ctx, args[i])
- args[i] = HandleBinaryLiteral(ctx, args[i], ec, funcName)
+ args[i] = HandleBinaryLiteral(ctx, args[i], ec, funcName, false)
case types.ETJson:
args[i] = WrapWithCastAsJSON(ctx, args[i])
// https://github.com/pingcap/tidb/issues/44196
// For decimal/datetime/timestamp/duration types, it is necessary to ensure that decimal are consistent with the output type,
// so adding a cast function here.
case types.ETDecimal, types.ETDatetime, types.ETTimestamp, types.ETDuration:
- if !args[i].GetType().Equal(argTps[i]) {
+ if !args[i].GetType(ctx.GetEvalCtx()).Equal(argTps[i]) {
args[i] = BuildCastFunction(ctx, args[i], argTps[i])
}
}
@@ -279,7 +279,7 @@ func newBaseBuiltinFuncWithFieldTypes(ctx BuildContext, funcName string, args []
bf.SetCoercibility(ec.Coer)
bf.SetRepertoire(ec.Repe)
// note this function must be called after wrap cast function to the args
- adjustNullFlagForReturnType(funcName, args, bf)
+ adjustNullFlagForReturnType(ctx.GetEvalCtx(), funcName, args, bf)
return bf, nil
}
@@ -902,6 +902,7 @@ var funcs = map[string]functionClass{
ast.JSONMergePreserve: &jsonMergePreserveFunctionClass{baseFunctionClass{ast.JSONMergePreserve, 2, -1}},
ast.JSONPretty: &jsonPrettyFunctionClass{baseFunctionClass{ast.JSONPretty, 1, 1}},
ast.JSONQuote: &jsonQuoteFunctionClass{baseFunctionClass{ast.JSONQuote, 1, 1}},
+ ast.JSONSchemaValid: &jsonSchemaValidFunctionClass{baseFunctionClass{ast.JSONSchemaValid, 2, 2}},
ast.JSONSearch: &jsonSearchFunctionClass{baseFunctionClass{ast.JSONSearch, 3, -1}},
ast.JSONStorageFree: &jsonStorageFreeFunctionClass{baseFunctionClass{ast.JSONStorageFree, 1, 1}},
ast.JSONStorageSize: &jsonStorageSizeFunctionClass{baseFunctionClass{ast.JSONStorageSize, 1, 1}},
diff --git a/pkg/expression/builtin_arithmetic.go b/pkg/expression/builtin_arithmetic.go
index 71415c1638ffc..3e9e14af32fb4 100644
--- a/pkg/expression/builtin_arithmetic.go
+++ b/pkg/expression/builtin_arithmetic.go
@@ -59,8 +59,8 @@ var (
)
// isConstantBinaryLiteral return true if expr is constant binary literal
-func isConstantBinaryLiteral(expr Expression) bool {
- if types.IsBinaryStr(expr.GetType()) {
+func isConstantBinaryLiteral(ctx EvalContext, expr Expression) bool {
+ if types.IsBinaryStr(expr.GetType(ctx)) {
if v, ok := expr.(*Constant); ok {
if k := v.Value.Kind(); k == types.KindBinaryLiteral {
return true
@@ -72,8 +72,8 @@ func isConstantBinaryLiteral(expr Expression) bool {
// numericContextResultType returns types.EvalType for numeric function's parameters.
// the returned types.EvalType should be one of: types.ETInt, types.ETDecimal, types.ETReal
-func numericContextResultType(expr Expression) types.EvalType {
- ft := expr.GetType()
+func numericContextResultType(ctx EvalContext, expr Expression) types.EvalType {
+ ft := expr.GetType(ctx)
if types.IsTypeTemporal(ft.GetType()) {
if ft.GetDecimal() > 0 {
return types.ETDecimal
@@ -83,7 +83,7 @@ func numericContextResultType(expr Expression) types.EvalType {
// to solve https://github.com/pingcap/tidb/issues/27698
// if expression is constant binary literal, like `0x1234`, `0b00011`, cast to integer
// for other binary str column related expression, like varbinary, cast to float/double.
- if isConstantBinaryLiteral(expr) || ft.GetType() == mysql.TypeBit {
+ if isConstantBinaryLiteral(ctx, expr) || ft.GetType() == mysql.TypeBit {
return types.ETInt
}
evalTp4Ft := types.ETReal
@@ -98,8 +98,8 @@ func numericContextResultType(expr Expression) types.EvalType {
// setFlenDecimal4RealOrDecimal is called to set proper `flen` and `decimal` of return
// type according to the two input parameter's types.
-func setFlenDecimal4RealOrDecimal(retTp *types.FieldType, arg0, arg1 Expression, isReal, isMultiply bool) {
- a, b := arg0.GetType(), arg1.GetType()
+func setFlenDecimal4RealOrDecimal(ctx EvalContext, retTp *types.FieldType, arg0, arg1 Expression, isReal, isMultiply bool) {
+ a, b := arg0.GetType(ctx), arg1.GetType(ctx)
if a.GetDecimal() != types.UnspecifiedLength && b.GetDecimal() != types.UnspecifiedLength {
retTp.SetDecimalUnderLimit(a.GetDecimal() + b.GetDecimal())
if !isMultiply {
@@ -166,13 +166,13 @@ func (c *arithmeticPlusFunctionClass) getFunction(ctx BuildContext, args []Expre
if err := c.verifyArgs(args); err != nil {
return nil, err
}
- lhsEvalTp, rhsEvalTp := numericContextResultType(args[0]), numericContextResultType(args[1])
+ lhsEvalTp, rhsEvalTp := numericContextResultType(ctx.GetEvalCtx(), args[0]), numericContextResultType(ctx.GetEvalCtx(), args[1])
if lhsEvalTp == types.ETReal || rhsEvalTp == types.ETReal {
bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETReal, types.ETReal, types.ETReal)
if err != nil {
return nil, err
}
- setFlenDecimal4RealOrDecimal(bf.tp, args[0], args[1], true, false)
+ setFlenDecimal4RealOrDecimal(ctx.GetEvalCtx(), bf.tp, args[0], args[1], true, false)
sig := &builtinArithmeticPlusRealSig{bf}
sig.setPbCode(tipb.ScalarFuncSig_PlusReal)
return sig, nil
@@ -181,7 +181,7 @@ func (c *arithmeticPlusFunctionClass) getFunction(ctx BuildContext, args []Expre
if err != nil {
return nil, err
}
- setFlenDecimal4RealOrDecimal(bf.tp, args[0], args[1], false, false)
+ setFlenDecimal4RealOrDecimal(ctx.GetEvalCtx(), bf.tp, args[0], args[1], false, false)
sig := &builtinArithmeticPlusDecimalSig{bf}
sig.setPbCode(tipb.ScalarFuncSig_PlusDecimal)
return sig, nil
@@ -190,7 +190,7 @@ func (c *arithmeticPlusFunctionClass) getFunction(ctx BuildContext, args []Expre
if err != nil {
return nil, err
}
- if mysql.HasUnsignedFlag(args[0].GetType().GetFlag()) || mysql.HasUnsignedFlag(args[1].GetType().GetFlag()) {
+ if mysql.HasUnsignedFlag(args[0].GetType(ctx.GetEvalCtx()).GetFlag()) || mysql.HasUnsignedFlag(args[1].GetType(ctx.GetEvalCtx()).GetFlag()) {
bf.tp.AddFlag(mysql.UnsignedFlag)
}
sig := &builtinArithmeticPlusIntSig{bf}
@@ -219,8 +219,8 @@ func (s *builtinArithmeticPlusIntSig) evalInt(ctx EvalContext, row chunk.Row) (v
return 0, isNull, err
}
- isLHSUnsigned := mysql.HasUnsignedFlag(s.args[0].GetType().GetFlag())
- isRHSUnsigned := mysql.HasUnsignedFlag(s.args[1].GetType().GetFlag())
+ isLHSUnsigned := mysql.HasUnsignedFlag(s.args[0].GetType(ctx).GetFlag())
+ isRHSUnsigned := mysql.HasUnsignedFlag(s.args[1].GetType(ctx).GetFlag())
switch {
case isLHSUnsigned && isRHSUnsigned:
@@ -316,13 +316,13 @@ func (c *arithmeticMinusFunctionClass) getFunction(ctx BuildContext, args []Expr
if err := c.verifyArgs(args); err != nil {
return nil, err
}
- lhsEvalTp, rhsEvalTp := numericContextResultType(args[0]), numericContextResultType(args[1])
+ lhsEvalTp, rhsEvalTp := numericContextResultType(ctx.GetEvalCtx(), args[0]), numericContextResultType(ctx.GetEvalCtx(), args[1])
if lhsEvalTp == types.ETReal || rhsEvalTp == types.ETReal {
bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETReal, types.ETReal, types.ETReal)
if err != nil {
return nil, err
}
- setFlenDecimal4RealOrDecimal(bf.tp, args[0], args[1], true, false)
+ setFlenDecimal4RealOrDecimal(ctx.GetEvalCtx(), bf.tp, args[0], args[1], true, false)
sig := &builtinArithmeticMinusRealSig{bf}
sig.setPbCode(tipb.ScalarFuncSig_MinusReal)
return sig, nil
@@ -331,7 +331,7 @@ func (c *arithmeticMinusFunctionClass) getFunction(ctx BuildContext, args []Expr
if err != nil {
return nil, err
}
- setFlenDecimal4RealOrDecimal(bf.tp, args[0], args[1], false, false)
+ setFlenDecimal4RealOrDecimal(ctx.GetEvalCtx(), bf.tp, args[0], args[1], false, false)
sig := &builtinArithmeticMinusDecimalSig{bf}
sig.setPbCode(tipb.ScalarFuncSig_MinusDecimal)
return sig, nil
@@ -340,7 +340,7 @@ func (c *arithmeticMinusFunctionClass) getFunction(ctx BuildContext, args []Expr
if err != nil {
return nil, err
}
- if (mysql.HasUnsignedFlag(args[0].GetType().GetFlag()) || mysql.HasUnsignedFlag(args[1].GetType().GetFlag())) && !ctx.GetEvalCtx().SQLMode().HasNoUnsignedSubtractionMode() {
+ if (mysql.HasUnsignedFlag(args[0].GetType(ctx.GetEvalCtx()).GetFlag()) || mysql.HasUnsignedFlag(args[1].GetType(ctx.GetEvalCtx()).GetFlag())) && !ctx.GetEvalCtx().SQLMode().HasNoUnsignedSubtractionMode() {
bf.tp.AddFlag(mysql.UnsignedFlag)
}
sig := &builtinArithmeticMinusIntSig{baseBuiltinFunc: bf}
@@ -424,8 +424,8 @@ func (s *builtinArithmeticMinusIntSig) evalInt(ctx EvalContext, row chunk.Row) (
return 0, isNull, err
}
forceToSigned := sqlMode(ctx).HasNoUnsignedSubtractionMode()
- isLHSUnsigned := mysql.HasUnsignedFlag(s.args[0].GetType().GetFlag())
- isRHSUnsigned := mysql.HasUnsignedFlag(s.args[1].GetType().GetFlag())
+ isLHSUnsigned := mysql.HasUnsignedFlag(s.args[0].GetType(ctx).GetFlag())
+ isRHSUnsigned := mysql.HasUnsignedFlag(s.args[1].GetType(ctx).GetFlag())
errType := "BIGINT UNSIGNED"
signed := forceToSigned || (!isLHSUnsigned && !isRHSUnsigned)
@@ -499,14 +499,14 @@ func (c *arithmeticMultiplyFunctionClass) getFunction(ctx BuildContext, args []E
if err := c.verifyArgs(args); err != nil {
return nil, err
}
- lhsTp, rhsTp := args[0].GetType(), args[1].GetType()
- lhsEvalTp, rhsEvalTp := numericContextResultType(args[0]), numericContextResultType(args[1])
+ lhsTp, rhsTp := args[0].GetType(ctx.GetEvalCtx()), args[1].GetType(ctx.GetEvalCtx())
+ lhsEvalTp, rhsEvalTp := numericContextResultType(ctx.GetEvalCtx(), args[0]), numericContextResultType(ctx.GetEvalCtx(), args[1])
if lhsEvalTp == types.ETReal || rhsEvalTp == types.ETReal {
bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETReal, types.ETReal, types.ETReal)
if err != nil {
return nil, err
}
- setFlenDecimal4RealOrDecimal(bf.tp, args[0], args[1], true, true)
+ setFlenDecimal4RealOrDecimal(ctx.GetEvalCtx(), bf.tp, args[0], args[1], true, true)
sig := &builtinArithmeticMultiplyRealSig{bf}
sig.setPbCode(tipb.ScalarFuncSig_MultiplyReal)
return sig, nil
@@ -515,7 +515,7 @@ func (c *arithmeticMultiplyFunctionClass) getFunction(ctx BuildContext, args []E
if err != nil {
return nil, err
}
- setFlenDecimal4RealOrDecimal(bf.tp, args[0], args[1], false, true)
+ setFlenDecimal4RealOrDecimal(ctx.GetEvalCtx(), bf.tp, args[0], args[1], false, true)
sig := &builtinArithmeticMultiplyDecimalSig{bf}
sig.setPbCode(tipb.ScalarFuncSig_MultiplyDecimal)
return sig, nil
@@ -645,8 +645,8 @@ func (c *arithmeticDivideFunctionClass) getFunction(ctx BuildContext, args []Exp
if err := c.verifyArgs(args); err != nil {
return nil, err
}
- lhsTp, rhsTp := args[0].GetType(), args[1].GetType()
- lhsEvalTp, rhsEvalTp := numericContextResultType(args[0]), numericContextResultType(args[1])
+ lhsTp, rhsTp := args[0].GetType(ctx.GetEvalCtx()), args[1].GetType(ctx.GetEvalCtx())
+ lhsEvalTp, rhsEvalTp := numericContextResultType(ctx.GetEvalCtx(), args[0]), numericContextResultType(ctx.GetEvalCtx(), args[1])
if lhsEvalTp == types.ETReal || rhsEvalTp == types.ETReal {
bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETReal, types.ETReal, types.ETReal)
if err != nil {
@@ -739,8 +739,8 @@ func (c *arithmeticIntDivideFunctionClass) getFunction(ctx BuildContext, args []
if err := c.verifyArgs(args); err != nil {
return nil, err
}
- lhsTp, rhsTp := args[0].GetType(), args[1].GetType()
- lhsEvalTp, rhsEvalTp := numericContextResultType(args[0]), numericContextResultType(args[1])
+ lhsTp, rhsTp := args[0].GetType(ctx.GetEvalCtx()), args[1].GetType(ctx.GetEvalCtx())
+ lhsEvalTp, rhsEvalTp := numericContextResultType(ctx.GetEvalCtx(), args[0]), numericContextResultType(ctx.GetEvalCtx(), args[1])
if lhsEvalTp == types.ETInt && rhsEvalTp == types.ETInt {
bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETInt, types.ETInt, types.ETInt)
if err != nil {
@@ -799,8 +799,8 @@ func (s *builtinArithmeticIntDivideIntSig) evalInt(ctx EvalContext, row chunk.Ro
ret int64
val uint64
)
- isLHSUnsigned := mysql.HasUnsignedFlag(s.args[0].GetType().GetFlag())
- isRHSUnsigned := mysql.HasUnsignedFlag(s.args[1].GetType().GetFlag())
+ isLHSUnsigned := mysql.HasUnsignedFlag(s.args[0].GetType(ctx).GetFlag())
+ isRHSUnsigned := mysql.HasUnsignedFlag(s.args[1].GetType(ctx).GetFlag())
switch {
case isLHSUnsigned && isRHSUnsigned:
@@ -844,8 +844,8 @@ func (s *builtinArithmeticIntDivideDecimalSig) evalInt(ctx EvalContext, row chun
return 0, true, err
}
- isLHSUnsigned := mysql.HasUnsignedFlag(s.args[0].GetType().GetFlag())
- isRHSUnsigned := mysql.HasUnsignedFlag(s.args[1].GetType().GetFlag())
+ isLHSUnsigned := mysql.HasUnsignedFlag(s.args[0].GetType(ctx).GetFlag())
+ isRHSUnsigned := mysql.HasUnsignedFlag(s.args[1].GetType(ctx).GetFlag())
if isLHSUnsigned || isRHSUnsigned {
val, err := c.ToUint()
@@ -898,8 +898,8 @@ func (c *arithmeticModFunctionClass) getFunction(ctx BuildContext, args []Expres
if err := c.verifyArgs(args); err != nil {
return nil, err
}
- lhsTp, rhsTp := args[0].GetType(), args[1].GetType()
- lhsEvalTp, rhsEvalTp := numericContextResultType(args[0]), numericContextResultType(args[1])
+ lhsTp, rhsTp := args[0].GetType(ctx.GetEvalCtx()), args[1].GetType(ctx.GetEvalCtx())
+ lhsEvalTp, rhsEvalTp := numericContextResultType(ctx.GetEvalCtx(), args[0]), numericContextResultType(ctx.GetEvalCtx(), args[1])
if lhsEvalTp == types.ETReal || rhsEvalTp == types.ETReal {
bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETReal, types.ETReal, types.ETReal)
if err != nil {
@@ -932,8 +932,8 @@ func (c *arithmeticModFunctionClass) getFunction(ctx BuildContext, args []Expres
if mysql.HasUnsignedFlag(lhsTp.GetFlag()) {
bf.tp.AddFlag(mysql.UnsignedFlag)
}
- isLHSUnsigned := mysql.HasUnsignedFlag(args[0].GetType().GetFlag())
- isRHSUnsigned := mysql.HasUnsignedFlag(args[1].GetType().GetFlag())
+ isLHSUnsigned := mysql.HasUnsignedFlag(args[0].GetType(ctx.GetEvalCtx()).GetFlag())
+ isRHSUnsigned := mysql.HasUnsignedFlag(args[1].GetType(ctx.GetEvalCtx()).GetFlag())
switch {
case isLHSUnsigned && isRHSUnsigned:
sig := &builtinArithmeticModIntUnsignedUnsignedSig{bf}
diff --git a/pkg/expression/builtin_arithmetic_test.go b/pkg/expression/builtin_arithmetic_test.go
index f19d33f14d02c..a8c541dc912e0 100644
--- a/pkg/expression/builtin_arithmetic_test.go
+++ b/pkg/expression/builtin_arithmetic_test.go
@@ -19,6 +19,7 @@ import (
"testing"
"time"
+ "github.com/pingcap/tidb/pkg/expression/contextstatic"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/testkit/testutil"
@@ -29,6 +30,7 @@ import (
)
func TestSetFlenDecimal4RealOrDecimal(t *testing.T) {
+ ctx := contextstatic.NewStaticEvalContext()
ret := &types.FieldType{}
a := &types.FieldType{}
a.SetDecimal(1)
@@ -37,25 +39,25 @@ func TestSetFlenDecimal4RealOrDecimal(t *testing.T) {
b := &types.FieldType{}
b.SetDecimal(0)
b.SetFlag(2)
- setFlenDecimal4RealOrDecimal(ret, &Constant{RetType: a}, &Constant{RetType: b}, true, false)
+ setFlenDecimal4RealOrDecimal(ctx, ret, &Constant{RetType: a}, &Constant{RetType: b}, true, false)
require.Equal(t, 1, ret.GetDecimal())
require.Equal(t, 4, ret.GetFlen())
b.SetFlen(65)
- setFlenDecimal4RealOrDecimal(ret, &Constant{RetType: a}, &Constant{RetType: b}, true, false)
+ setFlenDecimal4RealOrDecimal(ctx, ret, &Constant{RetType: a}, &Constant{RetType: b}, true, false)
require.Equal(t, 1, ret.GetDecimal())
require.Equal(t, mysql.MaxRealWidth, ret.GetFlen())
- setFlenDecimal4RealOrDecimal(ret, &Constant{RetType: a}, &Constant{RetType: b}, false, false)
+ setFlenDecimal4RealOrDecimal(ctx, ret, &Constant{RetType: a}, &Constant{RetType: b}, false, false)
require.Equal(t, 1, ret.GetDecimal())
require.Equal(t, mysql.MaxDecimalWidth, ret.GetFlen())
b.SetFlen(types.UnspecifiedLength)
- setFlenDecimal4RealOrDecimal(ret, &Constant{RetType: a}, &Constant{RetType: b}, true, false)
+ setFlenDecimal4RealOrDecimal(ctx, ret, &Constant{RetType: a}, &Constant{RetType: b}, true, false)
require.Equal(t, 1, ret.GetDecimal())
require.Equal(t, types.UnspecifiedLength, ret.GetFlen())
b.SetDecimal(types.UnspecifiedLength)
- setFlenDecimal4RealOrDecimal(ret, &Constant{RetType: a}, &Constant{RetType: b}, true, false)
+ setFlenDecimal4RealOrDecimal(ctx, ret, &Constant{RetType: a}, &Constant{RetType: b}, true, false)
require.Equal(t, types.UnspecifiedLength, ret.GetDecimal())
require.Equal(t, types.UnspecifiedLength, ret.GetFlen())
@@ -68,25 +70,25 @@ func TestSetFlenDecimal4RealOrDecimal(t *testing.T) {
b.SetDecimal(0)
b.SetFlen(2)
- setFlenDecimal4RealOrDecimal(ret, &Constant{RetType: a}, &Constant{RetType: b}, true, true)
+ setFlenDecimal4RealOrDecimal(ctx, ret, &Constant{RetType: a}, &Constant{RetType: b}, true, true)
require.Equal(t, 1, ret.GetDecimal())
require.Equal(t, 5, ret.GetFlen())
b.SetFlen(65)
- setFlenDecimal4RealOrDecimal(ret, &Constant{RetType: a}, &Constant{RetType: b}, true, true)
+ setFlenDecimal4RealOrDecimal(ctx, ret, &Constant{RetType: a}, &Constant{RetType: b}, true, true)
require.Equal(t, 1, ret.GetDecimal())
require.Equal(t, mysql.MaxRealWidth, ret.GetFlen())
- setFlenDecimal4RealOrDecimal(ret, &Constant{RetType: a}, &Constant{RetType: b}, false, true)
+ setFlenDecimal4RealOrDecimal(ctx, ret, &Constant{RetType: a}, &Constant{RetType: b}, false, true)
require.Equal(t, 1, ret.GetDecimal())
require.Equal(t, mysql.MaxDecimalWidth, ret.GetFlen())
b.SetFlen(types.UnspecifiedLength)
- setFlenDecimal4RealOrDecimal(ret, &Constant{RetType: a}, &Constant{RetType: b}, true, true)
+ setFlenDecimal4RealOrDecimal(ctx, ret, &Constant{RetType: a}, &Constant{RetType: b}, true, true)
require.Equal(t, 1, ret.GetDecimal())
require.Equal(t, types.UnspecifiedLength, ret.GetFlen())
b.SetDecimal(types.UnspecifiedLength)
- setFlenDecimal4RealOrDecimal(ret, &Constant{RetType: a}, &Constant{RetType: b}, true, true)
+ setFlenDecimal4RealOrDecimal(ctx, ret, &Constant{RetType: a}, &Constant{RetType: b}, true, true)
require.Equal(t, types.UnspecifiedLength, ret.GetDecimal())
require.Equal(t, types.UnspecifiedLength, ret.GetFlen())
}
diff --git a/pkg/expression/builtin_arithmetic_vec.go b/pkg/expression/builtin_arithmetic_vec.go
index f4eaa6d11af6d..4942112b8850e 100644
--- a/pkg/expression/builtin_arithmetic_vec.go
+++ b/pkg/expression/builtin_arithmetic_vec.go
@@ -384,8 +384,8 @@ func (b *builtinArithmeticMinusIntSig) vecEvalInt(ctx EvalContext, input *chunk.
resulti64s := result.Int64s()
forceToSigned := sqlMode(ctx).HasNoUnsignedSubtractionMode()
- isLHSUnsigned := mysql.HasUnsignedFlag(b.args[0].GetType().GetFlag())
- isRHSUnsigned := mysql.HasUnsignedFlag(b.args[1].GetType().GetFlag())
+ isLHSUnsigned := mysql.HasUnsignedFlag(b.args[0].GetType(ctx).GetFlag())
+ isRHSUnsigned := mysql.HasUnsignedFlag(b.args[1].GetType(ctx).GetFlag())
errType := "BIGINT UNSIGNED"
signed := forceToSigned || (!isLHSUnsigned && !isRHSUnsigned)
@@ -583,8 +583,8 @@ func (b *builtinArithmeticIntDivideDecimalSig) vecEvalInt(ctx EvalContext, input
num[i] = buf[i].Decimals()
}
- isLHSUnsigned := mysql.HasUnsignedFlag(b.args[0].GetType().GetFlag())
- isRHSUnsigned := mysql.HasUnsignedFlag(b.args[1].GetType().GetFlag())
+ isLHSUnsigned := mysql.HasUnsignedFlag(b.args[0].GetType(ctx).GetFlag())
+ isRHSUnsigned := mysql.HasUnsignedFlag(b.args[1].GetType(ctx).GetFlag())
isUnsigned := isLHSUnsigned || isRHSUnsigned
result.ResizeInt64(n, false)
@@ -745,8 +745,8 @@ func (b *builtinArithmeticIntDivideIntSig) vecEvalInt(ctx EvalContext, input *ch
rhsI64s := rhsBuf.Int64s()
resultI64s := result.Int64s()
- isLHSUnsigned := mysql.HasUnsignedFlag(b.args[0].GetType().GetFlag())
- isRHSUnsigned := mysql.HasUnsignedFlag(b.args[1].GetType().GetFlag())
+ isLHSUnsigned := mysql.HasUnsignedFlag(b.args[0].GetType(ctx).GetFlag())
+ isRHSUnsigned := mysql.HasUnsignedFlag(b.args[1].GetType(ctx).GetFlag())
switch {
case isLHSUnsigned && isRHSUnsigned:
@@ -875,8 +875,8 @@ func (b *builtinArithmeticPlusIntSig) vecEvalInt(ctx EvalContext, input *chunk.C
rhi64s := rh.Int64s()
resulti64s := result.Int64s()
- isLHSUnsigned := mysql.HasUnsignedFlag(b.args[0].GetType().GetFlag())
- isRHSUnsigned := mysql.HasUnsignedFlag(b.args[1].GetType().GetFlag())
+ isLHSUnsigned := mysql.HasUnsignedFlag(b.args[0].GetType(ctx).GetFlag())
+ isRHSUnsigned := mysql.HasUnsignedFlag(b.args[1].GetType(ctx).GetFlag())
switch {
case isLHSUnsigned && isRHSUnsigned:
diff --git a/pkg/expression/builtin_cast.go b/pkg/expression/builtin_cast.go
index f04e5b096b9a6..afd8030f2002c 100644
--- a/pkg/expression/builtin_cast.go
+++ b/pkg/expression/builtin_cast.go
@@ -124,12 +124,12 @@ func (c *castAsIntFunctionClass) getFunction(ctx BuildContext, args []Expression
return nil, err
}
bf := newBaseBuiltinCastFunc(b, c.inUnion)
- if args[0].GetType().Hybrid() || IsBinaryLiteral(args[0]) {
+ if args[0].GetType(ctx.GetEvalCtx()).Hybrid() || IsBinaryLiteral(args[0]) {
sig = &builtinCastIntAsIntSig{bf}
sig.setPbCode(tipb.ScalarFuncSig_CastIntAsInt)
return sig, nil
}
- argTp := args[0].GetType().EvalType()
+ argTp := args[0].GetType(ctx.GetEvalCtx()).EvalType()
switch argTp {
case types.ETInt:
sig = &builtinCastIntAsIntSig{bf}
@@ -180,10 +180,10 @@ func (c *castAsRealFunctionClass) getFunction(ctx BuildContext, args []Expressio
return sig, nil
}
var argTp types.EvalType
- if args[0].GetType().Hybrid() {
+ if args[0].GetType(ctx.GetEvalCtx()).Hybrid() {
argTp = types.ETInt
} else {
- argTp = args[0].GetType().EvalType()
+ argTp = args[0].GetType(ctx.GetEvalCtx()).EvalType()
}
switch argTp {
case types.ETInt:
@@ -194,7 +194,7 @@ func (c *castAsRealFunctionClass) getFunction(ctx BuildContext, args []Expressio
sig.setPbCode(tipb.ScalarFuncSig_CastRealAsReal)
case types.ETDecimal:
sig = &builtinCastDecimalAsRealSig{bf}
- PropagateType(types.ETReal, sig.getArgs()...)
+ PropagateType(ctx.GetEvalCtx(), types.ETReal, sig.getArgs()...)
sig.setPbCode(tipb.ScalarFuncSig_CastDecimalAsReal)
case types.ETDatetime, types.ETTimestamp:
sig = &builtinCastTimeAsRealSig{bf}
@@ -236,10 +236,10 @@ func (c *castAsDecimalFunctionClass) getFunction(ctx BuildContext, args []Expres
return sig, nil
}
var argTp types.EvalType
- if args[0].GetType().Hybrid() {
+ if args[0].GetType(ctx.GetEvalCtx()).Hybrid() {
argTp = types.ETInt
} else {
- argTp = args[0].GetType().EvalType()
+ argTp = args[0].GetType(ctx.GetEvalCtx()).EvalType()
}
switch argTp {
case types.ETInt:
@@ -283,18 +283,18 @@ func (c *castAsStringFunctionClass) getFunction(ctx BuildContext, args []Express
if err != nil {
return nil, err
}
- if args[0].GetType().Hybrid() {
+ if args[0].GetType(ctx.GetEvalCtx()).Hybrid() {
sig = &builtinCastStringAsStringSig{bf}
sig.setPbCode(tipb.ScalarFuncSig_CastStringAsString)
return sig, nil
}
- argTp := args[0].GetType().EvalType()
+ argTp := args[0].GetType(ctx.GetEvalCtx()).EvalType()
switch argTp {
case types.ETInt:
if bf.tp.GetFlen() == types.UnspecifiedLength {
// check https://github.com/pingcap/tidb/issues/44786
// set flen from integers may truncate integers, e.g. char(1) can not display -1[int(1)]
- switch args[0].GetType().GetType() {
+ switch args[0].GetType(ctx.GetEvalCtx()).GetType() {
case mysql.TypeTiny:
bf.tp.SetFlen(4)
case mysql.TypeShort:
@@ -305,7 +305,7 @@ func (c *castAsStringFunctionClass) getFunction(ctx BuildContext, args []Express
// set it to 11 as mysql
bf.tp.SetFlen(11)
default:
- bf.tp.SetFlen(args[0].GetType().GetFlen())
+ bf.tp.SetFlen(args[0].GetType(ctx.GetEvalCtx()).GetFlen())
}
}
sig = &builtinCastIntAsStringSig{bf}
@@ -328,7 +328,7 @@ func (c *castAsStringFunctionClass) getFunction(ctx BuildContext, args []Express
case types.ETString:
// When cast from binary to some other charsets, we should check if the binary is valid or not.
// so we build a from_binary function to do this check.
- bf.args[0] = HandleBinaryLiteral(ctx, args[0], &ExprCollation{Charset: c.tp.GetCharset(), Collation: c.tp.GetCollate()}, c.funcName)
+ bf.args[0] = HandleBinaryLiteral(ctx, args[0], &ExprCollation{Charset: c.tp.GetCharset(), Collation: c.tp.GetCollate()}, c.funcName, true)
sig = &builtinCastStringAsStringSig{bf}
sig.setPbCode(tipb.ScalarFuncSig_CastStringAsString)
default:
@@ -351,7 +351,7 @@ func (c *castAsTimeFunctionClass) getFunction(ctx BuildContext, args []Expressio
if err != nil {
return nil, err
}
- argTp := args[0].GetType().EvalType()
+ argTp := args[0].GetType(ctx.GetEvalCtx()).EvalType()
switch argTp {
case types.ETInt:
sig = &builtinCastIntAsTimeSig{bf}
@@ -394,7 +394,7 @@ func (c *castAsDurationFunctionClass) getFunction(ctx BuildContext, args []Expre
if err != nil {
return nil, err
}
- argTp := args[0].GetType().EvalType()
+ argTp := args[0].GetType(ctx.GetEvalCtx()).EvalType()
switch argTp {
case types.ETInt:
sig = &builtinCastIntAsDurationSig{bf}
@@ -429,12 +429,12 @@ type castAsArrayFunctionClass struct {
tp *types.FieldType
}
-func (c *castAsArrayFunctionClass) verifyArgs(args []Expression) error {
+func (c *castAsArrayFunctionClass) verifyArgs(ctx EvalContext, args []Expression) error {
if err := c.baseFunctionClass.verifyArgs(args); err != nil {
return err
}
- if args[0].GetType().EvalType() != types.ETJson {
+ if args[0].GetType(ctx).EvalType() != types.ETJson {
return ErrInvalidTypeForJSON.GenWithStackByArgs(1, "cast_as_array")
}
@@ -442,7 +442,7 @@ func (c *castAsArrayFunctionClass) verifyArgs(args []Expression) error {
}
func (c *castAsArrayFunctionClass) getFunction(ctx BuildContext, args []Expression) (sig builtinFunc, err error) {
- if err := c.verifyArgs(args); err != nil {
+ if err := c.verifyArgs(ctx.GetEvalCtx(), args); err != nil {
return nil, err
}
arrayType := c.tp.ArrayType()
@@ -454,7 +454,7 @@ func (c *castAsArrayFunctionClass) getFunction(ctx BuildContext, args []Expressi
return nil, ErrNotSupportedYet.GenWithStackByArgs("specifying charset for multi-valued index")
}
if arrayType.EvalType() == types.ETString && arrayType.GetFlen() == types.UnspecifiedLength {
- return nil, ErrNotSupportedYet.GenWithStackByArgs("CAST-ing data to array of char/binary BLOBs")
+ return nil, ErrNotSupportedYet.GenWithStackByArgs("CAST-ing data to array of char/binary BLOBs with unspecified length")
}
bf, err := newBaseBuiltinFunc(ctx, c.funcName, args, c.tp)
@@ -518,7 +518,7 @@ func (b *castJSONAsArrayFunctionSig) evalJSON(ctx EvalContext, row chunk.Row) (r
return types.CreateBinaryJSON(arrayVals), false, nil
}
-// ConvertJSON2Tp returns a function that can convert JSON to the specified type.
+// ConvertJSON2Tp converts JSON to the specified type.
func ConvertJSON2Tp(v types.BinaryJSON, targetType *types.FieldType) (any, error) {
convertFunc := convertJSON2Tp(targetType.EvalType())
if convertFunc == nil {
@@ -560,7 +560,7 @@ func convertJSON2Tp(evalType types.EvalType) func(*stmtctx.StatementContext, typ
if (tp.GetType() == mysql.TypeDatetime && item.TypeCode != types.JSONTypeCodeDatetime) || (tp.GetType() == mysql.TypeDate && item.TypeCode != types.JSONTypeCodeDate) {
return nil, ErrInvalidJSONForFuncIndex
}
- res := item.GetTime()
+ res := item.GetTimeWithFsp(tp.GetDecimal())
res.SetType(tp.GetType())
if tp.GetType() == mysql.TypeDate {
// Truncate hh:mm:ss part if the type is Date.
@@ -594,7 +594,7 @@ func (c *castAsJSONFunctionClass) getFunction(ctx BuildContext, args []Expressio
if err != nil {
return nil, err
}
- argTp := args[0].GetType().EvalType()
+ argTp := args[0].GetType(ctx.GetEvalCtx()).EvalType()
switch argTp {
case types.ETInt:
sig = &builtinCastIntAsJSONSig{bf}
@@ -660,7 +660,7 @@ func (b *builtinCastIntAsRealSig) evalReal(ctx EvalContext, row chunk.Row) (res
if isNull || err != nil {
return res, isNull, err
}
- if unsignedArgs0 := mysql.HasUnsignedFlag(b.args[0].GetType().GetFlag()); !mysql.HasUnsignedFlag(b.tp.GetFlag()) && !unsignedArgs0 {
+ if unsignedArgs0 := mysql.HasUnsignedFlag(b.args[0].GetType(ctx).GetFlag()); !mysql.HasUnsignedFlag(b.tp.GetFlag()) && !unsignedArgs0 {
res = float64(val)
} else if b.inUnion && !unsignedArgs0 && val < 0 {
// Round up to 0 if the value is negative but the expression eval type is unsigned in `UNION` statement
@@ -690,7 +690,7 @@ func (b *builtinCastIntAsDecimalSig) evalDecimal(ctx EvalContext, row chunk.Row)
if isNull || err != nil {
return res, isNull, err
}
- if unsignedArgs0 := mysql.HasUnsignedFlag(b.args[0].GetType().GetFlag()); !mysql.HasUnsignedFlag(b.tp.GetFlag()) && !unsignedArgs0 {
+ if unsignedArgs0 := mysql.HasUnsignedFlag(b.args[0].GetType(ctx).GetFlag()); !mysql.HasUnsignedFlag(b.tp.GetFlag()) && !unsignedArgs0 {
//revive:disable:empty-lines
res = types.NewDecFromInt(val)
// Round up to 0 if the value is negative but the expression eval type is unsigned in `UNION` statement
@@ -724,7 +724,7 @@ func (b *builtinCastIntAsStringSig) evalString(ctx EvalContext, row chunk.Row) (
if isNull || err != nil {
return res, isNull, err
}
- tp := b.args[0].GetType()
+ tp := b.args[0].GetType(ctx)
if !mysql.HasUnsignedFlag(tp.GetFlag()) {
res = strconv.FormatInt(val, 10)
} else {
@@ -756,7 +756,7 @@ func (b *builtinCastIntAsTimeSig) evalTime(ctx EvalContext, row chunk.Row) (res
return res, isNull, err
}
- if b.args[0].GetType().GetType() == mysql.TypeYear {
+ if b.args[0].GetType(ctx).GetType() == mysql.TypeYear {
res, err = types.ParseTimeFromYear(val)
} else {
res, err = types.ParseTimeFromNum(typeCtx(ctx), val, b.tp.GetType(), b.tp.GetDecimal())
@@ -813,9 +813,9 @@ func (b *builtinCastIntAsJSONSig) evalJSON(ctx EvalContext, row chunk.Row) (res
if isNull || err != nil {
return res, isNull, err
}
- if mysql.HasIsBooleanFlag(b.args[0].GetType().GetFlag()) {
+ if mysql.HasIsBooleanFlag(b.args[0].GetType(ctx).GetFlag()) {
res = types.CreateBinaryJSON(val != 0)
- } else if mysql.HasUnsignedFlag(b.args[0].GetType().GetFlag()) {
+ } else if mysql.HasUnsignedFlag(b.args[0].GetType(ctx).GetFlag()) {
res = types.CreateBinaryJSON(uint64(val))
} else {
res = types.CreateBinaryJSON(val)
@@ -878,7 +878,7 @@ func (b *builtinCastStringAsJSONSig) evalJSON(ctx EvalContext, row chunk.Row) (r
return res, isNull, err
}
- typ := b.args[0].GetType()
+ typ := b.args[0].GetType(ctx)
if types.IsBinaryStr(typ) {
buf := []byte(val)
if typ.GetType() == mysql.TypeString && typ.GetFlen() > 0 {
@@ -888,7 +888,7 @@ func (b *builtinCastStringAsJSONSig) evalJSON(ctx EvalContext, row chunk.Row) (r
}
res := types.CreateBinaryJSON(types.Opaque{
- TypeCode: b.args[0].GetType().GetType(),
+ TypeCode: b.args[0].GetType(ctx).GetType(),
Buf: buf,
})
@@ -1043,7 +1043,7 @@ func (b *builtinCastRealAsStringSig) evalString(ctx EvalContext, row chunk.Row)
}
bits := 64
- if b.args[0].GetType().GetType() == mysql.TypeFloat {
+ if b.args[0].GetType(ctx).GetType() == mysql.TypeFloat {
// b.args[0].EvalReal() casts the value from float32 to float64, for example:
// float32(208.867) is cast to float64(208.86700439)
// If we strconv.FormatFloat the value with 64bits, the result is incorrect!
@@ -1348,7 +1348,7 @@ func (*builtinCastStringAsIntSig) handleOverflow(ctx EvalContext, origRes int64,
}
func (b *builtinCastStringAsIntSig) evalInt(ctx EvalContext, row chunk.Row) (res int64, isNull bool, err error) {
- if b.args[0].GetType().Hybrid() || IsBinaryLiteral(b.args[0]) {
+ if b.args[0].GetType(ctx).Hybrid() || IsBinaryLiteral(b.args[0]) {
return b.args[0].EvalInt(ctx, row)
}
@@ -1929,7 +1929,7 @@ func (b *builtinCastJSONAsTimeSig) evalTime(ctx EvalContext, row chunk.Row) (res
switch val.TypeCode {
case types.JSONTypeCodeDate, types.JSONTypeCodeDatetime, types.JSONTypeCodeTimestamp:
- res = val.GetTime()
+ res = val.GetTimeWithFsp(b.tp.GetDecimal())
res.SetType(b.tp.GetType())
if b.tp.GetType() == mysql.TypeDate {
// Truncate hh:mm:ss part if the type is Date.
@@ -1991,7 +1991,7 @@ func (b *builtinCastJSONAsDurationSig) evalDuration(ctx EvalContext, row chunk.R
switch val.TypeCode {
case types.JSONTypeCodeDate, types.JSONTypeCodeDatetime, types.JSONTypeCodeTimestamp:
- time := val.GetTime()
+ time := val.GetTimeWithFsp(b.tp.GetDecimal())
res, err = time.ConvertToDuration()
if err != nil {
return res, false, err
@@ -2062,14 +2062,14 @@ func BuildCastFunction4Union(ctx BuildContext, expr Expression, tp *types.FieldT
// BuildCastCollationFunction builds a ScalarFunction which casts the collation.
func BuildCastCollationFunction(ctx BuildContext, expr Expression, ec *ExprCollation, enumOrSetRealTypeIsStr bool) Expression {
- if expr.GetType().EvalType() != types.ETString {
+ if expr.GetType(ctx.GetEvalCtx()).EvalType() != types.ETString {
return expr
}
- if expr.GetType().GetCollate() == ec.Collation {
+ if expr.GetType(ctx.GetEvalCtx()).GetCollate() == ec.Collation {
return expr
}
- tp := expr.GetType().Clone()
- if expr.GetType().Hybrid() {
+ tp := expr.GetType(ctx.GetEvalCtx()).Clone()
+ if expr.GetType(ctx.GetEvalCtx()).Hybrid() {
if !enumOrSetRealTypeIsStr {
return expr
}
@@ -2099,7 +2099,7 @@ func BuildCastFunction(ctx BuildContext, expr Expression, tp *types.FieldType) (
// BuildCastFunctionWithCheck builds a CAST ScalarFunction from the Expression and return error if any.
func BuildCastFunctionWithCheck(ctx BuildContext, expr Expression, tp *types.FieldType, inUnion bool) (res Expression, err error) {
- argType := expr.GetType()
+ argType := expr.GetType(ctx.GetEvalCtx())
// If source argument's nullable, then target type should be nullable
if !mysql.HasNotNullFlag(argType.GetFlag()) {
tp.DelFlag(mysql.NotNullFlag)
@@ -2125,8 +2125,8 @@ func BuildCastFunctionWithCheck(ctx BuildContext, expr Expression, tp *types.Fie
}
case types.ETString:
fc = &castAsStringFunctionClass{baseFunctionClass{ast.Cast, 1, 1}, tp}
- if expr.GetType().GetType() == mysql.TypeBit {
- tp.SetFlen((expr.GetType().GetFlen() + 7) / 8)
+ if expr.GetType(ctx.GetEvalCtx()).GetType() == mysql.TypeBit {
+ tp.SetFlen((expr.GetType(ctx.GetEvalCtx()).GetFlen() + 7) / 8)
}
}
f, err := fc.getFunction(ctx, []Expression{expr})
@@ -2147,36 +2147,36 @@ func BuildCastFunctionWithCheck(ctx BuildContext, expr Expression, tp *types.Fie
// WrapWithCastAsInt wraps `expr` with `cast` if the return type of expr is not
// type int, otherwise, returns `expr` directly.
func WrapWithCastAsInt(ctx BuildContext, expr Expression) Expression {
- if expr.GetType().GetType() == mysql.TypeEnum {
+ if expr.GetType(ctx.GetEvalCtx()).GetType() == mysql.TypeEnum {
if col, ok := expr.(*Column); ok {
col = col.Clone().(*Column)
col.RetType = col.RetType.Clone()
expr = col
}
- expr.GetType().AddFlag(mysql.EnumSetAsIntFlag)
+ expr.GetType(ctx.GetEvalCtx()).AddFlag(mysql.EnumSetAsIntFlag)
}
- if expr.GetType().EvalType() == types.ETInt {
+ if expr.GetType(ctx.GetEvalCtx()).EvalType() == types.ETInt {
return expr
}
tp := types.NewFieldType(mysql.TypeLonglong)
- tp.SetFlen(expr.GetType().GetFlen())
+ tp.SetFlen(expr.GetType(ctx.GetEvalCtx()).GetFlen())
tp.SetDecimal(0)
types.SetBinChsClnFlag(tp)
- tp.AddFlag(expr.GetType().GetFlag() & (mysql.UnsignedFlag | mysql.NotNullFlag))
+ tp.AddFlag(expr.GetType(ctx.GetEvalCtx()).GetFlag() & (mysql.UnsignedFlag | mysql.NotNullFlag))
return BuildCastFunction(ctx, expr, tp)
}
// WrapWithCastAsReal wraps `expr` with `cast` if the return type of expr is not
// type real, otherwise, returns `expr` directly.
func WrapWithCastAsReal(ctx BuildContext, expr Expression) Expression {
- if expr.GetType().EvalType() == types.ETReal {
+ if expr.GetType(ctx.GetEvalCtx()).EvalType() == types.ETReal {
return expr
}
tp := types.NewFieldType(mysql.TypeDouble)
tp.SetFlen(mysql.MaxRealWidth)
tp.SetDecimal(types.UnspecifiedLength)
types.SetBinChsClnFlag(tp)
- tp.AddFlag(expr.GetType().GetFlag() & (mysql.UnsignedFlag | mysql.NotNullFlag))
+ tp.AddFlag(expr.GetType(ctx.GetEvalCtx()).GetFlag() & (mysql.UnsignedFlag | mysql.NotNullFlag))
return BuildCastFunction(ctx, expr, tp)
}
@@ -2202,29 +2202,29 @@ func minimalDecimalLenForHoldingInteger(tp byte) int {
// WrapWithCastAsDecimal wraps `expr` with `cast` if the return type of expr is
// not type decimal, otherwise, returns `expr` directly.
func WrapWithCastAsDecimal(ctx BuildContext, expr Expression) Expression {
- if expr.GetType().EvalType() == types.ETDecimal {
+ if expr.GetType(ctx.GetEvalCtx()).EvalType() == types.ETDecimal {
return expr
}
tp := types.NewFieldType(mysql.TypeNewDecimal)
- tp.SetFlenUnderLimit(expr.GetType().GetFlen())
- tp.SetDecimalUnderLimit(expr.GetType().GetDecimal())
+ tp.SetFlenUnderLimit(expr.GetType(ctx.GetEvalCtx()).GetFlen())
+ tp.SetDecimalUnderLimit(expr.GetType(ctx.GetEvalCtx()).GetDecimal())
- if expr.GetType().EvalType() == types.ETInt {
- tp.SetFlen(minimalDecimalLenForHoldingInteger(expr.GetType().GetType()))
+ if expr.GetType(ctx.GetEvalCtx()).EvalType() == types.ETInt {
+ tp.SetFlen(minimalDecimalLenForHoldingInteger(expr.GetType(ctx.GetEvalCtx()).GetType()))
tp.SetDecimal(0)
}
if tp.GetFlen() == types.UnspecifiedLength || tp.GetFlen() > mysql.MaxDecimalWidth {
tp.SetFlen(mysql.MaxDecimalWidth)
}
types.SetBinChsClnFlag(tp)
- tp.AddFlag(expr.GetType().GetFlag() & (mysql.UnsignedFlag | mysql.NotNullFlag))
+ tp.AddFlag(expr.GetType(ctx.GetEvalCtx()).GetFlag() & (mysql.UnsignedFlag | mysql.NotNullFlag))
castExpr := BuildCastFunction(ctx, expr, tp)
// For const item, we can use find-grained precision and scale by the result.
if castExpr.ConstLevel() == ConstStrict {
val, isnull, err := castExpr.EvalDecimal(ctx.GetEvalCtx(), chunk.Row{})
if !isnull && err == nil {
precision, frac := val.PrecisionAndFrac()
- castTp := castExpr.GetType()
+ castTp := castExpr.GetType(ctx.GetEvalCtx())
castTp.SetDecimalUnderLimit(frac)
castTp.SetFlenUnderLimit(precision)
}
@@ -2235,13 +2235,13 @@ func WrapWithCastAsDecimal(ctx BuildContext, expr Expression) Expression {
// WrapWithCastAsString wraps `expr` with `cast` if the return type of expr is
// not type string, otherwise, returns `expr` directly.
func WrapWithCastAsString(ctx BuildContext, expr Expression) Expression {
- exprTp := expr.GetType()
+ exprTp := expr.GetType(ctx.GetEvalCtx())
if exprTp.EvalType() == types.ETString {
return expr
}
argLen := exprTp.GetFlen()
// If expr is decimal, we should take the decimal point ,negative sign and the leading zero(0.xxx)
- // into consideration, so we set `expr.GetType().GetFlen() + 3` as the `argLen`.
+ // into consideration, so we set `expr.GetType(ctx.GetEvalCtx()).GetFlen() + 3` as the `argLen`.
// Since the length of float and double is not accurate, we do not handle
// them.
if exprTp.GetType() == mysql.TypeNewDecimal && argLen != types.UnspecifiedFsp {
@@ -2283,21 +2283,21 @@ func WrapWithCastAsString(ctx BuildContext, expr Expression) Expression {
// WrapWithCastAsTime wraps `expr` with `cast` if the return type of expr is not
// same as type of the specified `tp` , otherwise, returns `expr` directly.
func WrapWithCastAsTime(ctx BuildContext, expr Expression, tp *types.FieldType) Expression {
- exprTp := expr.GetType().GetType()
+ exprTp := expr.GetType(ctx.GetEvalCtx()).GetType()
if tp.GetType() == exprTp {
return expr
} else if (exprTp == mysql.TypeDate || exprTp == mysql.TypeTimestamp) && tp.GetType() == mysql.TypeDatetime {
return expr
}
- switch x := expr.GetType().EvalType(); x {
+ switch x := expr.GetType(ctx.GetEvalCtx()).EvalType(); x {
case types.ETInt:
tp.SetDecimal(types.MinFsp)
case types.ETString, types.ETReal, types.ETJson:
tp.SetDecimal(types.MaxFsp)
case types.ETDatetime, types.ETTimestamp, types.ETDuration:
- tp.SetDecimal(expr.GetType().GetDecimal())
+ tp.SetDecimal(expr.GetType(ctx.GetEvalCtx()).GetDecimal())
case types.ETDecimal:
- tp.SetDecimal(expr.GetType().GetDecimal())
+ tp.SetDecimal(expr.GetType(ctx.GetEvalCtx()).GetDecimal())
if tp.GetDecimal() > types.MaxFsp {
tp.SetDecimal(types.MaxFsp)
}
@@ -2319,11 +2319,11 @@ func WrapWithCastAsTime(ctx BuildContext, expr Expression, tp *types.FieldType)
// WrapWithCastAsDuration wraps `expr` with `cast` if the return type of expr is
// not type duration, otherwise, returns `expr` directly.
func WrapWithCastAsDuration(ctx BuildContext, expr Expression) Expression {
- if expr.GetType().GetType() == mysql.TypeDuration {
+ if expr.GetType(ctx.GetEvalCtx()).GetType() == mysql.TypeDuration {
return expr
}
tp := types.NewFieldType(mysql.TypeDuration)
- switch x := expr.GetType(); x.GetType() {
+ switch x := expr.GetType(ctx.GetEvalCtx()); x.GetType() {
case mysql.TypeDatetime, mysql.TypeTimestamp, mysql.TypeDate:
tp.SetDecimal(x.GetDecimal())
default:
@@ -2339,7 +2339,7 @@ func WrapWithCastAsDuration(ctx BuildContext, expr Expression) Expression {
// WrapWithCastAsJSON wraps `expr` with `cast` if the return type of expr is not
// type json, otherwise, returns `expr` directly.
func WrapWithCastAsJSON(ctx BuildContext, expr Expression) Expression {
- if expr.GetType().GetType() == mysql.TypeJSON && !mysql.HasParseToJSONFlag(expr.GetType().GetFlag()) {
+ if expr.GetType(ctx.GetEvalCtx()).GetType() == mysql.TypeJSON && !mysql.HasParseToJSONFlag(expr.GetType(ctx.GetEvalCtx()).GetFlag()) {
return expr
}
tp := types.NewFieldTypeBuilder().SetType(mysql.TypeJSON).SetFlag(mysql.BinaryFlag).SetFlen(12582912).SetCharset(mysql.DefaultCharset).SetCollate(mysql.DefaultCollationName).BuildP()
@@ -2377,7 +2377,7 @@ func TryPushCastIntoControlFunctionForHybridType(ctx BuildContext, expr Expressi
args := sf.GetArgs()
switch sf.FuncName.L {
case ast.If:
- if isHybrid(args[1].GetType()) || isHybrid(args[2].GetType()) {
+ if isHybrid(args[1].GetType(ctx.GetEvalCtx())) || isHybrid(args[2].GetType(ctx.GetEvalCtx())) {
args[1] = wrapCastFunc(ctx, args[1])
args[2] = wrapCastFunc(ctx, args[2])
f, err := funcs[ast.If].getFunction(ctx, args)
@@ -2390,10 +2390,10 @@ func TryPushCastIntoControlFunctionForHybridType(ctx BuildContext, expr Expressi
case ast.Case:
hasHybrid := false
for i := 0; i < len(args)-1; i += 2 {
- hasHybrid = hasHybrid || isHybrid(args[i+1].GetType())
+ hasHybrid = hasHybrid || isHybrid(args[i+1].GetType(ctx.GetEvalCtx()))
}
if len(args)%2 == 1 {
- hasHybrid = hasHybrid || isHybrid(args[len(args)-1].GetType())
+ hasHybrid = hasHybrid || isHybrid(args[len(args)-1].GetType(ctx.GetEvalCtx()))
}
if !hasHybrid {
return expr
@@ -2414,7 +2414,7 @@ func TryPushCastIntoControlFunctionForHybridType(ctx BuildContext, expr Expressi
case ast.Elt:
hasHybrid := false
for i := 1; i < len(args); i++ {
- hasHybrid = hasHybrid || isHybrid(args[i].GetType())
+ hasHybrid = hasHybrid || isHybrid(args[i].GetType(ctx.GetEvalCtx()))
}
if !hasHybrid {
return expr
diff --git a/pkg/expression/builtin_cast_test.go b/pkg/expression/builtin_cast_test.go
index 97746afa52b07..e19bf9c8ae2f7 100644
--- a/pkg/expression/builtin_cast_test.go
+++ b/pkg/expression/builtin_cast_test.go
@@ -1242,7 +1242,7 @@ func TestWrapWithCastAsTypesClasses(t *testing.T) {
for i, c := range cases {
// Test wrapping with CastAsInt.
intExpr := WrapWithCastAsInt(ctx, c.expr)
- require.Equal(t, types.ETInt, intExpr.GetType().EvalType())
+ require.Equal(t, types.ETInt, intExpr.GetType(ctx).EvalType())
intRes, isNull, err := intExpr.EvalInt(ctx, c.row.ToRow())
require.NoErrorf(t, err, "cast[%v]: %#v", i, t)
require.Equal(t, false, isNull)
@@ -1250,7 +1250,7 @@ func TestWrapWithCastAsTypesClasses(t *testing.T) {
// Test wrapping with CastAsReal.
realExpr := WrapWithCastAsReal(ctx, c.expr)
- require.Equal(t, types.ETReal, realExpr.GetType().EvalType())
+ require.Equal(t, types.ETReal, realExpr.GetType(ctx).EvalType())
realRes, isNull, err := realExpr.EvalReal(ctx, c.row.ToRow())
require.NoError(t, err)
require.Equal(t, false, isNull)
@@ -1258,7 +1258,7 @@ func TestWrapWithCastAsTypesClasses(t *testing.T) {
// Test wrapping with CastAsDecimal.
decExpr := WrapWithCastAsDecimal(ctx, c.expr)
- require.Equal(t, types.ETDecimal, decExpr.GetType().EvalType())
+ require.Equal(t, types.ETDecimal, decExpr.GetType(ctx).EvalType())
decRes, isNull, err := decExpr.EvalDecimal(ctx, c.row.ToRow())
require.NoError(t, err, "case[%v]: %#v\n", i, t)
require.Equal(t, false, isNull)
@@ -1266,7 +1266,7 @@ func TestWrapWithCastAsTypesClasses(t *testing.T) {
// Test wrapping with CastAsString.
strExpr := WrapWithCastAsString(ctx, c.expr)
- require.True(t, strExpr.GetType().EvalType().IsStringKind())
+ require.True(t, strExpr.GetType(ctx).EvalType().IsStringKind())
strRes, isNull, err := strExpr.EvalString(ctx, c.row.ToRow())
require.NoError(t, err)
require.Equal(t, false, isNull)
@@ -1277,7 +1277,7 @@ func TestWrapWithCastAsTypesClasses(t *testing.T) {
// test cast unsigned int as string.
strExpr := WrapWithCastAsString(ctx, unsignedIntExpr)
- require.True(t, strExpr.GetType().EvalType().IsStringKind())
+ require.True(t, strExpr.GetType(ctx).EvalType().IsStringKind())
strRes, isNull, err := strExpr.EvalString(ctx, chunk.MutRowFromDatums([]types.Datum{types.NewUintDatum(math.MaxUint64)}).ToRow())
require.NoError(t, err)
require.Equal(t, strconv.FormatUint(math.MaxUint64, 10), strRes)
@@ -1290,7 +1290,7 @@ func TestWrapWithCastAsTypesClasses(t *testing.T) {
// test cast unsigned int as decimal.
decExpr := WrapWithCastAsDecimal(ctx, unsignedIntExpr)
- require.Equal(t, types.ETDecimal, decExpr.GetType().EvalType())
+ require.Equal(t, types.ETDecimal, decExpr.GetType(ctx).EvalType())
decRes, isNull, err := decExpr.EvalDecimal(ctx, chunk.MutRowFromDatums([]types.Datum{types.NewUintDatum(uint64(1234))}).ToRow())
require.NoError(t, err)
require.Equal(t, false, isNull)
@@ -1298,7 +1298,7 @@ func TestWrapWithCastAsTypesClasses(t *testing.T) {
// test cast unsigned int as Time.
timeExpr := WrapWithCastAsTime(ctx, unsignedIntExpr, types.NewFieldType(mysql.TypeDatetime))
- require.Equal(t, mysql.TypeDatetime, timeExpr.GetType().GetType())
+ require.Equal(t, mysql.TypeDatetime, timeExpr.GetType(ctx).GetType())
timeRes, isNull, err := timeExpr.EvalTime(ctx, chunk.MutRowFromDatums([]types.Datum{types.NewUintDatum(uint64(curTimeInt))}).ToRow())
require.NoError(t, err)
require.Equal(t, false, isNull)
@@ -1393,7 +1393,7 @@ func TestWrapWithCastAsString(t *testing.T) {
cases := []struct {
expr Expression
- err bool
+ warn bool
ret string
}{
{
@@ -1432,11 +1432,14 @@ func TestWrapWithCastAsString(t *testing.T) {
"-127",
},
}
+ lastWarningLen := 0
for _, c := range cases {
expr := BuildCastFunction(ctx, c.expr, types.NewFieldType(mysql.TypeVarString))
- res, _, err := expr.EvalString(ctx, chunk.Row{})
- if c.err {
- require.Error(t, err)
+ res, _, _ := expr.EvalString(ctx, chunk.Row{})
+ if c.warn {
+ warns := ctx.GetSessionVars().StmtCtx.GetWarnings()
+ require.Greater(t, len(warns), lastWarningLen)
+ lastWarningLen = len(warns)
} else {
require.Equal(t, c.ret, res)
}
@@ -1592,8 +1595,8 @@ func TestCastConstAsDecimalFieldType(t *testing.T) {
ctx := createContext(t)
for _, tc := range allTestCase {
expr := WrapWithCastAsDecimal(ctx, tc.input)
- require.Equal(t, tc.resultFlen, expr.GetType().GetFlen())
- require.Equal(t, tc.resultDecimal, expr.GetType().GetDecimal())
+ require.Equal(t, tc.resultFlen, expr.GetType(ctx).GetFlen())
+ require.Equal(t, tc.resultDecimal, expr.GetType(ctx).GetDecimal())
}
}
diff --git a/pkg/expression/builtin_cast_vec.go b/pkg/expression/builtin_cast_vec.go
index bce2383171338..b3fc74274f3f2 100644
--- a/pkg/expression/builtin_cast_vec.go
+++ b/pkg/expression/builtin_cast_vec.go
@@ -105,7 +105,7 @@ func (b *builtinCastIntAsRealSig) vecEvalReal(ctx EvalContext, input *chunk.Chun
rs := result.Float64s()
hasUnsignedFlag0 := mysql.HasUnsignedFlag(b.tp.GetFlag())
- hasUnsignedFlag1 := mysql.HasUnsignedFlag(b.args[0].GetType().GetFlag())
+ hasUnsignedFlag1 := mysql.HasUnsignedFlag(b.args[0].GetType(ctx).GetFlag())
for i := 0; i < n; i++ {
if result.IsNull(i) {
@@ -204,7 +204,7 @@ func (b *builtinCastRealAsStringSig) vecEvalString(ctx EvalContext, input *chunk
}
bits := 64
- if b.args[0].GetType().GetType() == mysql.TypeFloat {
+ if b.args[0].GetType(ctx).GetType() == mysql.TypeFloat {
// b.args[0].EvalReal() casts the value from float32 to float64, for example:
// float32(208.867) is cast to float64(208.86700439)
// If we strconv.FormatFloat the value with 64bits, the result is incorrect!
@@ -334,7 +334,7 @@ func (b *builtinCastDurationAsIntSig) vecEvalInt(ctx EvalContext, input *chunk.C
i64s := result.Int64s()
var duration types.Duration
ds := buf.GoDurations()
- fsp := b.args[0].GetType().GetDecimal()
+ fsp := b.args[0].GetType(ctx).GetDecimal()
isYear := b.tp.GetType() == mysql.TypeYear
tc := typeCtx(ctx)
for i := 0; i < n; i++ {
@@ -391,7 +391,7 @@ func (b *builtinCastIntAsTimeSig) vecEvalTime(ctx EvalContext, input *chunk.Chun
continue
}
- if b.args[0].GetType().GetType() == mysql.TypeYear {
+ if b.args[0].GetType(ctx).GetType() == mysql.TypeYear {
tm, err = types.ParseTimeFromYear(i64s[i])
} else {
tm, err = types.ParseTimeFromNum(tc, i64s[i], b.tp.GetType(), fsp)
@@ -508,7 +508,7 @@ func (b *builtinCastJSONAsTimeSig) vecEvalTime(ctx EvalContext, input *chunk.Chu
switch val.TypeCode {
case types.JSONTypeCodeDate, types.JSONTypeCodeDatetime, types.JSONTypeCodeTimestamp:
- tm := val.GetTime()
+ tm := val.GetTimeWithFsp(b.tp.GetDecimal())
times[i] = tm
times[i].SetType(b.tp.GetType())
if b.tp.GetType() == mysql.TypeDate {
@@ -704,7 +704,7 @@ func (b *builtinCastIntAsStringSig) vecEvalString(ctx EvalContext, input *chunk.
return err
}
- tp := b.args[0].GetType()
+ tp := b.args[0].GetType(ctx)
isUnsigned := mysql.HasUnsignedFlag(tp.GetFlag())
isYearType := tp.GetType() == mysql.TypeYear
result.ReserveString(n)
@@ -842,7 +842,7 @@ func (b *builtinCastStringAsJSONSig) vecEvalJSON(ctx EvalContext, input *chunk.C
}
result.ReserveJSON(n)
- typ := b.args[0].GetType()
+ typ := b.args[0].GetType(ctx)
if types.IsBinaryStr(typ) {
var res types.BinaryJSON
for i := 0; i < n; i++ {
@@ -860,7 +860,7 @@ func (b *builtinCastStringAsJSONSig) vecEvalJSON(ctx EvalContext, input *chunk.C
}
res = types.CreateBinaryJSON(types.Opaque{
- TypeCode: b.args[0].GetType().GetType(),
+ TypeCode: b.args[0].GetType(ctx).GetType(),
Buf: resultBuf,
})
result.AppendJSON(res)
@@ -942,7 +942,7 @@ func (*builtinCastStringAsIntSig) vectorized() bool {
func (b *builtinCastStringAsIntSig) vecEvalInt(ctx EvalContext, input *chunk.Chunk, result *chunk.Column) error {
n := input.NumRows()
- if b.args[0].GetType().Hybrid() || IsBinaryLiteral(b.args[0]) {
+ if b.args[0].GetType(ctx).Hybrid() || IsBinaryLiteral(b.args[0]) {
return b.args[0].VecEvalInt(ctx, input, result)
}
@@ -1059,7 +1059,7 @@ func (b *builtinCastDurationAsDecimalSig) vecEvalDecimal(ctx EvalContext, input
var duration types.Duration
ds := buf.GoDurations()
tc, ec := typeCtx(ctx), errCtx(ctx)
- fsp := b.args[0].GetType().GetDecimal()
+ fsp := b.args[0].GetType(ctx).GetDecimal()
if fsp, err = types.CheckFsp(fsp); err != nil {
return err
}
@@ -1094,7 +1094,7 @@ func (b *builtinCastIntAsDecimalSig) vecEvalDecimal(ctx EvalContext, input *chun
}
isUnsignedTp := mysql.HasUnsignedFlag(b.tp.GetFlag())
- isUnsignedArgs0 := mysql.HasUnsignedFlag(b.args[0].GetType().GetFlag())
+ isUnsignedArgs0 := mysql.HasUnsignedFlag(b.args[0].GetType(ctx).GetFlag())
nums := buf.Int64s()
result.ResizeDecimal(n, false)
result.MergeNulls(buf)
@@ -1140,7 +1140,7 @@ func (b *builtinCastIntAsJSONSig) vecEvalJSON(ctx EvalContext, input *chunk.Chun
}
nums := buf.Int64s()
result.ReserveJSON(n)
- if mysql.HasIsBooleanFlag(b.args[0].GetType().GetFlag()) {
+ if mysql.HasIsBooleanFlag(b.args[0].GetType(ctx).GetFlag()) {
for i := 0; i < n; i++ {
if buf.IsNull(i) {
result.AppendNull()
@@ -1148,7 +1148,7 @@ func (b *builtinCastIntAsJSONSig) vecEvalJSON(ctx EvalContext, input *chunk.Chun
result.AppendJSON(types.CreateBinaryJSON(nums[i] != 0))
}
}
- } else if mysql.HasUnsignedFlag(b.args[0].GetType().GetFlag()) {
+ } else if mysql.HasUnsignedFlag(b.args[0].GetType(ctx).GetFlag()) {
for i := 0; i < n; i++ {
if buf.IsNull(i) {
result.AppendNull()
@@ -1228,7 +1228,7 @@ func (b *builtinCastDurationAsRealSig) vecEvalReal(ctx EvalContext, input *chunk
f64s := result.Float64s()
var duration types.Duration
- fsp := b.args[0].GetType().GetDecimal()
+ fsp := b.args[0].GetType(ctx).GetDecimal()
if fsp, err = types.CheckFsp(fsp); err != nil {
return err
}
@@ -1402,7 +1402,7 @@ func (b *builtinCastDurationAsStringSig) vecEvalString(ctx EvalContext, input *c
var isNull bool
tc := typeCtx(ctx)
result.ReserveString(n)
- fsp := b.args[0].GetType().GetDecimal()
+ fsp := b.args[0].GetType(ctx).GetDecimal()
for i := 0; i < n; i++ {
if buf.IsNull(i) {
result.AppendNull()
@@ -1975,7 +1975,7 @@ func (b *builtinCastJSONAsDurationSig) vecEvalDuration(ctx EvalContext, input *c
switch val.TypeCode {
case types.JSONTypeCodeDate, types.JSONTypeCodeDatetime, types.JSONTypeCodeTimestamp:
- time := val.GetTime()
+ time := val.GetTimeWithFsp(b.tp.GetDecimal())
d, err := time.ConvertToDuration()
if err != nil {
return err
diff --git a/pkg/expression/builtin_compare.go b/pkg/expression/builtin_compare.go
index 54b39f099c7c1..442ae50912883 100644
--- a/pkg/expression/builtin_compare.go
+++ b/pkg/expression/builtin_compare.go
@@ -119,7 +119,7 @@ func (c *coalesceFunctionClass) getFunction(ctx BuildContext, args []Expression)
flag := uint(0)
for _, arg := range args {
- flag |= arg.GetType().GetFlag() & mysql.NotNullFlag
+ flag |= arg.GetType(ctx.GetEvalCtx()).GetFlag() & mysql.NotNullFlag
}
resultFieldType, err := InferType4ControlFuncs(ctx, c.funcName, args...)
@@ -328,28 +328,28 @@ func (b *builtinCoalesceJSONSig) evalJSON(ctx EvalContext, row chunk.Row) (res t
return res, isNull, err
}
-func aggregateType(args []Expression) *types.FieldType {
+func aggregateType(ctx EvalContext, args []Expression) *types.FieldType {
fieldTypes := make([]*types.FieldType, len(args))
for i := range fieldTypes {
- fieldTypes[i] = args[i].GetType()
+ fieldTypes[i] = args[i].GetType(ctx)
}
return types.AggFieldType(fieldTypes)
}
// ResolveType4Between resolves eval type for between expression.
-func ResolveType4Between(args [3]Expression) types.EvalType {
- cmpTp := args[0].GetType().EvalType()
+func ResolveType4Between(ctx EvalContext, args [3]Expression) types.EvalType {
+ cmpTp := args[0].GetType(ctx).EvalType()
for i := 1; i < 3; i++ {
- cmpTp = getBaseCmpType(cmpTp, args[i].GetType().EvalType(), nil, nil)
+ cmpTp = getBaseCmpType(cmpTp, args[i].GetType(ctx).EvalType(), nil, nil)
}
hasTemporal := false
if cmpTp == types.ETString {
- if args[0].GetType().GetType() == mysql.TypeDuration {
+ if args[0].GetType(ctx).GetType() == mysql.TypeDuration {
cmpTp = types.ETDuration
} else {
for _, arg := range args {
- if types.IsTypeTemporal(arg.GetType().GetType()) {
+ if types.IsTypeTemporal(arg.GetType(ctx).GetType()) {
hasTemporal = true
break
}
@@ -359,16 +359,16 @@ func ResolveType4Between(args [3]Expression) types.EvalType {
}
}
}
- if (args[0].GetType().EvalType() == types.ETInt || IsBinaryLiteral(args[0])) &&
- (args[1].GetType().EvalType() == types.ETInt || IsBinaryLiteral(args[1])) &&
- (args[2].GetType().EvalType() == types.ETInt || IsBinaryLiteral(args[2])) {
+ if (args[0].GetType(ctx).EvalType() == types.ETInt || IsBinaryLiteral(args[0])) &&
+ (args[1].GetType(ctx).EvalType() == types.ETInt || IsBinaryLiteral(args[1])) &&
+ (args[2].GetType(ctx).EvalType() == types.ETInt || IsBinaryLiteral(args[2])) {
return types.ETInt
}
return cmpTp
}
-// GLCmpStringMode represents Greatest/Least interal string comparison mode
+// GLCmpStringMode represents Greatest/Least integral string comparison mode
type GLCmpStringMode uint8
const (
@@ -393,12 +393,12 @@ const (
)
// resolveType4Extremum gets compare type for GREATEST and LEAST and BETWEEN (mainly for datetime).
-func resolveType4Extremum(args []Expression) (_ *types.FieldType, fieldTimeType GLRetTimeType, cmpStringMode GLCmpStringMode) {
- aggType := aggregateType(args)
+func resolveType4Extremum(ctx EvalContext, args []Expression) (_ *types.FieldType, fieldTimeType GLRetTimeType, cmpStringMode GLCmpStringMode) {
+ aggType := aggregateType(ctx, args)
var temporalItem *types.FieldType
if aggType.EvalType().IsStringKind() {
for i := range args {
- item := args[i].GetType()
+ item := args[i].GetType(ctx)
// Find the temporal value in the arguments but prefer DateTime value.
if types.IsTypeTemporal(item.GetType()) {
if temporalItem == nil || item.GetType() == mysql.TypeDatetime {
@@ -428,7 +428,7 @@ func resolveType4Extremum(args []Expression) (_ *types.FieldType, fieldTimeType
// unsupportedJSONComparison reports warnings while there is a JSON type in least/greatest function's arguments
func unsupportedJSONComparison(ctx BuildContext, args []Expression) {
for _, arg := range args {
- tp := arg.GetType().GetType()
+ tp := arg.GetType(ctx.GetEvalCtx()).GetType()
if tp == mysql.TypeJSON {
ctx.GetEvalCtx().AppendWarning(errUnsupportedJSONComparison)
break
@@ -444,11 +444,11 @@ func (c *greatestFunctionClass) getFunction(ctx BuildContext, args []Expression)
if err = c.verifyArgs(args); err != nil {
return nil, err
}
- resFieldType, fieldTimeType, cmpStringMode := resolveType4Extremum(args)
+ resFieldType, fieldTimeType, cmpStringMode := resolveType4Extremum(ctx.GetEvalCtx(), args)
resTp := resFieldType.EvalType()
argTp := resTp
if cmpStringMode != GLCmpStringDirectly {
- // Args are temporal and string mixed, we cast all args as string and parse it to temporal mannualy to compare.
+ // Args are temporal and string mixed, we cast all args as string and parse it to temporal manually to compare.
argTp = types.ETString
} else if resTp == types.ETJson {
unsupportedJSONComparison(ctx, args)
@@ -498,16 +498,16 @@ func (c *greatestFunctionClass) getFunction(ctx BuildContext, args []Expression)
}
}
- flen, decimal := fixFlenAndDecimalForGreatestAndLeast(args)
+ flen, decimal := fixFlenAndDecimalForGreatestAndLeast(ctx.GetEvalCtx(), args)
sig.getRetTp().SetFlenUnderLimit(flen)
sig.getRetTp().SetDecimalUnderLimit(decimal)
return sig, nil
}
-func fixFlenAndDecimalForGreatestAndLeast(args []Expression) (flen, decimal int) {
+func fixFlenAndDecimalForGreatestAndLeast(ctx EvalContext, args []Expression) (flen, decimal int) {
for _, arg := range args {
- argFlen, argDecimal := arg.GetType().GetFlen(), arg.GetType().GetDecimal()
+ argFlen, argDecimal := arg.GetType(ctx).GetFlen(), arg.GetType(ctx).GetDecimal()
if argFlen > flen {
flen = argFlen
}
@@ -757,11 +757,11 @@ func (c *leastFunctionClass) getFunction(ctx BuildContext, args []Expression) (s
if err = c.verifyArgs(args); err != nil {
return nil, err
}
- resFieldType, fieldTimeType, cmpStringMode := resolveType4Extremum(args)
+ resFieldType, fieldTimeType, cmpStringMode := resolveType4Extremum(ctx.GetEvalCtx(), args)
resTp := resFieldType.EvalType()
argTp := resTp
if cmpStringMode != GLCmpStringDirectly {
- // Args are temporal and string mixed, we cast all args as string and parse it to temporal mannualy to compare.
+ // Args are temporal and string mixed, we cast all args as string and parse it to temporal manually to compare.
argTp = types.ETString
} else if resTp == types.ETJson {
unsupportedJSONComparison(ctx, args)
@@ -810,7 +810,7 @@ func (c *leastFunctionClass) getFunction(ctx BuildContext, args []Expression) (s
sig.setPbCode(tipb.ScalarFuncSig_LeastTime)
}
}
- flen, decimal := fixFlenAndDecimalForGreatestAndLeast(args)
+ flen, decimal := fixFlenAndDecimalForGreatestAndLeast(ctx.GetEvalCtx(), args)
sig.getRetTp().SetFlenUnderLimit(flen)
sig.getRetTp().SetDecimalUnderLimit(decimal)
return sig, nil
@@ -1048,7 +1048,7 @@ func (c *intervalFunctionClass) getFunction(ctx BuildContext, args []Expression)
// https://github.com/mysql/mysql-server/blob/f8cdce86448a211511e8a039c62580ae16cb96f5/sql/item_cmpfunc.cc#L2713-L2788
// https://github.com/mysql/mysql-server/blob/f8cdce86448a211511e8a039c62580ae16cb96f5/sql/item_cmpfunc.cc#L2632-L2686
for i := range args {
- tp := args[i].GetType()
+ tp := args[i].GetType(ctx.GetEvalCtx())
if tp.EvalType() != types.ETInt {
allInt = false
}
@@ -1100,7 +1100,7 @@ func (b *builtinIntervalIntSig) evalInt(ctx EvalContext, row chunk.Row) (int64,
if isNull {
return -1, false, nil
}
- isUint1 := mysql.HasUnsignedFlag(b.args[0].GetType().GetFlag())
+ isUint1 := mysql.HasUnsignedFlag(b.args[0].GetType(ctx).GetFlag())
var idx int
if b.hasNullable {
idx, err = b.linearSearch(ctx, arg0, isUint1, b.args[1:], row)
@@ -1114,7 +1114,7 @@ func (b *builtinIntervalIntSig) evalInt(ctx EvalContext, row chunk.Row) (int64,
func (b *builtinIntervalIntSig) linearSearch(ctx EvalContext, target int64, isUint1 bool, args []Expression, row chunk.Row) (i int, err error) {
i = 0
for ; i < len(args); i++ {
- isUint2 := mysql.HasUnsignedFlag(args[i].GetType().GetFlag())
+ isUint2 := mysql.HasUnsignedFlag(args[i].GetType(ctx).GetFlag())
arg, isNull, err := args[i].EvalInt(ctx, row)
if err != nil {
return 0, err
@@ -1155,7 +1155,7 @@ func (b *builtinIntervalIntSig) binSearch(ctx EvalContext, target int64, isUint1
if isNull {
v = target
}
- isUint2 := mysql.HasUnsignedFlag(args[mid].GetType().GetFlag())
+ isUint2 := mysql.HasUnsignedFlag(args[mid].GetType(ctx).GetFlag())
switch {
case !isUint1 && !isUint2:
cmp = target < v
@@ -1283,8 +1283,8 @@ func getBaseCmpType(lhs, rhs types.EvalType, lft, rft *types.FieldType) types.Ev
// GetAccurateCmpType uses a more complex logic to decide the EvalType of the two args when compare with each other than
// getBaseCmpType does.
-func GetAccurateCmpType(lhs, rhs Expression) types.EvalType {
- lhsFieldType, rhsFieldType := lhs.GetType(), rhs.GetType()
+func GetAccurateCmpType(ctx EvalContext, lhs, rhs Expression) types.EvalType {
+ lhsFieldType, rhsFieldType := lhs.GetType(ctx), rhs.GetType(ctx)
lhsEvalType, rhsEvalType := lhsFieldType.EvalType(), rhsFieldType.EvalType()
cmpType := getBaseCmpType(lhsEvalType, rhsEvalType, lhsFieldType, rhsFieldType)
if (lhsEvalType.IsStringKind() && lhsFieldType.GetType() == mysql.TypeJSON) || (rhsEvalType.IsStringKind() && rhsFieldType.GetType() == mysql.TypeJSON) {
@@ -1315,8 +1315,8 @@ func GetAccurateCmpType(lhs, rhs Expression) types.EvalType {
Do comparison as decimal rather than float, in order not to lose precision.
)*/
cmpType = types.ETDecimal
- } else if isTemporalColumn(lhs) && isRHSConst ||
- isTemporalColumn(rhs) && isLHSConst {
+ } else if isTemporalColumn(ctx, lhs) && isRHSConst ||
+ isTemporalColumn(ctx, rhs) && isLHSConst {
/*
or
@@ -1328,7 +1328,7 @@ func GetAccurateCmpType(lhs, rhs Expression) types.EvalType {
if !isLHSColumn {
col = rhs.(*Column)
}
- if col.GetType().GetType() == mysql.TypeDuration {
+ if col.GetType(ctx).GetType() == mysql.TypeDuration {
cmpType = types.ETDuration
}
}
@@ -1338,7 +1338,7 @@ func GetAccurateCmpType(lhs, rhs Expression) types.EvalType {
// GetCmpFunction get the compare function according to two arguments.
func GetCmpFunction(ctx BuildContext, lhs, rhs Expression) CompareFunc {
- switch GetAccurateCmpType(lhs, rhs) {
+ switch GetAccurateCmpType(ctx.GetEvalCtx(), lhs, rhs) {
case types.ETInt:
return CompareInt
case types.ETReal:
@@ -1360,8 +1360,8 @@ func GetCmpFunction(ctx BuildContext, lhs, rhs Expression) CompareFunc {
// isTemporalColumn checks if a expression is a temporal column,
// temporal column indicates time column or duration column.
-func isTemporalColumn(expr Expression) bool {
- ft := expr.GetType()
+func isTemporalColumn(ctx EvalContext, expr Expression) bool {
+ ft := expr.GetType(ctx)
if _, isCol := expr.(*Column); !isCol {
return false
}
@@ -1373,14 +1373,14 @@ func isTemporalColumn(expr Expression) bool {
// tryToConvertConstantInt tries to convert a constant with other type to a int constant.
// isExceptional indicates whether the 'int column [cmp] const' might be true/false.
-// If isExceptional is true, ExecptionalVal is returned. Or, CorrectVal is returned.
+// If isExceptional is true, ExceptionalVal is returned. Or, CorrectVal is returned.
// CorrectVal: The computed result. If the constant can be converted to int without exception, return the val. Else return 'con'(the input).
// ExceptionalVal : It is used to get more information to check whether 'int column [cmp] const' is true/false
//
// If the op == LT,LE,GT,GE and it gets an Overflow when converting, return inf/-inf.
// If the op == EQ,NullEQ and the constant can never be equal to the int column, return ‘con’(the input, a non-int constant).
func tryToConvertConstantInt(ctx BuildContext, targetFieldType *types.FieldType, con *Constant) (_ *Constant, isExceptional bool) {
- if con.GetType().EvalType() == types.ETInt {
+ if con.GetType(ctx.GetEvalCtx()).EvalType() == types.ETInt {
return con, false
}
@@ -1412,7 +1412,7 @@ func tryToConvertConstantInt(ctx BuildContext, targetFieldType *types.FieldType,
// RefineComparedConstant changes a non-integer constant argument to its ceiling or floor result by the given op.
// isExceptional indicates whether the 'int column [cmp] const' might be true/false.
-// If isExceptional is true, ExecptionalVal is returned. Or, CorrectVal is returned.
+// If isExceptional is true, ExceptionalVal is returned. Or, CorrectVal is returned.
// CorrectVal: The computed result. If the constant can be converted to int without exception, return the val. Else return 'con'(the input).
// ExceptionalVal : It is used to get more information to check whether 'int column [cmp] const' is true/false
//
@@ -1428,7 +1428,10 @@ func RefineComparedConstant(ctx BuildContext, targetFieldType types.FieldType, c
targetFieldType = *types.NewFieldType(mysql.TypeLonglong)
}
var intDatum types.Datum
- intDatum, err = dt.ConvertTo(evalCtx.TypeCtx(), &targetFieldType)
+ // Disable AllowNegativeToUnsigned to make sure return 0 when underflow happens.
+ oriTypeCtx := evalCtx.TypeCtx()
+ newTypeCtx := oriTypeCtx.WithFlags(oriTypeCtx.Flags().WithAllowNegativeToUnsigned(false))
+ intDatum, err = dt.ConvertTo(newTypeCtx, &targetFieldType)
if err != nil {
if terror.ErrorEqual(err, types.ErrOverflow) {
return &Constant{
@@ -1464,7 +1467,7 @@ func RefineComparedConstant(ctx BuildContext, targetFieldType types.FieldType, c
return tryToConvertConstantInt(ctx, &targetFieldType, resultCon)
}
case opcode.NullEQ, opcode.EQ:
- switch con.GetType().EvalType() {
+ switch con.GetType(ctx.GetEvalCtx()).EvalType() {
// An integer value equal or NULL-safe equal to a float value which contains
// non-zero decimal digits is definitely false.
// e.g.,
@@ -1529,7 +1532,7 @@ func allowCmpArgsRefining4PlanCache(ctx BuildContext, args []Expression) (allowR
// case 1: year-expr const
// refine `year < 12` to `year < 2012` to guarantee the correctness.
// see https://github.com/pingcap/tidb/issues/41626 for more details.
- exprType := args[1-conIdx].GetType()
+ exprType := args[1-conIdx].GetType(ctx.GetEvalCtx())
exprEvalType := exprType.EvalType()
if exprType.GetType() == mysql.TypeYear {
ctx.SetSkipPlanCache(fmt.Sprintf("'%v' may be converted to INT", args[conIdx].String()))
@@ -1538,7 +1541,7 @@ func allowCmpArgsRefining4PlanCache(ctx BuildContext, args []Expression) (allowR
// case 2: int-expr string/float/double/decimal-const
// refine `int_key < 1.1` to `int_key < 2` to generate RangeScan instead of FullScan.
- conEvalType := args[conIdx].GetType().EvalType()
+ conEvalType := args[conIdx].GetType(ctx.GetEvalCtx()).EvalType()
if exprEvalType == types.ETInt &&
(conEvalType == types.ETString || conEvalType == types.ETReal || conEvalType == types.ETDecimal) {
ctx.SetSkipPlanCache(fmt.Sprintf("'%v' may be converted to INT", args[conIdx].String()))
@@ -1565,7 +1568,7 @@ func allowCmpArgsRefining4PlanCache(ctx BuildContext, args []Expression) (allowR
// This refining operation depends on the values of these args, but these values can change when using plan-cache.
// So we have to skip this operation or mark the plan as over-optimized when using plan-cache.
func (c *compareFunctionClass) refineArgs(ctx BuildContext, args []Expression) ([]Expression, error) {
- arg0Type, arg1Type := args[0].GetType(), args[1].GetType()
+ arg0Type, arg1Type := args[0].GetType(ctx.GetEvalCtx()), args[1].GetType(ctx.GetEvalCtx())
arg0EvalType, arg1EvalType := arg0Type.EvalType(), arg1Type.EvalType()
arg0IsInt := arg0EvalType == types.ETInt
arg1IsInt := arg1EvalType == types.ETInt
@@ -1602,7 +1605,7 @@ func (c *compareFunctionClass) refineArgs(ctx BuildContext, args []Expression) (
// TODO if the plan doesn't care about whether the result of the function is null or false, we don't need
// to check the NotNullFlag, then more optimizations can be enabled.
isExceptional = isExceptional && mysql.HasNotNullFlag(arg0Type.GetFlag())
- if isExceptional && arg1.GetType().EvalType() == types.ETInt {
+ if isExceptional && arg1.GetType(ctx.GetEvalCtx()).EvalType() == types.ETInt {
// Judge it is inf or -inf
// For int:
// inf: 01111111 & 1 == 1
@@ -1626,7 +1629,7 @@ func (c *compareFunctionClass) refineArgs(ctx BuildContext, args []Expression) (
// TODO if the plan doesn't care about whether the result of the function is null or false, we don't need
// to check the NotNullFlag, then more optimizations can be enabled.
isExceptional = isExceptional && mysql.HasNotNullFlag(arg1Type.GetFlag())
- if isExceptional && arg0.GetType().EvalType() == types.ETInt {
+ if isExceptional && arg0.GetType(ctx.GetEvalCtx()).EvalType() == types.ETInt {
if arg0.Value.GetInt64()&1 == 1 {
isNegativeInfinite = true
} else {
@@ -1701,7 +1704,7 @@ func (c *compareFunctionClass) refineNumericConstantCmpDatetime(ctx BuildContext
func (c *compareFunctionClass) refineArgsByUnsignedFlag(ctx BuildContext, args []Expression) []Expression {
// Only handle int cases, cause MySQL declares that `UNSIGNED` is deprecated for FLOAT, DOUBLE and DECIMAL types,
// and support for it would be removed in a future version.
- if args[0].GetType().EvalType() != types.ETInt || args[1].GetType().EvalType() != types.ETInt {
+ if args[0].GetType(ctx.GetEvalCtx()).EvalType() != types.ETInt || args[1].GetType(ctx.GetEvalCtx()).EvalType() != types.ETInt {
return args
}
colArgs := make([]*Column, 2)
@@ -1767,7 +1770,7 @@ func (c *compareFunctionClass) getFunction(ctx BuildContext, rawArgs []Expressio
if err != nil {
return nil, err
}
- cmpType := GetAccurateCmpType(args[0], args[1])
+ cmpType := GetAccurateCmpType(ctx.GetEvalCtx(), args[0], args[1])
sig, err = c.generateCmpSigs(ctx, args, cmpType)
return sig, err
}
@@ -1781,7 +1784,7 @@ func (c *compareFunctionClass) generateCmpSigs(ctx BuildContext, args []Expressi
if tp == types.ETJson {
// In compare, if we cast string to JSON, we shouldn't parse it.
for i := range args {
- DisableParseJSONFlag4Expr(args[i])
+ DisableParseJSONFlag4Expr(ctx.GetEvalCtx(), args[i])
}
}
bf.tp.SetFlen(1)
@@ -2565,7 +2568,7 @@ func (b *builtinNullEQIntSig) evalInt(ctx EvalContext, row chunk.Row) (val int64
if err != nil {
return 0, isNull1, err
}
- isUnsigned0, isUnsigned1 := mysql.HasUnsignedFlag(b.args[0].GetType().GetFlag()), mysql.HasUnsignedFlag(b.args[1].GetType().GetFlag())
+ isUnsigned0, isUnsigned1 := mysql.HasUnsignedFlag(b.args[0].GetType(ctx).GetFlag()), mysql.HasUnsignedFlag(b.args[1].GetType(ctx).GetFlag())
var res int64
switch {
case isNull0 && isNull1:
@@ -2875,7 +2878,7 @@ func CompareInt(sctx EvalContext, lhsArg, rhsArg Expression, lhsRow, rhsRow chun
return compareNull(isNull0, isNull1), true, nil
}
- isUnsigned0, isUnsigned1 := mysql.HasUnsignedFlag(lhsArg.GetType().GetFlag()), mysql.HasUnsignedFlag(rhsArg.GetType().GetFlag())
+ isUnsigned0, isUnsigned1 := mysql.HasUnsignedFlag(lhsArg.GetType(sctx).GetFlag()), mysql.HasUnsignedFlag(rhsArg.GetType(sctx).GetFlag())
return int64(types.CompareInt(arg0, isUnsigned0, arg1, isUnsigned1)), false, nil
}
diff --git a/pkg/expression/builtin_compare_test.go b/pkg/expression/builtin_compare_test.go
index a8493a8d96139..39310a042f5b7 100644
--- a/pkg/expression/builtin_compare_test.go
+++ b/pkg/expression/builtin_compare_test.go
@@ -137,8 +137,8 @@ func TestCompare(t *testing.T) {
bf, err := funcs[test.funcName].getFunction(ctx, primitiveValsToConstants(ctx, []any{test.arg0, test.arg1}))
require.NoError(t, err)
args := bf.getArgs()
- require.Equal(t, test.tp, args[0].GetType().GetType())
- require.Equal(t, test.tp, args[1].GetType().GetType())
+ require.Equal(t, test.tp, args[0].GetType(ctx).GetType())
+ require.Equal(t, test.tp, args[1].GetType(ctx).GetType())
res, err := evalBuiltinFunc(bf, ctx, chunk.Row{})
require.NoError(t, err)
require.False(t, res.IsNull())
@@ -151,24 +151,24 @@ func TestCompare(t *testing.T) {
bf, err := funcs[ast.LT].getFunction(ctx, []Expression{decimalCol, stringCon})
require.NoError(t, err)
args := bf.getArgs()
- require.Equal(t, mysql.TypeNewDecimal, args[0].GetType().GetType())
- require.Equal(t, mysql.TypeNewDecimal, args[1].GetType().GetType())
+ require.Equal(t, mysql.TypeNewDecimal, args[0].GetType(ctx).GetType())
+ require.Equal(t, mysql.TypeNewDecimal, args[1].GetType(ctx).GetType())
// test