Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

load data: physical mode part1 #42817

Merged
merged 20 commits into from
Apr 7, 2023
15 changes: 8 additions & 7 deletions br/pkg/lightning/mydump/loader.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,13 +69,14 @@ func (m *MDDatabaseMeta) GetSchema(ctx context.Context, store storage.ExternalSt

// MDTableMeta contains some parsed metadata for a table in the source by MyDumper Loader.
type MDTableMeta struct {
DB string
Name string
SchemaFile FileInfo
DataFiles []FileInfo
charSet string
TotalSize int64
IndexRatio float64
DB string
Name string
SchemaFile FileInfo
DataFiles []FileInfo
charSet string
TotalSize int64
IndexRatio float64
// default to true, and if we do precheck, this var is updated using data sampling result, so it's not accurate.
IsRowOrdered bool
}

Expand Down
33 changes: 30 additions & 3 deletions executor/importer/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -2,44 +2,70 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")

go_library(
name = "importer",
srcs = ["import.go"],
srcs = [
"chunk_process.go",
"engine_process.go",
"import.go",
"kv_encode.go",
"table_import.go",
],
importpath = "github.com/pingcap/tidb/executor/importer",
visibility = ["//visibility:public"],
deps = [
"//br/pkg/lightning/backend",
"//br/pkg/lightning/backend/encode",
"//br/pkg/lightning/backend/kv",
"//br/pkg/lightning/backend/local",
"//br/pkg/lightning/checkpoints",
"//br/pkg/lightning/common",
"//br/pkg/lightning/config",
"//br/pkg/lightning/log",
"//br/pkg/lightning/mydump",
"//br/pkg/lightning/verification",
"//br/pkg/storage",
"//config",
"//expression",
"//kv",
"//meta/autoid",
"//parser/ast",
"//parser/mysql",
"//parser/terror",
"//planner/core",
"//sessionctx",
"//sessionctx/variable",
"//table",
"//table/tables",
"//tablecodec",
"//types",
"//util",
"//util/chunk",
"//util/dbterror",
"//util/dbterror/exeerrors",
"//util/filter",
"//util/intest",
"//util/logutil",
"//util/stringutil",
"@com_github_docker_go_units//:go-units",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_log//:log",
"@com_github_tikv_client_go_v2//config",
"@com_github_tikv_client_go_v2//tikv",
"@org_uber_go_multierr//:multierr",
"@org_uber_go_zap//:zap",
],
)

go_test(
name = "importer_test",
timeout = "short",
srcs = ["import_test.go"],
srcs = [
"engine_process_test.go",
"import_test.go",
],
embed = [":importer"],
flaky = True,
race = "on",
shard_count = 4,
shard_count = 5,
deps = [
"//br/pkg/errors",
"//br/pkg/lightning/config",
Expand All @@ -50,6 +76,7 @@ go_test(
"//util/dbterror/exeerrors",
"//util/mock",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_log//:log",
"@com_github_stretchr_testify//require",
],
)
Loading