Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master' into add-default-literal
Browse files Browse the repository at this point in the history 
* upstream/master: (93 commits)
  disjoinset: add generic impl (pingcap#54917)
  planner: derive index filters for mv index paths (pingcap#54877)
  br: cli refactor backup error handling logic (pingcap#54697)
  expression: fix infinity loop in `timestampadd` (pingcap#54916)
  planner: import more expand test. (pingcap#54962)
  planner: use code-gen to generate CloneForPlanCache method for some operators (pingcap#54957)
  test: fix flaky test TestFailSchemaSyncer (pingcap#54958)
  planner: move logical show into logicalop pkg. (pingcap#54928)
  privilege: Remove TestAbnormalMySQLTable (pingcap#54925)
  resource_control: support unlimited keyword when setting the resource group (pingcap#54704)
  ddl: fix a data race on localRowCntListener Written() (pingcap#54484)
  lightning: fix SET SESSION on connection pool (pingcap#54927)
  planner: move logical mem-table to logicalop pkg. (pingcap#54903)
  table: Add `CachedTableSupport` and `TemporaryTableSupport` for `MutateContext` (pingcap#54900)
  executor: fix index_hash_join hang when context canceled (pingcap#54855)
  statistics: add metrics for unneeded analyze table (pingcap#54822)
  *: refine pipelined dml benchmarks (pingcap#54844)
  ddl: assign table IDs for jobs submitted to queue (pingcap#54880)
  *: avoid using Tables field of model.DBInfo, use API instead (pingcap#52302)
  planner: remove useless check (pingcap#54907)
  ...
  • Loading branch information
@morgo
morgo committed Jul 28, 2024
2 parents 888e362 + 7cdf918 commit 4424521
Show file tree
Hide file tree
Showing 521 changed files with 20,713 additions and 14,370 deletions.
60 changes: 30 additions & 30 deletions DEPS.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5984,13 +5984,13 @@ def go_deps():
name = "com_github_pingcap_kvproto",
build_file_proto_mode = "disable_global",
importpath = "github.com/pingcap/kvproto",
sha256 = "9c11f4c5354534ac08526d61d180bf873786fef9089b1f018fc7365350839207",
strip_prefix = "github.com/pingcap/[email protected]20240620063548-118a4cab53e4",
sha256 = "b1af34db24f2650cd2a687fa9c58bd746eb3ca76e08afe5dbe2ac569ce54b597",
strip_prefix = "github.com/pingcap/[email protected]20240716095229-5f7ffec83ea7",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240620063548-118a4cab53e4.zip",
"http://ats.apps.svc/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240620063548-118a4cab53e4.zip",
"https://cache.hawkingrei.com/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240620063548-118a4cab53e4.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240620063548-118a4cab53e4.zip",
"http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240716095229-5f7ffec83ea7.zip",
"http://ats.apps.svc/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240716095229-5f7ffec83ea7.zip",
"https://cache.hawkingrei.com/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240716095229-5f7ffec83ea7.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/kvproto/com_github_pingcap_kvproto-v0.0.0-20240716095229-5f7ffec83ea7.zip",
],
)
go_repository(
Expand Down Expand Up @@ -7193,13 +7193,13 @@ def go_deps():
name = "com_github_tikv_pd_client",
build_file_proto_mode = "disable_global",
importpath = "github.com/tikv/pd/client",
sha256 = "af957cdaccb24818d126f992ff3677b04adb2e80b53b898d5bab6f134a144120",
strip_prefix = "github.com/tikv/pd/[email protected]20240703065657-6b25787aff4d",
sha256 = "9ca4b7c7f070194aab4c537cbf58c39918341c59834fc2eb1d81d53865aaed36",
strip_prefix = "github.com/tikv/pd/[email protected]20240724130437-83f32e9221f2",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240703065657-6b25787aff4d.zip",
"http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240703065657-6b25787aff4d.zip",
"https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240703065657-6b25787aff4d.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240703065657-6b25787aff4d.zip",
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240724130437-83f32e9221f2.zip",
"http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240724130437-83f32e9221f2.zip",
"https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240724130437-83f32e9221f2.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240724130437-83f32e9221f2.zip",
],
)
go_repository(
Expand Down Expand Up @@ -10248,13 +10248,13 @@ def go_deps():
name = "org_golang_x_crypto",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/crypto",
sha256 = "7853b5097deae0de87f1361a0c63d01f09bf41a8e20f7033388426a961aa9d38",
strip_prefix = "golang.org/x/crypto@v0.24.0",
sha256 = "5cd21f5fcb21845389eed13ad39186f57c86db70f3cc6cf8a4904633f07d1bdb",
strip_prefix = "golang.org/x/crypto@v0.25.0",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.24.0.zip",
"http://ats.apps.svc/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.24.0.zip",
"https://cache.hawkingrei.com/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.24.0.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.24.0.zip",
"http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.25.0.zip",
"http://ats.apps.svc/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.25.0.zip",
"https://cache.hawkingrei.com/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.25.0.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.25.0.zip",
],
)
go_repository(
Expand Down Expand Up @@ -10339,13 +10339,13 @@ def go_deps():
name = "org_golang_x_net",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/net",
sha256 = "3ea90c14742269c7c1d5217bc1e3cf58a71552e5724fd40ffa69ba51a133adb0",
strip_prefix = "golang.org/x/net@v0.26.0",
sha256 = "0adf4068f2b9d2b2852288b77caf1f76dac61f960e691e7544191de9bfa36406",
strip_prefix = "golang.org/x/net@v0.27.0",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/net/org_golang_x_net-v0.26.0.zip",
"http://ats.apps.svc/gomod/golang.org/x/net/org_golang_x_net-v0.26.0.zip",
"https://cache.hawkingrei.com/gomod/golang.org/x/net/org_golang_x_net-v0.26.0.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/net/org_golang_x_net-v0.26.0.zip",
"http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/net/org_golang_x_net-v0.27.0.zip",
"http://ats.apps.svc/gomod/golang.org/x/net/org_golang_x_net-v0.27.0.zip",
"https://cache.hawkingrei.com/gomod/golang.org/x/net/org_golang_x_net-v0.27.0.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/net/org_golang_x_net-v0.27.0.zip",
],
)
go_repository(
Expand Down Expand Up @@ -10417,13 +10417,13 @@ def go_deps():
name = "org_golang_x_term",
build_file_proto_mode = "disable_global",
importpath = "golang.org/x/term",
sha256 = "3ba59c5c929c526e0bf016696db8ffec313b43876428677ecda4198312a6f696",
strip_prefix = "golang.org/x/term@v0.21.0",
sha256 = "0766e30db0cea597a7c87ad6a829df2a538939dc65b5ca65791dfd6e4f1244b4",
strip_prefix = "golang.org/x/term@v0.22.0",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/term/org_golang_x_term-v0.21.0.zip",
"http://ats.apps.svc/gomod/golang.org/x/term/org_golang_x_term-v0.21.0.zip",
"https://cache.hawkingrei.com/gomod/golang.org/x/term/org_golang_x_term-v0.21.0.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/term/org_golang_x_term-v0.21.0.zip",
"http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/term/org_golang_x_term-v0.22.0.zip",
"http://ats.apps.svc/gomod/golang.org/x/term/org_golang_x_term-v0.22.0.zip",
"https://cache.hawkingrei.com/gomod/golang.org/x/term/org_golang_x_term-v0.22.0.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/term/org_golang_x_term-v0.22.0.zip",
],
)
go_repository(
Expand Down
1 change: 1 addition & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -496,6 +496,7 @@ gen_mock: mockgen
tools/bin/mockgen -package mock github.com/pingcap/tidb/pkg/util/sqlexec RestrictedSQLExecutor > pkg/util/sqlexec/mock/restricted_sql_executor_mock.go
tools/bin/mockgen -package mockstorage github.com/pingcap/tidb/br/pkg/storage ExternalStorage > br/pkg/mock/storage/storage.go
tools/bin/mockgen -package mock github.com/pingcap/tidb/pkg/ddl SchemaLoader > pkg/ddl/mock/schema_loader_mock.go
tools/bin/mockgen -package mock github.com/pingcap/tidb/pkg/ddl/systable Manager > pkg/ddl/mock/systable_manager_mock.go

# There is no FreeBSD environment for GitHub actions. So cross-compile on Linux
# but that doesn't work with CGO_ENABLED=1, so disable cgo. The reason to have
Expand Down
1 change: 1 addition & 0 deletions br/pkg/aws/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ go_library(
deps = [
"//br/pkg/config",
"//br/pkg/glue",
"//br/pkg/utils",
"//pkg/util",
"@com_github_aws_aws_sdk_go//aws",
"@com_github_aws_aws_sdk_go//aws/awserr",
Expand Down
3 changes: 2 additions & 1 deletion br/pkg/aws/ebs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/pingcap/log"
"github.com/pingcap/tidb/br/pkg/config"
"github.com/pingcap/tidb/br/pkg/glue"
"github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/pkg/util"
"go.uber.org/atomic"
"go.uber.org/zap"
Expand Down Expand Up @@ -252,7 +253,7 @@ func (e *EC2Session) WaitSnapshotsCreated(snapIDMap map[string]string, progress
log.Info("snapshot completed", zap.String("id", *s.SnapshotId))
totalVolumeSize += *s.VolumeSize
} else if *s.State == ec2.SnapshotStateError {
log.Error("snapshot failed", zap.String("id", *s.SnapshotId), zap.String("error", (*s.StateMessage)))
log.Error("snapshot failed", zap.String("id", *s.SnapshotId), zap.String("error", utils.GetOrZero(s.StateMessage)))
return 0, errors.Errorf("snapshot %s failed", *s.SnapshotId)
} else {
log.Debug("snapshot creating...", zap.Stringer("snap", s))
Expand Down
19 changes: 19 additions & 0 deletions br/pkg/aws/ebs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,25 @@ func TestWaitSnapshotsCreated(t *testing.T) {
expectedSize: 0,
expectErr: true,
},
{
desc: "snapshot failed w/out state message",
snapshotsOutput: ec2.DescribeSnapshotsOutput{
Snapshots: []*ec2.Snapshot{
{
SnapshotId: awsapi.String("snap-1"),
VolumeSize: awsapi.Int64(1),
State: awsapi.String(ec2.SnapshotStateCompleted),
},
{
SnapshotId: awsapi.String("snap-2"),
State: awsapi.String(ec2.SnapshotStateError),
StateMessage: nil,
},
},
},
expectedSize: 0,
expectErr: true,
},
{
desc: "snapshots pending",
snapshotsOutput: ec2.DescribeSnapshotsOutput{
Expand Down
8 changes: 3 additions & 5 deletions br/pkg/backup/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -809,9 +809,7 @@ func BuildBackupRangeAndInitSchema(
return nil, nil, nil, nil
}
return ranges, NewBackupSchemas(func(storage kv.Storage, fn func(*model.DBInfo, *model.TableInfo)) error {
return BuildBackupSchemas(storage, tableFilter, backupTS, isFullBackup, func(dbInfo *model.DBInfo, tableInfo *model.TableInfo) {
fn(dbInfo, tableInfo)
})
return BuildBackupSchemas(storage, tableFilter, backupTS, isFullBackup, fn)
}, schemasNum), policies, nil
}

Expand Down Expand Up @@ -1235,9 +1233,9 @@ func (bc *Client) OnBackupResponse(
return txnlock.NewLock(lockErr), nil
}
}
res := errContext.HandleIgnorableError(errPb, storeID)
res := utils.HandleBackupError(errPb, storeID, errContext)
switch res.Strategy {
case utils.GiveUpStrategy:
case utils.StrategyGiveUp:
errMsg := res.Reason
if len(errMsg) <= 0 {
errMsg = errPb.Msg
Expand Down
1 change: 1 addition & 0 deletions br/pkg/errors/errors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ var (
ErrKVClusterIDMismatch = errors.Normalize("tikv cluster ID mismatch", errors.RFCCodeText("BR:KV:ErrKVClusterIDMismatch"))
ErrKVNotLeader = errors.Normalize("not leader", errors.RFCCodeText("BR:KV:ErrKVNotLeader"))
ErrKVNotTiKV = errors.Normalize("storage is not tikv", errors.RFCCodeText("BR:KV:ErrNotTiKVStorage"))
ErrKVDiskFull = errors.Normalize("disk is full", errors.RFCCodeText("BR:KV:ErrKVDiskFull"))

// ErrKVEpochNotMatch is the error raised when ingestion failed with "epoch
// not match". This error is retryable.
Expand Down
2 changes: 1 addition & 1 deletion br/pkg/gluetidb/glue.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ func (gs *tidbSession) CreateDatabase(ctx context.Context, schema *model.DBInfo)

// CreatePlacementPolicy implements glue.Session.
func (gs *tidbSession) CreatePlacementPolicy(ctx context.Context, policy *model.PolicyInfo) error {
d := domain.GetDomain(gs.se).DDL()
d := domain.GetDomain(gs.se).DDLExecutor()
gs.se.SetValue(sessionctx.QueryString, gs.showCreatePlacementPolicy(policy))
// the default behaviour is ignoring duplicated policy during restore.
return d.CreatePlacementPolicyWithInfo(gs.se, policy, ddl.OnExistIgnore)
Expand Down
18 changes: 9 additions & 9 deletions br/pkg/metautil/load_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,9 @@ func TestLoadBackupMeta(t *testing.T) {
mockDB := model.DBInfo{
ID: 1,
Name: dbName,
Tables: []*model.TableInfo{
mockTbl,
},
}
mockDB.Deprecated.Tables = []*model.TableInfo{
mockTbl,
}
dbBytes, err := json.Marshal(mockDB)
require.NoError(t, err)
Expand Down Expand Up @@ -140,9 +140,9 @@ func TestLoadBackupMetaPartionTable(t *testing.T) {
mockDB := model.DBInfo{
ID: 1,
Name: dbName,
Tables: []*model.TableInfo{
mockTbl,
},
}
mockDB.Deprecated.Tables = []*model.TableInfo{
mockTbl,
}
dbBytes, err := json.Marshal(mockDB)
require.NoError(t, err)
Expand Down Expand Up @@ -249,9 +249,9 @@ func buildBenchmarkBackupmeta(b *testing.B, dbName string, tableCount, fileCount
mockDB := model.DBInfo{
ID: 1,
Name: model.NewCIStr(dbName),
Tables: []*model.TableInfo{
mockTbl,
},
}
mockDB.Deprecated.Tables = []*model.TableInfo{
mockTbl,
}
dbBytes, err := json.Marshal(mockDB)
require.NoError(b, err)
Expand Down
6 changes: 5 additions & 1 deletion br/pkg/restore/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,13 @@ go_library(
deps = [
"//br/pkg/conn",
"//br/pkg/conn/util",
"//br/pkg/errors",
"//br/pkg/logutil",
"//br/pkg/pdutil",
"//br/pkg/utils",
"//pkg/domain",
"//pkg/kv",
"//pkg/meta",
"//pkg/parser/model",
"//pkg/util",
"@com_github_go_sql_driver_mysql//:mysql",
Expand Down Expand Up @@ -49,8 +52,9 @@ go_test(
"//br/pkg/mock",
"//br/pkg/pdutil",
"//br/pkg/utiltest",
"//pkg/infoschema",
"//pkg/kv",
"//pkg/parser/model",
"//pkg/session",
"@com_github_coreos_go_semver//semver",
"@com_github_pingcap_failpoint//:failpoint",
"@com_github_pingcap_kvproto//pkg/import_sstpb",
Expand Down
7 changes: 7 additions & 0 deletions br/pkg/restore/ingestrec/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,12 @@ go_library(
importpath = "github.com/pingcap/tidb/br/pkg/restore/ingestrec",
visibility = ["//visibility:public"],
deps = [
"//pkg/infoschema",
"//pkg/parser/model",
"//pkg/types",
"@com_github_pingcap_errors//:errors",
"@com_github_pingcap_log//:log",
"@org_uber_go_zap//:zap",
],
)

Expand All @@ -20,7 +23,11 @@ go_test(
shard_count = 3,
deps = [
":ingestrec",
"//pkg/kv",
"//pkg/meta",
"//pkg/parser/model",
"//pkg/session",
"//pkg/store/mockstore",
"@com_github_pingcap_errors//:errors",
"@com_github_stretchr_testify//require",
],
Expand Down
Loading

0 comments on commit 4424521

Please sign in to comment.