From 1f1f33e8a405565874e5d25bf0a36b433d42a2fd Mon Sep 17 00:00:00 2001 From: hillium Date: Fri, 27 Oct 2023 12:31:30 +0800 Subject: [PATCH 1/2] use the default value from flags Signed-off-by: hillium --- br/pkg/task/backup.go | 16 ++++++++ br/pkg/task/common.go | 10 +++++ br/pkg/task/common_test.go | 80 ++++++++++++++++++++++++++++++++++++++ br/pkg/task/restore.go | 17 ++++++++ pkg/executor/brie.go | 30 +++++++------- 5 files changed, 137 insertions(+), 16 deletions(-) diff --git a/br/pkg/task/backup.go b/br/pkg/task/backup.go index 4b49ed0e86700..a66dae921b00b 100644 --- a/br/pkg/task/backup.go +++ b/br/pkg/task/backup.go @@ -41,6 +41,7 @@ import ( "github.com/spf13/pflag" "github.com/tikv/client-go/v2/oracle" kvutil "github.com/tikv/client-go/v2/util" + "go.uber.org/multierr" "go.uber.org/zap" ) @@ -784,6 +785,21 @@ func ParseTSString(ts string, tzCheck bool) (uint64, error) { return oracle.GoTimeToTS(t1), nil } +func DefaultBackupConfig() BackupConfig { + fs := pflag.NewFlagSet("dummy", pflag.ContinueOnError) + DefineCommonFlags(fs) + DefineBackupFlags(fs) + cfg := BackupConfig{} + err := multierr.Combine( + cfg.ParseFromFlags(fs), + cfg.Config.ParseFromFlags(fs), + ) + if err != nil { + log.Panic("infallible operation failed.", zap.Error(err)) + } + return cfg +} + func parseCompressionType(s string) (backuppb.CompressionType, error) { var ct backuppb.CompressionType switch s { diff --git a/br/pkg/task/common.go b/br/pkg/task/common.go index 1aec04510bbc6..509b02e335b73 100644 --- a/br/pkg/task/common.go +++ b/br/pkg/task/common.go @@ -331,6 +331,16 @@ func HiddenFlagsForStream(flags *pflag.FlagSet) { storage.HiddenFlagsForStream(flags) } +func DefaultConfig() Config { + fs := pflag.NewFlagSet("dummy", pflag.ContinueOnError) + DefineCommonFlags(fs) + cfg := Config{} + if err := cfg.ParseFromFlags(fs); err != nil { + log.Panic("infallible operation failed.", zap.Error(err)) + } + return cfg +} + // DefineDatabaseFlags defines the required --db flag for `db` subcommand. func DefineDatabaseFlags(command *cobra.Command) { command.Flags().String(flagDatabase, "", "database name") diff --git a/br/pkg/task/common_test.go b/br/pkg/task/common_test.go index 0ee14c52f02ea..a5d43d3f2e7a9 100644 --- a/br/pkg/task/common_test.go +++ b/br/pkg/task/common_test.go @@ -9,7 +9,10 @@ import ( backup "github.com/pingcap/kvproto/pkg/brpb" "github.com/pingcap/kvproto/pkg/encryptionpb" + "github.com/pingcap/tidb/br/pkg/storage" + "github.com/pingcap/tidb/br/pkg/utils" "github.com/pingcap/tidb/pkg/config" + filter "github.com/pingcap/tidb/pkg/util/table-filter" "github.com/spf13/pflag" "github.com/stretchr/testify/require" ) @@ -157,3 +160,80 @@ func TestCheckCipherKey(t *testing.T) { } } } + +func must[T any](t T, err error) T { + if err != nil { + panic(err) + } + return t +} + +func expectedDefaultConfig() Config { + return Config{ + BackendOptions: storage.BackendOptions{S3: storage.S3BackendOptions{ForcePathStyle: true}}, + PD: []string{"127.0.0.1:2379"}, + ChecksumConcurrency: 4, + Checksum: true, + SendCreds: true, + CheckRequirements: true, + FilterStr: []string(nil), + TableFilter: filter.CaseInsensitive(must(filter.Parse([]string{"*.*"}))), + Schemas: map[string]struct{}{}, + Tables: map[string]struct{}{}, + SwitchModeInterval: 300000000000, + GRPCKeepaliveTime: 10000000000, + GRPCKeepaliveTimeout: 3000000000, + CipherInfo: backup.CipherInfo{CipherType: 1}, + MetadataDownloadBatchSize: 0x80, + } +} + +func expectedDefaultBackupConfig() BackupConfig { + return BackupConfig{ + Config: expectedDefaultConfig(), + GCTTL: utils.DefaultBRGCSafePointTTL, + CompressionConfig: CompressionConfig{ + CompressionType: backup.CompressionType_ZSTD, + }, + IgnoreStats: true, + UseBackupMetaV2: true, + UseCheckpoint: true, + } +} + +func expectedDefaultRestoreConfig() RestoreConfig { + defaultConfig := expectedDefaultConfig() + defaultConfig.Concurrency = defaultRestoreConcurrency + return RestoreConfig{ + Config: defaultConfig, + RestoreCommonConfig: RestoreCommonConfig{Online: false, + MergeSmallRegionSizeBytes: 0x6000000, + MergeSmallRegionKeyCount: 0xea600, + WithSysTable: false, + ResetSysUsers: []string{"cloud_admin", "root"}}, + NoSchema: false, + PDConcurrency: 0x1, + BatchFlushInterval: 16000000000, + DdlBatchSize: 0x80, + WithPlacementPolicy: "STRICT", + UseCheckpoint: true, + } +} + +func TestDefault(t *testing.T) { + def := DefaultConfig() + defaultConfig := expectedDefaultConfig() + require.Equal(t, defaultConfig, def) +} + +func TestDefaultBackup(t *testing.T) { + def := DefaultBackupConfig() + defaultConfig := expectedDefaultBackupConfig() + require.Equal(t, defaultConfig, def) +} + +func TestDefaultRestore(t *testing.T) { + def := DefaultRestoreConfig() + defaultConfig := expectedDefaultRestoreConfig() + require.Equal(t, defaultConfig, def) +} diff --git a/br/pkg/task/restore.go b/br/pkg/task/restore.go index b70788c821ef8..f51f3cacd8a81 100644 --- a/br/pkg/task/restore.go +++ b/br/pkg/task/restore.go @@ -569,6 +569,23 @@ func removeCheckpointDataForLogRestore(ctx context.Context, storageName string, return errors.Trace(checkpoint.RemoveCheckpointDataForLogRestore(ctx, s, taskName, clusterID)) } +func DefaultRestoreConfig() RestoreConfig { + fs := pflag.NewFlagSet("dummy", pflag.ContinueOnError) + DefineCommonFlags(fs) + DefineRestoreFlags(fs) + cfg := RestoreConfig{} + err := multierr.Combine( + cfg.ParseFromFlags(fs), + cfg.RestoreCommonConfig.ParseFromFlags(fs), + cfg.Config.ParseFromFlags(fs), + ) + if err != nil { + log.Panic("infallible failed.", zap.Error(err)) + } + + return cfg +} + // RunRestore starts a restore task inside the current goroutine. func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConfig) error { etcdCLI, err := dialEtcdWithCfg(c, cfg.Config) diff --git a/pkg/executor/brie.go b/pkg/executor/brie.go index e40493c3f44e2..953d725225a20 100644 --- a/pkg/executor/brie.go +++ b/pkg/executor/brie.go @@ -277,21 +277,15 @@ func (b *executorBuilder) buildBRIE(s *ast.BRIEStmt, schema *expression.Schema) } tidbCfg := config.GetGlobalConfig() - cfg := task.Config{ - TLS: task.TLSConfig{ - CA: tidbCfg.Security.ClusterSSLCA, - Cert: tidbCfg.Security.ClusterSSLCert, - Key: tidbCfg.Security.ClusterSSLKey, - }, - PD: strings.Split(tidbCfg.Path, ","), - Concurrency: 4, - Checksum: true, - SendCreds: true, - LogProgress: true, - CipherInfo: backuppb.CipherInfo{ - CipherType: encryptionpb.EncryptionMethod_PLAINTEXT, - }, + tlsCfg := task.TLSConfig{ + CA: tidbCfg.Security.ClusterSSLCA, + Cert: tidbCfg.Security.ClusterSSLCert, + Key: tidbCfg.Security.ClusterSSLKey, } + pds := strings.Split(tidbCfg.Path, ",") + cfg := task.DefaultConfig() + cfg.PD = pds + cfg.TLS = tlsCfg storageURL, err := storage.ParseRawURL(s.Storage) if err != nil { @@ -364,7 +358,9 @@ func (b *executorBuilder) buildBRIE(s *ast.BRIEStmt, schema *expression.Schema) switch s.Kind { case ast.BRIEKindBackup: - e.backupCfg = &task.BackupConfig{Config: cfg} + bcfg := task.DefaultBackupConfig() + bcfg.Config = cfg + e.backupCfg = &bcfg for _, opt := range s.Options { switch opt.Tp { @@ -392,7 +388,9 @@ func (b *executorBuilder) buildBRIE(s *ast.BRIEStmt, schema *expression.Schema) } case ast.BRIEKindRestore: - e.restoreCfg = &task.RestoreConfig{Config: cfg} + rcfg := task.DefaultRestoreConfig() + rcfg.Config = cfg + e.restoreCfg = &rcfg for _, opt := range s.Options { if opt.Tp == ast.BRIEOptionOnline { e.restoreCfg.Online = opt.UintValue != 0 From 438361fa9c86ae7fa6d8381c60f81cfa7670161b Mon Sep 17 00:00:00 2001 From: hillium Date: Fri, 27 Oct 2023 17:07:23 +0800 Subject: [PATCH 2/2] make bazel_prepare Signed-off-by: hillium --- br/pkg/task/BUILD.bazel | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/br/pkg/task/BUILD.bazel b/br/pkg/task/BUILD.bazel index 2013b2f3997bf..cac5b33d9d0bc 100644 --- a/br/pkg/task/BUILD.bazel +++ b/br/pkg/task/BUILD.bazel @@ -99,7 +99,7 @@ go_test( ], embed = [":task"], flaky = True, - shard_count = 18, + shard_count = 21, deps = [ "//br/pkg/conn", "//br/pkg/errors", @@ -112,6 +112,7 @@ go_test( "//pkg/parser/model", "//pkg/statistics/handle/util", "//pkg/tablecodec", + "//pkg/util/table-filter", "@com_github_golang_protobuf//proto", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_kvproto//pkg/brpb",