config: filter out all system schemas by default (#459)

lightning/common/util.go

@@ -317,3 +317,16 @@ type KvPair struct {
 func TableHasAutoRowID(info *model.TableInfo) bool {
 	return !info.PKIsHandle && !info.IsCommonHandle
 }
+
+// StringSliceEqual checks if two string slices are equal.
+func StringSliceEqual(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}

lightning/common/util_test.go

@@ -190,3 +190,16 @@ func (s *utilSuite) TestSQLWithRetry(c *C) {
 
 	c.Assert(mock.ExpectationsWereMet(), IsNil)
 }
+
+func (s *utilSuite) TestStringSliceEqual(c *C) {
+	c.Assert(common.StringSliceEqual(nil, nil), IsTrue)
+	c.Assert(common.StringSliceEqual(nil, []string{}), IsTrue)
+	c.Assert(common.StringSliceEqual(nil, []string{"a"}), IsFalse)
+	c.Assert(common.StringSliceEqual([]string{"a"}, nil), IsFalse)
+	c.Assert(common.StringSliceEqual([]string{"a"}, []string{"a"}), IsTrue)
+	c.Assert(common.StringSliceEqual([]string{"a"}, []string{"b"}), IsFalse)
+	c.Assert(common.StringSliceEqual([]string{"a", "b", "c"}, []string{"a", "b", "c"}), IsTrue)
+	c.Assert(common.StringSliceEqual([]string{"a"}, []string{"a", "b", "c"}), IsFalse)
+	c.Assert(common.StringSliceEqual([]string{"a", "b", "c"}, []string{"a", "b"}), IsFalse)
+	c.Assert(common.StringSliceEqual([]string{"a", "x", "y"}, []string{"a", "y", "x"}), IsFalse)
+}

lightning/config/config.go

@@ -67,6 +67,16 @@ const (
 var (
 	defaultConfigPaths    = []string{"tidb-lightning.toml", "conf/tidb-lightning.toml"}
 	supportedStorageTypes = []string{"file", "local", "s3", "noop"}
+
+	DefaultFilter = []string{
+		"*.*",
+		"!mysql.*",
+		"!sys.*",
+		"!INFORMATION_SCHEMA.*",
+		"!PERFORMANCE_SCHEMA.*",
+		"!METRICS_SCHEMA.*",
+		"!INSPECTION_SCHEMA.*",
+	}
 )
 
 type DBStore struct {
@@ -344,7 +354,7 @@ func NewConfig() *Config {
 			},
 			StrictFormat:  false,
 			MaxRegionSize: MaxRegionSize,
-			Filter:        []string{"*.*"},
+			Filter:        DefaultFilter,
 		},
 		TikvImporter: TikvImporter{
 			Backend:         BackendImporter,
@@ -547,7 +557,7 @@ func (cfg *Config) Adjust() error {
 	// mydumper.filter and black-white-list cannot co-exist.
 	if cfg.HasLegacyBlackWhiteList() {
 		log.L().Warn("the config `black-white-list` has been deprecated, please replace with `mydumper.filter`")
-		if !(len(cfg.Mydumper.Filter) == 1 && cfg.Mydumper.Filter[0] == "*.*") {
+		if !common.StringSliceEqual(cfg.Mydumper.Filter, DefaultFilter) {
 			return errors.New("invalid config: `mydumper.filter` and `black-white-list` cannot be simultaneously defined")
 		}
 	}

lightning/config/config_test.go

@@ -612,3 +612,21 @@ func (s *configTestSuite) TestCronEncodeDecode(c *C) {
 	c.Assert(cfg2.LoadFromTOML([]byte(confStr)), IsNil)
 	c.Assert(cfg2.Cron, DeepEquals, cfg.Cron)
 }
+
+func (s *configTestSuite) TestAdjustWithLegacyBlackWhiteList(c *C) {
+	cfg := config.NewConfig()
+	assignMinimalLegalValue(cfg)
+	c.Assert(cfg.Mydumper.Filter, DeepEquals, config.DefaultFilter)
+	c.Assert(cfg.HasLegacyBlackWhiteList(), IsFalse)
+
+	cfg.Mydumper.Filter = []string{"test.*"}
+	c.Assert(cfg.Adjust(), IsNil)
+	c.Assert(cfg.HasLegacyBlackWhiteList(), IsFalse)
+
+	cfg.BWList.DoDBs = []string{"test"}
+	c.Assert(cfg.Adjust(), ErrorMatches, "invalid config: `mydumper\\.filter` and `black-white-list` cannot be simultaneously defined")
+
+	cfg.Mydumper.Filter = config.DefaultFilter
+	c.Assert(cfg.Adjust(), IsNil)
+	c.Assert(cfg.HasLegacyBlackWhiteList(), IsTrue)
+}

lightning/config/global.go

@@ -99,7 +99,7 @@ func NewGlobalConfig() *GlobalConfig {
 			LogLevel:   "error",
 		},
 		Mydumper: GlobalMydumper{
-			Filter: []string{"*.*"},
+			Filter: DefaultFilter,
 		},
 		TikvImporter: GlobalImporter{
 			Backend: "importer",

tidb-lightning.toml

@@ -144,7 +144,7 @@ strict-format = false
 #default-file-rules = false
 
 # only import tables if the wildcard rules are matched. See documention for details.
-filter = ['*.*']
+filter = ['*.*', '!mysql.*', '!sys.*', '!INFORMATION_SCHEMA.*', '!PERFORMANCE_SCHEMA.*', '!METRICS_SCHEMA.*', '!INSPECTION_SCHEMA.*']
 
 # CSV files are imported according to MySQL's LOAD DATA INFILE rules.
 [mydumper.csv]