From fc07893960f9348c85c0fdee27d5b4579f3360ad Mon Sep 17 00:00:00 2001
From: kennytm <kennytm@gmail.com>
Date: Mon, 9 Nov 2020 17:51:19 +0800
Subject: [PATCH] config: filter out all system schemas by default

---
 lightning/common/util.go        | 13 +++++++++++++
 lightning/common/util_test.go   | 13 +++++++++++++
 lightning/config/config.go      | 14 ++++++++++++--
 lightning/config/config_test.go | 18 ++++++++++++++++++
 lightning/config/global.go      |  2 +-
 tidb-lightning.toml             |  2 +-
 6 files changed, 58 insertions(+), 4 deletions(-)

diff --git a/lightning/common/util.go b/lightning/common/util.go
index b483b0de3..96f532d0b 100644
--- a/lightning/common/util.go
+++ b/lightning/common/util.go
@@ -317,3 +317,16 @@ type KvPair struct {
 func TableHasAutoRowID(info *model.TableInfo) bool {
 	return !info.PKIsHandle && !info.IsCommonHandle
 }
+
+// StringSliceEqual checks if two string slices are equal.
+func StringSliceEqual(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}
diff --git a/lightning/common/util_test.go b/lightning/common/util_test.go
index c43149d18..907154068 100644
--- a/lightning/common/util_test.go
+++ b/lightning/common/util_test.go
@@ -190,3 +190,16 @@ func (s *utilSuite) TestSQLWithRetry(c *C) {
 
 	c.Assert(mock.ExpectationsWereMet(), IsNil)
 }
+
+func (s *utilSuite) TestStringSliceEqual(c *C) {
+	c.Assert(common.StringSliceEqual(nil, nil), IsTrue)
+	c.Assert(common.StringSliceEqual(nil, []string{}), IsTrue)
+	c.Assert(common.StringSliceEqual(nil, []string{"a"}), IsFalse)
+	c.Assert(common.StringSliceEqual([]string{"a"}, nil), IsFalse)
+	c.Assert(common.StringSliceEqual([]string{"a"}, []string{"a"}), IsTrue)
+	c.Assert(common.StringSliceEqual([]string{"a"}, []string{"b"}), IsFalse)
+	c.Assert(common.StringSliceEqual([]string{"a", "b", "c"}, []string{"a", "b", "c"}), IsTrue)
+	c.Assert(common.StringSliceEqual([]string{"a"}, []string{"a", "b", "c"}), IsFalse)
+	c.Assert(common.StringSliceEqual([]string{"a", "b", "c"}, []string{"a", "b"}), IsFalse)
+	c.Assert(common.StringSliceEqual([]string{"a", "x", "y"}, []string{"a", "y", "x"}), IsFalse)
+}
diff --git a/lightning/config/config.go b/lightning/config/config.go
index 098bea48e..4bbb28467 100644
--- a/lightning/config/config.go
+++ b/lightning/config/config.go
@@ -67,6 +67,16 @@ const (
 var (
 	defaultConfigPaths    = []string{"tidb-lightning.toml", "conf/tidb-lightning.toml"}
 	supportedStorageTypes = []string{"file", "local", "s3", "noop"}
+
+	DefaultFilter = []string{
+		"*.*",
+		"!mysql.*",
+		"!sys.*",
+		"!INFORMATION_SCHEMA.*",
+		"!PERFORMANCE_SCHEMA.*",
+		"!METRICS_SCHEMA.*",
+		"!INSPECTION_SCHEMA.*",
+	}
 )
 
 type DBStore struct {
@@ -344,7 +354,7 @@ func NewConfig() *Config {
 			},
 			StrictFormat:  false,
 			MaxRegionSize: MaxRegionSize,
-			Filter:        []string{"*.*"},
+			Filter:        DefaultFilter,
 		},
 		TikvImporter: TikvImporter{
 			Backend:         BackendImporter,
@@ -547,7 +557,7 @@ func (cfg *Config) Adjust() error {
 	// mydumper.filter and black-white-list cannot co-exist.
 	if cfg.HasLegacyBlackWhiteList() {
 		log.L().Warn("the config `black-white-list` has been deprecated, please replace with `mydumper.filter`")
-		if !(len(cfg.Mydumper.Filter) == 1 && cfg.Mydumper.Filter[0] == "*.*") {
+		if !common.StringSliceEqual(cfg.Mydumper.Filter, DefaultFilter) {
 			return errors.New("invalid config: `mydumper.filter` and `black-white-list` cannot be simultaneously defined")
 		}
 	}
diff --git a/lightning/config/config_test.go b/lightning/config/config_test.go
index 0110fcf2c..8cf67cd21 100644
--- a/lightning/config/config_test.go
+++ b/lightning/config/config_test.go
@@ -612,3 +612,21 @@ func (s *configTestSuite) TestCronEncodeDecode(c *C) {
 	c.Assert(cfg2.LoadFromTOML([]byte(confStr)), IsNil)
 	c.Assert(cfg2.Cron, DeepEquals, cfg.Cron)
 }
+
+func (s *configTestSuite) TestAdjustWithLegacyBlackWhiteList(c *C) {
+	cfg := config.NewConfig()
+	assignMinimalLegalValue(cfg)
+	c.Assert(cfg.Mydumper.Filter, DeepEquals, config.DefaultFilter)
+	c.Assert(cfg.HasLegacyBlackWhiteList(), IsFalse)
+
+	cfg.Mydumper.Filter = []string{"test.*"}
+	c.Assert(cfg.Adjust(), IsNil)
+	c.Assert(cfg.HasLegacyBlackWhiteList(), IsFalse)
+
+	cfg.BWList.DoDBs = []string{"test"}
+	c.Assert(cfg.Adjust(), ErrorMatches, "invalid config: `mydumper\\.filter` and `black-white-list` cannot be simultaneously defined")
+
+	cfg.Mydumper.Filter = config.DefaultFilter
+	c.Assert(cfg.Adjust(), IsNil)
+	c.Assert(cfg.HasLegacyBlackWhiteList(), IsTrue)
+}
diff --git a/lightning/config/global.go b/lightning/config/global.go
index 76ded7d30..eb1075fda 100644
--- a/lightning/config/global.go
+++ b/lightning/config/global.go
@@ -99,7 +99,7 @@ func NewGlobalConfig() *GlobalConfig {
 			LogLevel:   "error",
 		},
 		Mydumper: GlobalMydumper{
-			Filter: []string{"*.*"},
+			Filter: DefaultFilter,
 		},
 		TikvImporter: GlobalImporter{
 			Backend: "importer",
diff --git a/tidb-lightning.toml b/tidb-lightning.toml
index d7d5d1abb..2150581bc 100644
--- a/tidb-lightning.toml
+++ b/tidb-lightning.toml
@@ -144,7 +144,7 @@ strict-format = false
 #default-file-rules = false
 
 # only import tables if the wildcard rules are matched. See documention for details.
-filter = ['*.*']
+filter = ['*.*', '!mysql.*', '!sys.*', '!INFORMATION_SCHEMA.*', '!PERFORMANCE_SCHEMA.*', '!METRICS_SCHEMA.*', '!INSPECTION_SCHEMA.*']
 
 # CSV files are imported according to MySQL's LOAD DATA INFILE rules.
 [mydumper.csv]