Merge branch 'master' into lightning_check

br/pkg/lightning/backend/local/local_unix.go

@@ -25,8 +25,8 @@ import (
 )
 
 const (
-	// mininum max open files value
-	minRLimit = 1024
+	// maximum max open files value
+	maxRLimit = 1000000
 )
 
 func GetSystemRLimit() (Rlim_t, error) {
@@ -39,8 +39,8 @@ func GetSystemRLimit() (Rlim_t, error) {
 // In Local-backend, we need to read and write a lot of L0 SST files, so we need
 // to check system max open files limit.
 func VerifyRLimit(estimateMaxFiles Rlim_t) error {
-	if estimateMaxFiles < minRLimit {
-		estimateMaxFiles = minRLimit
+	if estimateMaxFiles > maxRLimit {
+		estimateMaxFiles = maxRLimit
 	}
 	var rLimit syscall.Rlimit
 	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)

br/pkg/lightning/lightning.go

@@ -667,7 +667,10 @@ func checkSystemRequirement(cfg *config.Config, dbsMeta []*mydump.MDDatabaseMeta
 
 		// region-concurrency: number of LocalWriters writing SST files.
 		// 2*totalSize/memCacheSize: number of Pebble MemCache files.
-		estimateMaxFiles := local.Rlim_t(cfg.App.RegionConcurrency) + local.Rlim_t(topNTotalSize)/local.Rlim_t(cfg.TikvImporter.EngineMemCacheSize)*2
+		maxDBFiles := topNTotalSize / int64(cfg.TikvImporter.LocalWriterMemCacheSize) * 2
+		// the pebble db and all import routine need upto maxDBFiles fds for read and write.
+		maxOpenDBFiles := maxDBFiles * (1 + int64(cfg.TikvImporter.RangeConcurrency))
+		estimateMaxFiles := local.Rlim_t(cfg.App.RegionConcurrency) + local.Rlim_t(maxOpenDBFiles)
 		if err := local.VerifyRLimit(estimateMaxFiles); err != nil {
 			return err
 		}

br/pkg/lightning/lightning_test.go

@@ -447,7 +447,8 @@ func (s *lightningServerSuite) TestCheckSystemRequirement(c *C) {
 	cfg.App.CheckRequirements = true
 	cfg.App.TableConcurrency = 4
 	cfg.TikvImporter.Backend = config.BackendLocal
-	cfg.TikvImporter.EngineMemCacheSize = 512 * units.MiB
+	cfg.TikvImporter.LocalWriterMemCacheSize = 128 * units.MiB
+	cfg.TikvImporter.RangeConcurrency = 16
 
 	dbMetas := []*mydump.MDDatabaseMeta{
 		{
@@ -485,23 +486,22 @@ func (s *lightningServerSuite) TestCheckSystemRequirement(c *C) {
 		},
 	}
 
-	// with max open files 1024, the max table size will be: 65536MB
-	err := failpoint.Enable("github.com/pingcap/tidb/br/pkg/lightning/backend/local/GetRlimitValue", "return(2049)")
+	err := failpoint.Enable("github.com/pingcap/tidb/br/pkg/lightning/backend/local/GetRlimitValue", "return(139439)")
 	c.Assert(err, IsNil)
 	err = failpoint.Enable("github.com/pingcap/tidb/br/pkg/lightning/backend/local/SetRlimitError", "return(true)")
 	c.Assert(err, IsNil)
 	defer func() {
 		_ = failpoint.Disable("github.com/pingcap/tidb/br/pkg/lightning/backend/local/SetRlimitError")
 	}()
-	// with this dbMetas, the estimated fds will be 2050, so should return error
+	// with this dbMetas, the estimated fds will be 139440, so should return error
 	err = checkSystemRequirement(cfg, dbMetas)
 	c.Assert(err, NotNil)
 
 	err = failpoint.Disable("github.com/pingcap/tidb/br/pkg/lightning/backend/local/GetRlimitValue")
 	c.Assert(err, IsNil)
 
 	// the min rlimit should be bigger than the default min value (16384)
-	err = failpoint.Enable("github.com/pingcap/tidb/br/pkg/lightning/backend/local/GetRlimitValue", "return(8200)")
+	err = failpoint.Enable("github.com/pingcap/tidb/br/pkg/lightning/backend/local/GetRlimitValue", "return(139440)")
 	defer func() {
 		_ = failpoint.Disable("github.com/pingcap/tidb/br/pkg/lightning/backend/local/GetRlimitValue")
 	}()

docs/design/2020-06-24-placement-rules-in-sql.md

@@ -318,7 +318,7 @@ This is what a label rule may look like:
 }
 ```
 
-It connects the table name `db1/tb` with the key range.
+It connects the table name `db1/tb1` with the key range.
 
 Now you need to connect the label with the database / table / partition name in the placement rules.