Complete lightning env initlize procdeure (#4)

1. Get max open file limition for lightning, default 1024
2. Set light sorted path：
   2.1 If sysVar DataDir is not a start with / data dir path, then set path to /tmp/lightning
   2.2 otherwise set path to DataDir + "lightning"
   Check whether the lightning sorted path is exist and a dir, if yes then keep use it.
   otherwise create new one.
3, Set the lightning dir quota, default 10G

Add one unit test

ddl/ddl.go

@@ -30,6 +30,7 @@ import (
 	"github.com/pingcap/errors"
 	"github.com/pingcap/failpoint"
 	"github.com/pingcap/tidb/config"
+	lit "github.com/pingcap/tidb/ddl/lightning"
 	"github.com/pingcap/tidb/ddl/util"
 	"github.com/pingcap/tidb/infoschema"
 	"github.com/pingcap/tidb/kv"
@@ -431,6 +432,12 @@ func (d *ddl) Start(ctxPool *pools.ResourcePool) error {
 	// Start some background routine to manage TiFlash replica.
 	d.wg.Run(d.PollTiFlashRoutine)
 
+	// Init Lighting Global environment. Once met error then the
+	err := lit.InitGolbalLightningBackendEnv()
+	if err != nil {
+		logutil.BgLogger().Warn("Lightning initialize failed ", zap.String("Error", err.Error()))
+	}
+
 	return nil
 }
 

ddl/index_lightning.go

@@ -24,7 +24,9 @@ import (
 // isAllowFastDDL is used to
 func isAllowFastDDL(storage kv.Storage) bool {
 	sessCtx := newContext(storage)
-	if sessCtx.GetSessionVars().TiDBFastDDL {
+	// Only when both TiDBFastDDL is set to on and Lightning env is inited successful,
+	// the add index could choose lightning path to do backfill procedure.   
+	if sessCtx.GetSessionVars().TiDBFastDDL && lit.GlobalLightningEnv.IsInited {
 		return true
 	} else {
 		return false

ddl/lightning/env.go

@@ -14,13 +14,15 @@
 package lightning
 
 import (
-	"fmt"
+	"os"
 	"syscall"
 
 	lcom "github.com/pingcap/tidb/br/pkg/lightning/common"
 	"github.com/pingcap/tidb/br/pkg/lightning/log"
 	tidbcfg "github.com/pingcap/tidb/config"
+	"github.com/pingcap/tidb/sessionctx/variable"
 	"github.com/pingcap/tidb/util/logutil"
+	"go.uber.org/zap"
 )
 
 const (
@@ -30,6 +32,7 @@ const (
 	_gb             = 1024 * _mb
 	flush_size      = 8 * _mb
 	RegionSizeStats = "RegionSizeStats"
+	maxMemLimation  = 128 * _gb
 )
 
 type ClusterInfo struct {
@@ -46,43 +49,87 @@ type LightningEnv struct {
 	BackendCache  *BackendCache
 	EngineManager *EngineManager
 	diskQuota     int64
+	IsInited      bool
 }
 
 var GlobalLightningEnv LightningEnv
 
 func init() {
 	GlobalLightningEnv.limit = 1024           // Init a default value 1024 for limit.
-	GlobalLightningEnv.diskQuota = 1024 * _mb // default disk quota set to 1 GB
+	GlobalLightningEnv.diskQuota = 10 * _gb // default disk quota set to 10 GB
 	var rLimit syscall.Rlimit
 	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
 	if err != nil {
-		logutil.BgLogger().Warn(fmt.Sprintf("GetSystemRLimit err:%s;use default 1024.", err.Error()))
+		logutil.BgLogger().Warn("Lightning: GetSystemRLimit err:", zap.String("OS error:", err.Error()), zap.String("Default: ", "1024."))
 	} else {
 		GlobalLightningEnv.limit = int64(rLimit.Cur)
 	}
+	GlobalLightningEnv.IsInited = false
 }
 
-func InitGolbalLightningBackendEnv() {
+func InitGolbalLightningBackendEnv() (err error) {
 	cfg := tidbcfg.GetGlobalConfig()
 	GlobalLightningEnv.PdAddr = cfg.AdvertiseAddress
 	GlobalLightningEnv.Port = cfg.Port
 	GlobalLightningEnv.Status = cfg.Status.StatusPort
-	// Todo need to get the specific lightning sort part from config.
-	GlobalLightningEnv.SortPath = ""
+	if err := GlobalLightningEnv.initSortPath(); err != nil {
+		GlobalLightningEnv.IsInited = false
+		log.L().Warn("Lightning init failed: ", zap.String("Os error", err.Error()))
+		return err
+	}
+	if GlobalLightningEnv.IsInited {
+		GlobalLightningEnv.parseDiskQuota()
+	}
 	// Todo need to set Memory limitation, temp set to 128 G
-	GlobalLightningEnv.LitMemRoot.init(128 * _gb)
+	GlobalLightningEnv.LitMemRoot.init(maxMemLimation)
 	log.SetAppLogger(logutil.BgLogger())
-	log.L().Info("Init global lightning backend environment finished.")
-
+	log.L().Info("Lightning: Init global lightning backend environment finished.")
+	GlobalLightningEnv.IsInited = true
+	return nil
 }
 
 func (l *LightningEnv) parseDiskQuota() {
 	sz, err := lcom.GetStorageSize(l.SortPath)
 	if err != nil {
-		fmt.Println(fmt.Sprintf("GetStorageSize err:%s;use default 10G.", err.Error()))
+		log.L().Warn("Lightning: GetStorageSize err:", zap.String("Os error:", err.Error()), zap.String("default size", "10G"))
+		l.diskQuota = 10 * _gb
 		return
 	}
-	if l.diskQuota > int64(sz.Available) {
-		l.diskQuota = int64(sz.Available)
+	l.diskQuota = int64(sz.Available)
+}
+
+// This init SortPath func will clean up the file in dir, the file exist will be keep as it is be
+func (l *LightningEnv) initSortPath() error {
+	shouldCreate := true
+    l.SortPath = GenLightningDataDir()
+	if info, err := os.Stat(l.SortPath); err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+	} else if info.IsDir() {
+		shouldCreate = false
+	}
+
+	if shouldCreate {
+		err := os.Mkdir(l.SortPath, 0o700)
+		if err != nil {
+			return err
+		}
 	}
+	return nil
 }
+
+
+// Generate lightning path in TiDB datadir. 
+func GenLightningDataDir() string {
+	dataDir := variable.GetSysVar(variable.DataDir)
+	var sortPath string
+	// If DataDir is not a dir path, then set lightning to /tmp/lightning
+	if string(dataDir.Value[0]) != "/" {
+		sortPath = "/tmp/lightning"
+	} else {
+		sortPath = dataDir.Value + "lightning"
+	}
+	return sortPath
+}
+

sessionctx/variable/sysvar_test.go

@@ -968,3 +968,28 @@ func TestTiDBQueryLogMaxLen(t *testing.T) {
 	require.Equal(t, val, fmt.Sprintf("%d", expected))
 	require.NoError(t, err)
 }
+
+func TestSetTIDBFastDDL(t *testing.T) {
+	vars := NewSessionVars()
+	mock := NewMockGlobalAccessor4Tests()
+	mock.SessionVars = vars
+	vars.GlobalVarsAccessor = mock
+    fastDDL := GetSysVar(TiDBFastDDL)
+
+	// Default off
+	require.Equal(t, fastDDL.Value, Off)
+
+	// Set to On
+	err := mock.SetGlobalSysVar(TiDBFastDDL, On)
+	require.NoError(t, err)
+	val, err1 := mock.GetGlobalSysVar(TiDBFastDDL)
+	require.NoError(t, err1)
+	require.Equal(t, On, val)
+
+	// Set to off
+	err = mock.SetGlobalSysVar(TiDBFastDDL, Off)
+	require.NoError(t, err)
+	val, err1 = mock.GetGlobalSysVar(TiDBFastDDL)
+	require.NoError(t, err1)
+	require.Equal(t, Off, val)
+}

sessionctx/variable/variable.go

@@ -612,4 +612,4 @@ type GlobalVarAccessor interface {
 	GetTiDBTableValue(name string) (string, error)
 	// SetTiDBTableValue sets a value+comment for the mysql.tidb key 'name'
 	SetTiDBTableValue(name, value, comment string) error
-}
+}

tidb-server/main.go

@@ -34,7 +34,6 @@ import (
 	"github.com/pingcap/tidb/bindinfo"
 	"github.com/pingcap/tidb/config"
 	"github.com/pingcap/tidb/ddl"
-	"github.com/pingcap/tidb/ddl/lightning"
 	"github.com/pingcap/tidb/domain"
 	"github.com/pingcap/tidb/executor"
 	"github.com/pingcap/tidb/kv"
@@ -202,7 +201,6 @@ func main() {
 	printInfo()
 	setupBinlogClient()
 	setupMetrics()
-	lightning.InitGolbalLightningBackendEnv()
 
 	storage, dom := createStoreAndDomain()
 	svr := createServer(storage, dom)