util, types: don't let SPM be affected by charset (#23161)

bindinfo/handle.go

@@ -25,7 +25,6 @@ import (
 
 	"github.com/pingcap/parser"
 	"github.com/pingcap/parser/ast"
-	"github.com/pingcap/parser/format"
 	"github.com/pingcap/parser/mysql"
 	"github.com/pingcap/parser/terror"
 	"github.com/pingcap/tidb/metrics"
@@ -682,14 +681,10 @@ func GenerateBindSQL(ctx context.Context, stmtNode ast.StmtNode, planHint string
 	// We need to evolve plan based on the current sql, not the original sql which may have different parameters.
 	// So here we would remove the hint and inject the current best plan hint.
 	hint.BindHint(stmtNode, &hint.HintsSet{})
-	var sb strings.Builder
-	restoreCtx := format.NewRestoreCtx(format.DefaultRestoreFlags, &sb)
-	restoreCtx.DefaultDB = defaultDB
-	err := stmtNode.Restore(restoreCtx)
-	if err != nil {
-		logutil.Logger(ctx).Debug("[sql-bind] restore SQL failed when generating bind SQL", zap.Error(err))
+	bindSQL := utilparser.RestoreWithDefaultDB(stmtNode, defaultDB)
+	if bindSQL == "" {
+		return ""
 	}
-	bindSQL := sb.String()
 	switch n := stmtNode.(type) {
 	case *ast.DeleteStmt:
 		deleteIdx := strings.Index(bindSQL, "DELETE")

go.mod

@@ -46,7 +46,7 @@ require (
 	github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989
 	github.com/pingcap/kvproto v0.0.0-20210308075244-560097d1309b
 	github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8
-	github.com/pingcap/parser v0.0.0-20210328063857-e44ba053d8bb
+	github.com/pingcap/parser v0.0.0-20210421190550-451a84cf120a
 	github.com/pingcap/sysutil v0.0.0-20201130064824-f0c8aa6a6966
 	github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible
 	github.com/pingcap/tipb v0.0.0-20200618092958-4fad48b4c8c3

go.sum

@@ -384,8 +384,8 @@ github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIf
 github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8 h1:M+DNpOu/I3uDmwee6vcnoPd6GgSMqND4gxvDQ/W584U=
 github.com/pingcap/log v0.0.0-20201112100606-8f1e84a3abc8/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
-github.com/pingcap/parser v0.0.0-20210328063857-e44ba053d8bb h1:7cytk8JDmvowyPv/206e2/OeGc6byGQSN9xfz0a4xQY=
-github.com/pingcap/parser v0.0.0-20210328063857-e44ba053d8bb/go.mod h1:GbEr2PgY72/4XqPZzmzstlOU/+il/wrjeTNFs6ihsSE=
+github.com/pingcap/parser v0.0.0-20210421190550-451a84cf120a h1:przqXMJBDMD2tHLRyFTCsHkfcme4l1m43bEb7tQOi7o=
+github.com/pingcap/parser v0.0.0-20210421190550-451a84cf120a/go.mod h1:GbEr2PgY72/4XqPZzmzstlOU/+il/wrjeTNFs6ihsSE=
 github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
 github.com/pingcap/sysutil v0.0.0-20201130064824-f0c8aa6a6966 h1:JI0wOAb8aQML0vAVLHcxTEEC0VIwrk6gtw3WjbHvJLA=
 github.com/pingcap/sysutil v0.0.0-20201130064824-f0c8aa6a6966/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=

meta/meta.go

@@ -67,6 +67,7 @@ var (
 	mRandomIDPrefix   = "TARID"
 	mBootstrapKey     = []byte("BootstrapKey")
 	mSchemaDiffPrefix = "Diff"
+	mMinorVersionKey  = []byte("MinorVersionKey")
 )
 
 var (
@@ -873,6 +874,18 @@ func (m *Meta) FinishBootstrap(version int64) error {
 	return errors.Trace(err)
 }
 
+// GetMinorVersion returns the minor version of the server which bootstrap the store.
+func (m *Meta) GetMinorVersion() (int64, error) {
+	value, err := m.txn.GetInt64(mMinorVersionKey)
+	return value, errors.Trace(err)
+}
+
+// SetMinorVersion sets the minor version.
+func (m *Meta) SetMinorVersion(version int64) error {
+	err := m.txn.Set(mMinorVersionKey, []byte(fmt.Sprintf("%d", version)))
+	return errors.Trace(err)
+}
+
 // UpdateDDLReorgStartHandle saves the job reorganization latest processed start handle for later resuming.
 func (m *Meta) UpdateDDLReorgStartHandle(job *model.Job, startHandle int64) error {
 	err := m.txn.HSet(mDDLJobReorgKey, m.reorgJobStartHandle(job.ID), []byte(strconv.FormatInt(startHandle, 10)))

session/bootstrap.go

@@ -335,6 +335,10 @@ const (
 	// The variable name in mysql.TiDB table.
 	// It is used for getting the version of the TiDB server which bootstrapped the store.
 	tidbServerVersionVar = "tidb_server_version"
+	// The variable name in mysql.TiDB table.
+	// It is used for getting the version of the TiDB server which bootstrapped the store.
+	// It's only used for release-4.0.
+	tidbServerMinorVersionVar = "tidb_4.0_minor_version"
 	// The variable name in mysql.tidb table and it will be used when we want to know
 	// system timezone.
 	tidbSystemTZ = "system_tz"
@@ -409,6 +413,10 @@ const (
 	// version52 change mysql.stats_histograms cm_sketch column from blob to blob(6291456)
 	// and forces tidb_multi_statement_mode=OFF when tidb_multi_statement_mode=WARN
 	version52 = 52
+
+	// Const for TiDB release-4.0 minorVersion
+	// minorVersion1 restore all SQL bindings.
+	minorVersion1 = 1
 )
 
 var (
@@ -465,6 +473,10 @@ var (
 		upgradeToVer51,
 		upgradeToVer52,
 	}
+
+	upgradeMinorVersion = []func(Session, int64){
+		upgradeToMinorVer1,
+	}
 )
 
 func checkBootstrapped(s Session) (bool, error) {
@@ -522,7 +534,9 @@ func getTiDBVar(s Session, name string) (sVal string, isNull bool, e error) {
 func upgrade(s Session) {
 	ver, err := getBootstrapVersion(s)
 	terror.MustNil(err)
-	if ver >= currentBootstrapVersion {
+	minorVer, err := getMinorVersion(s)
+	terror.MustNil(err)
+	if ver >= currentBootstrapVersion && minorVer >= currentMinorVersion {
 		// It is already bootstrapped/upgraded by a higher version TiDB server.
 		return
 	}
@@ -531,7 +545,16 @@ func upgrade(s Session) {
 		upgrade(s, ver)
 	}
 
-	updateBootstrapVer(s)
+	for _, minorUpgrade := range upgradeMinorVersion {
+		minorUpgrade(s, ver)
+	}
+
+	if ver < currentBootstrapVersion {
+		updateBootstrapVer(s)
+	}
+	if minorVer < currentMinorVersion {
+		updateMinorVer(s)
+	}
 	_, err = s.ExecuteInternal(context.Background(), "COMMIT")
 
 	if err != nil {
@@ -544,13 +567,19 @@ func upgrade(s Session) {
 		if err1 != nil {
 			logutil.BgLogger().Fatal("upgrade failed", zap.Error(err1))
 		}
-		if v >= currentBootstrapVersion {
+		minorV, err2 := getMinorVersion(s)
+		if err2 != nil {
+			logutil.BgLogger().Fatal("upgrade minor failed", zap.Error(err2))
+		}
+		if v >= currentBootstrapVersion && minorV >= currentMinorVersion {
 			// It is already bootstrapped/upgraded by a higher version TiDB server.
 			return
 		}
 		logutil.BgLogger().Fatal("[Upgrade] upgrade failed",
 			zap.Int64("from", ver),
 			zap.Int("to", currentBootstrapVersion),
+			zap.Int64("from minor", ver),
+			zap.Int("to minor", currentMinorVersion),
 			zap.Error(err))
 	}
 }
@@ -1147,10 +1176,7 @@ type bindInfo struct {
 	source     string
 }
 
-func upgradeToVer51(s Session, ver int64) {
-	if ver >= version51 {
-		return
-	}
+func updateGlobalBindings(s Session) {
 	bindMap := make(map[string]bindInfo)
 	h := &bindinfo.BindHandle{}
 	var err error
@@ -1172,7 +1198,7 @@ func upgradeToVer51(s Session, ver int64) {
 			WHERE source != 'builtin'
 			ORDER BY update_time DESC`)
 	if err != nil {
-		logutil.BgLogger().Fatal("upgradeToVer61 error", zap.Error(err))
+		logutil.BgLogger().Fatal("upgradeToVer52 error", zap.Error(err))
 	}
 	defer terror.Call(rs.Close)
 	req := rs.NewChunk()
@@ -1182,7 +1208,7 @@ func upgradeToVer51(s Session, ver int64) {
 	for {
 		err = rs.Next(context.TODO(), req)
 		if err != nil {
-			logutil.BgLogger().Fatal("upgradeToVer61 error", zap.Error(err))
+			logutil.BgLogger().Fatal("upgradeToVer52 error", zap.Error(err))
 		}
 		if req.NumRows() == 0 {
 			break
@@ -1205,6 +1231,13 @@ func upgradeToVer51(s Session, ver int64) {
 	}
 }
 
+func upgradeToVer51(s Session, ver int64) {
+	if ver >= version51 {
+		return
+	}
+	updateGlobalBindings(s)
+}
+
 func upgradeToVer52(s Session, ver int64) {
 	if ver >= version52 {
 		return
@@ -1248,6 +1281,13 @@ func updateBindInfo(iter *chunk.Iterator4Chunk, p *parser.Parser, bindMap map[st
 	}
 }
 
+func upgradeToMinorVer1(s Session, ver int64) {
+	if ver >= minorVersion1 {
+		return
+	}
+	updateGlobalBindings(s)
+}
+
 func writeMemoryQuotaQuery(s Session) {
 	comment := "memory_quota_query is 32GB by default in v3.0.x, 1GB by default in v4.0.x"
 	mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, %?) ON DUPLICATE KEY UPDATE VARIABLE_VALUE=%?`,
@@ -1275,6 +1315,23 @@ func getBootstrapVersion(s Session) (int64, error) {
 	return strconv.ParseInt(sVal, 10, 64)
 }
 
+func updateMinorVer(s Session) {
+	mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES (%?, %?, "TiDB minor version. Only for release-4.0.") ON DUPLICATE KEY UPDATE VARIABLE_VALUE=%?`,
+		mysql.SystemDB, mysql.TiDBTable, tidbServerMinorVersionVar, currentMinorVersion, currentMinorVersion,
+	)
+}
+
+func getMinorVersion(s Session) (int64, error) {
+	sVal, isNull, err := getTiDBVar(s, tidbServerMinorVersionVar)
+	if err != nil {
+		return 0, errors.Trace(err)
+	}
+	if isNull {
+		return 0, nil
+	}
+	return strconv.ParseInt(sVal, 10, 64)
+}
+
 // doDDLWorks executes DDL statements in bootstrap stage.
 func doDDLWorks(s Session) {
 	// Create a test database.
@@ -1357,6 +1414,10 @@ func doDMLWorks(s Session) {
 		mysql.SystemDB, mysql.TiDBTable, tidbServerVersionVar, currentBootstrapVersion,
 	)
 
+	mustExecute(s, `INSERT HIGH_PRIORITY INTO %n.%n VALUES(%?, %?, "TiDB minor version. Only for release-4.0.")`,
+		mysql.SystemDB, mysql.TiDBTable, tidbServerMinorVersionVar, currentMinorVersion,
+	)
+
 	writeSystemTZ(s)
 
 	writeNewCollationParameter(s, config.GetGlobalConfig().NewCollationsEnabledOnFirstBootstrap)

session/bootstrap_test.go

@@ -191,6 +191,76 @@ func (s *testBootstrapSuite) TestBootstrapWithError(c *C) {
 	c.Assert(r.Close(), IsNil)
 }
 
+func (s *testBootstrapSuite) TestMinorUpgrade(c *C) {
+	ctx := context.Background()
+	defer testleak.AfterTest(c)()
+	store, _ := newStoreWithBootstrap(c, s.dbName)
+	defer store.Close()
+	se := newSession(c, store, s.dbName)
+	mustExecSQL(c, se, "USE mysql;")
+
+	// bootstrap with currentBootstrapVersion
+	r := mustExecSQL(c, se, `SELECT VARIABLE_VALUE from mysql.TiDB where VARIABLE_NAME="tidb_4.0_minor_version";`)
+	req := r.NewChunk()
+	err := r.Next(ctx, req)
+	row := req.GetRow(0)
+	c.Assert(err, IsNil)
+	c.Assert(req.NumRows() == 0, IsFalse)
+	c.Assert(row.Len(), Equals, 1)
+	c.Assert(row.GetBytes(0), BytesEquals, []byte(fmt.Sprintf("%d", currentMinorVersion)))
+	c.Assert(r.Close(), IsNil)
+
+	se1 := newSession(c, store, s.dbName)
+	ver, err := getMinorVersion(se1)
+	c.Assert(err, IsNil)
+	c.Assert(ver, Equals, int64(currentMinorVersion))
+
+	// Do something to downgrade the store.
+	// downgrade meta bootstrap version
+	txn, err := store.Begin()
+	c.Assert(err, IsNil)
+	m := meta.NewMeta(txn)
+	err = m.FinishBootstrap(int64(1))
+	c.Assert(err, IsNil)
+	err = txn.Commit(context.Background())
+	c.Assert(err, IsNil)
+	mustExecSQL(c, se1, `delete from mysql.TiDB where VARIABLE_NAME="tidb_4.0_minor_version";`)
+	mustExecSQL(c, se1, fmt.Sprintf(`delete from mysql.global_variables where VARIABLE_NAME="%s";`,
+		variable.TiDBDistSQLScanConcurrency))
+	mustExecSQL(c, se1, `commit;`)
+	unsetStoreBootstrapped(store.UUID())
+	// Make sure the version is downgraded.
+	r = mustExecSQL(c, se1, `SELECT VARIABLE_VALUE from mysql.TiDB where VARIABLE_NAME="tidb_4.0_minor_version";`)
+	req = r.NewChunk()
+	err = r.Next(ctx, req)
+	c.Assert(err, IsNil)
+	c.Assert(req.NumRows() == 0, IsTrue)
+	c.Assert(r.Close(), IsNil)
+
+	ver, err = getMinorVersion(se1)
+	c.Assert(err, IsNil)
+	c.Assert(ver, Equals, int64(0))
+
+	// Create a new session then upgrade() will run automatically.
+	dom1, err := BootstrapSession(store)
+	c.Assert(err, IsNil)
+	defer dom1.Close()
+	se2 := newSession(c, store, s.dbName)
+	r = mustExecSQL(c, se2, `SELECT VARIABLE_VALUE from mysql.TiDB where VARIABLE_NAME="tidb_4.0_minor_version";`)
+	req = r.NewChunk()
+	err = r.Next(ctx, req)
+	c.Assert(err, IsNil)
+	c.Assert(req.NumRows() == 0, IsFalse)
+	row = req.GetRow(0)
+	c.Assert(row.Len(), Equals, 1)
+	c.Assert(row.GetBytes(0), BytesEquals, []byte(fmt.Sprintf("%d", currentMinorVersion)))
+	c.Assert(r.Close(), IsNil)
+
+	ver, err = getMinorVersion(se2)
+	c.Assert(err, IsNil)
+	c.Assert(ver, Equals, int64(currentMinorVersion))
+}
+
 // TestUpgrade tests upgrading
 func (s *testBootstrapSuite) TestUpgrade(c *C) {
 	ctx := context.Background()
@@ -447,17 +517,25 @@ func (s *testBootstrapSuite) TestUpdateBindInfo(c *C) {
 			bindText:     "select /*+ use_index(t, idxb) */ * from t where a > 1",
 			db:           "test",
 			originWithDB: "select * from test . t where a > ?",
-			bindWithDB:   "SELECT /*+ use_index(t idxb)*/ * FROM test.t WHERE a > 1",
+			bindWithDB:   "SELECT /*+ use_index(`t` `idxb`)*/ * FROM `test`.`t` WHERE `a` > 1",
 			deleteText:   "select * from test.t where a > 1",
 		},
 		{
 			originText:   "select count ( ? ), max ( a ) from t group by b",
 			bindText:     "select /*+ use_index(t, idx) */ count(1), max(a) from t group by b",
 			db:           "test",
 			originWithDB: "select count ( ? ) , max ( a ) from test . t group by b",
-			bindWithDB:   "SELECT /*+ use_index(t idx)*/ count(1),max(a) FROM test.t GROUP BY b",
+			bindWithDB:   "SELECT /*+ use_index(`t` `idx`)*/ count(1),max(`a`) FROM `test`.`t` GROUP BY `b`",
 			deleteText:   "select count(1), max(a) from test.t group by b",
 		},
+		{
+			originText:   "select * from `test` . `t` where `a` = (_charset) ?",
+			bindText:     "SELECT * FROM test.t WHERE a = _utf8\\'ab\\'",
+			db:           "test",
+			originWithDB: "select * from test . t where a = ?",
+			bindWithDB:   "SELECT * FROM `test`.`t` WHERE `a` = 'ab'",
+			deleteText:   "select * from test.t where a = 'c'",
+		},
 	}
 	defer testleak.AfterTest(c)()
 	ctx := context.Background()
@@ -473,7 +551,7 @@ func (s *testBootstrapSuite) TestUpdateBindInfo(c *C) {
 		)
 		mustExecSQL(c, se, sql)
 
-		upgradeToVer51(se, version50)
+		upgradeToVer52(se, version50)
 		r := mustExecSQL(c, se, `select original_sql, bind_sql, default_db, status from mysql.bind_info where source != 'builtin'`)
 		req := r.NewChunk()
 		c.Assert(r.Next(ctx, req), IsNil)
@@ -508,15 +586,15 @@ func (s *testBootstrapSuite) TestUpdateDuplicateBindInfo(c *C) {
 	// The latest one.
 	mustExecSQL(c, se, `insert into mysql.bind_info values('select * from test . t', 'select /*+ use_index(t, idx_b)*/ * from test.t', 'test', 'using', '2021-01-04 14:50:58.257', '2021-01-09 14:50:58.257', 'utf8', 'utf8_general_ci', 'manual')`)
 
-	upgradeToVer51(se, version50)
+	upgradeToVer52(se, version50)
 
 	r := mustExecSQL(c, se, `select original_sql, bind_sql, default_db, status, create_time from mysql.bind_info where source != 'builtin'`)
 	req := r.NewChunk()
 	c.Assert(r.Next(ctx, req), IsNil)
 	c.Assert(req.NumRows(), Equals, 1)
 	row := req.GetRow(0)
 	c.Assert(row.GetString(0), Equals, "select * from test . t")
-	c.Assert(row.GetString(1), Equals, "SELECT /*+ use_index(t idx_b)*/ * FROM test.t")
+	c.Assert(row.GetString(1), Equals, "SELECT /*+ use_index(`t` `idx_b`)*/ * FROM `test`.`t`")
 	c.Assert(row.GetString(2), Equals, "")
 	c.Assert(row.GetString(3), Equals, "using")
 	c.Assert(row.GetTime(4).String(), Equals, "2021-01-04 14:50:58.257")