*(all): update dependencies on tidb and pd. (pingcap#8126)

close pingcap#8110, ref pingcap#8115 (cherry picked from commit 5a38ed9)
asddongmen · Feb 15, 2023 · e675bd4 · e675bd4
1 parent a1a198e
commit e675bd4
Show file tree

Hide file tree

Showing 41 changed files with 1,681 additions and 1,201 deletions.
diff --git a/Makefile b/Makefile
@@ -40,14 +40,23 @@ ifeq (${CDC_ENABLE_VENDOR}, 1)
 GOVENDORFLAG := -mod=vendor
 endif
 
-GOBUILD  := CGO_ENABLED=0 $(GO) build $(BUILD_FLAG) -trimpath $(GOVENDORFLAG)
+# Since TiDB add a new dependency on github.com/cloudfoundry/gosigar,
+# We need to add CGO_ENABLED=1 to make it work when build TiCDC in Darwin OS.
+# These logic is to check if the OS is Darwin, if so, add CGO_ENABLED=1.
+# ref: https://github.com/cloudfoundry/gosigar/issues/58#issuecomment-1150925711
+# ref: https://github.com/pingcap/tidb/pull/39526#issuecomment-1407952955
+OS    := "$(shell go env GOOS)"
+ifeq (${OS}, "linux")
+	CGO := 0
+else ifeq (${OS}, "darwin")
+	CGO := 1
+endif
+
+GOBUILD  := CGO_ENABLED=$(CGO) $(GO) build $(BUILD_FLAG) -trimpath $(GOVENDORFLAG)
 GOBUILDNOVENDOR  := CGO_ENABLED=0 $(GO) build $(BUILD_FLAG) -trimpath
-GOTEST   := CGO_ENABLED=1 $(GO) test -p $(P) --race
+GOTEST   := CGO_ENABLED=1 $(GO) test -p $(P) --race --tags=intest
 GOTESTNORACE := CGO_ENABLED=1 $(GO) test -p $(P)
 
-ARCH  := "$(shell uname -s)"
-LINUX := "Linux"
-MAC   := "Darwin"
 CDC_PKG := github.com/pingcap/tiflow
 DM_PKG := github.com/pingcap/tiflow/dm
 ENGINE_PKG := github.com/pingcap/tiflow/engine
@@ -163,7 +172,7 @@ unit_test_in_verify_ci: check_failpoint_ctl tools/bin/gotestsum tools/bin/gocov
 	mkdir -p "$(TEST_DIR)"
 	$(FAILPOINT_ENABLE)
 	@export log_level=error;\
-	CGO_ENABLED=1 tools/bin/gotestsum --junitfile cdc-junit-report.xml -- -v -timeout 5m -p $(P) --race \
+	CGO_ENABLED=1 tools/bin/gotestsum --junitfile cdc-junit-report.xml -- -v -timeout 5m -p $(P) --race --tags=intest \
 	-covermode=atomic -coverprofile="$(TEST_DIR)/cov.unit.out" $(PACKAGES_TICDC) \
 	|| { $(FAILPOINT_DISABLE); exit 1; }
 	tools/bin/gocov convert "$(TEST_DIR)/cov.unit.out" | tools/bin/gocov-xml > cdc-coverage.xml

diff --git a/cdc/api/v2/changefeed_test.go b/cdc/api/v2/changefeed_test.go
@@ -235,7 +235,7 @@ func TestUpdateChangefeed(t *testing.T) {
 	cp.EXPECT().IsOwner().Return(true).AnyTimes()
 
 	// case 1 invalid id
-	invalidID := "#Invalid_"
+	invalidID := "Invalid_#"
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequestWithContext(context.Background(), update.method,
 		fmt.Sprintf(update.url, invalidID), nil)

diff --git a/cdc/entry/schema/snapshot.go b/cdc/entry/schema/snapshot.go
@@ -121,7 +121,7 @@ func NewSingleSnapshotFromMeta(meta *timeta.Meta, currentTs uint64, forceReplica
 	// meta is nil only in unit tests
 	if meta == nil {
 		snap := NewEmptySnapshot(forceReplicate)
-		snap.InitConcurrentDDLTables()
+		snap.InitPreExistingTables()
 		snap.inner.currentTs = currentTs
 		return snap, nil
 	}
@@ -210,12 +210,16 @@ const (
 	mdlCreateTable = "create table mysql.tidb_mdl_info(job_id BIGINT NOT NULL PRIMARY KEY, version BIGINT NOT NULL, table_ids text(65535));"
 )
 
-// InitConcurrentDDLTables imitates the creating table logic for concurrent DDL.
+// InitPreExistingTables initializes the pre-existing tables in an empty Snapshot.
 // Since v6.2.0, tables of concurrent DDL will be directly written as meta KV in
 // TiKV, without being written to history DDL jobs. So the Snapshot which is not
 // build from meta needs this method to handle history DDL.
-func (s *Snapshot) InitConcurrentDDLTables() {
-	tableIDs := [...]int64{ddl.JobTableID, ddl.ReorgTableID, ddl.HistoryTableID}
+// Since v6.5.0, Backfill tables is written as meta KV in TiKV, so the Snapshot
+// which is not build from meta needs this method to handle history DDL.
+// See:https://github.com/pingcap/tidb/pull/39616
+func (s *Snapshot) InitPreExistingTables() {
+	ddlJobTableIDs := [...]int64{ddl.JobTableID, ddl.ReorgTableID, ddl.HistoryTableID}
+	backfillTableIDs := [...]int64{ddl.BackfillTableID, ddl.BackfillHistoryTableID}
 
 	mysqlDBInfo := &timodel.DBInfo{
 		ID:      mysqlDBID,
@@ -231,10 +235,20 @@ func (s *Snapshot) InitConcurrentDDLTables() {
 		stmt, _ := p.ParseOneStmt(table.SQL, "", "")
 		tblInfo, _ := ddl.BuildTableInfoFromAST(stmt.(*ast.CreateTableStmt))
 		tblInfo.State = timodel.StatePublic
-		tblInfo.ID = tableIDs[i]
+		tblInfo.ID = ddlJobTableIDs[i]
 		wrapped := model.WrapTableInfo(mysqlDBID, mysql.SystemDB, dummyTS, tblInfo)
 		_ = s.inner.createTable(wrapped, dummyTS)
 	}
+
+	for i, table := range session.BackfillTables {
+		stmt, _ := p.ParseOneStmt(table.SQL, "", "")
+		tblInfo, _ := ddl.BuildTableInfoFromAST(stmt.(*ast.CreateTableStmt))
+		tblInfo.State = timodel.StatePublic
+		tblInfo.ID = backfillTableIDs[i]
+		wrapped := model.WrapTableInfo(mysqlDBID, mysql.SystemDB, dummyTS, tblInfo)
+		_ = s.inner.createTable(wrapped, dummyTS)
+	}
+
 	stmt, _ := p.ParseOneStmt(mdlCreateTable, "", "")
 	tblInfo, _ := ddl.BuildTableInfoFromAST(stmt.(*ast.CreateTableStmt))
 	tblInfo.State = timodel.StatePublic
@@ -514,8 +528,14 @@ func (s *Snapshot) DoHandleDDL(job *timodel.Job) error {
 }
 
 // TableCount counts tables in the snapshot. It's only for tests.
-func (s *Snapshot) TableCount(includeIneligible bool) (count int) {
-	s.IterTables(includeIneligible, func(i *model.TableInfo) { count += 1 })
+func (s *Snapshot) TableCount(includeIneligible bool,
+	filter func(schema, table string) bool,
+) (count int) {
+	s.IterTables(includeIneligible, func(i *model.TableInfo) {
+		if filter(i.TableName.Schema, i.TableName.Table) {
+			count++
+		}
+	})
 	return
 }
 
@@ -552,7 +572,6 @@ func (s *Snapshot) DumpToString() string {
 		schema, _ := s.inner.schemaByID(schemaID)
 		tableNames = append(tableNames, fmt.Sprintf("%s.%s:%d", schema.Name.O, table, target))
 	})
-
 	return fmt.Sprintf("%s\n%s\n%s\n%s\n%s",
 		strings.Join(schemas, "\t"),
 		strings.Join(tables, "\t"),

diff --git a/cdc/entry/schema/snapshot_test.go b/cdc/entry/schema/snapshot_test.go
@@ -199,9 +199,10 @@ func TestTable(t *testing.T) {
 			require.False(t, snap.IsIneligibleTableID(12))
 			require.False(t, snap.IsIneligibleTableID(12+65536))
 		}
-
 		// IterTables should get no available tables.
-		require.Equal(t, snap.TableCount(true), 0)
+		require.Equal(t, snap.TableCount(true, func(table, schema string) bool {
+			return true
+		}), 0)
 	}
 }
 

diff --git a/cdc/entry/schema_storage.go b/cdc/entry/schema_storage.go
@@ -74,7 +74,7 @@ func NewSchemaStorage(
 	)
 	if meta == nil {
 		snap = schema.NewEmptySnapshot(forceReplicate)
-		snap.InitConcurrentDDLTables()
+		snap.InitPreExistingTables()
 	} else {
 		snap, err = schema.NewSnapshotFromMeta(meta, startTs, forceReplicate)
 		if err != nil {
@@ -188,7 +188,7 @@ func (s *schemaStorageImpl) HandleDDLJob(job *timodel.Job) error {
 		snap = lastSnap.Copy()
 	} else {
 		snap = schema.NewEmptySnapshot(s.forceReplicate)
-		snap.InitConcurrentDDLTables()
+		snap.InitPreExistingTables()
 	}
 	if err := snap.HandleDDL(job); err != nil {
 		log.Error("handle DDL failed",

diff --git a/cdc/entry/schema_storage_test.go b/cdc/entry/schema_storage_test.go
@@ -618,7 +618,7 @@ func TestMultiVersionStorage(t *testing.T) {
 	require.False(t, exist)
 
 	lastSchemaTs := storage.DoGC(0)
-	// Snapshot.InitConcurrentDDLTables will create a schema with ts = 1
+	// Snapshot.InitPreExistingTables will create a schema with ts = 1
 	require.Equal(t, uint64(1), lastSchemaTs)
 
 	snap, err = storage.GetSnapshot(ctx, 100)
@@ -729,12 +729,21 @@ func TestExplicitTables(t *testing.T) {
 	snap3, err := schema.NewSnapshotFromMeta(meta2, ver2.Ver, true /* forceReplicate */)
 	require.Nil(t, err)
 
-	require.Equal(t, snap2.TableCount(true)-snap1.TableCount(true), 5)
-	// some system tables are also ineligible
-	require.GreaterOrEqual(t, snap2.TableCount(false), 4)
+	// we don't need to count system tables since TiCDC
+	// don't replicate them and TiDB change them frequently,
+	// so we don't need to consider them in the table count
+	systemTablesFilter := func(dbName, tableName string) bool {
+		return dbName != "mysql" && dbName != "information_schema"
+	}
+	require.Equal(t, 5, snap2.TableCount(true,
+		systemTablesFilter)-snap1.TableCount(true, systemTablesFilter))
+	// only test simple_test1 included
+	require.Equal(t, 1, snap2.TableCount(false, systemTablesFilter))
 
-	require.Equal(t, snap3.TableCount(true)-snap1.TableCount(true), 5)
-	require.Equal(t, snap3.TableCount(false), 45)
+	require.Equal(t, 5, snap3.TableCount(true,
+		systemTablesFilter)-snap1.TableCount(true, systemTablesFilter))
+	// since we create a snapshot from meta2 and forceReplicate is true, so all tables are included
+	require.Equal(t, 5, snap3.TableCount(false, systemTablesFilter))
 }
 
 /*

diff --git a/cdc/owner/schema_test.go b/cdc/owner/schema_test.go
@@ -95,7 +95,7 @@ func TestAllTables(t *testing.T) {
 	require.Equal(t, tableName, model.TableName{
 		Schema:  "test",
 		Table:   "t1",
-		TableID: 76,
+		TableID: 84,
 	})
 	// add ineligible table
 	require.Nil(t, schema.HandleDDL(helper.DDL2Job("create table test.t2(id int)")))
@@ -104,7 +104,7 @@ func TestAllTables(t *testing.T) {
 	require.Equal(t, tableName, model.TableName{
 		Schema:  "test",
 		Table:   "t1",
-		TableID: 76,
+		TableID: 84,
 	})
 }
 

diff --git a/cdc/server/server_test.go b/cdc/server/server_test.go
@@ -40,7 +40,7 @@ import (
 	"github.com/pingcap/tiflow/pkg/security"
 	"github.com/pingcap/tiflow/pkg/util"
 	"github.com/stretchr/testify/require"
-	"github.com/tikv/pd/pkg/tempurl"
+	"github.com/tikv/pd/pkg/utils/tempurl"
 	clientv3 "go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/server/v3/embed"
 	"golang.org/x/sync/errgroup"

diff --git a/cdc/sink/codec/maxwell/maxwell_message.go b/cdc/sink/codec/maxwell/maxwell_message.go
@@ -21,7 +21,7 @@ import (
 	"github.com/pingcap/tiflow/cdc/model"
 	"github.com/pingcap/tiflow/cdc/sink/codec/internal"
 	cerror "github.com/pingcap/tiflow/pkg/errors"
-	"github.com/tikv/pd/pkg/tsoutil"
+	"github.com/tikv/pd/pkg/utils/tsoutil"
 )
 
 type maxwellMessage struct {

diff --git a/dm/master/election_test.go b/dm/master/election_test.go
@@ -24,7 +24,7 @@ import (
 	"github.com/pingcap/tiflow/dm/pkg/etcdutil"
 	"github.com/pingcap/tiflow/dm/pkg/log"
 	"github.com/stretchr/testify/require"
-	"github.com/tikv/pd/pkg/tempurl"
+	"github.com/tikv/pd/pkg/utils/tempurl"
 )
 
 func TestFailToStartLeader(t *testing.T) {

diff --git a/dm/master/etcd_test.go b/dm/master/etcd_test.go
@@ -26,7 +26,7 @@ import (
 	"github.com/pingcap/tiflow/dm/pkg/log"
 	"github.com/pingcap/tiflow/dm/pkg/terror"
 	"github.com/pingcap/tiflow/dm/pkg/utils"
-	"github.com/tikv/pd/pkg/tempurl"
+	"github.com/tikv/pd/pkg/utils/tempurl"
 	clientv3 "go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/server/v3/embed"
 )

diff --git a/dm/master/openapi_view_test.go b/dm/master/openapi_view_test.go
@@ -43,7 +43,7 @@ import (
 	"github.com/pingcap/tiflow/dm/pkg/utils"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
-	"github.com/tikv/pd/pkg/tempurl"
+	"github.com/tikv/pd/pkg/utils/tempurl"
 )
 
 // some data for test.