tools: balance region simulator

tools/pd-simulator/main.go

@@ -41,18 +41,25 @@ import (
 )
 
 var (
-	pdAddr         = flag.String("pd", "", "pd address")
-	configFile     = flag.String("config", "conf/simconfig.toml", "config file")
-	caseName       = flag.String("case", "", "case name")
-	serverLogLevel = flag.String("serverLog", "fatal", "pd server log level.")
-	simLogLevel    = flag.String("simLog", "fatal", "simulator log level.")
+	pdAddr                      = flag.String("pd", "", "pd address")
+	configFile                  = flag.String("config", "conf/simconfig.toml", "config file")
+	caseName                    = flag.String("case", "", "case name")
+	serverLogLevel              = flag.String("serverLog", "fatal", "pd server log level.")
+	simLogLevel                 = flag.String("simLog", "fatal", "simulator log level.")
+	regionNum                   = flag.Int("regionNum", 0, "regionNum of one store")
+	storeNum                    = flag.Int("storeNum", 0, "storeNum")
+	enableTransferRegionCounter = flag.Bool("enableTransferRegionCounter", false, "enableTransferRegionCounter")
 )
 
 func main() {
 	flag.Parse()
 
 	simutil.InitLogger(*simLogLevel)
+	simutil.InitCaseConfig(*storeNum, *regionNum, *enableTransferRegionCounter)
 	statistics.Denoising = false
+	if simutil.CaseConfigure.EnableTransferRegionCounter {
+		simutil.TransferRegionCounter.Init()
+	}
 
 	if *caseName == "" {
 		if *pdAddr != "" {
@@ -172,6 +179,12 @@ EXIT:
 
 	fmt.Printf("%s [%s] total iteration: %d, time cost: %v\n", simResult, simCase, driver.TickCount(), time.Since(start))
 	driver.PrintStatistics()
+	if simutil.TransferRegionCounter.IsValid {
+		simutil.Logger.Info("Total: ")
+		simutil.TransferRegionCounter.PrintGraph()
+		simutil.TransferRegionCounter.Result()
+		simutil.TransferRegionCounter.PrintResult()
+	}
 
 	if simResult != "OK" {
 		os.Exit(1)

tools/pd-simulator/simulator/cases/add_nodes.go

@@ -15,6 +15,7 @@ package cases
 import (
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/pd/server/core"
+	"github.com/pingcap/pd/tools/pd-simulator/simulator/dto"
 	"github.com/pingcap/pd/tools/pd-simulator/simulator/simutil"
 	"go.uber.org/zap"
 )
@@ -47,7 +48,7 @@ func newAddNodes() *Case {
 		})
 	}
 
-	simCase.Checker = func(regions *core.RegionsInfo) bool {
+	simCase.Checker = func(regions *core.RegionsInfo, stats []dto.StoreStats) bool {
 		res := true
 		leaderCounts := make([]int, 0, 8)
 		regionCounts := make([]int, 0, 8)

tools/pd-simulator/simulator/cases/add_nodes_dynamic.go

@@ -15,6 +15,7 @@ package cases
 import (
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/pd/server/core"
+	"github.com/pingcap/pd/tools/pd-simulator/simulator/dto"
 	"github.com/pingcap/pd/tools/pd-simulator/simulator/simutil"
 	"go.uber.org/zap"
 )
@@ -65,7 +66,7 @@ func newAddNodesDynamic() *Case {
 	}
 	simCase.Events = []EventDescriptor{e}
 
-	simCase.Checker = func(regions *core.RegionsInfo) bool {
+	simCase.Checker = func(regions *core.RegionsInfo, stats []dto.StoreStats) bool {
 		res := true
 		leaderCounts := make([]int, 0, numNodes)
 		regionCounts := make([]int, 0, numNodes)

tools/pd-simulator/simulator/cases/balance_leader.go

@@ -16,6 +16,7 @@ package cases
 import (
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/pd/server/core"
+	"github.com/pingcap/pd/tools/pd-simulator/simulator/dto"
 	"github.com/pingcap/pd/tools/pd-simulator/simulator/simutil"
 	"go.uber.org/zap"
 )
@@ -48,7 +49,7 @@ func newBalanceLeader() *Case {
 		})
 	}
 
-	simCase.Checker = func(regions *core.RegionsInfo) bool {
+	simCase.Checker = func(regions *core.RegionsInfo, stats []dto.StoreStats) bool {
 		count1 := regions.GetStoreLeaderCount(1)
 		count2 := regions.GetStoreLeaderCount(2)
 		count3 := regions.GetStoreLeaderCount(3)

tools/pd-simulator/simulator/cases/cases.go

@@ -16,18 +16,21 @@ package cases
 import (
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/pd/server/core"
+	"github.com/pingcap/pd/tools/pd-simulator/simulator/dto"
 )
 
 // Store is used to simulate tikv.
 type Store struct {
-	ID           uint64
-	Status       metapb.StoreState
-	Labels       []*metapb.StoreLabel
-	Capacity     uint64
-	Available    uint64
-	LeaderWeight float32
-	RegionWeight float32
-	Version      string
+	ID             uint64
+	Status         metapb.StoreState
+	Labels         []*metapb.StoreLabel
+	Capacity       uint64
+	Available      uint64
+	LeaderWeight   float32
+	RegionWeight   float32
+	Version        string
+	LastAvailable  uint64
+	LastUpdateTime int64
 }
 
 // Region is used to simulate a region.
@@ -40,7 +43,7 @@ type Region struct {
 }
 
 // CheckerFunc checks if the scheduler is finished.
-type CheckerFunc func(*core.RegionsInfo) bool
+type CheckerFunc func(*core.RegionsInfo, []dto.StoreStats) bool
 
 // Case represents a test suite for simulator.
 type Case struct {
@@ -90,6 +93,7 @@ var IDAllocator idAllocator
 // CaseMap is a mapping of the cases to the their corresponding initialize functions.
 var CaseMap = map[string]func() *Case{
 	"balance-leader":       newBalanceLeader,
+	"balance-region-loop":  newRedundantBalanceRegion,
 	"add-nodes":            newAddNodes,
 	"add-nodes-dynamic":    newAddNodesDynamic,
 	"delete-nodes":         newDeleteNodes,

tools/pd-simulator/simulator/cases/delete_nodes.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/pd/server/core"
+	"github.com/pingcap/pd/tools/pd-simulator/simulator/dto"
 	"github.com/pingcap/pd/tools/pd-simulator/simulator/simutil"
 	"go.uber.org/zap"
 )
@@ -68,7 +69,7 @@ func newDeleteNodes() *Case {
 	}
 	simCase.Events = []EventDescriptor{e}
 
-	simCase.Checker = func(regions *core.RegionsInfo) bool {
+	simCase.Checker = func(regions *core.RegionsInfo, stats []dto.StoreStats) bool {
 		res := true
 		leaderCounts := make([]int, 0, numNodes)
 		regionCounts := make([]int, 0, numNodes)

tools/pd-simulator/simulator/cases/hot_read.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/pd/server/core"
+	"github.com/pingcap/pd/tools/pd-simulator/simulator/dto"
 	"github.com/pingcap/pd/tools/pd-simulator/simulator/simutil"
 	"go.uber.org/zap"
 )
@@ -69,7 +70,7 @@ func newHotRead() *Case {
 	}
 	simCase.Events = []EventDescriptor{e}
 	// Checker description
-	simCase.Checker = func(regions *core.RegionsInfo) bool {
+	simCase.Checker = func(regions *core.RegionsInfo, stats []dto.StoreStats) bool {
 		var leaderCount [5]int
 		for id := range readFlow {
 			leaderStore := regions.GetRegion(id).GetLeader().GetStoreId()

tools/pd-simulator/simulator/cases/hot_write.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/pd/server/core"
+	"github.com/pingcap/pd/tools/pd-simulator/simulator/dto"
 	"github.com/pingcap/pd/tools/pd-simulator/simulator/simutil"
 )
 
@@ -72,7 +73,7 @@ func newHotWrite() *Case {
 	simCase.Events = []EventDescriptor{e}
 
 	// Checker description
-	simCase.Checker = func(regions *core.RegionsInfo) bool {
+	simCase.Checker = func(regions *core.RegionsInfo, stats []dto.StoreStats) bool {
 		var leaderCount, peerCount [10]int
 		for id := range writeFlow {
 			region := regions.GetRegion(id)

tools/pd-simulator/simulator/cases/import_data.go

@@ -23,6 +23,7 @@ import (
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/pd/server/core"
 	"github.com/pingcap/pd/table"
+	"github.com/pingcap/pd/tools/pd-simulator/simulator/dto"
 	"github.com/pingcap/pd/tools/pd-simulator/simulator/simutil"
 )
 
@@ -79,7 +80,7 @@ func newImportData() *Case {
 	simCase.Events = []EventDescriptor{e}
 
 	// Checker description
-	simCase.Checker = func(regions *core.RegionsInfo) bool {
+	simCase.Checker = func(regions *core.RegionsInfo, stats []dto.StoreStats) bool {
 		leaderDist := make(map[uint64]int)
 		peerDist := make(map[uint64]int)
 		leaderTotal := 0

tools/pd-simulator/simulator/cases/makeup_down_replica.go

@@ -15,6 +15,7 @@ package cases
 import (
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/pd/server/core"
+	"github.com/pingcap/pd/tools/pd-simulator/simulator/dto"
 	"github.com/pingcap/pd/tools/pd-simulator/simulator/simutil"
 	"go.uber.org/zap"
 )
@@ -62,7 +63,7 @@ func newMakeupDownReplicas() *Case {
 	}
 	simCase.Events = []EventDescriptor{e}
 
-	simCase.Checker = func(regions *core.RegionsInfo) bool {
+	simCase.Checker = func(regions *core.RegionsInfo, stats []dto.StoreStats) bool {
 		sum := 0
 		regionCounts := make([]int, 0, 3)
 		for i := 1; i <= 4; i++ {

tools/pd-simulator/simulator/cases/redundant_balance_region.go

@@ -0,0 +1,89 @@
+// Copyright 2019 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// //     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cases
+
+import (
+	"github.com/pingcap/kvproto/pkg/metapb"
+	"github.com/pingcap/pd/server/core"
+	"github.com/pingcap/pd/tools/pd-simulator/simulator/dto"
+	"github.com/pingcap/pd/tools/pd-simulator/simulator/simutil"
+	"time"
+)
+
+func newRedundantBalanceRegion() *Case {
+	var simCase Case
+
+	storeNum := simutil.CaseConfigure.StoreNum
+	regionNum := simutil.CaseConfigure.RegionNum * storeNum / 3
+	if storeNum == 0 || regionNum == 0 {
+		storeNum, regionNum = 6, 4000
+	}
+
+	for i := 0; i < storeNum; i++ {
+		if i%2 == 1 {
+			simCase.Stores = append(simCase.Stores, &Store{
+				ID:        IDAllocator.nextID(),
+				Status:    metapb.StoreState_Up,
+				Capacity:  1 * TB,
+				Available: 980 * GB,
+				Version:   "2.1.0",
+			})
+		} else {
+			simCase.Stores = append(simCase.Stores, &Store{
+				ID:        IDAllocator.nextID(),
+				Status:    metapb.StoreState_Up,
+				Capacity:  1 * TB,
+				Available: 1 * TB,
+				Version:   "2.1.0",
+			})
+		}
+	}
+
+	for i := 0; i < regionNum; i++ {
+		peers := []*metapb.Peer{
+			{Id: IDAllocator.nextID(), StoreId: uint64(i%storeNum + 1)},
+			{Id: IDAllocator.nextID(), StoreId: uint64((i+1)%storeNum + 1)},
+			{Id: IDAllocator.nextID(), StoreId: uint64((i+2)%storeNum + 1)},
+		}
+		simCase.Regions = append(simCase.Regions, Region{
+			ID:     IDAllocator.nextID(),
+			Peers:  peers,
+			Leader: peers[0],
+			Size:   96 * MB,
+			Keys:   960000,
+		})
+	}
+
+	simCase.Checker = func(regions *core.RegionsInfo, stats []dto.StoreStats) bool {
+		res := true
+		curTime := time.Now().Unix()
+		for i := 0; i < storeNum; i++ {
+			sliceStats := stats[i]
+			simCase.Stores[i].Available = sliceStats.GetAvailable()
+			if curTime-simCase.Stores[i].LastUpdateTime > 60 {
+				if simCase.Stores[i].LastAvailable != simCase.Stores[i].Available {
+					res = false
+				}
+				if sliceStats.ToCompactionSize != 0 {
+					res = false
+				}
+				simCase.Stores[i].LastUpdateTime = curTime
+				simCase.Stores[i].LastAvailable = simCase.Stores[i].Available
+			} else {
+				res = false
+			}
+		}
+		return res
+	}
+	return &simCase
+}

tools/pd-simulator/simulator/cases/region_merge.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/pd/server/core"
+	"github.com/pingcap/pd/tools/pd-simulator/simulator/dto"
 	"github.com/pingcap/pd/tools/pd-simulator/simulator/simutil"
 	"go.uber.org/zap"
 )
@@ -52,7 +53,7 @@ func newRegionMerge() *Case {
 	}
 
 	// Checker description
-	simCase.Checker = func(regions *core.RegionsInfo) bool {
+	simCase.Checker = func(regions *core.RegionsInfo, stats []dto.StoreStats) bool {
 		count1 := regions.GetStoreRegionCount(1)
 		count2 := regions.GetStoreRegionCount(2)
 		count3 := regions.GetStoreRegionCount(3)

tools/pd-simulator/simulator/cases/region_split.go

@@ -16,6 +16,7 @@ package cases
 import (
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/pd/server/core"
+	"github.com/pingcap/pd/tools/pd-simulator/simulator/dto"
 	"github.com/pingcap/pd/tools/pd-simulator/simulator/simutil"
 	"go.uber.org/zap"
 )
@@ -55,7 +56,7 @@ func newRegionSplit() *Case {
 	simCase.Events = []EventDescriptor{e}
 
 	// Checker description
-	simCase.Checker = func(regions *core.RegionsInfo) bool {
+	simCase.Checker = func(regions *core.RegionsInfo, stats []dto.StoreStats) bool {
 		count1 := regions.GetStoreRegionCount(1)
 		count2 := regions.GetStoreRegionCount(2)
 		count3 := regions.GetStoreRegionCount(3)

tools/pd-simulator/simulator/drive.go

@@ -22,6 +22,7 @@ import (
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/pd/server/core"
 	"github.com/pingcap/pd/tools/pd-simulator/simulator/cases"
+	"github.com/pingcap/pd/tools/pd-simulator/simulator/dto"
 	"github.com/pingcap/pd/tools/pd-simulator/simulator/simutil"
 	"github.com/pkg/errors"
 )
@@ -116,7 +117,11 @@ func (d *Driver) Tick() {
 
 // Check checks if the simulation is completed.
 func (d *Driver) Check() bool {
-	return d.simCase.Checker(d.raftEngine.regionsInfo)
+	stats := make([]dto.StoreStats, len(d.conn.Nodes)+1)
+	for index, node := range d.conn.Nodes {
+		stats[index] = *node.stats
+	}
+	return d.simCase.Checker(d.raftEngine.regionsInfo, stats)
 }
 
 // PrintStatistics prints the statistics of the scheduler.

tools/pd-simulator/simulator/dto/stats.go

@@ -0,0 +1,22 @@
+// Copyright 2019 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dto
+
+import "github.com/pingcap/kvproto/pkg/pdpb"
+
+//StoreStats is to save store stats of node.
+type StoreStats struct {
+	pdpb.StoreStats
+	ToCompactionSize uint64
+}