diff --git a/server/schedule/placement/fit.go b/server/schedule/placement/fit.go index 69cff7d6947..c17f1d8b65b 100644 --- a/server/schedule/placement/fit.go +++ b/server/schedule/placement/fit.go @@ -18,7 +18,6 @@ import ( "sort" "github.com/pingcap/kvproto/pkg/metapb" - "github.com/pingcap/pd/v4/pkg/slice" "github.com/pingcap/pd/v4/server/core" ) @@ -115,48 +114,134 @@ func compareRuleFit(a, b *RuleFit) int { } } +// StoreSet represents the store container. +type StoreSet interface { + GetStores() []*core.StoreInfo + GetStore(id uint64) *core.StoreInfo +} + // FitRegion tries to fit peers of a region to the rules. -func FitRegion(stores core.StoreSetInformer, region *core.RegionInfo, rules []*Rule) *RegionFit { - peers := prepareFitPeers(stores, region) +func FitRegion(stores StoreSet, region *core.RegionInfo, rules []*Rule) *RegionFit { + w := newFitWorker(stores, region, rules) + w.run() + return &w.bestFit +} - var regionFit RegionFit - if len(rules) == 0 { - return ®ionFit - } - for _, rule := range rules { - rf := fitRule(peers, rule) - regionFit.RuleFits = append(regionFit.RuleFits, rf) - // Remove selected. - peers = filterPeersBy(peers, func(p *fitPeer) bool { - return slice.NoneOf(rf.Peers, func(i int) bool { return rf.Peers[i].Id == p.Peer.Id }) +type fitWorker struct { + bestFit RegionFit // update during execution + peers []*fitPeer // p.selected is updated during execution. + rules []*Rule +} + +func newFitWorker(stores StoreSet, region *core.RegionInfo, rules []*Rule) *fitWorker { + var peers []*fitPeer + for _, p := range region.GetPeers() { + peers = append(peers, &fitPeer{ + Peer: p, + store: stores.GetStore(p.GetStoreId()), + isLeader: region.GetLeader().GetId() == p.GetId(), }) } - for _, p := range peers { - regionFit.OrphanPeers = append(regionFit.OrphanPeers, p.Peer) + // Sort peers to keep the match result deterministic. + sort.Slice(peers, func(i, j int) bool { return peers[i].GetId() < peers[j].GetId() }) + + return &fitWorker{ + bestFit: RegionFit{RuleFits: make([]*RuleFit, len(rules))}, + peers: peers, + rules: rules, } - return ®ionFit } -func fitRule(peers []*fitPeer, rule *Rule) *RuleFit { - // Ignore peers that does not match label constraints, and that cannot be - // transformed to expected role type. - peers = filterPeersBy(peers, - func(p *fitPeer) bool { return MatchLabelConstraints(p.store, rule.LabelConstraints) }, - func(p *fitPeer) bool { return p.matchRoleLoose(rule.Role) }) +func (w *fitWorker) run() { + w.fitRule(0) + w.updateOrphanPeers(0) // All peers go to orphanList when RuleList is empty. +} + +// Pick the most suitable peer combination for the rule. +// Index specifies the position of the rule. +// returns true if it replaces `bestFit` with a better alternative. +func (w *fitWorker) fitRule(index int) bool { + if index >= len(w.rules) { + return false + } + // Only consider stores: + // 1. Match label constraints + // 2. Role match, or can match after transformed. + // 3. Not selected by other rules. + var candidates []*fitPeer + for _, p := range w.peers { + if MatchLabelConstraints(p.store, w.rules[index].LabelConstraints) && + p.matchRoleLoose(w.rules[index].Role) && + !p.selected { + candidates = append(candidates, p) + } + } - if len(peers) <= rule.Count { - return newRuleFit(rule, peers) + count := w.rules[index].Count + if len(candidates) < count { + count = len(candidates) } + return w.enumPeers(candidates, nil, index, count) +} - // TODO: brute force can be improved. - var best *RuleFit - iterPeers(peers, rule.Count, func(candidates []*fitPeer) { - rf := newRuleFit(rule, candidates) - if best == nil || compareRuleFit(rf, best) > 0 { - best = rf +// Recursively traverses all feasible peer combinations. +// For each combination, call `compareBest` to determine whether it is better +// than the existing option. +// Returns true if it replaces `bestFit` with a better alternative. +func (w *fitWorker) enumPeers(candidates, selected []*fitPeer, index int, count int) bool { + if len(selected) == count { + // We collect enough peers. End recursive. + return w.compareBest(selected, index) + } + + var better bool + for i, p := range candidates { + p.selected = true + better = w.enumPeers(candidates[i+1:], append(selected, p), index, count) || better + p.selected = false + } + return better +} + +// compareBest checks if the selected peers is better then previous best. +// Returns true if it replaces `bestFit` with a better alternative. +func (w *fitWorker) compareBest(selected []*fitPeer, index int) bool { + rf := newRuleFit(w.rules[index], selected) + cmp := 1 + if best := w.bestFit.RuleFits[index]; best != nil { + cmp = compareRuleFit(rf, best) + } + + switch cmp { + case 1: + w.bestFit.RuleFits[index] = rf + // Reset previous result after position index. + for i := index + 1; i < len(w.rules); i++ { + w.bestFit.RuleFits[i] = nil + } + w.fitRule(index + 1) + w.updateOrphanPeers(index + 1) + return true + case 0: + if w.fitRule(index + 1) { + w.bestFit.RuleFits[index] = rf + return true } - }) - return best + } + return false +} + +// determine the orphanPeers list based on fitPeer.selected flag. +func (w *fitWorker) updateOrphanPeers(index int) { + if index != len(w.rules) { + return + } + w.bestFit.OrphanPeers = w.bestFit.OrphanPeers[:0] + for _, p := range w.peers { + if !p.selected { + w.bestFit.OrphanPeers = append(w.bestFit.OrphanPeers, p.Peer) + } + } } func newRuleFit(rule *Rule, peers []*fitPeer) *RuleFit { @@ -174,6 +259,7 @@ type fitPeer struct { *metapb.Peer store *core.StoreInfo isLeader bool + selected bool } func (p *fitPeer) matchRoleStrict(role PeerRoleType) bool { @@ -197,46 +283,6 @@ func (p *fitPeer) matchRoleLoose(role PeerRoleType) bool { return role != Learner || p.IsLearner } -func prepareFitPeers(stores core.StoreSetInformer, region *core.RegionInfo) []*fitPeer { - var peers []*fitPeer - for _, p := range region.GetPeers() { - peers = append(peers, &fitPeer{ - Peer: p, - store: stores.GetStore(p.GetStoreId()), - isLeader: region.GetLeader().GetId() == p.GetId(), - }) - } - // Sort peers to keep the match result deterministic. - sort.Slice(peers, func(i, j int) bool { return peers[i].GetId() < peers[j].GetId() }) - return peers -} - -func filterPeersBy(peers []*fitPeer, preds ...func(*fitPeer) bool) (selected []*fitPeer) { - for _, p := range peers { - if slice.AllOf(preds, func(i int) bool { return preds[i](p) }) { - selected = append(selected, p) - } - } - return -} - -// Iterate all combinations of select N peers from the list. -func iterPeers(peers []*fitPeer, n int, f func([]*fitPeer)) { - out := make([]*fitPeer, n) - iterPeersRecr(peers, 0, out, func() { f(out) }) -} - -func iterPeersRecr(peers []*fitPeer, index int, out []*fitPeer, f func()) { - for i := index; i <= len(peers)-len(out); i++ { - out[0] = peers[i] - if len(out) > 1 { - iterPeersRecr(peers, i+1, out[1:], f) - } else { - f() - } - } -} - func isolationScore(peers []*fitPeer, labels []string) float64 { var score float64 if len(labels) == 0 || len(peers) <= 1 { diff --git a/server/schedule/placement/fit_test.go b/server/schedule/placement/fit_test.go index a0474adb78b..93a163c2b2c 100644 --- a/server/schedule/placement/fit_test.go +++ b/server/schedule/placement/fit_test.go @@ -15,7 +15,7 @@ package placement import ( "fmt" - "sort" + "strconv" "strings" . "github.com/pingcap/check" @@ -27,8 +27,8 @@ var _ = Suite(&testFitSuite{}) type testFitSuite struct{} -func (s *testFitSuite) makeStores() map[uint64]*core.StoreInfo { - stores := make(map[uint64]*core.StoreInfo) +func (s *testFitSuite) makeStores() StoreSet { + stores := core.NewStoresInfo() for zone := 1; zone <= 5; zone++ { for rack := 1; rack <= 5; rack++ { for host := 1; host <= 5; host++ { @@ -38,8 +38,9 @@ func (s *testFitSuite) makeStores() map[uint64]*core.StoreInfo { "zone": fmt.Sprintf("zone%d", zone), "rack": fmt.Sprintf("rack%d", rack), "host": fmt.Sprintf("host%d", host), + "id": fmt.Sprintf("id%d", x), } - stores[id] = core.NewStoreInfoWithLabel(id, 0, labels) + stores.SetStore(core.NewStoreInfoWithLabel(id, 0, labels)) } } } @@ -47,122 +48,110 @@ func (s *testFitSuite) makeStores() map[uint64]*core.StoreInfo { return stores } -func (s *testFitSuite) TestFitByLocation(c *C) { - stores := s.makeStores() +// example: "1111_leader,1234,2111_learner" +func (s *testFitSuite) makeRegion(def string) *core.RegionInfo { + var regionMeta metapb.Region + var leader *metapb.Peer + for _, peerDef := range strings.Split(def, ",") { + role, idStr := Follower, peerDef + if strings.Contains(peerDef, "_") { + splits := strings.Split(peerDef, "_") + idStr, role = splits[0], PeerRoleType(splits[1]) + } + id, _ := strconv.Atoi(idStr) + peer := &metapb.Peer{Id: uint64(id), StoreId: uint64(id), IsLearner: role == Learner} + regionMeta.Peers = append(regionMeta.Peers, peer) + if role == Leader { + leader = peer + } + } + return core.NewRegionInfo(®ionMeta, leader) +} - type Case struct { - // peers info - peerStoreID []uint64 - peerRole []PeerRoleType // default: all Followers - // rule - locationLabels string // default: "" - count int // default: len(peerStoreID) - role PeerRoleType // default: Voter - // expect result: - expectedPeers []uint64 // default: same as peerStoreID +// example: "3/voter/zone=zone1+zone2,rack=rack2/zone,rack,host" +// count role constraints location_labels +func (s *testFitSuite) makeRule(def string) *Rule { + var rule Rule + splits := strings.Split(def, "/") + rule.Count, _ = strconv.Atoi(splits[0]) + rule.Role = PeerRoleType(splits[1]) + // only support k=v type constraint + for _, c := range strings.Split(splits[2], ",") { + if c == "" { + break + } + kv := strings.Split(c, "=") + rule.LabelConstraints = append(rule.LabelConstraints, LabelConstraint{ + Key: kv[0], + Op: "in", + Values: strings.Split(kv[1], "+"), + }) } + rule.LocationLabels = strings.Split(splits[3], ",") + return &rule +} - cases := []Case{ +func (s *testFitSuite) checkPeerMatch(peers []*metapb.Peer, expect string) bool { + if len(peers) == 0 && expect == "" { + return true + } + + m := make(map[string]struct{}) + for _, p := range peers { + m[strconv.Itoa(int(p.Id))] = struct{}{} + } + expects := strings.Split(expect, ",") + if len(expects) != len(m) { + return false + } + for _, p := range expects { + delete(m, p) + } + return len(m) == 0 +} + +func (s *testFitSuite) TestFitRegion(c *C) { + stores := s.makeStores() + + cases := []struct { + region string + rules []string + fitPeers string + }{ // test count - {peerStoreID: []uint64{1111, 1112, 1113}, count: 1, expectedPeers: []uint64{1111}}, - {peerStoreID: []uint64{1111, 1112, 1113}, count: 2, expectedPeers: []uint64{1111, 1112}}, - {peerStoreID: []uint64{1111, 1112, 1113}, count: 3, expectedPeers: []uint64{1111, 1112, 1113}}, - {peerStoreID: []uint64{1111, 1112, 1113}, count: 5, expectedPeers: []uint64{1111, 1112, 1113}}, - // test isolation level - {peerStoreID: []uint64{1111}, locationLabels: "zone,rack,host"}, - {peerStoreID: []uint64{1111}, locationLabels: "zone,rack"}, - {peerStoreID: []uint64{1111}, locationLabels: "zone"}, - {peerStoreID: []uint64{1111}, locationLabels: ""}, - {peerStoreID: []uint64{1111, 2111}, locationLabels: "zone,rack,host"}, - {peerStoreID: []uint64{1111, 2222, 3333}, locationLabels: "zone,rack,host"}, - {peerStoreID: []uint64{1111, 1211, 3111}, locationLabels: "zone,rack,host"}, - {peerStoreID: []uint64{1111, 1121, 3111}, locationLabels: "zone,rack,host"}, - {peerStoreID: []uint64{1111, 1121, 1122}, locationLabels: "zone,rack,host"}, - // test best location - { - peerStoreID: []uint64{1111, 1112, 1113, 2111, 2222, 3222, 3333}, - locationLabels: "zone,rack,host", - count: 3, - expectedPeers: []uint64{1111, 2111, 3222}, - }, - { - peerStoreID: []uint64{1111, 1121, 1211, 2111, 2211}, - locationLabels: "zone,rack,host", - count: 3, - expectedPeers: []uint64{1111, 1211, 2111}, - }, - { - peerStoreID: []uint64{1111, 1211, 1311, 1411, 2111, 2211, 2311, 3111}, - locationLabels: "zone,rack,host", - count: 5, - expectedPeers: []uint64{1111, 1211, 2111, 2211, 3111}, - }, + {"1111,1112,1113", []string{"1/voter//"}, "1111"}, + {"1111,1112,1113", []string{"2/voter//"}, "1111,1112"}, + {"1111,1112,1113", []string{"3/voter//"}, "1111,1112,1113"}, + {"1111,1112,1113", []string{"5/voter//"}, "1111,1112,1113"}, + // best location + {"1111,1112,1113,2111,2222,3222,3333", []string{"3/voter//zone,rack,host"}, "1111,2111,3222"}, + {"1111,1121,1211,2111,2211", []string{"3/voter//zone,rack,host"}, "1111,1211,2111"}, + {"1111,1211,1311,1411,2111,2211,2311,3111", []string{"5/voter//zone,rack,host"}, "1111,1211,2111,2211,3111"}, // test role match - { - peerStoreID: []uint64{1111, 1112, 1113}, - peerRole: []PeerRoleType{Learner, Follower, Follower}, - count: 1, - expectedPeers: []uint64{1112}, - }, - { - peerStoreID: []uint64{1111, 1112, 1113}, - peerRole: []PeerRoleType{Learner, Follower, Follower}, - count: 2, - expectedPeers: []uint64{1112, 1113}, - }, - { - peerStoreID: []uint64{1111, 1112, 1113}, - peerRole: []PeerRoleType{Learner, Follower, Follower}, - count: 3, - expectedPeers: []uint64{1112, 1113, 1111}, - }, - { - peerStoreID: []uint64{1111, 1112, 1121, 1122, 1131, 1132, 1141, 1142}, - peerRole: []PeerRoleType{Follower, Learner, Learner, Learner, Learner, Follower, Follower, Follower}, - locationLabels: "zone,rack,host", - count: 3, - expectedPeers: []uint64{1111, 1132, 1141}, - }, + {"1111_learner,1112,1113", []string{"1/voter//"}, "1112"}, + {"1111_learner,1112,1113", []string{"2/voter//"}, "1112,1113"}, + {"1111_learner,1112,1113", []string{"3/voter//"}, "1111,1112,1113"}, + {"1111,1112_learner,1121_learner,1122_learner,1131_learner,1132,1141,1142", []string{"3/follower//zone,rack,host"}, "1111,1132,1141"}, + // test 2 rule + {"1111,1112,1113,1114", []string{"3/voter//", "1/voter/id=id1/"}, "1112,1113,1114/1111"}, + {"1111,2211,3111,3112", []string{"3/voter//zone", "1/voter/rack=rack2/"}, "1111,2211,3111//3112"}, + {"1111,2211,3111,3112", []string{"1/voter/rack=rack2/", "3/voter//zone"}, "2211/1111,3111,3112"}, } for _, cc := range cases { - var peers []*fitPeer - for i := range cc.peerStoreID { - role := Follower - if i < len(cc.peerRole) { - role = cc.peerRole[i] - } - peers = append(peers, &fitPeer{ - Peer: &metapb.Peer{Id: cc.peerStoreID[i], StoreId: cc.peerStoreID[i], IsLearner: role == Learner}, - store: stores[cc.peerStoreID[i]], - isLeader: role == Leader, - }) - } - - rule := &Rule{Count: len(cc.peerStoreID), Role: Voter} - if len(cc.locationLabels) > 0 { - rule.LocationLabels = strings.Split(cc.locationLabels, ",") - } - if cc.role != "" { - rule.Role = cc.role - } - if cc.count > 0 { - rule.Count = cc.count + region := s.makeRegion(cc.region) + var rules []*Rule + for _, r := range cc.rules { + rules = append(rules, s.makeRule(r)) } - c.Log("Peers:", peers) - c.Log("rule:", rule) - ruleFit := fitRule(peers, rule) - selectedIDs := make([]uint64, 0) - for _, p := range ruleFit.Peers { - selectedIDs = append(selectedIDs, p.GetId()) + rf := FitRegion(stores, region, rules) + expects := strings.Split(cc.fitPeers, "/") + for i, f := range rf.RuleFits { + c.Assert(s.checkPeerMatch(f.Peers, expects[i]), IsTrue) } - sort.Slice(selectedIDs, func(i, j int) bool { return selectedIDs[i] < selectedIDs[j] }) - expectedPeers := cc.expectedPeers - if len(expectedPeers) == 0 { - expectedPeers = cc.peerStoreID + if len(rf.RuleFits) < len(expects) { + c.Assert(s.checkPeerMatch(rf.OrphanPeers, expects[len(rf.RuleFits)]), IsTrue) } - sort.Slice(expectedPeers, func(i, j int) bool { return expectedPeers[i] < expectedPeers[j] }) - c.Assert(selectedIDs, DeepEquals, expectedPeers) } } @@ -185,7 +174,7 @@ func (s *testFitSuite) TestIsolationScore(c *C) { for _, id := range ids { peers = append(peers, &fitPeer{ Peer: &metapb.Peer{StoreId: id}, - store: stores[id], + store: stores.GetStore(id), }) } return peers diff --git a/server/schedule/placement/rule_manager.go b/server/schedule/placement/rule_manager.go index b749ece6c6a..5e82e3d47da 100644 --- a/server/schedule/placement/rule_manager.go +++ b/server/schedule/placement/rule_manager.go @@ -269,7 +269,7 @@ func (m *RuleManager) GetRulesForApplyRegion(region *core.RegionInfo) []*Rule { } // FitRegion fits a region to the rules it matches. -func (m *RuleManager) FitRegion(stores core.StoreSetInformer, region *core.RegionInfo) *RegionFit { +func (m *RuleManager) FitRegion(stores StoreSet, region *core.RegionInfo) *RegionFit { rules := m.GetRulesForApplyRegion(region) return FitRegion(stores, region, rules) }