Merge branch 'release-5.2' into cherry-pick-4347-to-release-5.2

metrics/alertmanager/pd.rules.yml

@@ -110,12 +110,12 @@ groups:
       summary: PD_pending_peer_region_count
 
   - alert: PD_leader_change
-    expr: count( changes(pd_server_tso{type="save"}[10m]) > 0 )   >= 2
+    expr: count( changes(pd_tso_events{type="save"}[10m]) > 0 )   >= 2
     for: 1m
     labels:
       env: ENV_LABELS_ENV
       level: warning
-      expr:  count( changes(pd_server_tso{type="save"}[10m]) > 0 )   >= 2
+      expr:  count( changes(pd_tso_events{type="save"}[10m]) > 0 )   >= 2
     annotations:
       description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
       value: '{{ $value }}'
@@ -134,12 +134,12 @@ groups:
       summary: TiKV_space_used_more_than_80%
 
   - alert: PD_system_time_slow
-    expr: changes(pd_server_tso{type="system_time_slow"}[10m]) >= 1
+    expr: changes(pd_tso_events{type="system_time_slow"}[10m]) >= 1
     for: 1m
     labels:
       env: ENV_LABELS_ENV
       level: warning
-      expr: changes(pd_server_tso{type="system_time_slow"}[10m]) >= 1
+      expr: changes(pd_tso_events{type="system_time_slow"}[10m]) >= 1
     annotations:
       description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'
       value: '{{ $value }}'

metrics/grafana/pd.json

@@ -1368,7 +1368,7 @@
             "rgba(237, 129, 40, 0.89)",
             "#d44a3a"
           ],
-          "datasource": "tidb-cluster",
+          "datasource": "${DS_TEST-CLUSTER}",
           "fieldConfig": {
             "defaults": {
               "custom": {}
@@ -7474,7 +7474,7 @@
             "x": 0,
             "y": 20
           },
-          "id": 1433,
+          "id": 1439,
           "legend": {
             "alignAsTable": true,
             "avg": true,

server/cluster/cluster.go

@@ -1269,6 +1269,7 @@ func (c *RaftCluster) resetMetrics() {
 func (c *RaftCluster) collectClusterMetrics() {
 	c.RLock()
 	if c.regionStats == nil {
+		c.RUnlock()
 		return
 	}
 	c.regionStats.Collect()
@@ -1281,7 +1282,6 @@ func (c *RaftCluster) collectClusterMetrics() {
 
 func (c *RaftCluster) resetClusterMetrics() {
 	c.RLock()
-
 	if c.regionStats == nil {
 		c.RUnlock()
 		return

server/core/store.go

@@ -332,7 +332,7 @@ func (s *StoreInfo) regionScoreV1(highSpaceRatio, lowSpaceRatio float64, delta i
 }
 
 func (s *StoreInfo) regionScoreV2(delta int64, lowSpaceRatio float64) float64 {
-	A := float64(s.GetAvgAvailable())
+	A := float64(s.GetAvgAvailable()) / gb
 	C := float64(s.GetCapacity()) / gb
 	R := float64(s.GetRegionSize() + delta)
 	if R < 0 {

server/core/store_test.go

@@ -126,3 +126,47 @@ func (s *testStoreSuite) TestLowSpaceRatio(c *C) {
 	store.rawStats.Available = store.rawStats.Capacity >> 2
 	c.Assert(store.IsLowSpace(0.8), Equals, false)
 }
+
+func (s *testStoreSuite) TestLowSpaceScoreV2(c *C) {
+	testdata := []struct {
+		bigger *StoreInfo
+		small  *StoreInfo
+	}{{
+		// store1 and store2 has same store available ratio and store1 less 50gb
+		bigger: NewStoreInfoWithAvailable(1, 20*gb, 100*gb, 1.4),
+		small:  NewStoreInfoWithAvailable(2, 200*gb, 1000*gb, 1.4),
+	}, {
+		// store1 and store2 has same available space and less than 50gb
+		bigger: NewStoreInfoWithAvailable(1, 10*gb, 1000*gb, 1.4),
+		small:  NewStoreInfoWithAvailable(2, 10*gb, 100*gb, 1.4),
+	}, {
+		// store1 and store2 has same available ratio less than 0.2
+		bigger: NewStoreInfoWithAvailable(1, 10*gb, 1000*gb, 1.4),
+		small:  NewStoreInfoWithAvailable(2, 1*gb, 100*gb, 1.4),
+	}, {
+		// store1 and store2 has same available ratio
+		// but the store1 ratio less than store2 ((50-10)/50=0.8<(200-100)/200=0.5)
+		bigger: NewStoreInfoWithAvailable(1, 10*gb, 100*gb, 1.4),
+		small:  NewStoreInfoWithAvailable(2, 100*gb, 1000*gb, 1.4),
+	}, {
+		// store1 and store2 has same usedSize and capacity
+		// but the bigger's amp is bigger
+		bigger: NewStoreInfoWithAvailable(1, 10*gb, 100*gb, 1.5),
+		small:  NewStoreInfoWithAvailable(2, 10*gb, 100*gb, 1.4),
+	}, {
+		// store1 and store2 has same capacity and regionSize（40g)
+		// but store1 has less available space size
+		bigger: NewStoreInfoWithAvailable(1, 60*gb, 100*gb, 1),
+		small:  NewStoreInfoWithAvailable(2, 80*gb, 100*gb, 2),
+	}, {
+		// store1 and store2 has same capacity and store2 (40g) has twice usedSize than store1 (20g)
+		// but store1 has higher amp, so store1(60g) has more regionSize (40g)
+		bigger: NewStoreInfoWithAvailable(1, 80*gb, 100*gb, 3),
+		small:  NewStoreInfoWithAvailable(2, 60*gb, 100*gb, 1),
+	}}
+	for _, v := range testdata {
+		score1 := v.bigger.regionScoreV2(0, 0.8)
+		score2 := v.small.regionScoreV2(0, 0.8)
+		c.Assert(score1, Greater, score2)
+	}
+}

server/core/test_util.go

@@ -83,6 +83,24 @@ func NewTestRegionInfo(start, end []byte) *RegionInfo {
 	}}
 }
 
+// NewStoreInfoWithAvailable is create with available and capacity
+func NewStoreInfoWithAvailable(id, available, capacity uint64, amp float64) *StoreInfo {
+	stats := &pdpb.StoreStats{}
+	stats.Capacity = capacity
+	stats.Available = available
+	usedSize := capacity - available
+	regionSize := (float64(usedSize) * amp) / mb
+	store := NewStoreInfo(
+		&metapb.Store{
+			Id: id,
+		},
+		SetStoreStats(stats),
+		SetRegionCount(int(regionSize/96)),
+		SetRegionSize(int64(regionSize)),
+	)
+	return store
+}
+
 // NewStoreInfoWithLabel is create a store with specified labels.
 func NewStoreInfoWithLabel(id uint64, regionCount int, labels map[string]string) *StoreInfo {
 	storeLabels := make([]*metapb.StoreLabel, 0, len(labels))

server/schedulers/evict_leader.go

@@ -240,7 +240,7 @@ func (s *evictLeaderScheduler) Schedule(cluster opt.Cluster) []*operator.Operato
 	s.conf.mu.RLock()
 	defer s.conf.mu.RUnlock()
 
-	return scheduleEvictLeaderBatch(s.GetName(), cluster, s.conf.StoreIDWithRanges, EvictLeaderBatchSize)
+	return scheduleEvictLeaderBatch(s.GetName(), s.GetType(), cluster, s.conf.StoreIDWithRanges, EvictLeaderBatchSize)
 }
 
 func uniqueAppendOperator(dst []*operator.Operator, src ...*operator.Operator) []*operator.Operator {
@@ -258,10 +258,10 @@ func uniqueAppendOperator(dst []*operator.Operator, src ...*operator.Operator) [
 	return dst
 }
 
-func scheduleEvictLeaderBatch(name string, cluster opt.Cluster, storeRanges map[uint64][]core.KeyRange, batchSize int) []*operator.Operator {
+func scheduleEvictLeaderBatch(name, typ string, cluster opt.Cluster, storeRanges map[uint64][]core.KeyRange, batchSize int) []*operator.Operator {
 	var ops []*operator.Operator
 	for i := 0; i < batchSize; i++ {
-		once := scheduleEvictLeaderOnce(name, cluster, storeRanges)
+		once := scheduleEvictLeaderOnce(name, typ, cluster, storeRanges)
 		// no more regions
 		if len(once) == 0 {
 			break
@@ -275,7 +275,7 @@ func scheduleEvictLeaderBatch(name string, cluster opt.Cluster, storeRanges map[
 	return ops
 }
 
-func scheduleEvictLeaderOnce(name string, cluster opt.Cluster, storeRanges map[uint64][]core.KeyRange) []*operator.Operator {
+func scheduleEvictLeaderOnce(name, typ string, cluster opt.Cluster, storeRanges map[uint64][]core.KeyRange) []*operator.Operator {
 	ops := make([]*operator.Operator, 0, len(storeRanges))
 	for id, ranges := range storeRanges {
 		var filters []filter.Filter
@@ -295,17 +295,17 @@ func scheduleEvictLeaderOnce(name string, cluster opt.Cluster, storeRanges map[u
 			for _, peer := range region.GetPendingPeers() {
 				unhealthyPeerStores[peer.GetStoreId()] = struct{}{}
 			}
-			filters = append(filters, filter.NewExcludedFilter(EvictLeaderName, nil, unhealthyPeerStores))
+			filters = append(filters, filter.NewExcludedFilter(name, nil, unhealthyPeerStores))
 		}
 
-		filters = append(filters, &filter.StoreStateFilter{ActionScope: EvictLeaderName, TransferLeader: true})
+		filters = append(filters, &filter.StoreStateFilter{ActionScope: name, TransferLeader: true})
 		target := filter.NewCandidates(cluster.GetFollowerStores(region)).
 			FilterTarget(cluster.GetOpts(), filters...).RandomPick()
 		if target == nil {
 			schedulerCounter.WithLabelValues(name, "no-target-store").Inc()
 			continue
 		}
-		op, err := operator.CreateTransferLeaderOperator(EvictLeaderType, cluster, region, region.GetLeader().GetStoreId(), target.GetID(), operator.OpLeader)
+		op, err := operator.CreateTransferLeaderOperator(typ, cluster, region, region.GetLeader().GetStoreId(), target.GetID(), operator.OpLeader)
 		if err != nil {
 			log.Debug("fail to create evict leader operator", errs.ZapError(err))
 			continue

server/schedulers/evict_slow_store.go

@@ -130,7 +130,7 @@ func (s *evictSlowStoreScheduler) schedulerEvictLeader(cluster opt.Cluster) []*o
 	storeMap := map[uint64][]core.KeyRange{
 		s.conf.EvictedStores[0]: {core.NewKeyRange("", "")},
 	}
-	return scheduleEvictLeaderBatch(s.GetName(), cluster, storeMap, EvictLeaderBatchSize)
+	return scheduleEvictLeaderBatch(s.GetName(), s.GetType(), cluster, storeMap, EvictLeaderBatchSize)
 }
 
 func (s *evictSlowStoreScheduler) IsScheduleAllowed(cluster opt.Cluster) bool {

server/schedulers/scheduler_test.go

@@ -640,6 +640,7 @@ func (s *testEvictSlowStoreSuite) TestEvictSlowStore(c *C) {
 	// Add evict leader scheduler to store 1
 	op := es.Schedule(tc)
 	testutil.CheckTransferLeader(c, op[0], operator.OpLeader, 1, 2)
+	c.Assert(op[0].Desc(), Equals, EvictSlowStoreType)
 	// Cannot balance leaders to store 1
 	op = bs.Schedule(tc)
 	c.Check(op, IsNil)