reduce heartbeat overlap operations

Signed-off-by: Ryan Leung <[email protected]>

server/core/basic_cluster.go

@@ -373,13 +373,6 @@ func (bc *BasicCluster) GetRangeCount(startKey, endKey []byte) int {
 	return bc.Regions.GetRangeCount(startKey, endKey)
 }
 
-// GetOverlaps returns the regions which are overlapped with the specified region range.
-func (bc *BasicCluster) GetOverlaps(region *RegionInfo) []*RegionInfo {
-	bc.Regions.mu.RLock()
-	defer bc.Regions.mu.RUnlock()
-	return bc.Regions.GetOverlaps(region)
-}
-
 // GetRegionSizeByRange scans regions intersecting [start key, end key), returns the total region size of this range.
 func (bc *BasicCluster) GetRegionSizeByRange(startKey, endKey []byte) int64 {
 	bc.Regions.mu.RLock()
@@ -419,7 +412,7 @@ func (bc *BasicCluster) GetStoresWriteRate() (storeIDs []uint64, bytesRates, key
 	return bc.getWriteRate(bc.Regions.GetStoreWriteRate)
 }
 
-func (bc *BasicCluster) getRelevantRegionsLocked(region *RegionInfo) (origin *RegionInfo, overlaps []*RegionInfo) {
+func (bc *BasicCluster) getRelevantRegionsLocked(region *RegionInfo) (origin *RegionInfo, overlaps []*regionItem) {
 	origin = bc.Regions.GetRegion(region.GetID())
 	if origin == nil || !bytes.Equal(origin.GetStartKey(), region.GetStartKey()) || !bytes.Equal(origin.GetEndKey(), region.GetEndKey()) {
 		overlaps = bc.Regions.GetOverlaps(region)
@@ -430,14 +423,14 @@ func (bc *BasicCluster) getRelevantRegionsLocked(region *RegionInfo) (origin *Re
 /* Regions write operations */
 
 // PreCheckPutRegion checks if the region is valid to put.
-func (bc *BasicCluster) PreCheckPutRegion(region *RegionInfo) (*RegionInfo, []*RegionInfo, error) {
+func (bc *BasicCluster) PreCheckPutRegion(region *RegionInfo) (*RegionInfo, []*regionItem, error) {
 	bc.Regions.mu.RLock()
 	origin, overlaps := bc.getRelevantRegionsLocked(region)
 	bc.Regions.mu.RUnlock()
 	return bc.check(region, origin, overlaps)
 }
 
-func (bc *BasicCluster) check(region, origin *RegionInfo, overlaps []*RegionInfo) (*RegionInfo, []*RegionInfo, error) {
+func (bc *BasicCluster) check(region, origin *RegionInfo, overlaps []*regionItem) (*RegionInfo, []*regionItem, error) {
 	for _, item := range overlaps {
 		// PD ignores stale regions' heartbeats, unless it is recreated recently by unsafe recover operation.
 		if region.GetRegionEpoch().GetVersion() < item.GetRegionEpoch().GetVersion() && !region.isRegionRecreated() {
@@ -480,7 +473,7 @@ func (bc *BasicCluster) AtomicCheckAndPutRegion(region *RegionInfo) ([]*RegionIn
 	if err != nil {
 		return nil, err
 	}
-	return bc.Regions.SetRegion(region, false), nil
+	return bc.Regions.SetRegion(region, true, overlaps...), nil
 }
 
 // PutRegion put a region.

server/core/rangetree/range_tree.go

@@ -44,11 +44,6 @@ func NewRangeTree(degree int, factory DebrisFactory) *RangeTree {
 	}
 }
 
-// Factory is the factory that generates some debris when updating items.
-func (r *RangeTree) Factory(startKey, endKey []byte, old RangeItem) []RangeItem {
-	return r.factory(startKey, endKey, old)
-}
-
 // Update insert the item and delete overlaps.
 func (r *RangeTree) Update(item RangeItem) []RangeItem {
 	overlaps := r.GetOverlaps(item)
@@ -57,23 +52,14 @@ func (r *RangeTree) Update(item RangeItem) []RangeItem {
 		children := r.factory(item.GetStartKey(), item.GetEndKey(), old)
 		for _, child := range children {
 			if c := bytes.Compare(child.GetStartKey(), child.GetEndKey()); c < 0 {
-				r.ReplaceOrInsert(child)
+				r.tree.ReplaceOrInsert(child)
 			} else if c > 0 && len(child.GetEndKey()) == 0 {
-				r.ReplaceOrInsert(child)
+				r.tree.ReplaceOrInsert(child)
 			}
 		}
 	}
-	r.ReplaceOrInsert(item)
-	return overlaps
-}
-
-// ReplaceOrInsert adds the given item to the tree.  If an item in the tree
-// already equals the given one, it is removed from the tree and returned.
-// Otherwise, nil is returned.
-//
-// nil cannot be added to the tree (will panic).
-func (r *RangeTree) ReplaceOrInsert(item RangeItem) {
 	r.tree.ReplaceOrInsert(item)
+	return overlaps
 }
 
 // GetOverlaps returns the range items that has some intersections with the given items.

server/core/region.go

@@ -729,7 +729,7 @@ func (r *RegionsInfo) GetRegion(regionID uint64) *RegionInfo {
 
 // SetRegion sets the RegionInfo to regionTree and regionMap, also update leaders and followers by region peers
 // overlaps: Other regions that overlap with the specified region, excluding itself.
-func (r *RegionsInfo) SetRegion(region *RegionInfo, passCheck bool, ol ...*RegionInfo) (overlaps []*RegionInfo) {
+func (r *RegionsInfo) SetRegion(region *RegionInfo, withOverlaps bool, ol ...*regionItem) (overlaps []*RegionInfo) {
 	var item *regionItem // Pointer to the *RegionInfo of this ID.
 	rangeChanged := true // This Region is new, or its range has changed.
 	if item = r.regions[region.GetID()]; item != nil {
@@ -739,6 +739,11 @@ func (r *RegionsInfo) SetRegion(region *RegionInfo, passCheck bool, ol ...*Regio
 		if rangeChanged {
 			// Delete itself in regionTree so that overlaps will not contain itself.
 			// Because the regionItem is reused, there is no need to delete it in the regionMap.
+			for i, o := range ol {
+				if o.GetID() == region.GetID() {
+					ol = append(ol[:i], ol[i+1:]...)
+				}
+			}
 			r.tree.remove(origin)
 		} else {
 			// If the range is not changed, only the statistical on the regionTree needs to be updated.
@@ -764,15 +769,7 @@ func (r *RegionsInfo) SetRegion(region *RegionInfo, passCheck bool, ol ...*Regio
 	}
 
 	if rangeChanged {
-		if passCheck {
-			if len(ol) != 0 {
-				overlaps = r.tree.updateWithOverlaps(item, ol)
-			} else {
-				r.tree.tree.ReplaceOrInsert(item)
-			}
-		} else {
-			overlaps = r.tree.update(item)
-		}
+		overlaps = r.tree.update(item, withOverlaps, ol...)
 		for _, old := range overlaps {
 			r.RemoveRegion(r.GetRegion(old.GetID()))
 		}
@@ -787,7 +784,7 @@ func (r *RegionsInfo) SetRegion(region *RegionInfo, passCheck bool, ol ...*Regio
 			store = newRegionTree()
 			peersMap[storeID] = store
 		}
-		store.update(item)
+		store.update(item, false)
 	}
 
 	// Add to leaders and followers.
@@ -849,8 +846,8 @@ func (r *RegionsInfo) updateSubTreeStat(origin *RegionInfo, region *RegionInfo)
 }
 
 // GetOverlaps returns the regions which are overlapped with the specified region range.
-func (r *RegionsInfo) GetOverlaps(region *RegionInfo) []*RegionInfo {
-	return r.tree.getOverlaps(region)
+func (r *RegionsInfo) GetOverlaps(region *RegionInfo) []*regionItem {
+	return r.tree.overlaps(&regionItem{RegionInfo: region})
 }
 
 // RemoveRegion removes RegionInfo from regionTree and regionMap

server/core/region_tree.go

@@ -80,17 +80,6 @@ func (t *regionTree) length() int {
 	return t.tree.Len()
 }
 
-// getOverlaps gets the regions which are overlapped with the specified region range.
-func (t *regionTree) getOverlaps(region *RegionInfo) []*RegionInfo {
-	item := &regionItem{RegionInfo: region}
-	result := t.overlaps(item)
-	overlaps := make([]*RegionInfo, len(result))
-	for i, r := range result {
-		overlaps[i] = r.RegionInfo
-	}
-	return overlaps
-}
-
 // GetOverlaps returns the range items that has some intersections with the given items.
 func (t *regionTree) overlaps(item *regionItem) []*regionItem {
 	// note that Find() gets the last item that is less or equal than the item.
@@ -118,51 +107,24 @@ func (t *regionTree) overlaps(item *regionItem) []*regionItem {
 // update updates the tree with the region.
 // It finds and deletes all the overlapped regions first, and then
 // insert the region.
-func (t *regionTree) update(item *regionItem) []*RegionInfo {
+func (t *regionTree) update(item *regionItem, withOverlaps bool, overlaps ...*regionItem) []*RegionInfo {
 	region := item.RegionInfo
-	t.totalSize += region.approximateSize
-	regionWriteBytesRate, regionWriteKeysRate := region.GetWriteRate()
+	t.totalSize += item.GetApproximateSize()
+	regionWriteBytesRate, regionWriteKeysRate := item.GetWriteRate()
 	t.totalWriteBytesRate += regionWriteBytesRate
 	t.totalWriteKeysRate += regionWriteKeysRate
 
-	overlaps := t.overlaps(item)
-	for _, old := range overlaps {
-		t.tree.Delete(old)
-	}
-	t.tree.ReplaceOrInsert(item)
-	result := make([]*RegionInfo, len(overlaps))
-	for i, overlap := range overlaps {
-		old := overlap.RegionInfo
-		result[i] = old
-		log.Debug("overlapping region",
-			zap.Uint64("region-id", old.GetID()),
-			logutil.ZapRedactStringer("delete-region", RegionToHexMeta(old.GetMeta())),
-			logutil.ZapRedactStringer("update-region", RegionToHexMeta(region.GetMeta())))
-		t.totalSize -= old.approximateSize
-		regionWriteBytesRate, regionWriteKeysRate = old.GetWriteRate()
-		t.totalWriteBytesRate -= regionWriteBytesRate
-		t.totalWriteKeysRate -= regionWriteKeysRate
+	if !withOverlaps {
+		overlaps = t.overlaps(item)
 	}
 
-	return result
-}
-
-// updateWithOverlaps updates the tree with the region with pre obtained overlaps.
-func (t *regionTree) updateWithOverlaps(item *regionItem, overlaps []*RegionInfo) []*RegionInfo {
-	region := item.RegionInfo
-	t.totalSize += region.approximateSize
-	regionWriteBytesRate, regionWriteKeysRate := region.GetWriteRate()
-	t.totalWriteBytesRate += regionWriteBytesRate
-	t.totalWriteKeysRate += regionWriteKeysRate
-
 	for _, old := range overlaps {
-		o := &regionItem{old}
-		t.tree.Delete(o)
+		t.tree.Delete(old)
 	}
 	t.tree.ReplaceOrInsert(item)
 	result := make([]*RegionInfo, len(overlaps))
 	for i, overlap := range overlaps {
-		old := overlap
+		old := overlap.RegionInfo
 		result[i] = old
 		log.Debug("overlapping region",
 			zap.Uint64("region-id", old.GetID()),

server/core/region_tree_test.go

@@ -356,7 +356,7 @@ func TestRandomRegionDiscontinuous(t *testing.T) {
 
 func updateNewItem(tree *regionTree, region *RegionInfo) {
 	item := &regionItem{RegionInfo: region}
-	tree.update(item)
+	tree.update(item, false)
 }
 
 func checkRandomRegion(re *require.Assertions, tree *regionTree, regions []*RegionInfo, ranges []KeyRange) {