kvcoord: Add metric to keep track of restarted ranges in rangefeed

Add a `distsender.rangefeed.restart_ranges` metric to keep track
of the number of ranges restarted due to transient error.

Epic: CRDB-25044
Release note: None

pkg/kv/kvclient/kvcoord/dist_sender.go

@@ -165,6 +165,12 @@ This counts the number of ranges with an active rangefeed that are performing ca
 		Measurement: "Ranges",
 		Unit:        metric.Unit_COUNT,
 	}
+	metaDistSenderRangefeedRestartRanges = metric.Metadata{
+		Name:        "distsender.rangefeed.restart_ranges",
+		Help:        `Number of ranges that were restarted due to transient errors`,
+		Measurement: "Ranges",
+		Unit:        metric.Unit_COUNT,
+	}
 	metaDistSenderRangefeedRestartStuck = metric.Metadata{
 		Name: "distsender.rangefeed.restart_stuck",
 		Help: `Number of times a rangefeed was restarted due to not receiving ` +
@@ -239,6 +245,7 @@ type DistSenderMetrics struct {
 	RangefeedRanges         *metric.Gauge
 	RangefeedCatchupRanges  *metric.Gauge
 	RangefeedErrorCatchup   *metric.Counter
+	RangefeedRestartRanges  *metric.Counter
 	RangefeedRestartStuck   *metric.Counter
 	MethodCounts            [kvpb.NumMethods]*metric.Counter
 	ErrCounts               [kvpb.NumErrors]*metric.Counter
@@ -260,6 +267,7 @@ func makeDistSenderMetrics() DistSenderMetrics {
 		RangefeedRanges:         metric.NewGauge(metaDistSenderRangefeedTotalRanges),
 		RangefeedCatchupRanges:  metric.NewGauge(metaDistSenderRangefeedCatchupRanges),
 		RangefeedErrorCatchup:   metric.NewCounter(metaDistSenderRangefeedErrorCatchupRanges),
+		RangefeedRestartRanges:  metric.NewCounter(metaDistSenderRangefeedRestartRanges),
 		RangefeedRestartStuck:   metric.NewCounter(metaDistSenderRangefeedRestartStuck),
 	}
 	for i := range m.MethodCounts {

pkg/kv/kvclient/kvcoord/dist_sender_mux_rangefeed.go

@@ -440,6 +440,8 @@ func (m *rangefeedMuxer) receiveEventsFromNode(
 func (m *rangefeedMuxer) restartActiveRangeFeed(
 	ctx context.Context, active *activeMuxRangeFeed, reason error,
 ) error {
+	m.ds.metrics.RangefeedRestartRanges.Inc(1)
+
 	if log.V(1) {
 		log.Infof(ctx, "RangeFeed %s@%s disconnected with last checkpoint %s ago: %v",
 			active.Span, active.StartAfter, timeutil.Since(active.Resolved.GoTime()), reason)

pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go

@@ -484,6 +484,7 @@ func (ds *DistSender) partialRangeFeed(
 		if err != nil {
 			return err
 		}
+		ds.metrics.RangefeedRestartRanges.Inc(1)
 		if errInfo.evict {
 			token.Evict(ctx)
 			token = rangecache.EvictionToken{}

pkg/ts/catalog/chart_catalog.go

@@ -304,6 +304,12 @@ var charts = []sectionDescription{
 					"distsender.rangefeed.restart_stuck",
 				},
 			},
+			{
+				Title: "Restarted Ranges",
+				Metrics: []string{
+					"distsender.rangefeed.restart_ranges",
+				},
+			},
 			{
 				Title: "RPCs",
 				Metrics: []string{