gossip: Don't check if the network should be tightened so often

Touches cockroachdb#51838. See the thread there for more detail, but the gist is
that tightenNetwork gets called hundreds of times per second (taking a
full mutex lock each time) on large clusters, even when nothing needs
tightening.

Release note (performance improvement): Avoid wasteful contention on the
gossip mutex caused by checking if the network needs tightening hundreds
of times per second.

pkg/gossip/gossip.go

@@ -131,6 +131,10 @@ const (
 	// StoreTTL is time-to-live for store-related info.
 	StoreTTL = 2 * StoresInterval
 
+	// gossipTightenInterval is how long to wait between tightenNetwork checks if
+	// we didn't need to tighten the last time we checked.
+	gossipTightenInterval = time.Second
+
 	unknownNodeID roachpb.NodeID = 0
 )
 
@@ -1462,12 +1466,23 @@ func jitteredInterval(interval time.Duration) time.Duration {
 func (g *Gossip) tightenNetwork(ctx context.Context) {
 	g.mu.Lock()
 	defer g.mu.Unlock()
+
+	now := time.Now()
+	if now.Before(g.mu.lastTighten.Add(gossipTightenInterval)) {
+		// It hasn't been long since we last tightened the network, so skip it.
+		return
+	}
+	g.mu.lastTighten = now
+
 	if g.outgoing.hasSpace() {
 		distantNodeID, distantHops := g.mu.is.mostDistant(g.hasOutgoingLocked)
 		log.VEventf(ctx, 2, "distantHops: %d from %d", distantHops, distantNodeID)
 		if distantHops <= maxHops {
 			return
 		}
+		// If tightening is needed, then reset lastTighten to avoid restricting how
+		// soon we try again.
+		g.mu.lastTighten = time.Time{}
 		if nodeAddr, err := g.getNodeIDAddress(distantNodeID, true /* locked */); err != nil || nodeAddr == nil {
 			log.Health.Errorf(ctx, "unable to get address for n%d: %s", distantNodeID, err)
 		} else {

pkg/gossip/server.go

@@ -55,6 +55,9 @@ type server struct {
 		// composable. There's an open proposal to add them:
 		// https://github.com/golang/go/issues/16620
 		ready chan struct{}
+		// The time at which we last checked if the network should be tightened.
+		// Used to avoid burning CPU and mutex cycles on checking too frequently.
+		lastTighten time.Time
 	}
 	tighten chan struct{} // Sent on when we may want to tighten the network
 
@@ -344,6 +347,11 @@ func (s *server) gossipReceiver(
 }
 
 func (s *server) maybeTightenLocked() {
+	now := time.Now()
+	if now.Before(s.mu.lastTighten.Add(gossipTightenInterval)) {
+		// It hasn't been long since we last tightened the network, so skip it.
+		return
+	}
 	select {
 	case s.tighten <- struct{}{}:
 	default: