eth/executionclient: rate limit multi client Healthy call (#2010)

* eth/executionclient: rate limit Healthy call

* use atomic instead of mutex

* use < instead of <=

eth/executionclient/defaults.go

@@ -8,7 +8,7 @@ const (
 	DefaultConnectionTimeout           = 10 * time.Second
 	DefaultReconnectionInitialInterval = 1 * time.Second
 	DefaultReconnectionMaxInterval     = 64 * time.Second
-	DefaultHealthInvalidationInterval  = 10 * time.Second // TODO: decide on this value, for now choosing a bit less than block interval
+	DefaultHealthInvalidationInterval  = 24 * time.Second // TODO: decide on this value, for now choosing the node prober interval but it should probably be a bit less than block interval
 	DefaultFollowDistance              = 8
 	// TODO ALAN: revert
 	DefaultHistoricalLogsBatchSize = 500

eth/executionclient/multi_client.go

@@ -71,6 +71,7 @@ type MultiClient struct {
 	clientsMu          []sync.Mutex           // clientsMu allow for lazy initialization of each client in `clients` slice in thread-safe manner (atomically)
 	clients            []SingleClientProvider // nil if not connected
 	currentClientIndex atomic.Int64
+	lastHealthy        atomic.Int64
 }
 
 // NewMulti creates a new instance of MultiClient.
@@ -278,6 +279,10 @@ func (mc *MultiClient) StreamLogs(ctx context.Context, fromBlock uint64) <-chan
 
 // Healthy returns if execution client is currently healthy: responds to requests and not in the syncing state.
 func (mc *MultiClient) Healthy(ctx context.Context) error {
+	if time.Since(time.Unix(mc.lastHealthy.Load(), 0)) < mc.healthInvalidationInterval {
+		return nil
+	}
+
 	healthyClients := atomic.Bool{}
 	p := pool.New().WithErrors().WithContext(ctx)
 
@@ -307,6 +312,7 @@ func (mc *MultiClient) Healthy(ctx context.Context) error {
 	}
 	err := p.Wait()
 	if healthyClients.Load() {
+		mc.lastHealthy.Store(time.Now().Unix())
 		return nil
 	}
 	return fmt.Errorf("no healthy clients: %w", err)