Fix some hive tests

turbo/engineapi/engine_helpers/fork_validator.go

@@ -129,9 +129,9 @@ func (fv *ForkValidator) FlushExtendingFork(tx kv.RwTx, accumulator *shards.Accu
 }
 
 // ValidatePayload returns whether a payload is valid or invalid, or if cannot be determined, it will be accepted.
-// if the payload extend the canonical chain, then we stack it in extendingFork without any unwind.
-// if the payload is a fork then we unwind to the point where the fork meet the canonical chain and we check if it is valid or not from there.
-// if for any reasons none of the action above can be performed due to lack of information, we accept the payload and avoid validation.
+// if the payload extends the canonical chain, then we stack it in extendingFork without any unwind.
+// if the payload is a fork then we unwind to the point where the fork meets the canonical chain, and there we check whether it is valid.
+// if for any reason none of the actions above can be performed due to lack of information, we accept the payload and avoid validation.
 func (fv *ForkValidator) ValidatePayload(tx kv.Tx, header *types.Header, body *types.RawBody, extendCanonical bool) (status engine_types.EngineStatus, latestValidHash libcommon.Hash, validationError error, criticalError error) {
 	fv.lock.Lock()
 	defer fv.lock.Unlock()

turbo/engineapi/engine_server.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/ledgerwatch/erigon-lib/chain"
 	libcommon "github.com/ledgerwatch/erigon-lib/common"
+	"github.com/ledgerwatch/erigon-lib/common/fixedgas"
 	"github.com/ledgerwatch/erigon-lib/common/hexutility"
 	"github.com/ledgerwatch/erigon-lib/gointerfaces/execution"
 	"github.com/ledgerwatch/erigon-lib/gointerfaces/txpool"
@@ -112,6 +113,39 @@ func (s *EngineServer) checkWithdrawalsPresence(time uint64, withdrawals []*type
 	return nil
 }
 
+func (s *EngineServer) validatePayloadBlobs(req *engine_types.ExecutionPayload,
+	expectedBlobHashes []libcommon.Hash, transactions *[]types.Transaction) (*engine_types.PayloadStatus, error) {
+	if expectedBlobHashes == nil {
+		return nil, &rpc.InvalidParamsError{Message: "nil blob hashes array"}
+	}
+	actualBlobHashes := []libcommon.Hash{}
+	for _, txn := range *transactions {
+		actualBlobHashes = append(actualBlobHashes, txn.GetBlobHashes()...)
+	}
+	if len(actualBlobHashes) > int(fixedgas.MaxBlobsPerBlock) || req.BlobGasUsed.Uint64() > fixedgas.MaxBlobGasPerBlock {
+		s.logger.Warn("[NewPayload] blobs/blobGasUsed exceeds max per block",
+			"count", len(actualBlobHashes), "BlobGasUsed", req.BlobGasUsed.Uint64())
+		bad, latestValidHash := s.hd.IsBadHeaderPoS(req.ParentHash)
+		if !bad {
+			latestValidHash = req.ParentHash
+		}
+		return &engine_types.PayloadStatus{
+			Status:          engine_types.InvalidStatus,
+			ValidationError: engine_types.NewStringifiedErrorFromString("blobs/blobgas exceeds max"),
+			LatestValidHash: &latestValidHash,
+		}, nil
+	}
+	if !reflect.DeepEqual(actualBlobHashes, expectedBlobHashes) {
+		s.logger.Warn("[NewPayload] mismatch in blob hashes",
+			"expectedBlobHashes", expectedBlobHashes, "actualBlobHashes", actualBlobHashes)
+		return &engine_types.PayloadStatus{
+			Status:          engine_types.InvalidStatus,
+			ValidationError: engine_types.NewStringifiedErrorFromString("mismatch in blob hashes"),
+		}, nil
+	}
+	return nil, nil
+}
+
 // EngineNewPayload validates and possibly executes payload
 func (s *EngineServer) newPayload(ctx context.Context, req *engine_types.ExecutionPayload,
 	expectedBlobHashes []libcommon.Hash, parentBeaconBlockRoot *libcommon.Hash, version clparams.StateVersion,
@@ -172,8 +206,6 @@ func (s *EngineServer) newPayload(ctx context.Context, req *engine_types.Executi
 
 	blockHash := req.BlockHash
 	if header.Hash() != blockHash {
-		m3, _ := header.MarshalJSON()
-		fmt.Println(string(m3))
 		s.logger.Error("[NewPayload] invalid block hash", "stated", blockHash, "actual", header.Hash())
 		return &engine_types.PayloadStatus{
 			Status:          engine_types.InvalidStatus,
@@ -200,20 +232,9 @@ func (s *EngineServer) newPayload(ctx context.Context, req *engine_types.Executi
 		}, nil
 	}
 	if version >= clparams.DenebVersion {
-		actualBlobHashes := []libcommon.Hash{}
-		for _, tx := range transactions {
-			actualBlobHashes = append(actualBlobHashes, tx.GetBlobHashes()...)
-		}
-		if expectedBlobHashes == nil {
-			return nil, &rpc.InvalidParamsError{Message: "nil blob hashes array"}
-		}
-		if !reflect.DeepEqual(actualBlobHashes, expectedBlobHashes) {
-			s.logger.Warn("[NewPayload] mismatch in blob hashes",
-				"expectedBlobHashes", expectedBlobHashes, "actualBlobHashes", actualBlobHashes)
-			return &engine_types.PayloadStatus{
-				Status:          engine_types.InvalidStatus,
-				ValidationError: engine_types.NewStringifiedErrorFromString("mismatch in blob hashes"),
-			}, nil
+		status, err := s.validatePayloadBlobs(req, expectedBlobHashes, &transactions)
+		if err != nil || status != nil {
+			return status, err
 		}
 	}
 
@@ -430,6 +451,28 @@ func (s *EngineServer) forkchoiceUpdated(ctx context.Context, forkchoiceState *e
 		}
 	}
 
+	if payloadAttributes != nil {
+		timestamp := uint64(payloadAttributes.Timestamp)
+		if !s.config.IsCancun(timestamp) && version >= clparams.DenebVersion { // V3 before cancun
+			if payloadAttributes.ParentBeaconBlockRoot == nil {
+				return nil, &rpc.InvalidParamsError{Message: "Beacon Root missing"}
+			}
+			return nil, &rpc.UnsupportedForkError{Message: "Unsupported fork"}
+		}
+		if s.config.IsCancun(timestamp) && version < clparams.DenebVersion { // Not V3 after cancun
+			if payloadAttributes.ParentBeaconBlockRoot != nil {
+				return nil, &rpc.InvalidParamsError{Message: "Unexpected Beacon Root"}
+			}
+			return nil, &rpc.UnsupportedForkError{Message: "Unsupported fork"}
+		}
+
+		if s.config.IsCancun(timestamp) && version >= clparams.DenebVersion {
+			if payloadAttributes.ParentBeaconBlockRoot == nil {
+				return nil, &rpc.InvalidParamsError{Message: "Beacon Root missing"}
+			}
+		}
+	}
+
 	// No need for payload building
 	if payloadAttributes == nil || status.Status != engine_types.ValidStatus {
 		return &engine_types.ForkChoiceUpdatedResponse{PayloadStatus: status}, nil
@@ -439,20 +482,6 @@ func (s *EngineServer) forkchoiceUpdated(ctx context.Context, forkchoiceState *e
 		return nil, fmt.Errorf("execution layer not running as a proposer. enable proposer by taking out the --proposer.disable flag on startup")
 	}
 
-	timestamp := uint64(payloadAttributes.Timestamp)
-	if !s.config.IsCancun(timestamp) && version >= clparams.DenebVersion { // V3 before cancun
-		if payloadAttributes.ParentBeaconBlockRoot == nil {
-			return nil, &rpc.InvalidParamsError{Message: "Beacon Root missing"}
-		}
-		return nil, &rpc.UnsupportedForkError{Message: "Unsupported fork"}
-	}
-	if s.config.IsCancun(timestamp) && version < clparams.DenebVersion { // Not V3 after cancun
-		if payloadAttributes.ParentBeaconBlockRoot != nil {
-			return nil, &rpc.InvalidParamsError{Message: "Beacon Root missing"}
-		}
-		return nil, &rpc.UnsupportedForkError{Message: "Unsupported fork"}
-	}
-
 	headHeader := s.chainRW.GetHeaderByHash(forkchoiceState.HeadHash)
 
 	if headHeader.Hash() != forkchoiceState.HeadHash {
@@ -468,6 +497,7 @@ func (s *EngineServer) forkchoiceUpdated(ctx context.Context, forkchoiceState *e
 		return &engine_types.ForkChoiceUpdatedResponse{PayloadStatus: status}, nil
 	}
 
+	timestamp := uint64(payloadAttributes.Timestamp)
 	if headHeader.Time >= timestamp {
 		return nil, &engine_helpers.InvalidPayloadAttributesErr
 	}
@@ -538,6 +568,8 @@ func (s *EngineServer) getPayloadBodiesByRange(ctx context.Context, start, count
 	return resp, nil
 }
 
+// Returns the most recent version of the payload(for the payloadID) at the time of receiving the call
+// See https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md#engine_getpayloadv1
 func (e *EngineServer) GetPayloadV1(ctx context.Context, payloadId hexutility.Bytes) (*engine_types.ExecutionPayload, error) {
 
 	decodedPayloadId := binary.BigEndian.Uint64(payloadId)
@@ -551,26 +583,38 @@ func (e *EngineServer) GetPayloadV1(ctx context.Context, payloadId hexutility.By
 	return response.ExecutionPayload, nil
 }
 
+// Same as [GetPayloadV1] with addition of blockValue
+// See https://github.com/ethereum/execution-apis/blob/main/src/engine/shanghai.md#engine_getpayloadv2
 func (e *EngineServer) GetPayloadV2(ctx context.Context, payloadID hexutility.Bytes) (*engine_types.GetPayloadResponse, error) {
 	decodedPayloadId := binary.BigEndian.Uint64(payloadID)
 	e.logger.Info("Received GetPayloadV2", "payloadId", decodedPayloadId)
 	return e.getPayload(ctx, decodedPayloadId, clparams.CapellaVersion)
 }
 
+// Same as [GetPayloadV2], with addition of blobsBundle containing valid blobs, commitments, proofs
+// See https://github.com/ethereum/execution-apis/blob/main/src/engine/cancun.md#engine_getpayloadv3
 func (e *EngineServer) GetPayloadV3(ctx context.Context, payloadID hexutility.Bytes) (*engine_types.GetPayloadResponse, error) {
 	decodedPayloadId := binary.BigEndian.Uint64(payloadID)
 	e.logger.Info("Received GetPayloadV3", "payloadId", decodedPayloadId)
 	return e.getPayload(ctx, decodedPayloadId, clparams.DenebVersion)
 }
 
+// Updates the forkchoice state after validating the headBlockHash
+// Additionally, builds and returns a unique identifier for an initial version of a payload
+// (asynchronously updated with transactions), if payloadAttributes is not nil and passes validation
+// See https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md#engine_forkchoiceupdatedv1
 func (e *EngineServer) ForkchoiceUpdatedV1(ctx context.Context, forkChoiceState *engine_types.ForkChoiceState, payloadAttributes *engine_types.PayloadAttributes) (*engine_types.ForkChoiceUpdatedResponse, error) {
 	return e.forkchoiceUpdated(ctx, forkChoiceState, payloadAttributes, clparams.BellatrixVersion)
 }
 
+// Same as, and a replacement for, [ForkchoiceUpdatedV1], post Shanghai
+// See https://github.com/ethereum/execution-apis/blob/main/src/engine/shanghai.md#engine_forkchoiceupdatedv2
 func (e *EngineServer) ForkchoiceUpdatedV2(ctx context.Context, forkChoiceState *engine_types.ForkChoiceState, payloadAttributes *engine_types.PayloadAttributes) (*engine_types.ForkChoiceUpdatedResponse, error) {
 	return e.forkchoiceUpdated(ctx, forkChoiceState, payloadAttributes, clparams.CapellaVersion)
 }
 
+// Successor of [ForkchoiceUpdatedV2] post Cancun, with stricter check on params
+// See https://github.com/ethereum/execution-apis/blob/main/src/engine/cancun.md#engine_forkchoiceupdatedv3
 func (e *EngineServer) ForkchoiceUpdatedV3(ctx context.Context, forkChoiceState *engine_types.ForkChoiceState, payloadAttributes *engine_types.PayloadAttributes) (*engine_types.ForkChoiceUpdatedResponse, error) {
 	return e.forkchoiceUpdated(ctx, forkChoiceState, payloadAttributes, clparams.DenebVersion)
 }
@@ -615,6 +659,8 @@ func (e *EngineServer) ExchangeTransitionConfigurationV1(ctx context.Context, be
 	}, nil
 }
 
+// Returns an array of execution payload bodies referenced by their block hashes
+// See https://github.com/ethereum/execution-apis/blob/main/src/engine/shanghai.md#engine_getpayloadbodiesbyhashv1
 func (e *EngineServer) GetPayloadBodiesByHashV1(ctx context.Context, hashes []libcommon.Hash) ([]*engine_types.ExecutionPayloadBodyV1, error) {
 	if len(hashes) > 1024 {
 		return nil, &engine_helpers.TooLargeRequestErr
@@ -623,6 +669,8 @@ func (e *EngineServer) GetPayloadBodiesByHashV1(ctx context.Context, hashes []li
 	return e.getPayloadBodiesByHash(ctx, hashes, clparams.DenebVersion)
 }
 
+// Returns an ordered (as per canonical chain) array of execution payload bodies, with corresponding execution block numbers from "start", up to "count"
+// See https://github.com/ethereum/execution-apis/blob/main/src/engine/shanghai.md#engine_getpayloadbodiesbyrangev1
 func (e *EngineServer) GetPayloadBodiesByRangeV1(ctx context.Context, start, count hexutil.Uint64) ([]*engine_types.ExecutionPayloadBodyV1, error) {
 	if start == 0 || count == 0 {
 		return nil, &rpc.InvalidParamsError{Message: fmt.Sprintf("invalid start or count, start: %v count: %v", start, count)}

turbo/engineapi/interface.go

@@ -2,7 +2,6 @@ package engineapi
 
 import (
 	"context"
-
 	"github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/erigon-lib/common/hexutility"
 	"github.com/ledgerwatch/erigon/common/hexutil"

turbo/stages/headerdownload/header_data_struct.go

@@ -27,7 +27,7 @@ const (
 	PersistedQueueID
 )
 
-// Link is a chain link that can be connect to other chain links
+// Link is a chain link that can be connected to other links in the chain
 // For a given link, parent link can be found by hd.links[link.header.ParentHash], and child links by link.next (there may be more than one child in case of forks)
 // Links encapsule block headers
 // Links can be either persistent or not. Persistent links encapsule headers that have already been saved to the database, but these links are still