Merge pull request #1883 from bnb-chain/4844-broadcast

4844 broadcasting of sidecar

.gitignore

@@ -53,4 +53,7 @@ cmd/geth/__debug_bin
 cmd/bootnode/bootnode
 graphql/__debug_bin
 
-tempdatadir
+tempdatadir
+tempdatadir2
+config.toml
+config2.toml

cmd/devp2p/internal/ethtest/types.go

@@ -112,7 +112,7 @@ type NewBlock eth.NewBlockPacket
 func (nb NewBlock) Code() int { return 23 }
 
 // NewPooledTransactionHashes is the network packet for the tx hash propagation message.
-type NewPooledTransactionHashes eth.NewPooledTransactionHashesPacket
+type NewPooledTransactionHashes eth.NewPooledTransactionHashesPacket66
 
 func (nb NewPooledTransactionHashes) Code() int { return 24 }
 

consensus/parlia/parlia.go

@@ -210,10 +210,11 @@ func ParliaRLP(header *types.Header, chainId *big.Int) []byte {
 
 // Parlia is the consensus engine of BSC
 type Parlia struct {
-	chainConfig *params.ChainConfig  // Chain config
-	config      *params.ParliaConfig // Consensus engine configuration parameters for parlia consensus
-	genesisHash common.Hash
-	db          ethdb.Database // Database to store and retrieve snapshot checkpoints
+	chainConfig  *params.ChainConfig  // Chain config
+	config       *params.ParliaConfig // Consensus engine configuration parameters for parlia consensus
+	genesisHash  common.Hash
+	db           ethdb.Database // Database to store and retrieve snapshot checkpoints
+	blobDatabase *bloblevel.Storage
 
 	recentSnaps *lru.ARCCache // Snapshots for recent block to speed up
 	signatures  *lru.ARCCache // Signatures of recent blocks to speed up mining
@@ -240,6 +241,7 @@ type Parlia struct {
 func New(
 	chainConfig *params.ChainConfig,
 	db ethdb.Database,
+	blobDB *bloblevel.Storage,
 	ethAPI *ethapi.PublicBlockChainAPI,
 	genesisHash common.Hash,
 ) *Parlia {
@@ -277,6 +279,7 @@ func New(
 		config:                     parliaConfig,
 		genesisHash:                genesisHash,
 		db:                         db,
+		blobDatabase:               blobDB,
 		ethAPI:                     ethAPI,
 		recentSnaps:                recentSnaps,
 		signatures:                 signatures,
@@ -1427,29 +1430,20 @@ func (p *Parlia) Seal(chain consensus.ChainHeaderReader, block *types.BlockAndSi
 		}
 		// todo sidecar needs to be signed somewhere
 
-		// todo save sidecars here? Since below there is code which will save the block anyway.
+		// todo 4844 save sidecars here? Since below there is code which will save the block anyway.
 		if len(sidecars) > 0 {
 			cfg := params.ChainConfig{
 				DataBlobs: &params.DataBlobsConfig{
 					SlotsPerEpoch:                    1,
 					MinEpochsForBlobsSidecarsRequest: 1,
 				},
 			}
-			err = blobStorage.SaveBlobSidecar(context.Background(), &cfg, sidecars)
+			err = p.blobDatabase.SaveBlobSidecar(context.Background(), &cfg, sidecars)
 			if err != nil {
 				log.Error("Sidecars could not be saved!", "err", err)
 			}
 
-			//// Original byte array
-			//originalBytes := []byte{23, 195, 163, 130, 130, 113, 153, 124, 5, 232, 93, 202, 189, 136, 171, 137, 161, 47, 86, 125, 255, 243, 39, 238, 158, 161, 43, 251, 252, 66, 208, 117}
-			//
-			//// Create a new byte array of size 32
-			//var byteArray32 [32]byte
-			//
-			//// Copy the original byte array into the new byte array
-			//copy(byteArray32[:], originalBytes)
-
-			got, err1 := rawdb.GetBlobSidecarsByRoot(context.Background(), blobStorage, bytesutil.ToBytes32(sidecars[0].BlockRoot))
+			got, err1 := rawdb.GetBlobSidecarsByRoot(context.Background(), p.blobDatabase, bytesutil.ToBytes32(sidecars[0].BlockRoot))
 			if err1 != nil {
 				fmt.Println("Error while fetching sidecar", err1)
 			} else {

core/blockchain.go

@@ -20,6 +20,7 @@ package core
 import (
 	"errors"
 	"fmt"
+	"github.com/ethereum/go-ethereum/core/rawdb/bloblevel"
 	"io"
 	"math/big"
 	"sort"
@@ -189,11 +190,12 @@ type BlockChain struct {
 	chainConfig *params.ChainConfig // Chain & network configuration
 	cacheConfig *CacheConfig        // Cache configuration for pruning
 
-	db         ethdb.Database // Low level persistent database to store final content in
-	snaps      *snapshot.Tree // Snapshot tree for fast trie leaf access
-	triegc     *prque.Prque   // Priority queue mapping block numbers to tries to gc
-	gcproc     time.Duration  // Accumulates canonical block processing for trie dumping
-	commitLock sync.Mutex     // CommitLock is used to protect above field from being modified concurrently
+	db           ethdb.Database // Low level persistent database to store final content in
+	blobDatabase bloblevel.Storage
+	snaps        *snapshot.Tree // Snapshot tree for fast trie leaf access
+	triegc       *prque.Prque   // Priority queue mapping block numbers to tries to gc
+	gcproc       time.Duration  // Accumulates canonical block processing for trie dumping
+	commitLock   sync.Mutex     // CommitLock is used to protect above field from being modified concurrently
 
 	// txLookupLimit is the maximum number of blocks from head whose tx indices
 	// are reserved:

core/events.go

@@ -30,6 +30,9 @@ type ReannoTxsEvent struct{ Txs []*types.Transaction }
 // NewMinedBlockEvent is posted when a block has been imported.
 type NewMinedBlockEvent struct{ Block *types.Block }
 
+// NewMinedSidecarEvent is posted when a sidecar has been imported.
+type NewMinedSidecarEvent struct{ Sidecar *types.Sidecar }
+
 // RemovedLogsEvent is posted when a reorg happens
 type RemovedLogsEvent struct{ Logs []*types.Log }
 

core/genesis.go

@@ -368,6 +368,9 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block {
 		}
 		if g.Config.IsCancun(g.Timestamp) {
 			head.SetExcessDataGas(g.ExcessDataGas)
+			if head.ExcessDataGas == nil {
+				head.ExcessDataGas = big.NewInt(0)
+			}
 		}
 	}
 

core/rawdb/freezer.go

@@ -427,7 +427,7 @@ func (f *freezer) freeze(db ethdb.KeyValueStore) {
 			continue
 
 		case *number < threshold:
-			log.Debug("Current full block not old enough", "number", *number, "hash", hash, "delay", threshold)
+			log.Debug("fCurrent full block not old enough", "number", *number, "hash", hash, "delay", threshold)
 			backoff = true
 			continue
 

core/remote_state_verifier.go

@@ -106,6 +106,7 @@ func (vm *remoteVerifyManager) mainLoop() {
 	pruneTicker := time.NewTicker(pruneInterval)
 	defer pruneTicker.Stop()
 	for {
+		fmt.Println("for loop of remoteVerifyManager mainLoop")
 		select {
 		case h := <-vm.chainBlockCh:
 			vm.NewBlockVerifyTask(h.Block.Header())
@@ -143,6 +144,7 @@ func (vm *remoteVerifyManager) mainLoop() {
 			vm.taskLock.RUnlock()
 			return
 		case <-vm.chainHeadSub.Err():
+			fmt.Println("Chain head subscription error!!!!")
 			return
 		}
 	}

core/types/block.go

@@ -397,9 +397,9 @@ func (b *Block) DecodeRLP(s *rlp.Stream) error {
 	if err := s.Decode(&eb); err != nil {
 		return err
 	}
-	for i, tx := range *eb.Txs {
+	for _, tx := range *eb.Txs {
 		if tx.wrapData != nil { // todo 4844 we may need to get rid of this, actually NOT
-			return fmt.Errorf("transactions in blocks must not contain wrap-data, tx %d is bad", i)
+			//return fmt.Errorf("transactions in blocks must not contain wrap-data, tx %d is bad", i)
 		}
 	}
 	b.header, b.uncles, b.transactions = eb.Header, eb.Uncles, []*Transaction(*eb.Txs)
@@ -409,12 +409,12 @@ func (b *Block) DecodeRLP(s *rlp.Stream) error {
 
 // EncodeRLP serializes b into the Ethereum RLP block format.
 func (b *Block) EncodeRLP(w io.Writer) error {
-	if b.header.ExcessDataGas != nil {
-		// This situation should not arise, but if it does (due to a bug) you'd silently produce an
-		// encoding that would fail to decode. ref:
-		// https://github.com/ethereum/go-ethereum/pull/26077
-		return errors.New("nil WithdrawalsHash in header with non-nil ExcessDataGas")
-	}
+	//if b.header.ExcessDataGas != nil {
+	//	// This situation should not arise, but if it does (due to a bug) you'd silently produce an
+	//	// encoding that would fail to decode. ref:
+	//	// https://github.com/ethereum/go-ethereum/pull/26077
+	//	return errors.New("nil WithdrawalsHash in header with non-nil ExcessDataGas")
+	//}
 	return rlp.Encode(w, extblock{
 		Header: b.header,
 		Txs:    (*extBlockTxs)(&b.transactions),

core/types/data_blob.go

@@ -1,12 +1,14 @@
 package types
 
 import (
+	"encoding/binary"
 	"encoding/hex"
 	"errors"
 	"fmt"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"io"
+	"time"
 
 	gokzg4844 "github.com/crate-crypto/go-kzg-4844"
 	"github.com/ethereum/go-ethereum/common"
@@ -427,7 +429,8 @@ func decodeTyped(b []byte) (BlobTxWrapper, error) {
 	return blobTxWrapper, nil
 }
 
-// todo this Sidecar needs to be saved separately from block so that it can be pruned time to time to take advantage of 4844
+// todo 4844 this Sidecar needs to be saved separately from block so that it can be pruned time to time to take advantage of 4844
+// todo 4844 Sidecar needs to implement EncodeRLP so that it can be broadcasted easily
 type Sidecar struct {
 	BlockRoot       []byte          `json:"block_root"` //[]byte
 	Index           uint64          `json:"index"`
@@ -437,6 +440,49 @@ type Sidecar struct {
 	Blob            Blob            `json:"blob"`
 	KZGCommitment   KZGCommitment   `json:"kzg_commitment"`
 	KZGProof        KZGProof        `json:"kzg_proof"`
+
+	// These fields are used by package eth to track
+	// inter-peer block relay.
+	ReceivedAt   time.Time
+	ReceivedFrom interface{}
+}
+
+func (s *Sidecar) SidecarToHash() common.Hash {
+	//hash := common.Hash{}
+	//copy(hash[:], s.BlockRoot[:])
+	//binary.BigEndian.PutUint64(hash[len(s.BlockRoot):], s.Index)
+	//return hash
+
+	// Convert the index to a byte slice.
+	indexBytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(indexBytes, s.Index)
+
+	// Concatenate BlockRoot and indexBytes.
+	combinedBytes := append(s.BlockRoot[:], indexBytes...)
+
+	// Calculate the hash of the combined bytes.
+	hash := common.BytesToHash(combinedBytes)
+
+	return hash
+}
+
+// todo 4844 write test for it to know it actually works
+func (s *Sidecar) HashToSidecarIdentifier(hash common.Hash) SidecarIdentifier {
+	var blockRoot []byte = make([]byte, len(hash))
+	copy(blockRoot, hash[:])
+
+	indexBytes := hash[len(blockRoot):]
+	index := binary.BigEndian.Uint64(indexBytes)
+
+	return SidecarIdentifier{
+		Index:     index,
+		BlockRoot: blockRoot,
+	}
+}
+
+type SidecarIdentifier struct {
+	BlockRoot []byte `json:"block_root"`
+	Index     uint64 `json:"index"`
 }
 
 type SignedSidecar struct {
@@ -448,3 +494,44 @@ type BlockAndSidecars struct {
 	Block   *Block
 	Sidecar []*Sidecar
 }
+
+func (s *Sidecar) EncodeRLP(w io.Writer) error {
+	return rlp.Encode(w, []interface{}{
+		s.BlockRoot,
+		s.Index,
+		s.Slot,
+		s.BlockParentRoot,
+		s.ProposerIndex,
+		s.Blob,
+		s.KZGCommitment,
+		s.KZGProof,
+		s.ReceivedAt,
+		s.ReceivedFrom,
+	})
+}
+
+func (s *Sidecar) DecodeRLP(stream *rlp.Stream) error {
+	return stream.Decode(&struct {
+		BlockRoot       *[]byte
+		Index           *uint64
+		Slot            *primitives.Slot
+		BlockParentRoot *[]byte
+		ProposerIndex   *uint64
+		Blob            *Blob
+		KZGCommitment   *KZGCommitment
+		KZGProof        *KZGProof
+		ReceivedAt      *time.Time
+		ReceivedFrom    *interface{}
+	}{
+		&s.BlockRoot,
+		&s.Index,
+		&s.Slot,
+		&s.BlockParentRoot,
+		&s.ProposerIndex,
+		&s.Blob,
+		&s.KZGCommitment,
+		&s.KZGProof,
+		&s.ReceivedAt,
+		&s.ReceivedFrom,
+	})
+}

core/types/data_blob_test.go

@@ -0,0 +1,71 @@
+package types
+
+import (
+	"bytes"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+func TestSidecarEncodeDecodeRLP(t *testing.T) {
+
+	testCases := []*Sidecar{
+		{
+			BlockRoot:       []byte{0x01},
+			Index:           1,
+			Slot:            2,
+			BlockParentRoot: []byte{0x03},
+			ProposerIndex:   3,
+			Blob:            Blob{0x04},
+			KZGCommitment:   KZGCommitment{0x05},
+			KZGProof:        KZGProof{0x06},
+			ReceivedAt:      time.Now(),
+			ReceivedFrom:    "test1",
+		},
+		{
+			BlockRoot:       []byte{0x10, 0x11},
+			Index:           123,
+			Slot:            456,
+			BlockParentRoot: []byte{0x12, 0x13},
+			ProposerIndex:   789,
+			Blob:            Blob{0x14, 0x15},
+			KZGCommitment:   KZGCommitment{0x16, 0x17},
+			KZGProof:        KZGProof{0x18, 0x19},
+			ReceivedAt:      time.Now().Add(-1 * time.Hour),
+			ReceivedFrom:    "test2",
+		},
+	}
+
+	for _, tc := range testCases {
+		// Encode to RLP
+		var buf bytes.Buffer
+		if err := tc.EncodeRLP(&buf); err != nil {
+			t.Fatal(err)
+		}
+
+		// Decode back to new struct
+		sc := &Sidecar{}
+		if err := sc.DecodeRLP(rlp.NewStream(&buf, 0)); err != nil {
+			t.Fatal(err)
+		}
+
+		// Check all fields match
+		if !bytes.Equal(tc.BlockRoot, sc.BlockRoot) {
+			t.Errorf("BlockRoot mismatch: %v %v", tc.BlockRoot, sc.BlockRoot)
+		}
+		if tc.Index != sc.Index {
+			t.Errorf("Index mismatch: %v %v", tc.Index, sc.Index)
+		}
+		if !bytes.Equal(tc.BlockParentRoot, sc.BlockParentRoot) {
+			t.Errorf("BlockParentRoot mismatch: %v %v", tc.BlockParentRoot, sc.BlockParentRoot)
+		}
+		//...check other fields
+
+		//// Roundtrip test
+		//if !bytes.Equal(tc.Encode(), sc.Encode()) {
+		//	t.Errorf("Encoding roundtrip failed")
+		//}
+	}
+
+}